dbt-labs · Jan 29, 2025 · Mar 4, 2025 · Mar 7, 2025 · Mar 7, 2025
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.2
+current_version = 1.9.3
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

diff --git a/.changes/1.9.3.md b/.changes/1.9.3.md
@@ -0,0 +1,5 @@
+## dbt-core 1.9.3 - March 07, 2025
+
+### Fixes
+
+- Fix microbatch execution to not block main thread nor hang ([#11243](https://github.com/dbt-labs/dbt-core/issues/11243), [#11306](https://github.com/dbt-labs/dbt-core/issues/11306))
diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml
@@ -61,7 +61,7 @@ body:
       label: Environment
       description: |
         examples:
-          - **OS**: Ubuntu 20.04
+          - **OS**: Ubuntu 24.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt-core**: 1.1.1 (`dbt --version`)
       value: |

diff --git a/.github/ISSUE_TEMPLATE/regression-report.yml b/.github/ISSUE_TEMPLATE/regression-report.yml
@@ -55,7 +55,7 @@ body:
       label: Environment
       description: |
         examples:
-          - **OS**: Ubuntu 20.04
+          - **OS**: Ubuntu 24.04
           - **Python**: 3.9.12 (`python3 --version`)
           - **dbt-core (working version)**: 1.1.1 (`dbt --version`)
           - **dbt-core (regression version)**: 1.2.0 (`dbt --version`)

diff --git a/.github/actions/setup-postgres-linux/action.yml b/.github/actions/setup-postgres-linux/action.yml
diff --git a/.github/actions/setup-postgres-linux/setup_db.sh b/.github/actions/setup-postgres-linux/setup_db.sh
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -152,7 +152,7 @@ jobs:
           echo "include=${INCLUDE_GROUPS}"
           echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
 
-  integration:
+  integration-postgres:
     name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
 
     runs-on: ${{ matrix.os }}
@@ -163,9 +163,8 @@ jobs:
       fail-fast: false
       matrix:
         python-version: [ "3.9", "3.10", "3.11", "3.12" ]
-        os: [ubuntu-20.04]
+        os: [ubuntu-latest]
         split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
-        include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
     env:
       TOXENV: integration
       DBT_INVOCATION_ENV: github-actions
@@ -178,6 +177,24 @@ jobs:
       DD_ENV: ci
       DD_SERVICE: ${{ github.event.repository.name }}
 
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: Check out the repository
         uses: actions/checkout@v4
@@ -187,9 +204,83 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Set up postgres (linux)
-        if: runner.os == 'Linux'
-        uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
+
+      - name: Install python tools
+        run: |
+          python -m pip install --user --upgrade pip
+          python -m pip --version
+          python -m pip install tox
+          tox --version
+
+      - name: Run integration tests
+        uses: nick-fields/retry@v3
+        with:
+          timeout_minutes: 30
+          max_attempts: 3
+          command: tox -- --ddtrace
+        env:
+          PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
+
+      - name: Get current date
+        if: always()
+        id: date
+        run: |
+          CURRENT_DATE=$(date +'%Y-%m-%dT%H_%M_%S') # no colons allowed for artifacts
+          echo "date=$CURRENT_DATE" >> $GITHUB_OUTPUT
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: logs_${{ matrix.python-version }}_${{ matrix.os }}_${{ matrix.split-group }}_${{ steps.date.outputs.date }}
+          path: ./logs
+
+      - name: Upload Integration Test Coverage to Codecov
+        if: ${{ matrix.python-version == '3.11' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: integration
+
+  integration-mac-windows:
+    name: (${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}
+
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    needs:
+      - integration-metadata
+    strategy:
+      fail-fast: false
+      matrix:
+        split-group: ${{ fromJson(needs.integration-metadata.outputs.split-groups) }}
+        # this include is where we add the mac and windows os
+        include: ${{ fromJson(needs.integration-metadata.outputs.include) }}
+    env:
+      TOXENV: integration
+      DBT_INVOCATION_ENV: github-actions
+      DBT_TEST_USER_1: dbt_test_user_1
+      DBT_TEST_USER_2: dbt_test_user_2
+      DBT_TEST_USER_3: dbt_test_user_3
+      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
+      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+      DD_SITE: datadoghq.com
+      DD_ENV: ci
+      DD_SERVICE: ${{ github.event.repository.name }}
+
+    steps:
+      - name: Check out the repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
 
       - name: Set up postgres (macos)
         if: runner.os == 'macOS'
@@ -239,10 +330,10 @@ jobs:
     if: ${{ always() }}
     name: Integration Test Suite
     runs-on: ubuntu-latest
-    needs: integration
+    needs: [integration-mac-windows, integration-postgres]
     steps:
       - name: "Integration Tests Failed"
-        if: ${{ contains(needs.integration.result, 'failure') || contains(needs.integration.result, 'cancelled') }}
+        if: ${{ contains(needs.integration-mac-windows.result, 'failure') || contains(needs.integration-mac-windows.result, 'cancelled') || contains(needs.integration-postgres.result, 'failure') || contains(needs.integration-postgres.result, 'cancelled') }}
         # when this is true the next step won't execute
         run: |
           echo "::notice title='Integration test suite failed'"

diff --git a/.github/workflows/structured-logging-schema-check.yml b/.github/workflows/structured-logging-schema-check.yml
@@ -45,7 +45,7 @@ jobs:
   # run the performance measurements on the current or default branch
   test-schema:
     name: Test Log Schema
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     timeout-minutes: 30
     needs:
       - integration-metadata
@@ -67,6 +67,24 @@ jobs:
       DBT_TEST_USER_2: dbt_test_user_2
       DBT_TEST_USER_3: dbt_test_user_3
 
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
     steps:
       - name: checkout dev
         uses: actions/checkout@v4
@@ -85,8 +103,13 @@ jobs:
           pip install tox
           tox --version
 
-      - name: Set up postgres
-        uses: ./.github/actions/setup-postgres-linux
+      - name: Run postgres setup script
+        run: |
+          ./test/setup_db.sh
+        env:
+          PGHOST: localhost
+          PGPORT: 5432
+          PGPASSWORD: password
 
       - name: ls
         run: ls

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,14 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-core/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-core 1.9.3 - March 07, 2025
+
+### Fixes
+
+- Fix microbatch execution to not block main thread nor hang ([#11243](https://github.com/dbt-labs/dbt-core/issues/11243), [#11306](https://github.com/dbt-labs/dbt-core/issues/11306))
+
+
+
 ## dbt-core 1.9.2 - January 29, 2025
 
 ### Fixes
@@ -18,7 +26,6 @@
 ### Contributors
 - [@internetcoffeephone](https://github.com/internetcoffeephone) ([#10556](https://github.com/dbt-labs/dbt-core/issues/10556), [#11098](https://github.com/dbt-labs/dbt-core/issues/11098))
 
-
 ## dbt-core 1.9.1 - December 16, 2024
 
 ### Fixes

diff --git a/core/dbt/graph/thread_pool.py b/core/dbt/graph/thread_pool.py
@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from multiprocessing.pool import ThreadPool
+
+
+class DbtThreadPool(ThreadPool):
+    """A ThreadPool that tracks whether or not it's been closed"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.closed = False
+
+    def close(self):
+        self.closed = True
+        super().close()
+
+    def is_closed(self):
+        return self.closed
diff --git a/core/dbt/materializations/incremental/microbatch.py b/core/dbt/materializations/incremental/microbatch.py
@@ -100,7 +100,8 @@ def build_batches(self, start: datetime, end: datetime) -> List[BatchType]:
 
         return batches
 
-    def build_jinja_context_for_batch(self, incremental_batch: bool) -> Dict[str, Any]:
+    @staticmethod
+    def build_jinja_context_for_batch(model: ModelNode, incremental_batch: bool) -> Dict[str, Any]:
         """
         Create context with entries that reflect microbatch model + incremental execution state
 
@@ -109,9 +110,9 @@ def build_jinja_context_for_batch(self, incremental_batch: bool) -> Dict[str, An
         jinja_context: Dict[str, Any] = {}
 
         # Microbatch model properties
-        jinja_context["model"] = self.model.to_dict()
-        jinja_context["sql"] = self.model.compiled_code
-        jinja_context["compiled_code"] = self.model.compiled_code
+        jinja_context["model"] = model.to_dict()
+        jinja_context["sql"] = model.compiled_code
+        jinja_context["compiled_code"] = model.compiled_code
 
         # Add incremental context variables for batches running incrementally
         if incremental_batch:

diff --git a/core/dbt/task/build.py b/core/dbt/task/build.py
@@ -169,7 +169,8 @@ def call_model_and_unit_tests_runner(self, node, pool) -> RunResult:
             runner.do_skip(cause=cause)
 
         if isinstance(runner, MicrobatchModelRunner):
-            return self.handle_microbatch_model(runner, pool)
+            runner.set_parent_task(self)
+            runner.set_pool(pool)
 
         return self.call_runner(runner)
 
@@ -184,10 +185,11 @@ def handle_job_queue_node(self, node, pool, callback):
             runner.do_skip(cause=cause)
 
         if isinstance(runner, MicrobatchModelRunner):
-            callback(self.handle_microbatch_model(runner, pool))
-        else:
-            args = [runner]
-            self._submit(pool, args, callback)
+            runner.set_parent_task(self)
+            runner.set_pool(pool)
+
+        args = [runner]
+        self._submit(pool, args, callback)
 
     # Make a map of model unique_ids to selected unit test unique_ids,
     # for processing before the model.