build(deps): bump https://github.com/astral-sh/ruff-pre-commit from v… #11331
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| on: | |
| push: # Run on pushes to the default branch | |
| branches: [main] | |
| # Also run on pull requests originating from forks. Every test job here is secret-dependent | |
| # (integration tests, kubernetes, code coverage) and depends on the Authorize job, which requires | |
| # manually approving the workflow run for external PRs. Jobs that need no secrets (type-check, | |
| # unit, telemetry and performance tests) run in the separate, unprivileged test-unprivileged.yml | |
| # workflow without that approval; only Check-changed-files is duplicated across both. | |
| pull_request_target: | |
| branches: [main, 'release-*'] # zizmor: ignore[dangerous-triggers] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| SCARF_NO_ANALYTICS: "true" | |
| jobs: | |
| Authorize: | |
| environment: ${{ github.event_name == 'pull_request_target' && | |
| github.event.pull_request.head.repo.full_name != github.repository && | |
| 'external' || 'internal' }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - run: true | |
| # Skip the integration and kubernetes tests when only non-code files changed. | |
| Check-changed-files: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| code_changed: ${{ steps.check.outputs.code_changed }} | |
| steps: | |
| # Only the `push` branch below uses the working tree (`git diff`). For pull_request / | |
| # pull_request_target the changed-file list comes from the GitHub API, so we must NOT | |
| # check out untrusted fork code in this ungated job — that is the pull_request_target | |
| # "pwn request" risk actions/checkout v7 refuses by default. | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| if: github.event_name == 'push' | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.sha }} | |
| persist-credentials: false | |
| - id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then | |
| # Use GitHub API to list changed files — works reliably for fork PRs | |
| # where git diff may fail because the fork's commits aren't in local history. | |
| PR_NUMBER="${{ github.event.pull_request.number }}" | |
| if ! FILES=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename' 2>/tmp/gh_api_err); then | |
| echo "::warning::Failed to list PR changed files via GitHub API, falling back to running all tests. Error: $(cat /tmp/gh_api_err)" | |
| echo "code_changed=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| else | |
| BASE="${{ github.event.before }}" | |
| HEAD="${{ github.sha }}" | |
| # Handle cases where BASE is missing or all zeros (e.g., first push, force push) | |
| if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then | |
| echo "code_changed=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Compute changed files; if git diff fails, assume code changed to be safe | |
| if ! FILES=$(git diff --name-only "$BASE" "$HEAD" 2>/dev/null); then | |
| echo "code_changed=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| fi | |
| CODE_CHANGED=false | |
| # If we can't determine changed files, run tests to be safe | |
| if [ -z "$FILES" ]; then | |
| CODE_CHANGED=true | |
| fi | |
| while IFS= read -r f; do | |
| [ -z "$f" ] && continue | |
| case "$f" in | |
| # Documentation and metadata | |
| README.rst|README.md) ;; | |
| CHANGELOG.rst) ;; | |
| CODE_OF_CONDUCT.md) ;; | |
| CONTRIBUTING.md) ;; | |
| LICENSE) ;; | |
| SECURITY.rst) ;; | |
| PRIVACY_NOTICE.rst) ;; | |
| CODEOWNERS) ;; | |
| CLAUDE.md) ;; | |
| AGENTS.md) ;; | |
| docs/*) ;; | |
| # GitHub and CI config unrelated to tests | |
| .github/pull_request_template.md) ;; | |
| .github/ISSUE_TEMPLATE/*) ;; | |
| .github/dependabot.yml) ;; | |
| .github/workflows/docs.yml) ;; | |
| .github/workflows/docs-build.yml) ;; | |
| .github/workflows/stale.yml) ;; | |
| .github/workflows/actionlint.yml) ;; | |
| .github/workflows/codeql.yml) ;; | |
| .github/workflows/zizmor.yml) ;; | |
| .github/workflows/deploy.yml) ;; | |
| # Tooling config that does not affect test outcomes | |
| .gitignore) ;; | |
| .tiltignore) ;; | |
| Tiltfile) ;; | |
| .codespell-ignore-words) ;; | |
| .pre-commit-config.yaml) ;; | |
| .airflow-registry.yaml) ;; | |
| *) CODE_CHANGED=true; break ;; | |
| esac | |
| done <<< "$FILES" | |
| echo "code_changed=$CODE_CHANGED" >> "$GITHUB_OUTPUT" | |
| Run-Integration-Tests: | |
| needs: [Authorize, Check-changed-files] | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| # Diagnostic: cap the job at 60 min so a hang does not run until GitHub's 6h default | |
| # timeout. This is a job-level backstop and does not itself produce a pytest traceback. | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| airflow-version: ["2.9", "2.10", "2.11", "3.0", "3.1", "3.2", "3.3"] | |
| dbt-version: [ "1.12" ] | |
| split-group: [1, 2, 3] | |
| exclude: | |
| # Apache Airflow versions prior to 3.1.0 have not been tested with Python 3.13. | |
| - python-version: "3.13" | |
| airflow-version: "2.9" | |
| - python-version: "3.13" | |
| airflow-version: "2.10" | |
| - python-version: "3.13" | |
| airflow-version: "2.11" | |
| - python-version: "3.13" | |
| airflow-version: "3.0" | |
| # Python 3.14 is only pinned in the Airflow 3.2/3.3 dbt-1.12 lockfiles. | |
| - python-version: "3.14" | |
| airflow-version: "2.9" | |
| - python-version: "3.14" | |
| airflow-version: "2.10" | |
| - python-version: "3.14" | |
| airflow-version: "2.11" | |
| - python-version: "3.14" | |
| airflow-version: "3.0" | |
| - python-version: "3.14" | |
| airflow-version: "3.1" | |
| services: | |
| postgres: | |
| image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}--${{ matrix.dbt-version }}${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install libxml2/libxslt headers for lxml sdist build | |
| # lxml==5.4.0 (pinned in the dbt-1.12 lockfiles) has no cp314 wheel and must build | |
| # from source on Python 3.14; ubuntu-latest doesn't ship the dev headers it needs. | |
| if: matrix.python-version == '3.14' | |
| run: sudo apt-get update && sudo apt-get install -y libxml2-dev libxslt1-dev | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt ${{ matrix.dbt-version }} (split ${{ matrix.split-group }}/3) | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-setup | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration | |
| env: | |
| PYTEST_SPLITS: 3 | |
| PYTEST_SPLIT_GROUP: ${{ matrix.split-group }} | |
| AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 | |
| AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS: 0 | |
| AIRFLOW__COSMOS__ENABLE_LAX_SELECTOR_PARSING: 0 | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 120 | |
| AIRFLOW__CORE__DAG_FILE_PROCESSOR_TIMEOUT: 120 | |
| AIRFLOW__DAG_PROCESSOR__DAGBAG_IMPORT_TIMEOUT: 120 | |
| AIRFLOW__DAG_PROCESSOR__DAG_FILE_PROCESSOR_TIMEOUT: 120 | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} | |
| AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} | |
| AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} | |
| DATABRICKS_HOST: mock | |
| DATABRICKS_WAREHOUSE_ID: mock | |
| DATABRICKS_TOKEN: mock | |
| DATABRICKS_CLUSTER_ID: mock | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn | |
| AIRFLOW_CONN_DUCKDB_DEFAULT: "duckdb:///?host=''" | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-integration-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-split${{ matrix.split-group }} | |
| path: .coverage | |
| include-hidden-files: true | |
| Run-Integration-Tests-Expensive: | |
| needs: [Authorize, Check-changed-files] | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| airflow-version: ["2.9"] | |
| dbt-version: ["1.9"] | |
| services: | |
| postgres: | |
| image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt ${{ matrix.dbt-version }} | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-setup | |
| DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-expensive | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} | |
| AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} | |
| AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} | |
| DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 | |
| COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-integration-expensive-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} | |
| DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
| Run-Integration-Tests-DBT-1-5-4: | |
| needs: [Authorize, Check-changed-files] | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ "3.11" ] | |
| airflow-version: [ "2.10", "3.0" ] | |
| dbt-version: [ "1.5" ] | |
| services: | |
| postgres: | |
| image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: integration-dbt-1-5-4-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt 1.5.4 | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbt-1-5-4 | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} | |
| AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} | |
| AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| AIRFLOW__COSMOS__ENABLE_CACHE: 0 | |
| COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} | |
| DATABRICKS_CLUSTER_ID: mock | |
| DATABRICKS_HOST: mock | |
| DATABRICKS_WAREHOUSE_ID: mock | |
| DATABRICKS_TOKEN: mock | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn | |
| DBT_PROJECT_NAME: "altered_jaffle_shop" # The syntax of jaffle_shop is not supported in dbt 1.5.4, only in dbt >= 1.10 | |
| TEST_SINGLE_DAG: "basic_cosmos_task_group.py" # There are circumstances when we only need to run a single DAG, but we were parsing all the DAGs. This change is to avoid parsing all the DAGs. | |
| Run-Integration-Tests-DBT-Async: | |
| needs: [Authorize, Check-changed-files] | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ "3.11" ] | |
| airflow-version: [ "2.11", "3.0" ] | |
| dbt-version: ["1.6", "1.7", "1.8", "1.9", "1.10", "1.11"] | |
| services: | |
| postgres: | |
| image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: integration-dbt-async-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt ${{ matrix.dbt-version }} | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbt-async | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-integration-dbt-async-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| DBT_PROJECT_NAME: "altered_jaffle_shop" # The syntax of jaffle_shop is not supported in dbt 1.5.4, only in dbt >= 1.10 | |
| AIRFLOW__COSMOS__ENABLE_DATASET_ALIAS: 0 # The altered_jaffle_shop project has non-ASCII characters which raise an exception in earlier versions of Airflow, as documented in https://github.com/astronomer/astronomer-cosmos/issues/1802 | |
| AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} | |
| AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 | |
| AIRFLOW__COSMOS__ENABLE_CACHE: 0 | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" | |
| AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn | |
| DBT_ADAPTER_VERSION: ${{ matrix.dbt-version }} | |
| # The following are only required because the profiles.yml file references them. They are not used by the tests selected by this job. | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 | |
| Run-Integration-dbt-fusion-Tests: | |
| needs: [Authorize, Check-changed-files] | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11"] | |
| airflow-version: ["2.10", "3.0"] | |
| dbt-version: ["2.0"] # dbt Fusion | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: integration-dbtf-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}--${{ matrix.dbt-version }}${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Set RESOURCE_PREFIX without periods | |
| id: set-resource-prefix | |
| run: | | |
| PYTHON_VER=$(echo "${{ matrix.python-version }}" | tr -d '.') | |
| AIRFLOW_VER=$(echo "${{ matrix.airflow-version }}" | tr -d '.') | |
| DBT_VER=$(echo "${{ matrix.dbt-version }}" | tr -d '.') | |
| PREFIX="COSMOS_${{ github.run_id }}_PY${PYTHON_VER}_AF${AIRFLOW_VER}_DBT${DBT_VER}" | |
| echo "prefix=${PREFIX}" >> "$GITHUB_OUTPUT" | |
| - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt ${{ matrix.dbt-version }} | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbtf-setup | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbtf | |
| env: | |
| AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0 | |
| AIRFLOW__COSMOS__ENABLE_CACHE_DBT_YAML_SELECTORS: 0 | |
| AIRFLOW__COSMOS__ENABLE_LAX_SELECTOR_PARSING: 0 | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} | |
| SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
| SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }} | |
| SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} | |
| SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} | |
| AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} | |
| RESOURCE_PREFIX: ${{ steps.set-resource-prefix.outputs.prefix }} | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-integration-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| - name: Clean up Snowflake resources | |
| if: always() | |
| run: | | |
| pip install snowflake-connector-python | |
| # Trigger a python script to delete the resources | |
| python scripts/ci_dbtf_delete_snowflake_resources.py | |
| env: | |
| SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} | |
| SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }} | |
| SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
| SNOWFLAKE_SCHEMA: ${{ secrets.SNOWFLAKE_SCHEMA }} | |
| SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }} | |
| SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }} | |
| RESOURCE_PREFIX: ${{ steps.set-resource-prefix.outputs.prefix }} | |
| Run-Kubernetes-Tests: | |
| needs: [Authorize, Check-changed-files] | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: [ "3.12" ] | |
| airflow-version: [ "2.10", "3.0" ] | |
| dbt-version: [ "1.11" ] | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: coverage-integration-kubernetes-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Create KinD cluster | |
| uses: container-tools/kind-action@0ad70e2299366b0e1552c7240f4e4567148f723e # v2.0.4 | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Run kubernetes tests | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-kubernetes-setup | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-kubernetes | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} | |
| DATABRICKS_CLUSTER_ID: mock | |
| DATABRICKS_HOST: mock | |
| DATABRICKS_WAREHOUSE_ID: mock | |
| DATABRICKS_TOKEN: mock | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-integration-kubernetes-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| Code-Coverage: | |
| if: github.event.action != 'labeled' | |
| needs: | |
| - Authorize | |
| - Run-Integration-Tests | |
| - Run-Integration-Tests-Expensive | |
| - Run-Integration-Tests-DBT-Async | |
| - Run-Integration-dbt-fusion-Tests | |
| - Run-Kubernetes-Tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| # Required to read artifacts from the sibling test-unprivileged.yml run. | |
| actions: read | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| # Safe under pull_request_target: this job runs only after the Authorize job's | |
| # owner-review environment gate approves the run (see the Authorize job above). | |
| allow-unsafe-pr-checkout: true | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.11" | |
| - name: Install coverage | |
| run: | | |
| pip3 install coverage | |
| - name: Download coverage artifacts from this run | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ./coverage | |
| pattern: coverage-* | |
| # The unit, telemetry and dbt-loom tests run in the unprivileged test-unprivileged.yml | |
| # workflow, so their coverage lands in a separate workflow run for the same commit. Locate | |
| # that run by head SHA and wait until those coverage artifacts are available. (The performance | |
| # tests run in that same workflow but upload no coverage, so they are irrelevant here — we | |
| # wait only on the coverage-producing artifacts, never on the whole run, so the longer-running | |
| # performance matrix does not hold this step up.) | |
| # | |
| # By the time this job runs, those artifacts are expected to already exist: this job only | |
| # starts after its `needs` (the integration and kubernetes jobs) complete, which take | |
| # ~5-18 min, whereas the unprivileged unit/telemetry/dbt-loom tests run in parallel from the | |
| # same commit and finish well within that window. So the first attempt below almost always | |
| # finds the artifacts and breaks immediately. The retry loop is a safety gate for the rare | |
| # race where this job reaches this step before the unprivileged run has uploaded them; it | |
| # waits up to 5 minutes (15 attempts x 20s) before failing rather than silently dropping | |
| # coverage. | |
| - name: Locate unprivileged test run for this commit | |
| id: unprivileged-run | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}" | |
| RUN_ID="" | |
| FOUND="" | |
| for attempt in $(seq 1 15); do | |
| RUN_ID=$(gh api \ | |
| "repos/${{ github.repository }}/actions/workflows/test-unprivileged.yml/runs?head_sha=${HEAD_SHA}&per_page=1" \ | |
| --jq '.workflow_runs[0].id // ""') | |
| if [ -n "$RUN_ID" ]; then | |
| NAMES=$(gh api \ | |
| "repos/${{ github.repository }}/actions/runs/${RUN_ID}/artifacts?per_page=100" \ | |
| --jq '.artifacts[].name') | |
| if echo "$NAMES" | grep -q '^coverage-unit-test-' \ | |
| && echo "$NAMES" | grep -q '^coverage-telemetry-test-' \ | |
| && echo "$NAMES" | grep -q '^coverage-integration-dbt-loom-test-'; then | |
| echo "Found unit, telemetry and dbt-loom coverage artifacts in run ${RUN_ID}." | |
| FOUND=1 | |
| break | |
| fi | |
| fi | |
| echo "Waiting for test-unprivileged.yml coverage artifacts (attempt ${attempt}/15)…" | |
| sleep 20 | |
| done | |
| if [ -z "$RUN_ID" ]; then | |
| echo "::error::No test-unprivileged.yml run found for ${HEAD_SHA}; its coverage would be missing." | |
| exit 1 | |
| fi | |
| if [ -z "$FOUND" ]; then | |
| echo "::error::test-unprivileged.yml run ${RUN_ID} did not publish all expected coverage artifacts (unit, telemetry, dbt-loom) after waiting; refusing to upload partial coverage." | |
| exit 1 | |
| fi | |
| echo "run_id=${RUN_ID}" >> "$GITHUB_OUTPUT" | |
| - name: Download unit, telemetry and dbt-loom coverage from the unprivileged run | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: ./coverage | |
| pattern: coverage-* | |
| run-id: ${{ steps.unprivileged-run.outputs.run_id }} | |
| github-token: ${{ github.token }} | |
| - name: Combine coverage | |
| run: | | |
| coverage combine ./coverage/coverage*/.coverage | |
| coverage report | |
| coverage xml | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6.0.2 | |
| with: | |
| fail_ci_if_error: true | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: coverage.xml |