Use boto3 for Kubernetes dbt docs S3 uploads #685
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test-unprivileged | |
| on: | |
| push: # Run on pushes to the default branch so coverage on main stays current. | |
| branches: [main] | |
| # Unprivileged test jobs that execute untrusted PR code but require no secrets. Running them | |
| # under `pull_request` (not `pull_request_target`) means fork PRs get a read-only token and no | |
| # access to repository secrets, so checking out and running the PR's code is not a privileged | |
| # operation. This preserves fast feedback for forks without a manual approval gate, while | |
| # avoiding the privileged-checkout and cache-poisoning patterns that `pull_request_target` | |
| # would introduce. Jobs that need secrets (integration, kubernetes, coverage upload) stay in | |
| # test.yml, gated behind the Authorize environment. | |
| pull_request: | |
| branches: [main, 'release-*'] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| SCARF_NO_ANALYTICS: "true" | |
| jobs: | |
| # Skip unit/telemetry/performance tests when only non-code files changed. | |
| Check-changed-files: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| code_changed: ${{ steps.check.outputs.code_changed }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.pull_request.head.sha || github.sha }} | |
| persist-credentials: false | |
| - id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then | |
| # Use GitHub API to list changed files — works reliably for fork PRs | |
| # where git diff may fail because the fork's commits aren't in local history. | |
| PR_NUMBER="${{ github.event.pull_request.number }}" | |
| if ! FILES=$(gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename' 2>/tmp/gh_api_err); then | |
| echo "::warning::Failed to list PR changed files via GitHub API, falling back to running all tests. Error: $(cat /tmp/gh_api_err)" | |
| echo "code_changed=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| else | |
| BASE="${{ github.event.before }}" | |
| HEAD="${{ github.sha }}" | |
| # Handle cases where BASE is missing or all zeros (e.g., first push, force push) | |
| if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then | |
| echo "code_changed=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| # Compute changed files; if git diff fails, assume code changed to be safe | |
| if ! FILES=$(git diff --name-only "$BASE" "$HEAD" 2>/dev/null); then | |
| echo "code_changed=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| fi | |
| CODE_CHANGED=false | |
| # If we can't determine changed files, run tests to be safe | |
| if [ -z "$FILES" ]; then | |
| CODE_CHANGED=true | |
| fi | |
| while IFS= read -r f; do | |
| [ -z "$f" ] && continue | |
| case "$f" in | |
| # Documentation and metadata | |
| README.rst|README.md) ;; | |
| CHANGELOG.rst) ;; | |
| CODE_OF_CONDUCT.md) ;; | |
| CONTRIBUTING.md) ;; | |
| LICENSE) ;; | |
| SECURITY.rst) ;; | |
| PRIVACY_NOTICE.rst) ;; | |
| CODEOWNERS) ;; | |
| CLAUDE.md) ;; | |
| AGENTS.md) ;; | |
| docs/*) ;; | |
| # GitHub and CI config unrelated to tests | |
| .github/pull_request_template.md) ;; | |
| .github/ISSUE_TEMPLATE/*) ;; | |
| .github/dependabot.yml) ;; | |
| .github/workflows/docs.yml) ;; | |
| .github/workflows/docs-build.yml) ;; | |
| .github/workflows/stale.yml) ;; | |
| .github/workflows/actionlint.yml) ;; | |
| .github/workflows/codeql.yml) ;; | |
| .github/workflows/zizmor.yml) ;; | |
| .github/workflows/deploy.yml) ;; | |
| # Tooling config that does not affect test outcomes | |
| .gitignore) ;; | |
| .tiltignore) ;; | |
| Tiltfile) ;; | |
| .codespell-ignore-words) ;; | |
| .pre-commit-config.yaml) ;; | |
| .airflow-registry.yaml) ;; | |
| *) CODE_CHANGED=true; break ;; | |
| esac | |
| done <<< "$FILES" | |
| echo "code_changed=$CODE_CHANGED" >> "$GITHUB_OUTPUT" | |
| Type-Check: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: "3.10" | |
| architecture: "x64" | |
| - run: pip3 install "hatch>=1.14.2" | |
| - run: hatch run tests.py3.10-3.1-1.9:type-check | |
| Run-Unit-Tests: | |
| needs: Check-changed-files | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] | |
| airflow-version: ["2.9", "2.10", "2.11", "3.0", "3.1", "3.2", "3.3"] | |
| dbt-version: ["1.11", "1.12"] | |
| exclude: | |
| # Apache Airflow versions prior to 3.1.0 have not been tested with Python 3.13. | |
| - python-version: "3.13" | |
| airflow-version: "2.9" | |
| - python-version: "3.13" | |
| airflow-version: "2.10" | |
| - python-version: "3.13" | |
| airflow-version: "2.11" | |
| - python-version: "3.13" | |
| airflow-version: "3.0" | |
| # Python 3.14 is only pinned in the Airflow 3.2/3.3 dbt-1.12 lockfiles: dbt-core | |
| # 1.11 has no Python 3.14 classifier, and older Airflow versions aren't pinned either. | |
| - python-version: "3.14" | |
| airflow-version: "2.9" | |
| - python-version: "3.14" | |
| airflow-version: "2.10" | |
| - python-version: "3.14" | |
| airflow-version: "2.11" | |
| - python-version: "3.14" | |
| airflow-version: "3.0" | |
| - python-version: "3.14" | |
| airflow-version: "3.1" | |
| - python-version: "3.14" | |
| dbt-version: "1.11" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: unit-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install libxml2/libxslt headers for lxml sdist build | |
| # lxml==5.4.0 (pinned in the dbt-1.12 lockfiles) has no cp314 wheel and must build | |
| # from source on Python 3.14; ubuntu-latest doesn't ship the dev headers it needs. | |
| if: matrix.python-version == '3.14' | |
| run: sudo apt-get update && sudo apt-get install -y libxml2-dev libxslt1-dev | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt ${{ matrix.dbt-version }} | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-cov | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-unit-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| Run-Telemetry-Tests: | |
| needs: Check-changed-files | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11"] | |
| airflow-version: ["2.10"] | |
| dbt-version: ["1.9"] | |
| env: | |
| SCARF_NO_ANALYTICS: "false" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: telemetry-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Run telemetry tests | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-telemetry | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-telemetry-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| Run-Performance-Tests: | |
| needs: Check-changed-files | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11"] | |
| airflow-version: ["2.10", "3.0"] | |
| dbt-version: ["1.9"] | |
| num-models: [1, 10, 50, 100, 500] | |
| services: | |
| postgres: | |
| image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: perf-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Run performance tests against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt ${{ matrix.dbt-version }} | |
| id: run-performance-tests | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-performance-setup | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-performance | |
| # read the performance results and set them as an env var for the next step | |
| # format: NUM_MODELS={num_models}\nTIME={end - start}\n | |
| cat /tmp/performance_results.txt > "$GITHUB_STEP_SUMMARY" | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 180.0 | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 | |
| MODEL_COUNT: ${{ matrix.num-models }} | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| Run-Integration-Tests-dbt-Loom: | |
| # Dedicated job for the cross-project example DAGs (cross_project_manifest_dag.py, | |
| # cross_project_dbt_ls_dag.py, and cross_project_bidirectional_dag.py), which | |
| # exercise dbt-loom. Carved out of Run-Integration-Tests so the loom job's dbt + | |
| # loom versions can move independently of the main matrix. It uses no secrets, so | |
| # it runs here in the unprivileged workflow rather than under pull_request_target. | |
| needs: Check-changed-files | |
| if: needs.Check-changed-files.outputs.code_changed == 'true' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.11"] | |
| airflow-version: ["3.2"] | |
| dbt-version: ["1.11"] | |
| services: | |
| postgres: | |
| image: postgres@sha256:4cd697181d4bd3ddc41a09012f339fa8cb5a8cd3d8b30130ea8378c176b6c494 # 14.18 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| ports: | |
| - 5432:5432 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| persist-credentials: false | |
| - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/uv | |
| key: integration-dbt-loom-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}-${{ hashFiles('pyproject.toml') }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install packages and dependencies | |
| run: | | |
| python -m pip install uv | |
| uv pip install --system "hatch>=1.14.2" | |
| hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} run pip freeze | |
| - name: Test cross-project dbt-loom example DAGs (manifest, dbt ls, bidirectional) | |
| run: | | |
| hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }}:test-integration-dbt-loom | |
| - name: Upload coverage to GitHub | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: coverage-integration-dbt-loom-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ matrix.dbt-version }} | |
| path: .coverage | |
| include-hidden-files: true | |
| env: | |
| AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ | |
| PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH | |
| AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 | |
| AIRFLOW__COSMOS__ENABLE_CACHE: 0 | |
| DBT_ADAPTER_VERSION: ${{ matrix.dbt-version }} | |
| # cross_project_manifest_dag's profiles.yml references these | |
| AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres | |
| POSTGRES_HOST: localhost | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_SCHEMA: public | |
| POSTGRES_PORT: 5432 |