feat(CI): Keep local and Github CI test in sync with cargo xtask
#701
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # ------------ | |
| # Quick start | |
| # ------------ | |
| # | |
| # Reproduce this workflow locally: | |
| # cargo xtask ci workflow rust | |
| # | |
| # Run one CI step locally: | |
| # cargo xtask ci step clippy | |
| # | |
| # ------------- | |
| # Architecture | |
| # ------------- | |
| # | |
| # Each atomic CI check lives in an individual `xtask` step. The GitHub workflow | |
| # is only a thin wrapper that calls a list of `xtask` steps. A `xtask` step looks like: | |
| # | |
| # cargo xtask ci step <step> | |
| # | |
| # `xtask` also provides local orchestration for this `rust.yml` workflow. It similarly | |
| # calls the same list of `xtask` steps. For example the below command is a local | |
| # reproducer for this Github Workflow: | |
| # | |
| # cargo xtask ci workflow rust | |
| # | |
| # Example: | |
| # | |
| # (rust.yml) | |
| # ... | |
| # clippy: | |
| # ... | |
| # steps: | |
| # - name: Run clippy | |
| # run: cargo xtask ci step clippy # <--- Atomic CI check item | |
| # ... | |
| # | |
| # | |
| # ---------------------- | |
| # Adding a new check | |
| # ---------------------- | |
| # | |
| # First wrap the atomic check in an xtask step: | |
| # cargo xtask ci step new-check | |
| # | |
| # Then add that step to the Rust workflow group in `xtask/src/job_groups.rs` so | |
| # local reproduction includes it: | |
| # cargo xtask ci workflow rust # it should include 'new-check' | |
| # | |
| # ---------------------- | |
| # Job dependency graph | |
| # ---------------------- | |
| # | |
| # (@ means no `needs` dependency.) | |
| # | |
| # @ -> linux-build-lib | |
| # -> linux-datafusion-common-features | |
| # -> linux-datafusion-substrait-features | |
| # -> linux-datafusion-proto-features | |
| # -> linux-cargo-check-datafusion | |
| # -> linux-cargo-check-datafusion-functions | |
| # -> linux-test | |
| # -> linux-test-datafusion-cli | |
| # -> linux-test-example | |
| # -> linux-test-doc | |
| # -> linux-rustdoc | |
| # -> verify-benchmark-results | |
| # -> sqllogictest-postgres | |
| # -> sqllogictest-substrait | |
| # -> clippy | |
| # -> cargo-toml-formatting-checks | |
| # -> config-docs-check | |
| # -> examples-docs-check | |
| # | |
| # @ -> linux-wasm-pack | |
| # @ -> macos-aarch64 | |
| # @ -> vendor | |
| # @ -> check-fmt | |
| # @ -> msrv | |
| # | |
| # ------------ | |
| # Runner info | |
| # ------------ | |
| # | |
| # Some jobs use Runs-On to run on ASF infrastructure: | |
| # https://datafusion.apache.org/contributor-guide/#ci-runners | |
| name: Rust | |
| concurrency: | |
| group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| on: | |
| push: | |
| branches-ignore: | |
| - 'gh-readonly-queue/**' | |
| - 'dependabot/**' | |
| paths-ignore: | |
| - "docs/**" | |
| - "**.md" | |
| - ".github/ISSUE_TEMPLATE/**" | |
| - ".github/pull_request_template.md" | |
| pull_request: | |
| # manual trigger | |
| # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow | |
| workflow_dispatch: | |
| merge_group: | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-build-lib: | |
| name: linux build test | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| shared-key: "amd-ci-check" # this job uses it's own cache becase check has a separate cache and we need it to be fast as it blocks other jobs | |
| save-if: ${{ github.ref_name == 'main' }} | |
| - name: Prepare cargo build | |
| run: cargo xtask ci step rust-check workspace | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-datafusion-common-features: | |
| name: cargo check datafusion-common features | |
| needs: linux-build-lib | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Check datafusion-common features | |
| run: cargo xtask ci step rust-check datafusion-common | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-datafusion-substrait-features: | |
| name: cargo check datafusion-substrait features | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| save-if: false # set in linux-test | |
| shared-key: "amd-ci" | |
| - name: Install cmake | |
| run: | | |
| # note the builder setup runs apt-get update / installs protobuf compiler | |
| apt-get install -y cmake | |
| - name: Check datafusion-substrait features | |
| run: cargo xtask ci step rust-check datafusion-substrait | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-datafusion-proto-features: | |
| name: cargo check datafusion-proto features | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Check datafusion-proto features | |
| run: cargo xtask ci step rust-check datafusion-proto | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-cargo-check-datafusion: | |
| name: cargo check datafusion features | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| save-if: false # set in linux-test | |
| shared-key: "amd-ci" | |
| - name: Check datafusion features | |
| run: cargo xtask ci step rust-check datafusion | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-cargo-check-datafusion-functions: | |
| name: cargo check datafusion-functions features | |
| needs: linux-build-lib | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Check datafusion-functions features | |
| run: cargo xtask ci step rust-check datafusion-functions | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-test: | |
| name: cargo test (amd64) | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| volumes: | |
| - /usr/local:/host/usr/local | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| save-if: ${{ github.ref_name == 'main' }} | |
| shared-key: "amd-ci" | |
| - name: Run tests (excluding doctests and datafusion-cli) | |
| run: cargo xtask ci step rust-test | |
| - name: Verify Working Directory Clean | |
| run: cargo xtask ci step verify-clean | |
| - name: Verify Working Directory Clean (No Untracked Files) | |
| run: cargo xtask ci step verify-clean --untracked | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-test-datafusion-cli: | |
| name: cargo test datafusion-cli (amd64) | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| run: rustup toolchain install stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| save-if: false # set in linux-test | |
| shared-key: "amd-ci" | |
| - name: Run tests (excluding doctests) | |
| run: cargo xtask ci step rust-test datafusion-cli | |
| - name: Verify Working Directory Clean | |
| run: cargo xtask ci step verify-clean | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-test-example: | |
| name: cargo examples (amd64) | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| save-if: ${{ github.ref_name == 'main' }} | |
| shared-key: "amd-ci-linux-test-example" | |
| - name: Run examples | |
| run: cargo xtask ci step rust-examples | |
| - name: Verify Working Directory Clean | |
| run: cargo xtask ci step verify-clean | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-test-doc: | |
| name: cargo test doc (amd64) | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Run doctests | |
| run: cargo xtask ci step rust-doctest | |
| - name: Verify Working Directory Clean | |
| run: cargo xtask ci step verify-clean | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-rustdoc: | |
| name: cargo doc | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Run cargo doc | |
| run: cargo xtask ci step rust-doc | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| linux-wasm-pack: | |
| name: build and run with wasm-pack | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup for wasm32 | |
| run: | | |
| rustup target add wasm32-unknown-unknown | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y -qq clang | |
| - name: Setup wasm-pack | |
| uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 | |
| with: | |
| tool: wasm-pack | |
| - name: Run tests with headless mode | |
| run: cargo xtask ci step rust-wasm | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| verify-benchmark-results: | |
| name: verify benchmark results (amd64) | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Verify that benchmark queries return expected results | |
| run: cargo xtask ci step rust-benchmark-results | |
| - name: Verify Working Directory Clean | |
| run: cargo xtask ci step verify-clean | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| sqllogictest-postgres: | |
| name: "Run sqllogictest with Postgres runner" | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| services: | |
| postgres: | |
| image: postgres:15 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: db_test | |
| POSTGRES_INITDB_ARGS: --encoding=UTF-8 --lc-collate=C --lc-ctype=C | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Run sqllogictest | |
| run: cargo xtask ci step rust-sqllogictest postgres | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in `xtask/src/steps.rs`. | |
| sqllogictest-substrait: | |
| name: "Run sqllogictest in Substrait round-trip mode" | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Run sqllogictest | |
| run: cargo xtask ci step rust-sqllogictest substrait | |
| # Temporarily commenting out the Windows flow, the reason is enormously slow running build | |
| # Waiting for new Windows 2025 github runner | |
| # Details: https://github.com/apache/datafusion/issues/13726 | |
| # | |
| # windows: | |
| # name: cargo test (win64) | |
| # runs-on: windows-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # submodules: true | |
| # - name: Setup Rust toolchain | |
| # uses: ./.github/actions/setup-windows-builder | |
| # - name: Run tests (excluding doctests) | |
| # shell: bash | |
| # run: | | |
| # export PATH=$PATH:$HOME/d/protoc/bin | |
| # cargo test --lib --tests --bins --features avro,json,backtrace | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| macos-aarch64: | |
| name: cargo test (macos-aarch64) | |
| runs-on: macos-15 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-macos-aarch64-builder | |
| - name: Run datafusion-ffi tests | |
| shell: bash | |
| run: cargo xtask ci step rust-test datafusion-ffi | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| vendor: | |
| name: Verify Vendored Code | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Run gen | |
| run: cargo xtask ci step rust-vendor | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| check-fmt: | |
| name: Check cargo fmt | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Run | |
| run: cargo xtask ci step fmt | |
| # Coverage job disabled due to | |
| # https://github.com/apache/datafusion/issues/3678 | |
| # coverage: | |
| # name: coverage | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # submodules: true | |
| # - name: Install protobuf compiler | |
| # shell: bash | |
| # run: | | |
| # mkdir -p $HOME/d/protoc | |
| # cd $HOME/d/protoc | |
| # export PROTO_ZIP="protoc-21.4-linux-x86_64.zip" | |
| # curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP | |
| # unzip $PROTO_ZIP | |
| # export PATH=$PATH:$HOME/d/protoc/bin | |
| # protoc --version | |
| # - name: Setup Rust toolchain | |
| # run: | | |
| # rustup toolchain install stable | |
| # rustup default stable | |
| # rustup component add rustfmt clippy | |
| # - name: Cache Cargo | |
| # uses: actions/cache@v4 | |
| # with: | |
| # path: /home/runner/.cargo | |
| # # this key is not equal because the user is different than on a container (runner vs github) | |
| # key: cargo-coverage-cache3- | |
| # - name: Run coverage | |
| # run: | | |
| # export PATH=$PATH:$HOME/d/protoc/bin | |
| # rustup toolchain install stable | |
| # rustup default stable | |
| # cargo install --version 0.20.1 cargo-tarpaulin | |
| # cargo tarpaulin --all --out Xml | |
| # - name: Report coverage | |
| # continue-on-error: true | |
| # run: bash <(curl -s https://codecov.io/bash) | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| clippy: | |
| name: clippy | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Rust Dependency Cache | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| save-if: ${{ github.ref_name == 'main' }} | |
| shared-key: "amd-ci-clippy" | |
| - name: Run clippy | |
| run: cargo xtask ci step clippy | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| cargo-toml-formatting-checks: | |
| name: check Cargo.toml formatting | |
| needs: linux-build-lib | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - name: Check Cargo.toml formatting | |
| run: cargo xtask ci step rust-toml-format | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| config-docs-check: | |
| name: check configs.md and ***_functions.md is up-to-date | |
| needs: linux-build-lib | |
| runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }} | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| with: | |
| rust-version: stable | |
| - uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version: "20" | |
| - name: Check if generated docs have been modified | |
| run: cargo xtask ci step rust-config-docs | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| examples-docs-check: | |
| name: check example README is up-to-date | |
| needs: linux-build-lib | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Mark repository as safe for git | |
| # Required for git commands inside container (avoids "dubious ownership" error) | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Set up Node.js (required for prettier) | |
| # doc_prettier_check.sh uses npx to run prettier for Markdown formatting | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: '18' | |
| - name: Run examples docs check script | |
| run: cargo xtask ci step rust-examples-docs | |
| # Keep step-specific comments in the corresponding `StepInfo` entry in | |
| # `xtask/src/steps.rs`. | |
| msrv: | |
| name: Verify MSRV (Min Supported Rust Version) | |
| runs-on: ubuntu-latest | |
| container: | |
| image: amd64/rust | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Setup Rust toolchain | |
| uses: ./.github/actions/setup-builder | |
| - name: Install cargo-msrv | |
| uses: taiki-e/install-action@9bcaee1dcae34154180f412e2fa69355a7cda9f6 # v2.82.6 | |
| with: | |
| tool: cargo-msrv | |
| - name: Check datafusion | |
| run: cargo xtask ci step rust-msrv datafusion | |
| - name: Check datafusion-substrait | |
| run: cargo xtask ci step rust-msrv datafusion-substrait | |
| - name: Check datafusion-proto | |
| run: cargo xtask ci step rust-msrv datafusion-proto | |
| # When adding a new step, see the "Adding a new check" comment section at the top of | |
| # this file. |