ci.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: CI

on:
  push:
    branches:
      - main
      - release/**
    paths-ignore:
      - '**.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/PULL_REQUEST_TEMPLATE.md'
      - '.asf.yaml'
      - '.licenserc.yaml'
      - '.commitlintrc.yaml'
      - 'LICENSE'
      - 'NOTICE'
  pull_request:
    branches:
      - main
    paths-ignore:
      - '**.md'
      - '.github/ISSUE_TEMPLATE/**'
      - '.github/PULL_REQUEST_TEMPLATE.md'
      - '.asf.yaml'
      - '.licenserc.yaml'
      - '.commitlintrc.yaml'
      - 'LICENSE'
      - 'NOTICE'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  rust-tests:
    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu-24.04 ]
    runs-on: ${{ matrix.os }}
    container:
      image: xd009642/tarpaulin:0.32.7
      options: --security-opt seccomp=unconfined
    steps:
      - uses: actions/checkout@v6
      - name: Install system dependencies
        run: |
          apt-get update
          apt-get install -y protobuf-compiler
      - name: Cache dependencies
        uses: Swatinem/rust-cache@v2
        with:
          # Only cache cargo registry and git, not target directory
          # Tarpaulin needs clean builds for accurate coverage
          cache-targets: false
          # Reduce cache size by cleaning old artifacts
          cache-all-crates: false
      - name: Rust unit tests with coverage report
        run: make coverage-xml
      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        with:
          name: cov-report-rust-tests-${{ runner.os }}
          path: ./cov-reports
          if-no-files-found: 'error'

  rust-tests-windows:
    runs-on: windows-2025
    # disable until https://github.com/apache/hudi-rs/issues/445 is resolved
    if: false    
    steps:
      - uses: actions/checkout@v6
      - name: Run unit tests on Windows with no coverage report
        run: cargo test --no-fail-fast --all-targets --all-features --workspace

  python-tests:
    strategy:
      fail-fast: false
      matrix:
        # TODO: add windows
        os: [ ubuntu-24.04, ubuntu-22.04-arm, macos-15 ]
        python-version: [ '3.10', '3.13' ]
        exclude:
          - os: ubuntu-24.04
            python-version: '3.10'
          - os: ubuntu-22.04-arm
            python-version: '3.13'
          - os: macos-15
            python-version: '3.10'
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v6

      - name: Install protobuf compiler (Linux)
        if: runner.os == 'Linux'
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler

      - name: Install protobuf compiler (macOS)
        if: runner.os == 'macOS'
        run: brew install protobuf

      # Cache Rust dependencies and build artifacts
      - name: Cache Rust dependencies
        uses: Swatinem/rust-cache@v2
        with:
          # Cache based on Python version and OS for better hit rate
          shared-key: "python-binding-${{ matrix.os }}-py${{ matrix.python-version }}"
          # Save cache even on failure to speed up retries
          save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'pull_request' }}

      - name: install uv and set the python version
        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
        with:
          version: "0.7.19"
          python-version: ${{ matrix.python-version }}
          enable-cache: true
          cache-dependency-glob: "python/pyproject.toml"

      - name: Setup Python venv and build
        run: |
          make setup-venv
          source .venv/bin/activate
          make build
          make develop
          
      - name: Python unit tests
        # Only compute coverage for one matrix combination to save time
        if: ${{ !((matrix.os == 'ubuntu-24.04') && (matrix.python-version == '3.13')) }}
        run: |
          source .venv/bin/activate
          pytest -v
          
      - name: Python unit tests with coverage report
        # Only compute coverage for ubuntu-24.04 with Python 3.13
        if: ${{ (matrix.os == 'ubuntu-24.04') && (matrix.python-version == '3.13') }}
        run: |
          source .venv/bin/activate
          coverage run --include 'python/hudi/*' -m pytest -v
          coverage xml --data-file=.coverage -o ./cov-reports/cov-report-python-tests-${{ join(matrix.*, '-') }}.xml
          
      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        if: ${{ (matrix.os == 'ubuntu-24.04') && (matrix.python-version == '3.13') }}
        with:
          name: cov-report-python-tests-${{ join(matrix.*, '-') }}
          path: ./cov-reports
          if-no-files-found: 'error'
          
      - name: Upload wheel files
        uses: actions/upload-artifact@v7
        with:
          name: python-artifact-${{ join(matrix.*, '-') }}
          path: ./target/wheels/*.whl
          if-no-files-found: 'error'

  integration-tests:
    strategy:
        fail-fast: false
        matrix:
          app-path: [ 'datafusion', 'hudi-file-group-api/cpp', 'hudi-table-api/rust' , 'hudi-table-api/python' ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v6
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
      - name: Pre-pull base images
        run: |
          docker pull quay.io/minio/minio:latest
      - name: Prepare cache directories with correct permissions
        run: |
          mkdir -p .cargo/registry .cargo/git target python/target .uv-cache
          sudo chown -R $(id -u):$(id -g) .cargo target python/target .uv-cache
      - name: Cache integration build artifacts (cargo/target/uv)
        uses: actions/cache@v5
        with:
          path: |
            target
            .cargo/registry
            .cargo/git
            python/target
            .uv-cache
          key: integration-${{ runner.os }}-${{ hashFiles('**/Cargo.lock', 'python/Cargo.toml', 'demo/infra/runner/Dockerfile') }}
          restore-keys: |
            integration-${{ runner.os }}-
      - name: Integration test - ${{ matrix.app-path }}
        run: |
          cd demo
          ./ci_run.sh ${{ matrix.app-path }}
      - name: Fix cache ownership before save
        if: always()
        run: |
          sudo chown -R $(id -u):$(id -g) $GITHUB_WORKSPACE/.cargo $GITHUB_WORKSPACE/target $GITHUB_WORKSPACE/python/target $GITHUB_WORKSPACE/.uv-cache || true

  publish-coverage:
    name: Publish coverage reports to codecov.io
    runs-on: ubuntu-latest
    needs: [ rust-tests, python-tests ]
    if: github.event.pull_request.user.login != 'dependabot[bot]'
    steps:
      - uses: actions/checkout@v6
      - uses: actions/download-artifact@v8
        with:
          pattern: cov-report-*
          merge-multiple: true
          path: ./cov-reports
      - name: Upload coverage reports to codecov.io
        uses: codecov/codecov-action@v6
        with:
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}
          directory: ./cov-reports