test_component.yml

# Copyright Advanced Micro Devices, Inc.
# SPDX-License-Identifier: MIT

name: Test component

on:
  workflow_call:
    inputs:
      artifact_run_id:
        type: string
        default: ""
      artifact_group:
        type: string
      amdgpu_families:
        type: string
      amdgpu_targets:
        type: string
        default: ""
      test_runs_on:
        type: string
      platform:
        type: string
      component:
        type: string
      release_type:
        type: string
        default: ci
      repository:
        description: "Repository to checkout. Defaults to github.repository."
        type: string
      ref:
        description: "Branch, tag, or SHA to checkout. Defaults to the triggering ref."
        type: string
      build_variant:
        description: "Build variant (e.g., release, asan). Used to skip tests incompatible with certain builds."
        type: string
        default: "release"
      default_container_image:
        type: string
        default: "ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:4150afe4759d14822f0e3f8930e1124f26e11f68b5c7b91ec9a02b20b1ebbb98"


permissions:
  contents: read

jobs:
  notify_quartz_start:
    if: ${{ vars.NOTIFY_QUARTZ_ENABLED == 'true' }}
    name: "Quartz - started - test ROCm component"
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    continue-on-error: true
    steps:
      - name: "Quartz - started - test ROCm component"
        uses: ROCm/Quartz/.github/actions/notify_quartz@b47e20b7539b407072a0c2249ab5e8d9983fac4e # notify_quartz/v1.1.0
        with:
          run_phase: started
          reporting_workflow: test_component.yml
          workflow_inputs: ${{ toJSON(inputs) }}
          gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }}
          gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }}

  test_component:
    name: >-
      Test ${{ fromJSON(inputs.component).job_name }}
      (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }})
      (${{ inputs.amdgpu_families }})
      ${{ fromJSON(inputs.component).expect_failure == true && '(xfail)' || '' }}
    runs-on: ${{ inputs.test_runs_on }}
    continue-on-error: ${{ fromJSON(inputs.component).expect_failure == true }}
    timeout-minutes: 210
    outputs:
      summary: ${{ steps.summary.outputs.summary }}
    container:
      # If the component has specified an alternate image, honor that option.
      # Otherwise use the default image.
      image: >-
        ${{ inputs.platform == 'linux' &&
            (fromJSON(inputs.component).container_image || inputs.default_container_image)
            || null
        }}
      # Container options are configured in build_tools/github_actions/fetch_test_configurations.py
      # and include base options (ipc, user, ulimit, seccomp), GPU options (devices, groups),
      # and any job-specific options
      options: ${{ fromJSON(inputs.component).container_options }}
    strategy:
      fail-fast: false
      matrix:
        # The shard array is based on "total_shards" from "fetch_test_configurations.py"
        # The test executable will shard based on the array. (ex: [1, 2, 3, 4] = four test shards)
        shard: ${{ fromJSON(inputs.component).shard_arr }}
    defaults:
      run:
        shell: bash
    env:
      VENV_DIR: ${{ github.workspace }}/.venv
      ARTIFACT_RUN_ID: "${{ inputs.artifact_run_id != '' && inputs.artifact_run_id || github.run_id }}"
      OUTPUT_ARTIFACTS_DIR: "./build"
      THEROCK_BIN_DIR: "./build/bin"
      AMDGPU_FAMILIES: ${{ inputs.amdgpu_families }}
      AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }}
      ARTIFACT_GROUP: ${{ inputs.artifact_group }}
      BUILD_VARIANT: ${{ inputs.build_variant }}
      # Benchmark results database API endpoints for performance metrics collection
      # NOTE: These secrets are only required for benchmark results submission in nightly CI runs.
      # For PR/push workflows, secret retrieval will fail gracefully and benchmarks will skip API submission.
      BENCHMARK_DB_URL: ${{ secrets.BENCHMARK_DB_URL }}
      BENCHMARK_DB_FALLBACK_URL: ${{ secrets.BENCHMARK_DB_FALLBACK_URL }}
    steps:
      - name: "Fetch 'build_tools' from repository"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ${{ inputs.repository || github.repository }}
          ref: ${{ inputs.ref }}
          sparse-checkout: build_tools
          path: "prejob"

      - name: Pre-job cleanup processes on Windows
        if: ${{ runner.os == 'Windows' }}
        timeout-minutes: 5
        shell: powershell
        run: . '${{ github.workspace }}\prejob\build_tools\github_actions\cleanup_processes.ps1'

      - name: Pre-job cleanup GPU processes on Linux
        if: ${{ runner.os == 'Linux' }}
        timeout-minutes: 5
        run: bash ./prejob/build_tools/github_actions/cleanup_processes.sh

      - name: Checking repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: ${{ inputs.repository || github.repository }}
          ref: ${{ inputs.ref }}

      - name: Run setup test environment workflow
        timeout-minutes: 15
        env:
          RETRY_THIS_STEP: "true"
          STEP_TIMEOUT_MINUTES: "15" # should match timeout-minutes specified above at this step
        uses: './.github/actions/setup_test_environment'
        with:
          ARTIFACT_RUN_ID: ${{ env.ARTIFACT_RUN_ID }}
          # TODO(#3381): revert back to `inputs.artifact_group` once issue is resolved
          ARTIFACT_GROUP: ${{ inputs.amdgpu_families }}
          AMDGPU_TARGETS: ${{ inputs.amdgpu_targets }}
          OUTPUT_ARTIFACTS_DIR: ${{ env.OUTPUT_ARTIFACTS_DIR }}
          VENV_DIR: ${{ env.VENV_DIR }}
          FETCH_ARTIFACT_ARGS: ${{ fromJSON(inputs.component).fetch_artifact_args }}
          RELEASE_TYPE: ${{ inputs.release_type }}

      # safe.directory must be set before Runner Health Status
      - name: Adjust git config
        run: |
          git config --global --add safe.directory $PWD
          git config fetch.parallel 10

      - name: Runner health status
        run: |
          python ./build_tools/health_status.py

      - name: Driver / GPU sanity check
        timeout-minutes: 3
        run: |
          python ./build_tools/print_driver_gpu_info.py

      - name: Setup Additional Requirements
        if: ${{ fromJSON(inputs.component).additional_requirements_files != '' }}
        run: |
          python ./build_tools/install_additional_requirements.py \
            --requirements-files ${{ join(fromJSON(inputs.component).additional_requirements_files, ',') }}

        # TODO(#3755): Clean this up after final sanitizer environment variable flags are determined
      - name: Enable sanitizer-specific test flags
        if: ${{ contains(inputs.artifact_group, 'asan') }}
        run: |
          echo "ASAN_OPTIONS=detect_odr_violation=0:quarantine_size_mb=600" >> $GITHUB_ENV
          echo "HSA_XNACK=1" >> $GITHUB_ENV
          echo "ASAN_SYMBOLIZER_PATH=${{ env.OUTPUT_ARTIFACTS_DIR }}/llvm/bin/llvm-symbolizer" >> $GITHUB_ENV

      # Set thread limits from KUBE_CPU_REQUEST (Kubernetes-injected) to avoid oversubscription.
      # Must be done at runtime since container env vars aren't in GitHub's env context.
      - name: Set thread limits
        run: |
          if [[ -n "${KUBE_CPU_REQUEST:-}" ]]; then
            CPU_COUNT="${KUBE_CPU_REQUEST%.*}"
            echo "OPENBLAS_NUM_THREADS=${CPU_COUNT}" >> $GITHUB_ENV
            echo "OMP_NUM_THREADS=${CPU_COUNT}" >> $GITHUB_ENV
            echo "Thread limits set: OPENBLAS_NUM_THREADS=${CPU_COUNT}, OMP_NUM_THREADS=${CPU_COUNT} (from KUBE_CPU_REQUEST=${KUBE_CPU_REQUEST})"
          else
            echo "KUBE_CPU_REQUEST not set, thread limits not configured"
          fi

      - name: Test
        id: test
        timeout-minutes: ${{ fromJSON(inputs.component).timeout_minutes }}
        env:
          SHARD_INDEX: ${{ matrix.shard }}
          TOTAL_SHARDS: ${{ fromJSON(inputs.component).total_shards }}
          TEST_TYPE: ${{ fromJSON(inputs.component).test_type }}
          TEST_COMPONENT: ${{ fromJSON(inputs.component).job_name }}
          ROCM_KPACK_DEBUG: "1"
          # Windows hip-tests (PAL=1, ROCR=0) set this via matrix; other components
          # leave it empty. #3587
          # An empty GPU_ENABLE_PAL is misread as ROCr on Windows (atoi("")==0),
          # whose Windows memory path OOMs some tests (e.g. rocsparse bsr*0 graph_test).
          # Default unset components to the platform backend, mirroring the runtime fix
          # in ROCm/rocm-systems#8050 (Windows -> PAL, Linux -> ROCr).
          GPU_ENABLE_PAL: ${{ fromJSON(inputs.component).gpu_enable_pal || (inputs.platform == 'windows' && '1' || '0') }}
          RETRY_THIS_STEP: "true"
          RETRY_COUNT: "1"
          RETRY_DELAY: "7"
          STEP_TIMEOUT_MINUTES: ${{ fromJSON(inputs.component).timeout_minutes }}
          STEP_NAME: >-
            Test ${{ fromJSON(inputs.component).job_name }}
            (shard ${{ matrix.shard }}/${{ fromJSON(inputs.component).total_shards }})
            (${{ inputs.amdgpu_families }})
            ${{ fromJSON(inputs.component).expect_failure == true && '(xfail)' || '' }}
        run: |
          ${{ fromJSON(inputs.component).test_script }}

      - name: Print test reproduction command
        if: ${{ failure() && steps.test.outcome == 'failure' }}
        run: |
          python ./build_tools/github_actions/reproduce_test_failure.py \
            --run-id "${{ env.ARTIFACT_RUN_ID }}" \
            --repository "${{ github.repository }}" \
            --amdgpu-family "${{ inputs.amdgpu_families }}" \
            --amdgpu-targets "${{ inputs.amdgpu_targets }}" \
            --test-script "${{ fromJSON(inputs.component).test_script }}" \
            --shard-index "${{ matrix.shard }}" \
            --total-shards "${{ fromJSON(inputs.component).total_shards }}" \
            --test-type "${{ fromJSON(inputs.component).test_type }}" \
            --print-cmd \
            ${{ fromJSON(inputs.component).fetch_artifact_args && format('--fetch-artifact-args="{0}"', fromJSON(inputs.component).fetch_artifact_args) || '' }}

      - name: Post-job cleanup GPU processes on Linux
        if: ${{ always() && runner.os == 'Linux' }}
        timeout-minutes: 5
        run: bash ./build_tools/github_actions/cleanup_processes.sh

      - name: Post-job cleanup processes on Windows
        if: ${{ always() && runner.os == 'Windows' }}
        timeout-minutes: 5
        shell: powershell
        run: . '${{ github.workspace }}\build_tools\github_actions\cleanup_processes.ps1'

      - name: Capture job summary
        id: summary
        if: always()
        run: python build_tools/github_actions/capture_job_summary.py

  notify_quartz_completed:
    if: ${{ always() && vars.NOTIFY_QUARTZ_ENABLED == 'true' }}
    needs: [test_component]
    name: "Quartz - completed - test ROCm component"
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    continue-on-error: true
    steps:
      - name: "Quartz - completed - test ROCm component"
        uses: ROCm/Quartz/.github/actions/notify_quartz@b47e20b7539b407072a0c2249ab5e8d9983fac4e # notify_quartz/v1.1.0
        with:
          run_phase: completed
          reporting_workflow: test_component.yml
          workflow_inputs: ${{ toJSON(inputs) }}
          workflow_captured_outputs: ${{ toJSON(needs) }}
          gh_app_client_id: ${{ secrets.GH_APP_HAULY_CID }}
          gh_app_private_key: ${{ secrets.GH_APP_HAULY_PRIVATE_KEY }}