uat-run.yaml

# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: UAT Run

# A distinct run title per dispatch so the nightly controller can identify the
# run it dispatched — `gh workflow run` does not return a run id, so the
# controller matches on this display title. The controller supplies a unique
# dispatch_key (its run id + cell index) that is appended here, so a concurrent
# manual dispatch of the same reservation/version can never be mistaken for the
# controller's run. A manual dispatch omits the key and the suffix is dropped.
# Quoted so YAML does not treat the ` #` in the dispatch_key suffix as a comment.
#
# The intent (training|inference) is rendered so nightly/daytime runs are
# self-describing in the Actions list. It defaults to `training` on both the
# dispatch and call paths, so it is always present. The deployer is rendered
# as `[<deployer>]` ONLY when it is non-empty and non-default (i.e. argocd
# today) so nightly/daytime titles, which never dispatch a non-default
# deployer, are unchanged. IMPORTANT: the nightly (uat-nightly-batch.yaml)
# and daytime (uat-daytime.yaml) controllers rebuild this exact title to
# resolve the run they dispatched — keep their `title=` construction in
# lockstep with this format string; if either controller ever starts passing
# `-f deployer=<x>` (x != helmfile), also add the `[<x>]` segment there.
# Folded scalar (>-): each newline below joins as a single space, which
# corresponds exactly to the literal single-space separators between the
# segments (`UAT · reservation · intent · @ · version`). Line-break points
# are chosen so no newline lands INSIDE an adjacent expression pair where
# the absence of a literal separator is load-bearing (e.g. the intent+
# deployer pair — deployer renders ' [<x>]' or '', its leading space is the
# separator on its own — and version+dispatch_key, same shape). Keeps the
# rendered title identical to the pre-fold single-line form.
run-name: >-
  UAT ${{ inputs.reservation }}
  ${{ inputs.intent }}${{ inputs.deployer != 'helmfile' && format(' [{0}]', inputs.deployer) || '' }}
  @ ${{ inputs.aicr_version || 'main' }}${{ inputs.dispatch_key != '' && format(' #{0}', inputs.dispatch_key) || '' }}

# The shared UAT dispatch surface and reservation-lease owner (#1274, DC1).
# A human (workflow_dispatch) or the nightly batch (workflow_call) requests a
# run against a named reservation; this workflow holds the per-reservation
# concurrency lease, resolves the reservation row from the checked-in registry
# (infra/uat/reservations.yaml) via the uat-broker helper, and dispatches the
# cloud-appropriate reusable pipeline. Because every run for a reservation
# routes through here and shares the lease, contending runs QUEUE instead of
# racing the hardware.
#
# DC2 (#1275) adds two orthogonal dials, both forwarded to the cloud pipeline:
#   - intent (training|inference) selects the per-intent test config +
#     recipe criteria.
#   - lifecycle (nightly|daytime-up|daytime-down) selects the cluster
#     lifecycle: the nightly provision→CUJ→teardown, the daytime
#     provision-and-hold (stable lease-tagged name; holds on success, tears
#     down on failure or cancellation), or the evening teardown of that held
#     daytime cluster.
on:
  workflow_dispatch:
    inputs:
      reservation:
        description: 'Reservation name from infra/uat/reservations.yaml (e.g. aws-h100, gcp-h100, azure-h100).'
        type: string
        required: true
      aicr_version:
        description: 'Released aicr version to test (e.g. v1.2.3); empty = build from source (main tip).'
        type: string
        default: ''
      dispatch_key:
        description: 'Unique correlation key set by the nightly controller so it can find this run; leave empty for manual runs.'
        type: string
        default: ''
      intent:
        description: 'Recipe intent — selects the per-intent test config (h100-<intent>-config.yaml) and recipe criteria.'
        type: choice
        options: [training, inference]
        default: training
      lifecycle:
        description: 'Cluster lifecycle. nightly: provision→CUJ→teardown. daytime-up: stable-named cluster, holds on success, tears down on failure or cancellation. daytime-down: tear down cluster.'
        type: choice
        options: [nightly, daytime-up, daytime-down]
        default: nightly
      slot:
        description: 'Daytime cluster slot (ADR-017) — the <slot> in aicr-uat-day-<slug>-<slot>-. Only slot 0 is provisioned today.'
        type: string
        default: '0'
      skip_delete:
        description: 'Skip cluster teardown (manual debugging only; ignored for daytime lifecycles). A retained cluster becomes eligible for janitor reclamation ~24h after the run finishes.'
        type: boolean
        default: false
      skip_tests:
        description: 'Skip UAT test execution (manual debugging only).'
        type: boolean
        default: false
      deployer:
        # helmfile = default via <accelerator>-<intent>-config.yaml (the
        # existing filename shape every cell has always used, kept for
        # backwards-compat). argocd = <accelerator>-<intent>-argocd-config.yaml
        # (e.g. h100-training-argocd-config.yaml). Only aws-h100 training
        # ships an argocd variant today; other cells fail with a missing
        # test-config. See issue #2194. (actionlint rejects an empty
        # string in a choice's options, so `helmfile` is the explicit
        # default value here; the TEST_CONFIG resolver in uat-aws.yaml
        # maps it to the unsuffixed filename.)
        description: 'Deployer variant. See workflow file for the variant matrix.'
        type: choice
        options: [helmfile, argocd]
        default: helmfile
  workflow_call:
    inputs:
      reservation:
        type: string
        required: true
      aicr_version:
        type: string
        default: ''
      dispatch_key:
        type: string
        default: ''
      intent:
        type: string
        default: training
      lifecycle:
        type: string
        default: nightly
      slot:
        type: string
        default: '0'
      skip_delete:
        type: boolean
        default: false
      skip_tests:
        type: boolean
        default: false
      deployer:
        type: string
        default: helmfile

permissions:
  contents: read

# The reservation lease. Every run targeting a given reservation serializes on
# this group, so a contender waits rather than racing the reservation;
# cancel-in-progress:false keeps an in-progress run (and its teardown) from
# being killed.
#
# NOTE: this is a single-slot lease, not a full queue. GitHub concurrency holds
# at most one in-progress + one PENDING run per group; a third contender
# supersedes (cancels) the older pending one — cancel-in-progress:false only
# protects the in-progress run, not the pending slot. Acceptable at launch (two
# reservations, at most cron + one ad-hoc each); preserving every request would
# need the deferred standing broker (see docs/contributor/uat.md, #1264).
#
# github.event.inputs is used for the dispatch path because the inputs context
# is unreliable in a top-level concurrency group on workflow_dispatch
# (community #45734/#35341); inputs.* covers the workflow_call path, where
# github.event.inputs is empty.
concurrency:
  group: uat-${{ github.event.inputs.reservation || inputs.reservation }}
  cancel-in-progress: false

jobs:
  # Resolve the reservation row to the cloud + on-disk config paths the
  # cloud-specific pipeline consumes. A typo'd reservation name fails here
  # (uat-broker exits NOT_FOUND), so the registry is the single source of truth.
  resolve:
    if: github.repository == 'nvidia/aicr'
    runs-on: ubuntu-latest
    outputs:
      cloud: ${{ steps.row.outputs.cloud }}
      # Daytime cluster-name discovery slug (ADR-017); forwarded to the cloud
      # pipelines that derive aicr-uat-day-<slug>-<slot>-<run_id>.
      slug: ${{ steps.row.outputs.slug }}
      cluster_config_path: ${{ steps.row.outputs['cluster-config-path'] }}
      test_config_dir: ${{ steps.row.outputs['test-config-dir'] }}
      # Accelerator selects the per-accelerator test-config filename
      # (<accelerator>-<intent>-config.yaml) in the AWS pipeline. The AWS
      # account (tenancy) is NOT plumbed here: the AWS pipeline derives it from
      # the cluster-config's .deployment.tenancy, the single source of truth.
      accelerator: ${{ steps.row.outputs.accelerator }}
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
        with:
          persist-credentials: false
      - name: Load versions
        id: versions
        uses: ./.github/actions/load-versions
      - name: Setup Go
        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e  # v7.0.0
        with:
          go-version: '${{ steps.versions.outputs.go }}'
          cache: true
          cache-dependency-path: |
            go.sum
            vendor/modules.txt
      - name: Resolve reservation row
        id: row
        env:
          GOFLAGS: -mod=vendor
          # Pass via env (not inline interpolation) to avoid expression
          # injection through the dispatch input.
          RESERVATION: ${{ inputs.reservation }}
          SLOT: ${{ inputs.slot }}
        run: |
          set -euo pipefail
          # Only slot 0 is provisioned today (ADR-017). Reject any other value at
          # this single dispatch surface — before it is forwarded to a cloud
          # pipeline — so a daytime-up with slot=1 cannot stand up a second daytime
          # cluster (and an unbounded value cannot blow the GKE name budget) until
          # multi-slot capacity and scheduling land.
          if [[ "${SLOT}" != "0" ]]; then
            echo "::error::unsupported slot '${SLOT}' (only slot 0 is enabled today)"; exit 1
          fi
          go build -o ./bin/uat-broker ./tools/uat-broker
          ./bin/uat-broker reservations --name "$RESERVATION" >> "$GITHUB_OUTPUT"

  # One of run-aws / run-gcp / run-azure / run-kind fires based on the resolved
  # cloud (the registry validates cloud membership, so at most one matches; the
  # unmapped-cloud job below fails closed if a registry cloud has no
  # run-<cloud> job here yet). The caller job grants the permission SUPERSET
  # its callee needs — a reusable workflow's jobs are ceilinged by the calling
  # job's permissions, and the called pipeline does OIDC, GHCR push, and
  # (nested) evidence-ingest GCS WIF.
  run-aws:
    needs: resolve
    if: always() && needs.resolve.outputs.cloud == 'aws'
    # Superset ceiling for the nested pipeline (a reusable workflow's jobs
    # cannot exceed the caller's permissions).
    permissions:
      contents: read  # checkout inside the called pipeline
      actions: write  # nested uat-{aws,gcp} watchers + evidence-ingest dashboard dispatch
      id-token: write  # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish
      packages: write  # push validator + snapshot-agent images to GHCR
    uses: ./.github/workflows/uat-aws.yaml
    with:
      reservation: ${{ inputs.reservation }}
      aicr_version: ${{ inputs.aicr_version }}
      intent: ${{ inputs.intent }}
      lifecycle: ${{ inputs.lifecycle }}
      # Daytime cluster-name axes (ADR-017): slug from the resolved registry row,
      # slot from the dispatch input. Together with github.run_id they form
      # aicr-uat-day-<slug>-<slot>-<run_id>.
      slug: ${{ needs.resolve.outputs.slug }}
      slot: ${{ inputs.slot }}
      cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
      test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
      accelerator: ${{ needs.resolve.outputs.accelerator }}
      skip_delete: ${{ inputs.skip_delete }}
      skip_tests: ${{ inputs.skip_tests }}
      deployer: ${{ inputs.deployer }}

  run-gcp:
    needs: resolve
    # deployer=argocd is aws-h100-scoped today (issue #2194): uat-gcp.yaml declares
    # no deployer input, so passing one here would be silently ignored (green
    # helmfile run against a request the user did not make). Skip this job when
    # a non-helmfile deployer is dispatched; the unsupported-deployer-for-cloud
    # guard job below emits the red-workflow signal for that combination.
    if: always() && needs.resolve.outputs.cloud == 'gcp' && inputs.deployer == 'helmfile'
    # Superset ceiling for the nested pipeline (a reusable workflow's jobs
    # cannot exceed the caller's permissions).
    permissions:
      contents: read  # checkout inside the called pipeline
      actions: write  # nested uat-{aws,gcp} watchers + evidence-ingest dashboard dispatch
      id-token: write  # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish
      packages: write  # push validator + snapshot-agent images to GHCR
    uses: ./.github/workflows/uat-gcp.yaml
    with:
      reservation: ${{ inputs.reservation }}
      aicr_version: ${{ inputs.aicr_version }}
      intent: ${{ inputs.intent }}
      lifecycle: ${{ inputs.lifecycle }}
      # Daytime cluster-name axes (ADR-017): slug from the resolved registry row,
      # slot from the dispatch input. Together with github.run_id they form
      # aicr-uat-day-<slug>-<slot>-<run_id>.
      slug: ${{ needs.resolve.outputs.slug }}
      slot: ${{ inputs.slot }}
      cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
      test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
      skip_delete: ${{ inputs.skip_delete }}
      skip_tests: ${{ inputs.skip_tests }}

  run-azure:
    needs: resolve
    # See run-gcp for the deployer-scope rationale.
    if: always() && needs.resolve.outputs.cloud == 'azure' && inputs.deployer == 'helmfile'
    # Superset ceiling for the nested pipeline (a reusable workflow's jobs
    # cannot exceed the caller's permissions).
    permissions:
      contents: read  # checkout inside the called pipeline
      actions: write  # nested uat-* watchers + evidence-ingest dashboard dispatch
      id-token: write  # cloud OIDC/WIF: provision, keyless cosign, and GCS evidence publish
      packages: write  # push validator + snapshot-agent images to GHCR
    uses: ./.github/workflows/uat-azure.yaml
    with:
      reservation: ${{ inputs.reservation }}
      aicr_version: ${{ inputs.aicr_version }}
      intent: ${{ inputs.intent }}
      lifecycle: ${{ inputs.lifecycle }}
      # Daytime cluster-name axes (ADR-017): slug from the resolved registry row,
      # slot from the dispatch input. Together with github.run_id they form
      # aicr-uat-day-<slug>-<slot>-<run_id>.
      slug: ${{ needs.resolve.outputs.slug }}
      slot: ${{ inputs.slot }}
      cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
      test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
      skip_delete: ${{ inputs.skip_delete }}
      skip_tests: ${{ inputs.skip_tests }}

  # nvkind real-silicon lane (DC5 #1278). "kind" is not a cloud but a
  # self-hosted GPU-runner lane; it slots into the same dispatch model so it
  # rides the nightly batch and shares tests/uat/kind/run + tests/uat/lib. It
  # provisions on the runner (no cloud actuator), so it needs no cloud OIDC —
  # but it DOES do keyless cosign (id-token), GHCR push (packages), and the
  # nested evidence-ingest GCS WIF, so the permission superset matches the
  # cloud lanes.
  run-kind:
    needs: resolve
    # See run-gcp for the deployer-scope rationale.
    if: needs.resolve.outputs.cloud == 'kind' && inputs.deployer == 'helmfile'
    permissions:
      contents: read  # checkout inside the called pipeline
      actions: write  # nested evidence-ingest dashboard-publish dispatch (a reusable workflow cannot elevate the caller token, so grant the ceiling here)
      id-token: write  # keyless cosign + GCS evidence-publish WIF
      packages: write  # push validator/agent + evidence images to GHCR
    uses: ./.github/workflows/uat-kind.yaml
    with:
      reservation: ${{ inputs.reservation }}
      aicr_version: ${{ inputs.aicr_version }}
      intent: ${{ inputs.intent }}
      lifecycle: ${{ inputs.lifecycle }}
      cluster_config_path: ${{ needs.resolve.outputs.cluster_config_path }}
      test_config_dir: ${{ needs.resolve.outputs.test_config_dir }}
      skip_delete: ${{ inputs.skip_delete }}
      skip_tests: ${{ inputs.skip_tests }}

  # Fail closed on a dispatched deployer variant this cell doesn't yet wire.
  # Today only run-aws forwards the `deployer` input to its reusable pipeline;
  # gcp/azure/kind reusable workflows declare no such input and would run the
  # helmfile lane if we let them fire. The run-*.if:s above skip those jobs
  # when a non-helmfile deployer is dispatched to a non-aws cell; this job
  # provides the paired red-workflow signal so the requester never sees a
  # green run for a request that was silently downgraded.
  # Mirrors the unmapped-cloud pattern for the same "no silent no-op" reason.
  unsupported-deployer-for-cloud:
    needs: resolve
    if: >-
      inputs.deployer != 'helmfile' &&
      needs.resolve.outputs.cloud != 'aws'
    runs-on: ubuntu-latest
    steps:
      - name: Fail on unsupported (deployer, cloud) combination
        env:
          CLOUD: ${{ needs.resolve.outputs.cloud }}
          DEPLOYER: ${{ inputs.deployer }}
        run: |
          set -euo pipefail
          echo "::error::deployer='${DEPLOYER}' is only wired for cloud='aws' today"
          echo "::error::(this reservation resolved to cloud='${CLOUD}'). Only run-aws"
          echo "::error::forwards the deployer input to its reusable pipeline; other"
          echo "::error::run-<cloud> jobs would run the helmfile lane silently. See"
          echo "::error::issue #2194 for the coverage roadmap. The run-<cloud>.if:"
          echo "::error::guards above skipped the dispatched cell to prevent that"
          echo "::error::silent downgrade."
          exit 1

  # Fail closed on an unmapped cloud. The broker validates cloud membership
  # against its own set, but a cloud can be added there (and to the registry)
  # before its run-<cloud> job exists here — without this job the dispatch
  # would silently no-op: the lease is taken, no pipeline runs, and the
  # requester sees a green run that did nothing.
  unmapped-cloud:
    needs: resolve
    if: >-
      needs.resolve.outputs.cloud != 'aws' &&
      needs.resolve.outputs.cloud != 'gcp' &&
      needs.resolve.outputs.cloud != 'azure' &&
      needs.resolve.outputs.cloud != 'kind'
    runs-on: ubuntu-latest
    steps:
      - name: Fail on unmapped cloud
        env:
          CLOUD: ${{ needs.resolve.outputs.cloud }}
        run: |
          set -euo pipefail
          echo "::error::reservation resolved to cloud '${CLOUD}', which has no run-<cloud> job in uat-run.yaml — add one (see run-azure for the shape)."
          exit 1