run-rancher-signoff-lane.yml

name: Run Rancher Sign-Off Lane
run-name: Run ${{ inputs.rancher_version }} / ${{ inputs.lane }}

on:
  workflow_dispatch:
    inputs:
      rancher_version:
        description: Target Rancher alpha or RC, for example v2.15.0-rc2.
        required: true
        type: string
      lane:
        description: Validation lane to run.
        required: true
        default: framework-regression
        type: choice
        options:
          - framework-regression
          - webhook-fresh-install
          - webhook-upgrade
          - webhook-candidate-on-previous
      previous_rancher_version:
        description: Previous Rancher release. Leave blank to resolve automatically.
        required: false
        type: string
      webhook_image:
        description: Candidate webhook image. Leave blank to use Docker Hub image from target build.yaml.
        required: false
        type: string
      signing_policy:
        description: Signing policy to include in the plan.
        required: false
        default: auto
        type: choice
        options:
          - auto
          - required
          - report-only
          - skip
      keep_infra_on_failure:
        description: Skip cleanup if setup fails. Use only for manual debugging.
        required: true
        default: false
        type: boolean
      run_rancher_tests:
        description: Clone rancher/tests and run the lane's test suites after the lane is ready.
        required: true
        default: true
        type: boolean
      rancher_tests_ref:
        description: rancher/tests ref to clone. Defaults to main.
        required: false
        type: string
      rke2_server_count:
        description: RKE2 server nodes per HA. Use 1 for single-server, 3 for default HA, or 5 for a larger HA layout.
        required: true
        default: "3"
        type: choice
        options:
          - "1"
          - "3"
          - "5"

permissions:
  contents: read
  id-token: write

concurrency:
  group: run-rancher-signoff-lane-${{ inputs.rancher_version }}-${{ inputs.lane }}
  cancel-in-progress: false

jobs:
  run-lane:
    name: ${{ inputs.rancher_version }} / ${{ inputs.lane }}
    runs-on: ubuntu-latest
    environment: rancher-signoff
    env:
      GH_TOKEN: ${{ github.token }}
      TF_IN_AUTOMATION: "true"
      TF_INPUT: "false"
      TF_STATE_BUCKET: ${{ vars.TF_STATE_BUCKET }}
      TF_STATE_LOCK_TABLE: ${{ vars.TF_STATE_LOCK_TABLE }}
      TF_STATE_REGION: ${{ vars.TF_STATE_REGION }}
      AWS_REGION: ${{ vars.AWS_REGION }}
      AWS_VPC: ${{ vars.AWS_VPC }}
      AWS_SUBNET_A: ${{ vars.AWS_SUBNET_A }}
      AWS_SUBNET_B: ${{ vars.AWS_SUBNET_B }}
      AWS_SUBNET_C: ${{ vars.AWS_SUBNET_C }}
      AWS_AMI: ${{ vars.AWS_AMI }}
      AWS_PREFIX: ${{ vars.AWS_PREFIX }}
      AWS_SUBNET_ID: ${{ vars.AWS_SUBNET_ID }}
      AWS_SECURITY_GROUP_ID: ${{ vars.AWS_SECURITY_GROUP_ID }}
      AWS_PEM_KEY_NAME: ${{ vars.AWS_PEM_KEY_NAME }}
      AWS_ROUTE53_FQDN: ${{ vars.AWS_ROUTE53_FQDN }}
      RANCHER_BOOTSTRAP_PASSWORD: ${{ secrets.RANCHER_BOOTSTRAP_PASSWORD }}
      LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
      RANCHER_TESTS_REF: ${{ vars.RANCHER_TESTS_REF }}
      RANCHER_TEST_SUITE_SETTLE_SECONDS: ${{ vars.RANCHER_TEST_SUITE_SETTLE_SECONDS }}
      RKE2_SERVER_COUNT: ${{ inputs.rke2_server_count }}
      OWNER_FIRST_NAME: ${{ vars.OWNER_FIRST_NAME }}
      OWNER_LAST_NAME: ${{ vars.OWNER_LAST_NAME }}
      SIGNOFF_LANE: ${{ inputs.lane }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
        with:
          ref: ${{ github.ref_name }}

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
        with:
          role-to-assume: ${{ secrets.AWS_AUTOMATION_ROLE_ARN }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Mask runtime secrets
        run: |
          set -euo pipefail
          for value in \
            "$RANCHER_BOOTSTRAP_PASSWORD" \
            "$LINODE_TOKEN" \
            "$DOCKERHUB_PASSWORD"; do
            if [ -n "$value" ]; then
              echo "::add-mask::$value"
            fi
          done

      - name: Set up Go
        uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
        with:
          go-version-file: go.mod

      - name: Set up Terraform
        uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
        with:
          terraform_version: 1.13.5
          terraform_wrapper: false

      - name: Install kubectl
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y apt-transport-https ca-certificates curl gpg
          sudo install -d -m 0755 /etc/apt/keyrings
          curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.34/deb/Release.key | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
          echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.34/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
          sudo apt-get update
          sudo apt-get install -y kubectl

      - name: Set up Helm 3
        uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1
        with:
          version: v3.21.3

      - name: Verify CLI versions
        run: |
          set -euo pipefail
          helm_version="$(helm version --template '{{ .Version }}')"
          if [ "$helm_version" != "v3.21.3" ]; then
            echo "Expected Helm v3.21.3, found $helm_version" >&2
            exit 1
          fi
          helm version
          kubectl version --client

      - name: Build automation tools
        run: |
          go build -o "$RUNNER_TEMP/signoff-plan" ./automation/signoff-plan
          go build -o "$RUNNER_TEMP/render-tool-config" ./automation/render-tool-config
          go build -o "$RUNNER_TEMP/render-signoff-report" ./automation/render-signoff-report
          go build -o "$RUNNER_TEMP/verify-webhook-signing" ./automation/verify-webhook-signing

      - name: Generate sign-off plan
        env:
          INPUT_PREVIOUS_RANCHER_VERSION: ${{ github.event.inputs.previous_rancher_version || '' }}
          INPUT_WEBHOOK_IMAGE: ${{ github.event.inputs.webhook_image || '' }}
        run: |
          set -euo pipefail

          args=(
            "-rancher-version" "${{ inputs.rancher_version }}"
            "-signing-policy" "${{ inputs.signing_policy }}"
            "-run-id" "${{ github.run_id }}"
            "-ignore-ledger"
            "-output" "signoff-plan.json"
          )
          if [ -n "${AWS_PREFIX:-}" ]; then
            args+=("-aws-base-prefix" "$AWS_PREFIX")
          fi
          if [ -n "$INPUT_PREVIOUS_RANCHER_VERSION" ]; then
            args+=("-previous-rancher-version" "$INPUT_PREVIOUS_RANCHER_VERSION")
          fi
          if [ -n "$INPUT_WEBHOOK_IMAGE" ]; then
            args+=("-webhook-image" "$INPUT_WEBHOOK_IMAGE")
          fi

          "$RUNNER_TEMP/signoff-plan" "${args[@]}"
          echo "RANCHER_WEBHOOK_CHART_VERSION=$(jq -r '.target_webhook_build' signoff-plan.json)" >> "$GITHUB_ENV"

      - name: Install slsactl
        run: |
          set -euo pipefail

          signing_policy="$(jq -r '.signing_policy // "skip"' signoff-plan.json)"
          if [ "$signing_policy" = "skip" ]; then
            echo "Signing policy is skip; slsactl is not required"
            exit 0
          fi

          version="0.1.31"
          os_name="linux"
          arch="$(uname -m)"
          case "$arch" in
            x86_64) arch="amd64" ;;
            aarch64|arm64) arch="arm64" ;;
            *)
              echo "Unsupported slsactl architecture: $arch"
              exit 1
              ;;
          esac

          asset="slsactl_${version}_${os_name}_${arch}.tar.gz"
          base_url="https://github.com/rancherlabs/slsactl/releases/download/v${version}"
          workdir="$RUNNER_TEMP/slsactl"
          mkdir -p "$workdir"
          curl --fail --silent --show-error --location --retry 5 --retry-all-errors --retry-delay 5 \
            --connect-timeout 20 --max-time 600 \
            --output "$workdir/$asset" \
            "$base_url/$asset"
          curl --fail --silent --show-error --location --retry 5 --retry-all-errors --retry-delay 5 \
            --connect-timeout 20 --max-time 120 \
            --output "$workdir/checksums.txt" \
            "$base_url/slsactl_${version}_checksums.txt"
          grep "  ${asset}$" "$workdir/checksums.txt" > "$workdir/${asset}.sha256"
          (
            cd "$workdir"
            sha256sum -c "${asset}.sha256"
          )
          mkdir -p "$workdir/bin"
          tar -xzf "$workdir/$asset" -C "$workdir/bin"
          sudo install -m 0755 "$workdir/bin/slsactl" /usr/local/bin/slsactl
          slsactl version

      - name: Verify webhook image signing
        run: |
          "$RUNNER_TEMP/verify-webhook-signing" \
            -plan signoff-plan.json \
            -output automation-output/webhook-signing.json \
            -timeout 5m

      - name: Render lane config
        id: render_config
        run: |
          "$RUNNER_TEMP/render-tool-config" \
            -plan signoff-plan.json \
            -lane "${{ inputs.lane }}" \
            -output tool-config.yml \
            -env-output lane.env
          cat lane.env >> "$GITHUB_ENV"

      - name: Validate generated config
        run: |
          test -s tool-config.yml
          test -n "${TF_STATE_KEY:-}"
          echo "TF state key: ${TF_STATE_KEY}"
          echo "Rancher upgrade version: ${RANCHER_UPGRADE_VERSION:-none}"

      - name: Prepare isolated run workspace
        run: |
          set -euo pipefail

          run_id="${{ github.run_id }}"
          safe_run_id="$(printf '%s' "$run_id" | tr -c 'A-Za-z0-9_.-' '_' | tr '[:upper:]' '[:lower:]')"
          run_root="$GITHUB_WORKSPACE/automation-output/runs/$safe_run_id"
          module_dir="$run_root/terraform/module"

          rm -rf "$module_dir"
          mkdir -p "$module_dir" "$run_root/ha" "$run_root/terraform"
          cp -R modules/aws/. "$module_dir/"
          rm -rf \
            "$module_dir/.terraform" \
            "$module_dir/.terraform.lock.hcl" \
            "$module_dir/backend.tf" \
            "$module_dir/terraform.tfvars" \
            "$module_dir/terraform.tfstate" \
            "$module_dir/terraform.tfstate.backup"
          find "$module_dir" \( -name '*.tfstate' -o -name '.terraform.*' \) -type f -delete

          {
            echo "HA_RANCHER_RUN_ID=$safe_run_id"
            echo "HA_RANCHER_HA_OUTPUT_ROOT=$run_root/ha"
            echo "HA_RANCHER_TF_MODULE_DIR=$module_dir"
            echo "HA_RANCHER_TF_DATA_DIR=$run_root/terraform/.terraform"
          } >> "$GITHUB_ENV"

      - name: Run lane setup
        id: setup
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHaSetup$' -timeout 90m ./terratest

      - name: Wait for lane readiness
        id: ready
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHAWaitReady$' -timeout 35m ./terratest

      - name: Export local suite env
        id: local_suite_env
        if: ${{ inputs.lane == 'framework-regression' }}
        run: go test -v -run '^TestHAWriteLocalSuiteEnv$' -timeout 5m ./terratest

      - name: Validate Linode token
        if: ${{ inputs.lane != 'framework-regression' }}
        run: test -n "${LINODE_TOKEN:-}"

      - name: Provision downstream Linode K3s
        id: downstream
        if: ${{ inputs.lane != 'framework-regression' }}
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHAProvisionLinodeDownstream$' -timeout 20m ./terratest

      - name: Override local webhook image
        id: local_webhook
        if: ${{ inputs.lane == 'webhook-candidate-on-previous' }}
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHAOverrideLocalWebhook$' -timeout 30m ./terratest

      - name: Override downstream webhook image
        id: downstream_webhook
        if: ${{ inputs.lane == 'webhook-candidate-on-previous' }}
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHAOverrideDownstreamWebhook$' -timeout 20m ./terratest

      - name: Run Rancher upgrade
        id: upgrade
        if: ${{ inputs.lane == 'webhook-upgrade' }}
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHAUpgradeRancher$' -timeout 90m ./terratest

      - name: Wait for webhook chart rollout
        id: webhook_chart
        if: ${{ inputs.lane == 'webhook-upgrade' && inputs.run_rancher_tests == true }}
        run: .github/scripts/run-with-cancel-cleanup.sh go test -v -run '^TestHAWaitWebhookChartVersion$' -timeout 20m ./terratest

      - name: Run Rancher tests
        id: rancher_tests
        if: ${{ inputs.run_rancher_tests == true }}
        env:
          INPUT_RANCHER_TESTS_REF: ${{ github.event.inputs.rancher_tests_ref || '' }}
        run: |
          .github/scripts/run-with-cancel-cleanup.sh bash -euo pipefail <<'SCRIPT'

          env_file="automation-output/downstream-ha-1.env"
          suites=("TestWebhookTestSuite")
          version="${{ inputs.rancher_version }}"
          version="${version#v}"
          IFS='.' read -r rancher_major rancher_minor _ <<< "$version"
          if [ "${{ inputs.lane }}" = "framework-regression" ]; then
            env_file="automation-output/local-suite-ha-1.env"
            suites=(
              "TestConfigMapTestSuite"
              "TestSchemaChangesTestSuite"
              "TestNodeAnnotationsTestSuite"
            )
            if [ "$rancher_major" -eq 2 ] && [ "$rancher_minor" -le 11 ]; then
              suites+=("TestVaiTestSuite/TestVaiDisabled")
            else
              suites+=("TestVaiTestSuite/TestVaiEnabled")
            fi
          fi
          if [ "${{ inputs.lane }}" != "framework-regression" ] && \
             [ "${{ inputs.lane }}" != "webhook-candidate-on-previous" ] && \
             { [ "$rancher_major" -gt 2 ] || { [ "$rancher_major" -eq 2 ] && [ "$rancher_minor" -ge 14 ]; }; }; then
            suites+=("TestWebhookSecuritySettingsTestSuite")
          fi
          if [ ! -s "$env_file" ]; then
            echo "Expected suite env file $env_file"
            exit 1
          fi

          set -a
          # shellcheck disable=SC1090
          . "$env_file"
          set +a
          echo "::add-mask::$RANCHER_ADMIN_TOKEN"

          rancher_host="${RANCHER_HOST#https://}"
          rancher_host="${rancher_host#http://}"
          rancher_host="${rancher_host%/}"
          export RANCHER_HOST="$rancher_host"
          echo "::add-mask::$RANCHER_HOST"
          echo "::add-mask::https://${RANCHER_HOST}"

          curl --fail --silent --show-error --insecure \
            -X PUT "https://${RANCHER_HOST}/v3/settings/server-url" \
            -H "Authorization: Bearer ${RANCHER_ADMIN_TOKEN}" \
            -H "Content-Type: application/json" \
            --data-binary "{\"name\":\"server-url\",\"value\":\"https://${RANCHER_HOST}\"}"

          curl --fail --silent --show-error --insecure \
            -H "Authorization: Bearer ${RANCHER_ADMIN_TOKEN}" \
            "https://${RANCHER_HOST}/v3" >/dev/null
          echo "Verified Rancher API access for rancher/tests"

          shell_image="$(
            curl --fail --silent --show-error --insecure \
              -H "Authorization: Bearer ${RANCHER_ADMIN_TOKEN}" \
              "https://${RANCHER_HOST}/v3/settings/shell-image" | jq -r '.value // ""'
          )"
          echo "Rancher shell-image setting: ${shell_image:-<empty>}"
          shell_tag="${shell_image##*:}"
          if [ -n "$shell_image" ] && [ "$shell_image" != "$shell_tag" ]; then
            if docker manifest inspect "$shell_image" >/dev/null 2>&1; then
              echo "Rancher shell-image manifest is reachable from the GitHub runner"
            else
              echo "Rancher shell-image manifest was not reachable from the GitHub runner"
            fi
            for candidate in "stgregistry.suse.com/rancher/shell:${shell_tag}" "registry.rancher.com/rancher/shell:${shell_tag}" "docker.io/rancher/shell:${shell_tag}"; do
              docker manifest inspect "$candidate" >/dev/null 2>&1 && echo "Available shell helper candidate: $candidate"
            done
          fi

          probe_image="${shell_image:-rancher/shell:v0.7.0}"
          case "$probe_image" in
            rancher/shell:*|docker.io/rancher/shell:*|index.docker.io/rancher/shell:*)
              shell_tag="${probe_image##*:}"
              for candidate in "registry.rancher.com/rancher/shell:${shell_tag}" "stgregistry.suse.com/rancher/shell:${shell_tag}"; do
                if docker manifest inspect "$candidate" >/dev/null 2>&1; then
                  echo "Using non-Docker-Hub Rancher shell helper image for CI: $candidate"
                  curl --fail --silent --show-error --insecure \
                    -X PUT "https://${RANCHER_HOST}/v3/settings/shell-image" \
                    -H "Authorization: Bearer ${RANCHER_ADMIN_TOKEN}" \
                    -H "Content-Type: application/json" \
                    --data-binary "{\"name\":\"shell-image\",\"value\":\"${candidate}\"}"
                  shell_image="$candidate"
                  probe_image="$candidate"
                  break
                fi
              done
              ;;
          esac

          local_kubeconfig="${HA_RANCHER_HA_OUTPUT_ROOT}/high-availability-1/kube_config.yaml"
          if [ ! -s "$local_kubeconfig" ]; then
            echo "Expected local kubeconfig for shell helper probe at $local_kubeconfig"
            exit 1
          fi
          probe_base="rr-shell-probe-${HA_RANCHER_RUN_ID:-${{ github.run_id }}}"
          probe_base="$(printf '%s' "$probe_base" | tr -c 'a-zA-Z0-9-' '-' | tr '[:upper:]' '[:lower:]' | cut -c1-45)"
          probe_ok=0
          for probe_attempt in 1 2 3; do
            probe="${probe_base}-${probe_attempt}"
            probe_sa="${probe}-sa"
            probe_crb="${probe}-crb"
            echo "Probing Rancher shell helper pod startup with $probe_image (attempt $probe_attempt/3)"
            kubectl --kubeconfig "$local_kubeconfig" delete job "$probe" -n kube-system --ignore-not-found=true >/dev/null 2>&1 || true
            kubectl --kubeconfig "$local_kubeconfig" delete clusterrolebinding "$probe_crb" --ignore-not-found=true >/dev/null 2>&1 || true
            kubectl --kubeconfig "$local_kubeconfig" delete serviceaccount "$probe_sa" -n kube-system --ignore-not-found=true >/dev/null 2>&1 || true
            cat <<EOF | kubectl --kubeconfig "$local_kubeconfig" apply -f -
          apiVersion: v1
          kind: ServiceAccount
          metadata:
            name: ${probe_sa}
            namespace: kube-system
          ---
          apiVersion: rbac.authorization.k8s.io/v1
          kind: ClusterRoleBinding
          metadata:
            name: ${probe_crb}
          subjects:
          - kind: ServiceAccount
            name: ${probe_sa}
            namespace: kube-system
          roleRef:
            apiGroup: rbac.authorization.k8s.io
            kind: ClusterRole
            name: cluster-admin
          ---
          apiVersion: batch/v1
          kind: Job
          metadata:
            name: ${probe}
            namespace: kube-system
          spec:
            ttlSecondsAfterFinished: 300
            backoffLimit: 0
            template:
              metadata:
                labels:
                  app: ${probe}
              spec:
                serviceAccountName: ${probe_sa}
                restartPolicy: Never
                containers:
                - name: ${probe}
                  image: ${probe_image}
                  imagePullPolicy: Always
                  securityContext:
                    runAsUser: 0
                    runAsGroup: 0
                  command: ["kubectl", "get", "ns", "kube-system"]
          EOF
            if kubectl --kubeconfig "$local_kubeconfig" wait --for=condition=complete --timeout=180s "job/${probe}" -n kube-system; then
              kubectl --kubeconfig "$local_kubeconfig" logs "job/${probe}" -n kube-system --all-containers=true || true
              probe_ok=1
              kubectl --kubeconfig "$local_kubeconfig" delete job "$probe" -n kube-system --ignore-not-found=true >/dev/null 2>&1 || true
              kubectl --kubeconfig "$local_kubeconfig" delete clusterrolebinding "$probe_crb" --ignore-not-found=true >/dev/null 2>&1 || true
              kubectl --kubeconfig "$local_kubeconfig" delete serviceaccount "$probe_sa" -n kube-system --ignore-not-found=true >/dev/null 2>&1 || true
              break
            fi
            echo "Shell helper probe attempt $probe_attempt did not complete"
            kubectl --kubeconfig "$local_kubeconfig" get pods -n kube-system -l "app=${probe}" -o wide || true
            kubectl --kubeconfig "$local_kubeconfig" describe pod -n kube-system -l "app=${probe}" || true
            kubectl --kubeconfig "$local_kubeconfig" logs "job/${probe}" -n kube-system --all-containers=true || true
            kubectl --kubeconfig "$local_kubeconfig" get events -n kube-system --sort-by=.lastTimestamp | grep -E "${probe}|Pulling|Pulled|Failed|BackOff|ErrImage|ImagePull|PodInitializing" | tail -80 || true
            kubectl --kubeconfig "$local_kubeconfig" delete job "$probe" -n kube-system --ignore-not-found=true >/dev/null 2>&1 || true
            kubectl --kubeconfig "$local_kubeconfig" delete clusterrolebinding "$probe_crb" --ignore-not-found=true >/dev/null 2>&1 || true
            kubectl --kubeconfig "$local_kubeconfig" delete serviceaccount "$probe_sa" -n kube-system --ignore-not-found=true >/dev/null 2>&1 || true
            sleep 45
          done
          if [ "$probe_ok" -ne 1 ]; then
            echo "Rancher shell helper probe failed after 3 attempts"
            exit 1
          fi

          mkdir -p automation-output
          mkdir -p test-results

          go install gotest.tools/gotestsum@latest
          go_bin="$(go env GOPATH)/bin"
          export PATH="$go_bin:$PATH"
          echo "$go_bin" >> "$GITHUB_PATH"

          tests_ref="${INPUT_RANCHER_TESTS_REF:-${RANCHER_TESTS_REF:-main}}"
          tests_dir="$RUNNER_TEMP/rancher-tests"
          git clone --depth 1 --branch "$tests_ref" https://github.com/rancher/tests.git "$tests_dir"

          config_path="$RUNNER_TEMP/cattle-config.yaml"
          cat > "$config_path" <<EOF
          rancher:
            host: "$RANCHER_HOST"
            adminToken: "$RANCHER_ADMIN_TOKEN"
            cleanup: true
            insecure: true
            clusterName: "$CLUSTER_NAME"
          EOF
          chmod 600 "$config_path"

          results_json="automation-output/rancher-test-results.json"
          results_tmp="$(mktemp)"
          printf '[]' > "$results_tmp"

          append_result() {
            suite="$1"
            package="$2"
            test_run="$3"
            junit="$4"
            conclusion="$5"
            tmp="$(mktemp)"
            jq \
              --arg suite "$suite" \
              --arg package "$package" \
              --arg test_run "$test_run" \
              --arg junit "$junit" \
              --arg conclusion "$conclusion" \
              '. + [{suite: $suite, package: $package, test_run: $test_run, junit: $junit, conclusion: $conclusion}]' \
              "$results_tmp" > "$tmp"
            mv "$tmp" "$results_tmp"
          }

          failed=0
          suite_settle_seconds="${RANCHER_TEST_SUITE_SETTLE_SECONDS:-30}"
          suite_index=0

          for suite in "${suites[@]}"; do
            suite_index=$((suite_index + 1))
            if [ "$suite_index" -gt 1 ] && [ "$suite_settle_seconds" -gt 0 ]; then
              echo "Settling for ${suite_settle_seconds}s before running $suite"
              sleep "$suite_settle_seconds"
            fi

            case "$suite" in
              TestWebhookTestSuite)
                package="./validation/charts"
                test_run="TestWebhookTestSuite"
                tags="validation,infra.any,cluster.any"
                timeout_args=()
                junit_name="TestWebhookTestSuite"
                if [ "${{ inputs.lane }}" = "webhook-candidate-on-previous" ]; then
                  test_run="TestWebhookTestSuite/TestWebhookChart/Verify_(webhook_pod_logs|the_count_of_webhook_is_greater_than_zero)"
                  junit_name="TestWebhookTestSuite-WebhookOverride"
                fi
                ;;
              TestConfigMapTestSuite)
                package="./validation/configmaps"
                test_run="TestConfigMapTestSuite"
                tags="validation,infra.any,cluster.any"
                timeout_args=()
                junit_name="TestConfigMapTestSuite"
                ;;
              TestSchemaChangesTestSuite)
                package="./validation/schemas"
                test_run="TestSchemaChangesTestSuite"
                tags="validation,infra.any,cluster.any"
                timeout_args=()
                junit_name="TestSchemaChangesTestSuite"
                ;;
              TestNodeAnnotationsTestSuite)
                package="./validation/nodeannotations"
                test_run="TestNodeAnnotationsTestSuite"
                tags="validation,infra.any,cluster.any"
                timeout_args=()
                junit_name="TestNodeAnnotationsTestSuite"
                ;;
              TestVaiTestSuite/TestVaiEnabled)
                package="./validation/steve/vai"
                test_run="TestVaiTestSuite/TestVaiEnabled"
                tags="validation,infra.any,cluster.any,extended"
                timeout_args=("-timeout" "30m")
                junit_name="TestVaiTestSuite-TestVaiEnabled"
                ;;
              TestVaiTestSuite/TestVaiDisabled)
                package="./validation/steve/vai"
                test_run="TestVaiTestSuite/TestVaiDisabled"
                tags="validation,infra.any,cluster.any,extended"
                timeout_args=("-timeout" "30m")
                junit_name="TestVaiTestSuite-TestVaiDisabled"
                ;;
              TestWebhookSecuritySettingsTestSuite)
                package="./validation/charts"
                test_run="TestWebhookSecuritySettingsTestSuite"
                tags="validation,infra.any,cluster.any"
                timeout_args=()
                junit_name="TestWebhookSecuritySettingsTestSuite"
                ;;
              *)
                echo "Unknown rancher/tests target $suite"
                exit 1
                ;;
            esac

            junit="$GITHUB_WORKSPACE/test-results/${junit_name}.xml"
            suite_log="$GITHUB_WORKSPACE/test-results/${junit_name}.log"
            echo "Running rancher/tests $package -run $test_run against cluster $CLUSTER_NAME"
            run_suite() {
              (
                cd "$tests_dir"
                CATTLE_TEST_CONFIG="$config_path" gotestsum \
                  --format standard-verbose \
                  --junitfile "$junit" \
                  -- \
                  -v "${timeout_args[@]}" -tags="$tags" "$package" -run "$test_run" 2>&1 | tee "$suite_log"
                exit "${PIPESTATUS[0]}"
              )
            }
            set +e
            run_suite
            exit_code=$?
            set -e
            transient_retry_reason=""
            if [ "$exit_code" -ne 0 ] && grep -Eq "error streaming pod logs for pod .*kubectl-|image can't be pulled|PodInitializing" "$suite_log"; then
              transient_retry_reason="Rancher shell helper pod startup failure"
            elif [ "$exit_code" -ne 0 ] &&
              grep -Fq 'expected: "404"' "$suite_log" &&
              grep -Fq 'actual  : "503"' "$suite_log"; then
              transient_retry_reason="schema endpoint returned a temporary 503 while waiting for CRD deletion to settle"
            fi
            if [ -n "$transient_retry_reason" ]; then
              echo "Detected $transient_retry_reason in $suite; waiting 90s and retrying once"
              sleep 90
              set +e
              run_suite
              exit_code=$?
              set -e
            fi

            conclusion="success"
            if [ "$exit_code" -ne 0 ]; then
              conclusion="failure"
              failed=1
            fi
            append_result "$test_run" "$package" "$test_run" "test-results/${junit_name}.xml" "$conclusion"
          done

          jq -n \
            --arg repo "https://github.com/rancher/tests.git" \
            --arg ref "$tests_ref" \
            --arg lane "${{ inputs.lane }}" \
            --arg rancher_version "${{ inputs.rancher_version }}" \
            --slurpfile results "$results_tmp" \
            '{repo: $repo, ref: $ref, lane: $lane, rancher_version: $rancher_version, results: $results[0]}' \
            > "$results_json"

          if [ "$failed" -ne 0 ]; then
            exit 1
          fi
          SCRIPT

      - name: Refresh AWS credentials before cleanup
        if: ${{ always() }}
        uses: aws-actions/configure-aws-credentials@ec61189d14ec14c8efccab744f656cffd0e33f37 # v6.1.0
        with:
          role-to-assume: ${{ secrets.AWS_AUTOMATION_ROLE_ARN }}
          aws-region: ${{ vars.AWS_REGION }}

      - name: Delete downstream Linode K3s
        id: delete_downstream
        if: ${{ always() && steps.render_config.outcome == 'success' && inputs.lane != 'framework-regression' && (inputs.keep_infra_on_failure == false || (steps.setup.outcome == 'success' && steps.ready.outcome == 'success' && steps.downstream.outcome == 'success' && (inputs.lane != 'webhook-upgrade' || steps.upgrade.outcome == 'success') && (inputs.lane != 'webhook-candidate-on-previous' || (steps.local_webhook.outcome == 'success' && steps.downstream_webhook.outcome == 'success')) && (inputs.run_rancher_tests == false || steps.rancher_tests.outcome == 'success'))) }}
        run: go test -v -run '^TestHADeleteLinodeDownstream$' -timeout 25m ./terratest

      - name: Run lane cleanup
        id: cleanup
        if: ${{ always() && steps.render_config.outcome == 'success' && (inputs.keep_infra_on_failure == false || (steps.setup.outcome == 'success' && steps.ready.outcome == 'success' && (inputs.lane == 'framework-regression' || steps.downstream.outcome == 'success') && (inputs.lane != 'webhook-upgrade' || steps.upgrade.outcome == 'success') && (inputs.lane != 'webhook-candidate-on-previous' || (steps.local_webhook.outcome == 'success' && steps.downstream_webhook.outcome == 'success')) && (inputs.run_rancher_tests == false || steps.rancher_tests.outcome == 'success'))) }}
        run: go test -v -run '^TestHACleanup$' -timeout 30m ./terratest

      - name: Render sign-off report
        if: always()
        continue-on-error: true
        run: |
          "$RUNNER_TEMP/render-signoff-report" \
            -plan signoff-plan.json \
            -lane "${{ inputs.lane }}" \
            -output automation-output/signoff-report.md

      - name: Write lane receipt
        id: lane_receipt
        if: always()
        env:
          TARGET_RANCHER_VERSION: ${{ inputs.rancher_version }}
          PREVIOUS_RANCHER_VERSION: ${{ github.event.inputs.previous_rancher_version || '' }}
          WEBHOOK_IMAGE: ${{ github.event.inputs.webhook_image || '' }}
          SIGNING_POLICY: ${{ inputs.signing_policy }}
          SETUP_OUTCOME: ${{ steps.setup.outcome }}
          READY_OUTCOME: ${{ steps.ready.outcome }}
          LOCAL_SUITE_ENV_OUTCOME: ${{ steps.local_suite_env.outcome }}
          DOWNSTREAM_OUTCOME: ${{ steps.downstream.outcome }}
          LOCAL_WEBHOOK_OUTCOME: ${{ steps.local_webhook.outcome }}
          DOWNSTREAM_WEBHOOK_OUTCOME: ${{ steps.downstream_webhook.outcome }}
          UPGRADE_OUTCOME: ${{ steps.upgrade.outcome }}
          WEBHOOK_CHART_OUTCOME: ${{ steps.webhook_chart.outcome }}
          RANCHER_TESTS_OUTCOME: ${{ steps.rancher_tests.outcome }}
          DELETE_DOWNSTREAM_OUTCOME: ${{ steps.delete_downstream.outcome }}
          CLEANUP_OUTCOME: ${{ steps.cleanup.outcome }}
        run: |
          set -euo pipefail

          mkdir -p automation-output/public
          plan_json='{}'
          lane_plan='{}'
          if [ -s signoff-plan.json ]; then
            plan_json="$(jq -c '.' signoff-plan.json)"
            lane_plan="$(jq -c --arg lane "${{ inputs.lane }}" '((.lanes // []) | map(select(.name == $lane)) | first) // {}' signoff-plan.json)"
          else
            target_json='{}'
            if [ -s signoff-targets.json ]; then
              target_json="$(jq -c --arg version "$TARGET_RANCHER_VERSION" '
                ((.targets // [])
                  | map(select(.rancher_version == $version))
                  | first) // {}
              ' signoff-targets.json)"
            fi
            plan_json="$(jq -n -c \
              --argjson target "$target_json" \
              --arg target_version "$TARGET_RANCHER_VERSION" \
              --arg previous_version "$PREVIOUS_RANCHER_VERSION" \
              --arg webhook_image "$WEBHOOK_IMAGE" \
              --arg signing_policy "$SIGNING_POLICY" \
              '{
                target_version: ($target.rancher_version // $target_version),
                previous_version: (if $previous_version != "" then $previous_version else ($target.previous_rancher_version // null) end),
                webhook_image: (if $webhook_image != "" then $webhook_image else ($target.webhook_image // null) end),
                signing_policy: (if $signing_policy != "" and $signing_policy != "auto" then $signing_policy else ($target.signing_policy // $signing_policy) end)
              }')"
          fi
          tests_json='[]'
          if [ -s automation-output/rancher-test-results.json ]; then
            tests_json="$(jq -c '.results // [] | map({
              suite: .suite,
              package: .package,
              status: .conclusion
            })' automation-output/rancher-test-results.json)"
          fi

          jq -n \
            --argjson plan "$plan_json" \
            --argjson lane_plan "$lane_plan" \
            --argjson tests "$tests_json" \
            --arg lane "${{ inputs.lane }}" \
            --arg status "${{ job.status }}" \
            --arg completed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
            --arg rke2_server_count "${{ inputs.rke2_server_count }}" \
            --arg setup "$SETUP_OUTCOME" \
            --arg ready "$READY_OUTCOME" \
            --arg local_suite_env "$LOCAL_SUITE_ENV_OUTCOME" \
            --arg downstream "$DOWNSTREAM_OUTCOME" \
            --arg local_webhook "$LOCAL_WEBHOOK_OUTCOME" \
            --arg downstream_webhook "$DOWNSTREAM_WEBHOOK_OUTCOME" \
            --arg upgrade "$UPGRADE_OUTCOME" \
            --arg webhook_chart "$WEBHOOK_CHART_OUTCOME" \
            --arg rancher_tests "$RANCHER_TESTS_OUTCOME" \
            --arg delete_downstream "$DELETE_DOWNSTREAM_OUTCOME" \
            --arg cleanup "$CLEANUP_OUTCOME" \
            '{
              target_version: ($plan.target_version // ""),
              previous_version: ($plan.previous_version // null),
              lane: $lane,
              status: $status,
              completed_at: $completed_at,
              rke2_server_count: ($rke2_server_count | tonumber),
              webhook_image: ($plan.webhook_image // null),
              terraform_state_key: ($lane_plan.terraform_state_key // ""),
              aws_prefix: ($lane_plan.aws_prefix // ""),
              steps: {
                setup: $setup,
                ready: $ready,
                local_suite_env: $local_suite_env,
                downstream: $downstream,
                local_webhook: $local_webhook,
                downstream_webhook: $downstream_webhook,
                upgrade: $upgrade,
                webhook_chart: $webhook_chart,
                rancher_tests: $rancher_tests,
                delete_downstream: $delete_downstream,
                cleanup: $cleanup
              },
              tests: $tests
            }' > automation-output/public/signoff-lane-receipt.json

      - name: Upload lane receipt
        if: ${{ always() && steps.lane_receipt.outcome == 'success' }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
        with:
          name: signoff-lane-${{ inputs.rancher_version }}-${{ inputs.lane }}
          path: automation-output/public/signoff-lane-receipt.json

      - name: Write step summary
        if: always()
        run: |
          {
            echo "## Rancher sign-off lane"
            echo ""
            echo "- Lane: \`${{ inputs.lane }}\`"
            echo "- Rancher version: \`${{ inputs.rancher_version }}\`"
            echo "- Upgrade version: \`${RANCHER_UPGRADE_VERSION:-none}\`"
            echo "- RKE2 server count: \`${{ inputs.rke2_server_count }}\`"
            echo "- Terraform state key: \`${TF_STATE_KEY:-unset}\`"
            echo "- Setup outcome: \`${{ steps.setup.outcome }}\`"
            echo "- Readiness outcome: \`${{ steps.ready.outcome }}\`"
            echo "- Local suite env outcome: \`${{ steps.local_suite_env.outcome }}\`"
            echo "- Downstream outcome: \`${{ steps.downstream.outcome }}\`"
            echo "- Local webhook override outcome: \`${{ steps.local_webhook.outcome }}\`"
            echo "- Downstream webhook override outcome: \`${{ steps.downstream_webhook.outcome }}\`"
            echo "- Upgrade outcome: \`${{ steps.upgrade.outcome }}\`"
            echo "- Rancher tests outcome: \`${{ steps.rancher_tests.outcome }}\`"
            echo "- Cleanup requested: \`${{ steps.render_config.outcome == 'success' && (inputs.keep_infra_on_failure == false || (steps.setup.outcome == 'success' && steps.ready.outcome == 'success' && (inputs.lane == 'framework-regression' || steps.downstream.outcome == 'success') && (inputs.lane != 'webhook-upgrade' || steps.upgrade.outcome == 'success') && (inputs.lane != 'webhook-candidate-on-previous' || (steps.local_webhook.outcome == 'success' && steps.downstream_webhook.outcome == 'success')) && (inputs.run_rancher_tests == false || steps.rancher_tests.outcome == 'success'))) }}\`"
          } >> "$GITHUB_STEP_SUMMARY"