validate_vllm.yml

name: Validate New vLLM Release

on:
  workflow_dispatch:
    inputs:
      lite:
        description: 'Run in lite mode (smallest model only)'
        type: boolean
        default: false
  pull_request:
    types: [opened, synchronize, reopened, labeled]
  merge_group:
  schedule:
    # Sunday at 6:00 PM UTC
    - cron: "0 18 * * 0"

permissions:
  contents: write
  pull-requests: write

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  LEMONADE_CI_MODE: "True"
  PYTHONIOENCODING: utf-8
  HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
  LITE_MODE: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' || (github.event_name == 'workflow_dispatch' && inputs.lite == true) }}

jobs:
  # ========================================================================
  # Discover the latest vllm-rocm release
  # ========================================================================
  get-latest-releases:
    name: Get latest releases
    # Only the auto-update runs resolve upstream releases. Everywhere else the
    # build validates the pins committed in backend_versions.json, so a bad
    # upstream release can't fail a merge or hide a broken pin.
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    outputs:
      vllm_rocm_release: ${{ steps.vllm_rocm.outputs.release }}
    steps:
      - name: Get latest vllm-rocm release
        id: vllm_rocm
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          RELEASE=$(gh api repos/lemonade-sdk/vllm-rocm/releases/latest --jq '.tag_name')
          echo "Latest vllm-rocm release: $RELEASE"
          echo "release=$RELEASE" >> "$GITHUB_OUTPUT"

  # ========================================================================
  # Build binaries with updated backend version
  # ========================================================================
  build:
    name: Build (Linux)
    needs: get-latest-releases
    # Label a PR `ci:upgrades` to opt it back in.
    if: (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:upgrades')) && always() && !failure() && !cancelled()
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Update backend_versions.json
        if: needs.get-latest-releases.result == 'success'
        env:
          VLLM_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.vllm_rocm_release }}
        run: |
          set -e
          python3 - << 'PY'
          import json, os
          path = "src/cpp/resources/backend_versions.json"
          with open(path) as f:
              data = json.load(f)
          release = os.environ["VLLM_ROCM_RELEASE"]
          old = data["vllm"].get("rocm", "")
          data["vllm"]["rocm"] = release
          with open(path, "w") as f:
              json.dump(data, f, indent=2)
              f.write("\n")
          print(f"vllm.rocm: {old} -> {release}")
          PY

      - name: Build Lemonade server
        run: |
          set -e
          ./setup.sh
          # Skip the web app; CI only exercises the backend.
          cmake -DBUILD_WEB_APP=OFF --preset default
          cmake --build --preset default
          test -f build/lemond || { echo "lemond not found!"; exit 1; }
          echo "Build successful!"

      - name: Upload build artifacts
        uses: actions/upload-artifact@v7
        with:
          name: vllm-build
          path: |
            build/lemond
            build/lemonade
            build/resources/
          retention-days: 7

  # ========================================================================
  # Validate backend on self-hosted runner
  # ========================================================================
  validate:
    name: Validate vllm (rocm)
    needs: [get-latest-releases, build]
    if: always() && needs.build.result == 'success'
    runs-on: [self-hosted, stx-halo, Linux, lemon-prod]
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true

      - name: Download build artifacts
        uses: actions/download-artifact@v7
        with:
          name: vllm-build
          path: build

      - name: Verify binaries
        run: |
          set -e
          chmod +x build/lemond build/lemonade
          ./build/lemond --version
          echo "Binaries verified!"

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run validation with lemond
        env:
          HOME: ${{ github.workspace }}/.home
        run: |
          set -e
          mkdir -p "$HOME"
          echo "HOME=$HOME" >> "$GITHUB_ENV"

          CACHE_DIR="$(pwd)/ci-cache-vllm"
          LOGS_DIR="$(pwd)/server-logs-rocm"
          mkdir -p "$CACHE_DIR" "$LOGS_DIR"
          mkdir -p "$RUNNER_TEMP/xdg-runtime"
          chmod 700 "$RUNNER_TEMP/xdg-runtime"
          echo "XDG_RUNTIME_DIR=$RUNNER_TEMP/xdg-runtime" >> "$GITHUB_ENV"
          export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime"

          # This validate job runs on a shared self-hosted runner, so the default
          # port 13305 may already be held by another job. lemond now refuses to
          # bind an in-use port, so start on a job-unique free port and pass it to
          # validate_vllm.py and the shutdown call.
          pick_free_port() {
            if command -v python3 >/dev/null 2>&1; then
              python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()' && return 0
            fi
            for _ in $(seq 1 50); do
              local p=$(( (RANDOM % 20000) + 20000 ))
              if ! ss -ltn 2>/dev/null | grep -q ":$p "; then echo "$p"; return 0; fi
            done
            return 1
          }

          LEMOND_PID=""
          started=0
          for start_attempt in $(seq 1 5); do
            PORT=$(pick_free_port) || { echo "ERROR: could not find a free port"; exit 1; }
            echo "Attempt $start_attempt: starting lemond on 127.0.0.1:$PORT..."
            ./build/lemond "$CACHE_DIR" --port "$PORT" --host 127.0.0.1 \
              > "$LOGS_DIR/lemond.stdout.log" 2> "$LOGS_DIR/lemond.stderr.log" &
            LEMOND_PID=$!
            for i in $(seq 1 30); do
              if curl -sf "http://127.0.0.1:$PORT/api/v1/health" > /dev/null 2>&1; then
                echo "lemond is healthy on port $PORT (PID $LEMOND_PID)"
                started=1
                break
              fi
              if ! kill -0 "$LEMOND_PID" 2>/dev/null; then
                echo "lemond (PID $LEMOND_PID) exited prematurely on port $PORT"
                cat "$LOGS_DIR/lemond.stderr.log" 2>/dev/null || true
                break
              fi
              sleep 2
            done
            [ "$started" = "1" ] && break
            echo "Start attempt $start_attempt failed; retrying on a new port..."
            kill -9 "$LEMOND_PID" 2>/dev/null || true
            sleep 1
          done
          if [ "$started" != "1" ]; then
            echo "ERROR: lemond failed to start after multiple attempts"
            exit 1
          fi

          EXIT_CODE=0
          VALIDATION_ARGS=(
            test/validate_vllm.py
            --backend rocm
            --port "$PORT"
            --output vllm_validation_rocm.json
            --logs-dir "$LOGS_DIR"
          )
          if [ "${LITE_MODE}" = "true" ]; then
            VALIDATION_ARGS+=(--lite)
          fi

          .venv/bin/python "${VALIDATION_ARGS[@]}" || EXIT_CODE=$?

          # Shut the server down cleanly so logs flush.
          curl -sf -X POST "http://127.0.0.1:$PORT/internal/shutdown" || true
          sleep 2
          kill $LEMOND_PID 2>/dev/null || true

          exit $EXIT_CODE

      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: validation-results-rocm
          path: vllm_validation_rocm.json
          retention-days: 30

      - name: Upload server logs
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: server-logs-rocm
          path: server-logs-rocm/
          retention-days: 30
          if-no-files-found: ignore

      - name: Cleanup
        if: always()
        run: |
          pkill -9 -f lemond 2>/dev/null || true
          pkill -9 -f "vllm" 2>/dev/null || true
          pkill -9 -f "resource_tracker" 2>/dev/null || true

  # ========================================================================
  # Create PR if validation passed (schedule or workflow_dispatch only)
  # ========================================================================
  create-pr:
    name: Create update PR
    needs: [get-latest-releases, validate]
    runs-on: ubuntu-latest
    if: >-
      (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') &&
      needs.validate.result == 'success'
    steps:
      - uses: actions/checkout@v5

      - name: Download validation results
        uses: actions/download-artifact@v7
        with:
          pattern: validation-results-*
          merge-multiple: true

      - name: Update backend_versions.json
        env:
          VLLM_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.vllm_rocm_release }}
        run: |
          python3 - << 'PY'
          import json, os
          path = 'src/cpp/resources/backend_versions.json'
          with open(path) as f:
              data = json.load(f)
          rocm = os.environ['VLLM_ROCM_RELEASE']
          data['vllm']['rocm'] = rocm
          with open(path, 'w') as f:
              json.dump(data, f, indent=2)
              f.write('\n')
          print(f"Updated: vllm.rocm = {rocm}")
          PY

      - name: Generate PR body
        env:
          VLLM_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.vllm_rocm_release }}
        run: |
          python3 << 'PYEOF'
          import json, os

          lines = []
          rocm = os.environ.get("VLLM_ROCM_RELEASE", "unknown")

          lines.append("## Auto-update vLLM backend")
          lines.append("")
          lines.append(f"- **vllm-rocm**: `{rocm}`")
          lines.append("")
          lines.append("## Validation Results")
          lines.append("")
          lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |")
          lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|")

          results_file = "vllm_validation_rocm.json"
          if not os.path.isfile(results_file):
              lines.append("| _rocm: no results_ | | | | | | |")
          else:
              with open(results_file) as f:
                  results = json.load(f)
              for r in results:
                  status = "PASS" if r["pass"] else "FAIL"
                  resp = str(r.get("response", ""))[:80].replace("|", "\\|").replace("\n", " ")
                  ttft = r.get("time_to_first_token", "N/A")
                  if isinstance(ttft, float):
                      ttft = f"{ttft:.3f}s"
                  tps = r.get("tokens_per_second", "N/A")
                  if isinstance(tps, float):
                      tps = f"{tps:.1f}"
                  lines.append(
                      f"| {r['model']} (rocm) | {status} | {resp} | "
                      f"{r.get('input_tokens', 'N/A')} | {r.get('output_tokens', 'N/A')} | "
                      f"{ttft} | {tps} |"
                  )

          lines.append("")
          lines.append("---")
          lines.append("*Auto-generated by validate_vllm workflow*")

          with open("pr_body.md", "w") as f:
              f.write("\n".join(lines))
          PYEOF

      - name: Create Pull Request
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          VLLM_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.vllm_rocm_release }}
        run: |
          BRANCH="auto/vllm-update-${VLLM_ROCM_RELEASE}"

          EXISTING_PR=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || echo "")
          if [ -n "$EXISTING_PR" ]; then
            echo "PR #$EXISTING_PR already exists for branch $BRANCH. Skipping."
            exit 0
          fi

          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          if git diff --quiet src/cpp/resources/backend_versions.json; then
            echo "backend_versions.json is unchanged — nothing to update."
            exit 0
          fi

          git checkout -b "$BRANCH"
          git add src/cpp/resources/backend_versions.json
          git commit -m "Update vllm-rocm to ${VLLM_ROCM_RELEASE}"
          git push origin "$BRANCH"

          gh pr create \
            --title "Update vllm-rocm to ${VLLM_ROCM_RELEASE}" \
            --body-file pr_body.md \
            --base main \
            --head "$BRANCH"

  # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran
  # successfully (otherwise the merge is blocked), and 2. these jobs do not need
  # to run on ordinary pull request pushes.
  validation-gate:
    name: vLLM validation
    needs: [build, validate]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check gated jobs
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...}
          # Fail if any job broke. In the merge queue, also fail if any never ran.
          echo "$NEEDS"
          broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS")
          if [ -n "$broke" ]; then
            echo "FAILED: $broke"
            exit 1
          fi
          if [ "${{ github.event_name }}" = "merge_group" ]; then
            absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS")
            if [ -n "$absent" ]; then
              echo "DID NOT RUN IN MERGE QUEUE: $absent"
              exit 1
            fi
          fi