cpp_server_build_test_release.yml

name: C++ Server Build, Test, and Release 🚀

on:
  push:
    branches: ["release-v*"]
    tags:
      - v*
  # `labeled` is here so the ci:* escape hatches take effect without a push.
  # Adding `types` replaces the defaults, so the three default activity types
  # have to be listed explicitly. `unlabeled` is deliberately absent: removing
  # a label only ever drops work, and including it would re-run everything.
  pull_request:
    types: [opened, synchronize, reopened, labeled]
  merge_group:
  workflow_dispatch:
    inputs:
      enable_signing:
        description: 'Enable MSI signing with SignPath (for testing)'
        required: false
        default: false
        type: boolean
      disable_macos_signing:
        description: 'Force the unsigned macOS path (reproduces a fork PR, which has no Apple secrets). Skips the self-hosted inference rigs.'
        required: false
        default: false
        type: boolean

permissions:
  contents: write
  actions: read  # Required for SignPath to read workflow/job details

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  LEMONADE_DISABLE_SYSTEMD_JOURNAL: "1"

jobs:
  # ========================================================================
  # BUILD JOBS - Run on rai-160-sdk workers
  # ========================================================================

  build-lemonade-server-installer:
    name: Build Lemonade Server Installer
    runs-on: windows-latest
    outputs:
      unsigned-artifact-id: ${{ steps.upload-unsigned-msi.outputs.artifact-id }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Install CMake if not available
        uses: ./.github/actions/install-cmake-windows

      - name: Install WiX Toolset
        uses: ./.github/actions/install-wix

      - name: Cache FetchContent dependencies
        uses: actions/cache@v5
        with:
          path: build/_deps
          # vs2026 marker: VS 2022-built _deps are generator-incompatible with the
          # VS 2026 configure, so they must not be restored across the toolchain bump.
          key: fetchcontent-windows-vs2026-${{ hashFiles('CMakeLists.txt') }}
          restore-keys: |
            fetchcontent-windows-vs2026-

      - name: Run setup.ps1 to configure environment
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          .\setup.ps1

      - name: Build installers
        shell: PowerShell
        env:
          # The Tauri desktop app (cargo + webpack) is roughly a third of this
          # build, and it only ends up in lemonade.msi. Every downstream test
          # job installs lemonade-server-minimal.msi, so on the PR and
          # merge-queue paths the full installer is pure critical-path latency;
          # build-lemonade-desktop-installer covers it off the critical path.
          BUILD_FULL_INSTALLER: ${{ startsWith(github.ref, 'refs/tags/v') || inputs.enable_signing == true }}
        run: |
          $ErrorActionPreference = "Stop"

          $full = $env:BUILD_FULL_INSTALLER -eq 'true'
          $target = if ($full) { "wix_installers" } else { "wix_installer_minimal" }
          # wix_installers is the only target that owns two wix build runs, and
          # it is only ever built here on the release and signing paths — so a
          # parallel configuration used there would first run for real during a
          # release. Build it serially; the two MSIs take seconds either way.
          $nodes = if ($full) { "/m:1" } else { "/m:2" }
          $expected = @(
              "build\Release\lemond.exe",
              "build\Release\LemonadeServer.exe",
              "build\resources\web-app\index.html",
              "lemonade-server-minimal.msi"
          )
          if ($full) { $expected += "build\app\lemonade-app.exe", "lemonade.msi" }

          Write-Host "Building target $target..." -ForegroundColor Cyan

          # Build by directory (not --preset) so we follow whichever generator
          # setup.ps1 auto-detected for the installed Visual Studio version.
          # /m lets MSBuild overlap independent projects — the lemonade CLI and
          # LemonadeServer otherwise wait for lemonade-server-core to finish.
          # Capped at 2 because /MP already spawns one cl.exe per core inside
          # each project, and the two multiply: unbounded, a 4-core runner can
          # reach 16 concurrent compilers and run itself out of heap.
          # Passed straight through rather than via --parallel, which makes
          # CMake set CL_MPCount=1 and cancel the /MP in CMakeLists.txt.
          cmake --build build --config Release --target $target -- $nodes
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          # Verify outputs
          $failures = @()
          $expected | ForEach-Object {
              if (-not (Test-Path $_)) {
                  Write-Host "ERROR: $_ not found!" -ForegroundColor Red
                  $failures += $_
              }
          }

          if ($failures.Count -gt 0) { exit 1 }
          Write-Host "Build and packaging successful!" -ForegroundColor Green

      - name: Run MCP client unit test on Windows
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          # Exercise the Windows-only process, quoting, environment-block, and
          # pipe/handle paths that Linux and macOS cannot cover.
          cmake --build build --config Release --target test_mcp_client_config
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          ctest --test-dir build -C Release --output-on-failure -R "^mcp_client_config$"
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

      - name: Upload Lemonade Server Installers
        id: upload-unsigned-msi
        uses: actions/upload-artifact@v7
        with:
          name: Lemonade_Server_MSI
          # Glob rather than two literal paths: lemonade.msi is only built on
          # the release/signing path (see BUILD_FULL_INSTALLER above).
          path: lemonade*.msi
          retention-days: 7

  build-lemonade-desktop-installer:
    name: Build Lemonade Desktop Installer
    # Packaging-only variant: it builds the Tauri desktop app and lemonade.msi,
    # which no test job installs. The merge queue gates every merge, so PR
    # pushes skip it and any desktop/full-MSI break still blocks at merge-queue
    # time. It reports through the `Packaging builds` gate, so it takes that
    # gate's `ci:distros` label rather than introducing a second label for one
    # check. The release paths are excluded because the build above already
    # produces lemonade.msi there; that first clause is the negation of
    # BUILD_FULL_INSTALLER above, spelled the same way so the two stay
    # provably complementary.
    if: >-
      !(startsWith(github.ref, 'refs/tags/v') || inputs.enable_signing == true) &&
      (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:distros'))
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Install CMake if not available
        uses: ./.github/actions/install-cmake-windows

      - name: Install WiX Toolset
        uses: ./.github/actions/install-wix

      - name: Restore FetchContent dependencies
        # Restore-only, and deliberately sharing build-lemonade-server-installer's
        # namespace: both jobs run bare setup.ps1, so they configure the same
        # preset and produce an identical build/_deps. That job runs
        # concurrently and owns the save; saving from here too would just race
        # it for the same key and log a reservation failure every cold run.
        # Best-effort by construction: on a run that changes CMakeLists.txt
        # neither job can hit the exact key, so this one falls back through
        # restore-keys or builds the dependencies cold.
        uses: actions/cache/restore@v5
        with:
          path: build/_deps
          key: fetchcontent-windows-vs2026-${{ hashFiles('CMakeLists.txt') }}
          restore-keys: |
            fetchcontent-windows-vs2026-

      - name: Run setup.ps1 to configure environment
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          .\setup.ps1

      - name: Build desktop installer
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          # /m:1, not /m:2: wix_installer_full depends on both web-app and
          # tauri-app, and they both touch src/app — web-app robocopies the whole
          # tree while tauri-app runs npm ci inside it. Run concurrently, robocopy
          # hits files npm is rewriting and retries under its default
          # /R:1000000 /W:30, hanging the job rather than failing.
          cmake --build build --config Release --target wix_installer_full -- /m:1
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          $failures = @()
          @(
              "build\app\lemonade-app.exe",
              "build\resources\web-app\index.html",
              "lemonade.msi"
          ) | ForEach-Object {
              if (-not (Test-Path $_)) {
                  Write-Host "ERROR: $_ not found!" -ForegroundColor Red
                  $failures += $_
              }
          }
          if ($failures.Count -gt 0) { exit 1 }
          Write-Host "Desktop installer built successfully!" -ForegroundColor Green

      - name: Upload Lemonade Desktop Installer
        # Without this the MSI is built and discarded, so nothing installs it
        # until a tag. Note this is a new artifact name, not a restoration of
        # Lemonade_Server_MSI: that one now carries only the minimal MSI off the
        # release path. No in-repo consumer reads lemonade.msi from it.
        uses: actions/upload-artifact@v7
        with:
          name: Lemonade_Desktop_MSI
          path: lemonade.msi
          retention-days: 7

  tauri-compile-check:
    name: Compile-check the Tauri host (${{ matrix.os }})
    # The Tauri host is Rust, which `npm run typecheck` does not touch. Both
    # jobs that build it — the Windows desktop installer and the macOS .dmg —
    # defer to the merge queue, so without this a broken src-tauri compiles
    # nowhere until a PR is already queued.
    #
    # Windows is not optional here: webview_shim.rs and lib.rs gate code behind
    # cfg(target_os = "windows"), and webview2-com/windows are Windows-only
    # dependencies, so a Linux-only check would miss exactly the code whose
    # packaging job was deferred. cargo check rather than a full build — this
    # is a compile canary, not a packaging step, so it catches type and borrow
    # errors but not link failures or asset embedding: `custom-protocol` stays
    # off, which is also why it needs no built renderer. Those still surface in
    # the desktop installer and .dmg jobs at merge-queue time.
    #
    # Deliberately uncached: a cold check measures ~1.5 min, and the Actions
    # cache is a 10 GB LRU pool shared with the FetchContent caches the Windows
    # installer jobs restore from.
    #
    # Lives here rather than in Docs And Style so it can report through the
    # `Packaging builds` gate, alongside the desktop installer it stands in for.
    # A new job name would otherwise block nothing until someone registered it
    # in branch protection.
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
    steps:
      - uses: actions/checkout@v5

      - name: Install Tauri build dependencies
        if: runner.os == 'Linux'
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev \
            libayatana-appindicator3-dev librsvg2-dev patchelf

      - name: Compile-check the Tauri host
        working-directory: src/app/src-tauri
        run: cargo check --locked --all-targets

  sign-msi-installers:
    name: Sign MSI Installers with SignPath
    runs-on: windows-latest
    needs: build-lemonade-server-installer
    # Sign on tag pushes (releases) or when manually enabled via workflow_dispatch
    if: startsWith(github.ref, 'refs/tags/v') || inputs.enable_signing == true
    steps:
      - name: Sign MSI Installers with SignPath
        id: sign-msi
        uses: signpath/github-action-submit-signing-request@v2
        with:
          api-token: '${{ secrets.SIGNPATH_API_TOKEN }}'
          organization-id: '8103545b-7814-4edc-86d6-a91dc2a2291b'
          project-slug: 'lemonade'
          signing-policy-slug: 'release-signing'
          github-artifact-id: '${{ needs.build-lemonade-server-installer.outputs.unsigned-artifact-id }}'
          wait-for-completion: true
          wait-for-completion-timeout-in-seconds: 3600
          output-artifact-directory: 'signed-msi'
          parameters: |
            version: "${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'test' }}"

      - name: Verify Signed MSI Files
        shell: PowerShell
        run: |
          Write-Host "Verifying signed MSI files..." -ForegroundColor Cyan

          if (-not (Test-Path "signed-msi\lemonade-server-minimal.msi")) {
              Write-Host "ERROR: Signed lemonade-server-minimal.msi not found!" -ForegroundColor Red
              exit 1
          }

          if (-not (Test-Path "signed-msi\lemonade.msi")) {
              Write-Host "ERROR: Signed lemonade.msi not found!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Signed MSI files verified!" -ForegroundColor Green
          Get-ChildItem -Path "signed-msi" -Recurse | Format-Table Name, Length

      - name: Upload Signed MSI Installers
        uses: actions/upload-artifact@v7
        with:
          name: Lemonade_Server_MSI_Signed
          path: |
            signed-msi/lemonade-server-minimal.msi
            signed-msi/lemonade.msi
          retention-days: 7

  build-lemonade-deb:
    name: Build Lemonade .deb Package
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Use the compiler cache primed into the build image
        # Release builds opt out so their -fdebug-prefix-map keeps the package
        # version dpkg derives from the changelog. Everywhere else it is pinned
        # to the value the prime recorded, which is what makes the primed
        # objects reusable across commits at all. release-v* is included
        # because those pushes also produce .deb and -dbgsym artifacts.
        if: ${{ !startsWith(github.ref, 'refs/tags/') && !startsWith(github.ref, 'refs/heads/release-v') }}
        shell: bash
        run: |
          set -euo pipefail
          # The prime writes this file itself, so an image built without it —
          # including one published before the prime existed — is detected
          # rather than assumed. Compiling without a cache is the safe fallback.
          debugpath_file=/opt/ccache-debugpath
          if [ ! -f "$debugpath_file" ]; then
            echo "No primed compiler cache in this build image; compiling without it."
            exit 0
          fi
          {
            echo "CMAKE_C_COMPILER_LAUNCHER=ccache"
            echo "CMAKE_CXX_COMPILER_LAUNCHER=ccache"
            echo "DEB_BUILD_DEBUGPATH=$(cat "$debugpath_file")"
          } >> "$GITHUB_ENV"
          ccache -z

      - name: Prepare Debian build
        id: get_version
        uses: ./.github/actions/prepare-debian-build
        with:
          release: '24.04'
          codename: 'noble'

      - name: Build Debian package
        id: build_deb
        uses: ./.github/actions/build-debian-package
        with:
          package-type: binary
          deb-version: ${{ steps.get_version.outputs.version }}

      - name: Upload .deb package
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-deb
          path: ${{ steps.build_deb.outputs.output-dir }}/*.deb
          retention-days: 7

      - name: Report compiler cache hit rate
        # Set only by the opt-in step above, so this covers both the release
        # path and an image published before the prime existed.
        if: ${{ always() && env.CMAKE_CXX_COMPILER_LAUNCHER == 'ccache' }}
        shell: bash
        run: |
          set -euo pipefail
          ccache -s
          # Zero hits against a primed image means the prime and this job
          # disagree on the compile command line — most likely the workspace
          # path, which GitHub derives from the repository name and the prime
          # hardcodes. Warn rather than fail: a change to a widely included
          # header legitimately invalidates every entry.
          stats="$(ccache --print-stats 2>/dev/null || true)"
          if ! grep -q '^direct_cache_hit' <<< "$stats"; then
            echo "Could not read ccache counters; skipping the hit-rate check."
            exit 0
          fi
          hits="$(awk -F'\t' '/^(direct|preprocessed)_cache_hit\t/ { n += $2 } END { print n+0 }' <<< "$stats")"
          echo "Cache hits this build: ${hits}"
          if [ "$hits" -eq 0 ]; then
            echo "::warning title=Primed ccache produced no hits::The build image ships a primed cache but nothing matched. Check that the prime path in .ubuntu/Dockerfile still matches this job's workspace."
          fi

  cpp-unit-tests:
    name: C++ unit tests
    # Own job rather than a step of build-lemonade-deb: it shares nothing with
    # the packaging build (different configure, different flags), so as a step
    # it just added its full build time to the critical path that gates every
    # .deb test job. Same build-environment image, so the configure resolves
    # the same system dependencies.
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - uses: actions/checkout@v5

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Install build dependencies
        # As a step of build-lemonade-deb this inherited the build-dep install
        # that job performs. Standalone it would rely on whatever the nightly
        # image baked in, so a newly added Build-Depends would fail here — and
        # not in the .deb job, which installs them — until the image caught up.
        shell: bash
        run: |
          set -euo pipefail
          apt-get update && apt-get build-dep ./contrib -y

      - name: Build and run C++ unit tests
        shell: bash
        run: |
          set -euo pipefail
          cmake --preset default
          cmake --build --preset default --target cpp-ci-tests
          ctest --test-dir build --output-on-failure --no-tests=error -L "^cpp-ci$"

  build-lemonade-debian13:
    name: Build Lemonade .deb Package (Debian 13)${{ matrix.job-suffix }}
    # Packaging-only variant: the merge queue gates every merge, so PR pushes
    # skip it. Any Debian 13 build break still blocks at merge-queue time.
    # Label a PR `ci:distros` to opt it back in.
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:distros')
    runs-on: ${{ matrix.runs-on }}
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    container:
      image: debian:trixie
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - arch-name: X86_64
            runs-on: ubuntu-latest
            deb-arch: amd64
            artifact-suffix: x86_64
            job-suffix: ""
          - arch-name: ARM64
            runs-on: ubuntu-24.04-arm
            deb-arch: arm64
            artifact-suffix: arm64
            job-suffix: " (ARM64)"
    steps:
      - name: Enable Trixie backports
        run: |
          echo "deb http://deb.debian.org/debian trixie-backports main" > /etc/apt/sources.list.d/trixie-backports.list
      - name: Install git
        run: |
          set -e
          apt-get update
          apt-get install -y git
      - name: Install from backports
        run: |
          set -e
          apt-get install -y -t trixie-backports libcpp-httplib-dev node-markdown-it-texmath

      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Prepare Debian build
        id: get_version
        uses: ./.github/actions/prepare-debian-build
        with:
          release: '13'
          codename: 'trixie'

      - name: Build Debian package
        id: build_deb
        uses: ./.github/actions/build-debian-package
        with:
          package-type: binary
          deb-version: ${{ steps.get_version.outputs.version }}

      - name: Rename server .deb for release
        id: rename
        shell: bash
        run: |
          set -e
          VERSION="${{ steps.get_version.outputs.version }}"
          OUTPUT_DIR="${{ steps.build_deb.outputs.output-dir }}"
          SRC=$(ls "$OUTPUT_DIR"/lemonade-server_*_${{ matrix.deb-arch }}.deb | head -n1)
          if [ -z "$SRC" ]; then
            echo "ERROR: server .deb not found in $OUTPUT_DIR"
            ls -la "$OUTPUT_DIR"
            exit 1
          fi
          DEST="$OUTPUT_DIR/lemonade-server_${VERSION}-debian13_${{ matrix.deb-arch }}.deb"
          mv "$SRC" "$DEST"
          echo "Renamed $SRC -> $DEST"
          echo "deb-path=$DEST" >> $GITHUB_OUTPUT

      - name: Upload .deb package
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-debian13-${{ matrix.artifact-suffix }}
          path: ${{ steps.rename.outputs.deb-path }}
          retention-days: 7

  test-lemonade-debian13-smoke:
    name: Smoke test .deb (Debian 13)${{ matrix.job-suffix }}
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:distros')
    runs-on: ${{ matrix.runs-on }}
    needs: build-lemonade-debian13
    container:
      image: debian:trixie
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - arch-name: X86_64
            runs-on: ubuntu-latest
            deb-arch: amd64
            artifact-suffix: x86_64
            job-suffix: ""
          - arch-name: ARM64
            runs-on: ubuntu-24.04-arm
            deb-arch: arm64
            artifact-suffix: arm64
            job-suffix: " (ARM64)"
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-debian13.outputs.version }}
    steps:
      - name: Download Lemonade .deb Package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-debian13-${{ matrix.artifact-suffix }}
          path: .

      - name: Install and smoke test
        shell: bash
        run: |
          set -e
          DEB_FILE=$(ls lemonade-server_*-debian13_${{ matrix.deb-arch }}.deb 2>/dev/null | head -n1)
          if [ -z "$DEB_FILE" ]; then
            echo "ERROR: .deb file not found"
            ls -la *.deb 2>/dev/null || echo "No .deb files found in current directory"
            exit 1
          fi
          echo "Debian 13 ${{ matrix.arch-name }} version: $(cat /etc/os-release | grep PRETTY_NAME)"
          echo "Testing artifact: $DEB_FILE"

          echo "deb http://deb.debian.org/debian trixie-backports main" > /etc/apt/sources.list.d/trixie-backports.list
          apt-get update
          apt-get install -y curl
          apt-get install -y -t trixie-backports ./"$DEB_FILE"

          /usr/bin/lemond --version
          /usr/bin/lemonade --version
          echo "Package installed successfully"

          mkdir -p "$RUNNER_TEMP/xdg-runtime"
          chmod 700 "$RUNNER_TEMP/xdg-runtime"
          export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime"
          /usr/bin/lemond "$(pwd)" > "$RUNNER_TEMP/lemonade-server.log" 2>&1 &
          SERVER_PID=$!
          echo "Server started (PID $SERVER_PID)"

          for i in $(seq 1 30); do
            if curl -sf http://127.0.0.1:13305/live > /dev/null 2>&1; then
              echo "Server is running and healthy on port 13305"
              break
            fi
            if ! kill -0 $SERVER_PID 2>/dev/null; then
              echo "ERROR: Server process exited prematurely"
              echo "=== Server log ==="
              cat "$RUNNER_TEMP/lemonade-server.log" 2>/dev/null || echo "(no log)"
              exit 1
            fi
            echo "Waiting for server... ($i/30)"
            sleep 2
          done
          kill $SERVER_PID 2>/dev/null || true

  build-lemonade-rpm:
    name: Build Lemonade .rpm - Fedora ${{ matrix.fedora-version }}${{ matrix.job-suffix }}
    # Packaging-only variant: the merge queue gates every merge, so PR pushes
    # skip it. Any Fedora build break still blocks at merge-queue time.
    # Label a PR `ci:distros` to opt it back in.
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:distros')
    runs-on: ${{ matrix.runs-on }}
    container:
      image: fedora:${{ matrix.fedora-version }}

    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        fedora-version: ["43", "44"]
        runs-on: ["ubuntu-latest", "ubuntu-24.04-arm"]
        include:
          - runs-on: ubuntu-latest
            arch-name: X86_64
            rpm-arch: x86_64
            artifact-suffix: x86_64
            job-suffix: ""
          - runs-on: ubuntu-24.04-arm
            arch-name: ARM64
            rpm-arch: aarch64
            artifact-suffix: arm64
            job-suffix: " (ARM64)"

    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Install RPM packaging tools
        shell: bash
        run: |
          set -e
          dnf install -y rpm-build

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Build Linux .rpm package
        shell: bash
        run: |
          set -e

          echo "Running setup.sh to configure build environment..."
          bash setup.sh

          # setup.sh configures with testing on, and the build below has no
          # --target, so every unit-test binary would be built and then left out
          # of the .rpm. Same reasoning as BUILD_TESTING=OFF in contrib/debian/rules.
          echo "Reconfiguring without the unit-test targets..."
          cmake --preset default -DBUILD_TESTING=OFF

          echo "Building lemond and lemonade for Fedora ${{ matrix.fedora-version }} (${{ matrix.arch-name }})..."
          cmake --build --preset default

          cd build

          echo "Creating .rpm package with CPack..."
          cpack -G RPM -V

          CPACK_RPM="lemonade-server-${LEMONADE_VERSION}.${{ matrix.rpm-arch }}.rpm"
          RPM_FILE="lemonade-server-${LEMONADE_VERSION}-fc${{ matrix.fedora-version }}.${{ matrix.rpm-arch }}.rpm"

          if [ ! -f "$CPACK_RPM" ]; then
              echo "ERROR: .rpm package not created!"
              echo "Contents of build directory:"
              ls -lR .
              exit 1
          fi

          mv "$CPACK_RPM" "$RPM_FILE"

          echo "Package information:"
          rpm -qip "$RPM_FILE"

      - name: Upload .rpm package
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-rpm-fedora-${{ matrix.fedora-version }}-${{ matrix.artifact-suffix }}
          path: build/lemonade-server-${{ env.LEMONADE_VERSION }}-fc${{ matrix.fedora-version }}.${{ matrix.rpm-arch }}.rpm
          retention-days: 7

  build-lemonade-macos-dmg:
    name: Build Lemonade macOS .dmg (with Tauri App)
    # macOS runners are the scarcest GitHub-hosted resource (org-wide cap: 5
    # concurrent) and observed queue waits reach 40+ minutes. Build Embeddable
    # Lemonade (macOS) stays on PR pushes as the AppleClang compile canary; the
    # Tauri/dmg/pkg surface is packaging-only, so it gates at merge-queue time.
    # Label a PR `ci:macos` to opt it back in.
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:macos')
    runs-on: macos-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    env:
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Check signing secrets
        id: check_signing
        shell: bash
        env:
          APP_CONNECT_KEY: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
        run: |
          if [ "${{ inputs.disable_macos_signing }}" == "true" ]; then
            echo "has_signing=false" >> $GITHUB_OUTPUT
            echo "Signing disabled by workflow_dispatch input - exercising the unsigned path"
          elif [ -n "$APP_CONNECT_KEY" ]; then
            echo "has_signing=true" >> $GITHUB_OUTPUT
            echo "Signing secrets available - will build signed and notarized .pkg"
          else
            echo "has_signing=false" >> $GITHUB_OUTPUT
            echo "No signing secrets - will build an unsigned .pkg without notarizing"
          fi

      - name: Setup macOS Keychain
        if: steps.check_signing.outputs.has_signing == 'true'
        uses: ./.github/actions/setup-macos-keychain
        with:
          dev-signing-key: ${{ secrets.MACOS_DEV_SIGNING_IDENTITY_KEY }}
          inst-signing-key: ${{ secrets.MACOS_INST_SIGNING_IDENTITY_KEY }}
          certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
          api-key: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
          api-key-id: '3WFZZ8F948'
          api-issuer-id: '2e545619-8206-4d14-9ba9-ef23eff841b2'

      - name: Build macOS .dmg
        uses: ./.github/actions/build-macos-dmg
        with:
          include-tauri: 'true'
          sign: ${{ steps.check_signing.outputs.has_signing }}

      - name: Run C++ unit tests
        shell: bash
        run: |
          set -euo pipefail
          cmake --build --preset default --target cpp-ci-tests
          ctest --test-dir build --output-on-failure --no-tests=error -L "^cpp-ci$"
      - name: Upload .pkg package
        # Two test jobs download this unconditionally; a unit-test failure above
        # must not turn into a confusing "artifact not found" downstream.
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: build/*.pkg
          retention-days: 7

      - name: Cleanup keychain
        if: always() && steps.check_signing.outputs.has_signing == 'true'
        shell: bash
        run: |
          if [ -n "$SIGNING_KEYCHAIN_PATH" ] && [ -f "$SIGNING_KEYCHAIN_PATH" ]; then
            echo "Cleaning up temporary keychain..."
            security delete-keychain "$SIGNING_KEYCHAIN_PATH" 2>/dev/null || true
          fi

  build-lemonade-embeddable-linux:
    name: Build Embeddable Lemonade (Linux${{ matrix.job-suffix }})
    runs-on: ${{ matrix.runs-on }}
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - arch-name: X86_64
            runs-on: ubuntu-latest
            platform: ubuntu-x64
            artifact-suffix: x86_64
            job-suffix: ""
          - arch-name: ARM64
            runs-on: ubuntu-24.04-arm
            platform: ubuntu-arm64
            artifact-suffix: arm64
            job-suffix: " ARM64"
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Install minimal build dependencies
        run: |
          set -e
          sudo apt-get update
          sudo apt-get install -y cmake ninja-build g++ pkg-config libssl-dev libdrm-dev

      - name: Build embeddable archive
        shell: bash
        run: |
          set -e

          # Do not use setup.sh — we intentionally avoid installing system
          # libraries so that FetchContent statically links all deps, making
          # the embeddable binary portable across Ubuntu versions.
          cmake --preset default -DBUILD_WEB_APP=OFF
          cmake --build --preset default --target embeddable

          ARCHIVE="build/lemonade-embeddable-${LEMONADE_VERSION}-${{ matrix.platform }}.tar.gz"
          test -f "$ARCHIVE"
          echo "Archive contents:"
          tar tzf "$ARCHIVE"
          ls -lh "$ARCHIVE"

      - name: Upload embeddable archive
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-embeddable-linux-${{ matrix.artifact-suffix }}
          path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-${{ matrix.platform }}.tar.gz
          retention-days: 7

  build-lemonade-linux-arm64:
    name: Build Lemonade (Linux ARM64)
    runs-on: ubuntu-24.04-arm
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Install build dependencies
        run: |
          set -e
          sudo apt-get update
          sudo apt-get install -y cmake ninja-build g++ pkg-config libssl-dev libdrm-dev

      - name: Build
        run: |
          set -e
          cmake --preset default -DBUILD_WEB_APP=OFF
          cmake --build --preset default --target lemond lemonade

      - name: Upload build artifacts
        uses: actions/upload-artifact@v7
        with:
          name: lemonade-linux-arm64-build
          path: |
            build/lemond
            build/lemonade
            build/resources/
          retention-days: 7

  build-lemonade-embeddable-windows:
    name: Build Embeddable Lemonade (Windows)
    runs-on: windows-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Install CMake if not available
        uses: ./.github/actions/install-cmake-windows

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Cache FetchContent dependencies
        uses: actions/cache@v4
        with:
          path: build/_deps
          # vs2026 marker: see installer job — VS 2022-built _deps cannot be
          # reused by the VS 2026 configure.
          key: fetchcontent-windows-embeddable-vs2026-${{ hashFiles('CMakeLists.txt') }}
          restore-keys: fetchcontent-windows-embeddable-vs2026-

      - name: Build embeddable archive
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          cmake --preset vs18 -DBUILD_WEB_APP=OFF
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          cmake --build build --config Release --target embeddable -- /m:2
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          $archive = "build\lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64.zip"
          if (-not (Test-Path $archive)) { throw "Archive not found: $archive" }
          Write-Host "Archive created:" -ForegroundColor Green
          Get-ChildItem $archive

      - name: Upload embeddable archive
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-embeddable-windows
          path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-windows-x64.zip
          retention-days: 7

  build-lemonade-embeddable-macos:
    name: Build Embeddable Lemonade (macOS)
    runs-on: macos-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Install minimal build dependencies
        shell: bash
        run: |
          set -e
          # macos-latest already ships cmake; install ninja if missing.
          # We intentionally avoid setup.sh so FetchContent statically links
          # deps where possible, matching the Linux/Windows embeddable jobs.
          if ! command -v ninja >/dev/null 2>&1; then
            brew install ninja
          fi

      - name: Build embeddable archive
        shell: bash
        run: |
          set -e

          cmake --preset default -DBUILD_WEB_APP=OFF
          cmake --build --preset default --target embeddable

          ARCHIVE="build/lemonade-embeddable-${LEMONADE_VERSION}-macos-arm64.tar.gz"
          test -f "$ARCHIVE"
          echo "Archive contents:"
          tar tzf "$ARCHIVE"
          ls -lh "$ARCHIVE"

      - name: Upload embeddable archive
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-embeddable-macos
          path: build/lemonade-embeddable-${{ env.LEMONADE_VERSION }}-macos-arm64.tar.gz
          retention-days: 7


  # ========================================================================
  # TEST JOBS - Inference tests on self-hosted runners
  # ========================================================================

  test-exe-inference:
    name: Test .exe - ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    needs: build-lemonade-server-installer
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch).
    # Self-hosted rig capacity is the binding constraint, so the merge queue runs
    # these instead of PR pushes. Label a PR `ci:backends` to opt it back in.
    if: >-
      !startsWith(github.ref, 'refs/tags/') &&
      inputs.enable_signing != true &&
      inputs.disable_macos_signing != true &&
      (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:backends'))
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - name: llamacpp
            script: server_llm.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan rocm"
            runner: [Windows, vulkan, rocm, lemon-prod]
          - name: omni
            script: server_omni.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan"
            runner: [Windows, vulkan, stx-halo, lemon-prod]
          - name: ryzenai
            script: server_llm.py
            extra_args: "--wrapped-server ryzenai"
            backends: "cpu hybrid npu"
            runner: [Windows, xdna2, lemon-prod]
          - name: flm
            script: server_llm.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [Windows, xdna2, lemon-prod]
          - name: whisper
            script: server_whisper.py
            extra_args: "--wrapped-server whispercpp"
            backends: "cpu npu rocm"
            runner: [Windows, stx-halo, lemon-prod]
          - name: flm-whisper
            script: server_whisper.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [Windows, xdna2, lemon-prod]
          - name: moonshine
            script: server_moonshine.py
            extra_args: "--wrapped-server moonshine"
            backends: "cpu"
            runner: [self-hosted, Windows, lemon-prod]
          - name: classify-onnxruntime
            script: server_classify.py
            extra_args: "--wrapped-server onnxruntime"
            backends: "cpu"
            runner: [self-hosted, Windows, lemon-prod]
          - name: router-onnxruntime
            script: server_router.py
            extra_args: "--wrapped-server onnxruntime"
            backends: "cpu"
            runner: [self-hosted, Windows, lemon-prod]
          - name: stable-diffusion
            script: server_sd.py
            extra_args: ""
            backends: "cpu rocm vulkan"
            runner: [Windows, rocm, vulkan, lemon-prod]
          - name: audio-gen-thinksound
            script: server_audio_generation.py
            extra_args: "--wrapped-server thinksound"
            backends: "vulkan rocm"
            runner: [Windows, rocm, vulkan, lemon-prod]
          - name: audio-gen-acestep
            script: server_audio_generation.py
            extra_args: "--wrapped-server acestep"
            backends: "vulkan rocm"
            runner: [Windows, rocm, vulkan, lemon-prod]
          - name: text-to-speech
            script: server_tts.py
            extra_args: ""
            backends: ""
            runner: [self-hosted, Windows, lemon-prod]
          - name: 3d-trellis
            script: server_3d.py
            extra_args: "--wrapped-server trellis"
            backends: "vulkan rocm"
            runner: [Windows, rocm, vulkan, lemon-prod]
          - name: tts-openmoss
            script: server_tts_openmoss.py
            extra_args: "--wrapped-server openmoss"
            backends: "vulkan rocm"
            runner: [Windows, rocm, vulkan, lemon-prod]
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      - name: Prepare Windows runner for long paths and stale workspace caches
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Continue"

          # 1. Kill any running/orphaned processes to release file locks
          $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer")
          foreach ($p in $patterns) {
              Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue
          }

          # 2. Force-delete stale in-tree legacy paths that could lock git checkout

          # Local composite actions are unavailable before actions/checkout,
          # so this minimal long-path-safe cleanup stays inline. It removes
          # stale workspace cache/install dirs that can otherwise make
          # checkout clean/reset fail on persistent self-hosted Windows runners.

          Write-Host "Enabling Git long path support before checkout..." -ForegroundColor Cyan
          git config --global core.longpaths true

          try {
            New-ItemProperty `
              -Path "HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem" `
              -Name "LongPathsEnabled" `
              -Value 1 `
              -PropertyType DWord `
              -Force `
              -ErrorAction Stop | Out-Null
            Write-Host "Windows long paths enabled in registry." -ForegroundColor Green
          } catch {
            Write-Host "Could not set Windows long-path registry flag; continuing with Git core.longpaths=true: $_" -ForegroundColor Yellow
          }

          function Remove-LongPathTree {
            param([string]$Path)
            if (-not $Path -or -not (Test-Path -LiteralPath $Path)) { return }

            $fullPath = [System.IO.Path]::GetFullPath($Path)
            if ($fullPath.StartsWith("\\")) {
              $longPath = "\\?\UNC\" + $fullPath.Substring(2)
            } else {
              $longPath = "\\?\" + $fullPath
            }

            Write-Host "Removing stale path: $fullPath" -ForegroundColor Yellow
            cmd /c rmdir /s /q "$longPath" 2>$null
            if (Test-Path -LiteralPath $Path) {
              Remove-Item -LiteralPath $Path -Recurse -Force -ErrorAction SilentlyContinue
            }
          }

          # These legacy locations were inside the repository workspace and could
          # make actions/checkout fail to clean/reset the repository on Windows,
          # especially after TheRock created very deep bin/ci-cache/bin/therock trees.
          if ($env:GITHUB_WORKSPACE) {
            Remove-LongPathTree (Join-Path $env:GITHUB_WORKSPACE "ci-cache")
            Remove-LongPathTree (Join-Path $env:GITHUB_WORKSPACE "hf-cache")
            Remove-LongPathTree (Join-Path $env:GITHUB_WORKSPACE "lemonade_server_install")
          }

      - uses: actions/checkout@v5
        with:
          clean: true
          fetch-depth: 0

      - name: Setup & sanitize Windows CI environment
        uses: ./.github/actions/setup-windows-ci-env
        with:
          matrix-name: ${{ matrix.name }}

      - name: Install and Verify Lemonade Server
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Install FLM backend for FLM wrapped-server tests
        if: ${{ runner.os == 'Windows' && contains(matrix.extra_args, '--wrapped-server flm') }}
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $lemonadeExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade.exe"
          Write-Host "Installing FLM backend for CI inference tests..." -ForegroundColor Cyan
          & $lemonadeExe backends install flm:npu
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

      - name: Run tests
        shell: PowerShell
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          $ErrorActionPreference = "Stop"
          $venvPython = ".\.venv\Scripts\python.exe"
          $serverExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade.exe"
          $extraArgs = "${{ matrix.extra_args }}" -split " " | Where-Object { $_ }
          $backends = "${{ matrix.backends }}" -split " " | Where-Object { $_ }

          function Invoke-TestBackend {
            param(
              [string]$Backend,
              [int]$MaxAttempts = 1
            )

            for ($attempt = 1; $attempt -le $MaxAttempts; $attempt++) {
              Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $Backend (attempt $attempt/$MaxAttempts)" -ForegroundColor Cyan
              & $venvPython test/${{ matrix.script }} @extraArgs --backend $Backend --cli-binary $serverExe
              $testExitCode = $LASTEXITCODE

              if ($testExitCode -eq 0) {
                return
              }

              if ($attempt -lt $MaxAttempts) {
                Write-Host "::warning title=Retrying transient backend test::test/${{ matrix.script }} --backend $Backend failed with exit code $testExitCode; cleaning sd-server state and retrying."

                Get-Process sd-server -ErrorAction SilentlyContinue |
                  Stop-Process -Force -ErrorAction SilentlyContinue
                Start-Sleep -Seconds 20

                try {
                  Invoke-RestMethod `
                    -Uri "http://localhost:13305/api/v1/unload" `
                    -Method Post `
                    -ContentType "application/json" `
                    -Body "{}" `
                    -TimeoutSec 30 | Out-Null
                } catch {
                  Write-Host "Unload before retry failed or server had no model loaded: $_" -ForegroundColor Yellow
                }

                Start-Sleep -Seconds 10
                continue
              }

              exit $testExitCode
            }
          }

          if ($backends.Count -eq 0) {
            Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }}" -ForegroundColor Cyan
            & $venvPython test/${{ matrix.script }} @extraArgs --cli-binary $serverExe
            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          } else {
            foreach ($backend in $backends) {
              $maxAttempts = 1
              if ("${{ matrix.name }}" -eq "stable-diffusion" -and $backend -eq "rocm") {
                $maxAttempts = 3
              }

              Invoke-TestBackend -Backend $backend -MaxAttempts $maxAttempts
            }
          }

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-exe-${{ matrix.name }}

      - name: Teardown Windows CI environment
        if: always()
        uses: ./.github/actions/teardown-windows-ci-env


  test-deb-inference:
    name: Test .deb - ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    needs: build-lemonade-deb
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch).
    # Self-hosted rig capacity is the binding constraint, so the merge queue runs
    # these instead of PR pushes. Label a PR `ci:backends` to opt it back in.
    if: >-
      !startsWith(github.ref, 'refs/tags/') &&
      inputs.enable_signing != true &&
      inputs.disable_macos_signing != true &&
      (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:backends'))
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - name: llamacpp
            script: server_llm.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan rocm"
            runner: [Linux, vulkan, rocm, lemon-prod]
          - name: omni
            script: server_omni.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan"
            runner: [Linux, vulkan, stx-halo, lemon-prod]
          - name: stable-diffusion
            script: server_sd.py
            extra_args: ""
            backends: "cpu rocm vulkan"
            runner: [Linux, vulkan, rocm, lemon-prod]
          - name: thenoise
            script: server_thenoise.py
            extra_args: "--wrapped-server thenoise"
            backends: "rocm"
            runner: [Linux, rocm, stx-halo, lemon-prod]
          - name: audio-gen-thinksound
            script: server_audio_generation.py
            extra_args: "--wrapped-server thinksound"
            backends: "vulkan rocm"
            runner: [Linux, vulkan, rocm, lemon-prod]
          - name: audio-gen-acestep
            script: server_audio_generation.py
            extra_args: "--wrapped-server acestep"
            backends: "vulkan rocm"
            runner: [Linux, vulkan, rocm, lemon-prod]
          - name: whisper
            script: server_whisper.py
            extra_args: "--wrapped-server whispercpp"
            backends: "cpu vulkan rocm"
            runner: [Linux, rocm, xdna2, lemon-prod]
          - name: moonshine
            script: server_moonshine.py
            extra_args: "--wrapped-server moonshine"
            backends: "cpu"
            runner: [self-hosted, Linux, lemon-prod]
          - name: flm
            script: server_llm.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [Linux, xdna2, lemon-prod]
          - name: text-to-speech
            script: server_tts.py
            extra_args: ""
            backends: ""
            runner: [self-hosted, Linux, lemon-prod]
          - name: 3d-trellis
            script: server_3d.py
            extra_args: "--wrapped-server trellis"
            backends: "vulkan rocm"
            runner: [Linux, vulkan, rocm, lemon-prod]
          - name: tts-openmoss
            script: server_tts_openmoss.py
            extra_args: "--wrapped-server openmoss"
            backends: "vulkan rocm"
            runner: [Linux, vulkan, rocm, lemon-prod]
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Cleanup processes
        uses: ./.github/actions/cleanup-processes-linux

      - name: Set HF_HOME environment variable
        run: |
          # Persistent HF cache outside the workspace so models survive across
          # jobs — the checkout clean wipes untracked workspace files (#2103)
          HF_HOME="$HOME/.cache/lemonade-ci/huggingface"
          mkdir -p "$HF_HOME"
          echo "HF_HOME=$HF_HOME" >> $GITHUB_ENV

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}
          # These jobs run directly on shared self-hosted runners (no container
          # network isolation), so the default port can be held by a concurrent
          # job. Use a job-unique free port to avoid the collision.
          port: auto

      - name: Install FLM backend for FLM wrapped-server tests
        if: ${{ contains(matrix.extra_args, '--wrapped-server flm') }}
        run: |
          set -e
          echo "Installing FLM backend for CI inference tests..."
          lemonade backends install flm:npu

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run tests
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          if [ -z "${{ matrix.backends }}" ]; then
            echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }}"
            .venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --cli-binary lemonade
          else
            for backend in ${{ matrix.backends }}; do
              echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend"
              .venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend --cli-binary lemonade
            done
          fi

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-deb-${{ matrix.name }}

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-linux

  test-rpm-package:
    name: Test .rpm - Fedora ${{ matrix.fedora-version }}${{ matrix.job-suffix }}
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:distros')
    runs-on: ${{ matrix.runs-on }}
    needs: build-lemonade-rpm
    container:
      image: fedora:${{ matrix.fedora-version }}

    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        fedora-version: ["43", "44"]
        runs-on: ["ubuntu-latest", "ubuntu-24.04-arm"]
        include:
          - runs-on: ubuntu-latest
            arch-name: X86_64
            rpm-arch: x86_64
            artifact-suffix: x86_64
            job-suffix: ""
          - runs-on: ubuntu-24.04-arm
            arch-name: ARM64
            rpm-arch: aarch64
            artifact-suffix: arm64
            job-suffix: " (ARM64)"

    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
    steps:
      - name: Download Lemonade .rpm Package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-rpm-fedora-${{ matrix.fedora-version }}-${{ matrix.artifact-suffix }}
          path: .

      - name: Install and verify Lemonade (.rpm)
        shell: bash
        run: |
          set -e

          RPM_FILE="lemonade-server-${LEMONADE_VERSION}-fc${{ matrix.fedora-version }}.${{ matrix.rpm-arch }}.rpm"

          if [ ! -f "$RPM_FILE" ]; then
              echo "ERROR: .rpm file not found: $RPM_FILE"
              ls -la *.rpm 2>/dev/null || echo "No .rpm files found in current directory"
              exit 1
          fi

          dnf install -y shadow-utils "$RPM_FILE"

          echo "Installed package information:"
          rpm -qi lemonade-server

          echo "Installed file list:"
          rpm -ql lemonade-server | sort

          test -f /opt/bin/lemonade
          test -f /opt/bin/lemond

          /opt/bin/lemonade --version
          /opt/bin/lemond --version

          rpm -qpl "$RPM_FILE" | grep -Fx "/usr/lib/systemd/user/lemond.service"
          rpm -qpl --dump "$RPM_FILE" \
            | grep -E '^/usr/lib/systemd/user/lemond\.service ' \
            | grep -F "/opt/lib/systemd/user/lemond.service"

          # Verify tray is NOT in the server RPM
          echo "Verifying server-only RPM content..."
          if rpm -qpl "$RPM_FILE" | grep -Fxq "/opt/bin/lemonade-tray"; then
              echo "ERROR: lemonade-tray binary found in server RPM"
              exit 1
          fi
          if rpm -qpl "$RPM_FILE" | grep -Fxq "/usr/bin/lemonade-tray"; then
              echo "ERROR: lemonade-tray symlink found in server RPM"
              exit 1
          fi

          # Verify no desktop dependency leakage
          if rpm -qpR "$RPM_FILE" | grep -qiE "gtk|appindicator|ayatana"; then
              echo "ERROR: server RPM requires desktop dependencies (GTK/AppIndicator/Ayatana)"
              rpm -qpR "$RPM_FILE" | grep -iE "gtk|appindicator|ayatana"
              exit 1
          fi

  # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran
  # successfully (otherwise the merge is blocked), and 2. these jobs do not need
  # to run on ordinary pull request pushes.
  packaging-gate:
    name: Packaging builds
    needs: [build-lemonade-debian13, test-lemonade-debian13-smoke, build-lemonade-rpm, test-rpm-package, build-lemonade-desktop-installer, tauri-compile-check]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check gated jobs
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...}
          # Fail if any job broke. In the merge queue, also fail if any never ran.
          echo "$NEEDS"
          broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS")
          if [ -n "$broke" ]; then
            echo "FAILED: $broke"
            exit 1
          fi
          if [ "${{ github.event_name }}" = "merge_group" ]; then
            absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS")
            if [ -n "$absent" ]; then
              echo "DID NOT RUN IN MERGE QUEUE: $absent"
              exit 1
            fi
          fi

  # Same gate pattern as packaging-gate, for the self-hosted inference rigs
  # that PR pushes skip. The job name below belongs in main's required status
  # checks — that registration is what makes the merge-queue strictness bind,
  # so renaming the job requires a matching branch-protection update.
  backends-gate:
    name: Inference backend tests
    needs: [build-lemonade-server-installer, build-lemonade-deb, cpp-unit-tests, test-exe-inference, test-deb-inference]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check gated jobs
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...}
          # Fail if any job broke. In the merge queue, also fail if any never ran.
          echo "$NEEDS"
          broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS")
          if [ -n "$broke" ]; then
            echo "FAILED: $broke"
            exit 1
          fi
          if [ "${{ github.event_name }}" = "merge_group" ]; then
            absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS")
            if [ -n "$absent" ]; then
              echo "DID NOT RUN IN MERGE QUEUE: $absent"
              exit 1
            fi
          fi

  # Same gate pattern as packaging-gate, for the macOS jobs that PR pushes skip.
  macos-gate:
    name: macOS builds
    needs: [build-lemonade-macos-dmg, test-cli-endpoints-macos, test-embeddable-macos, test-dmg-inference]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - name: Check gated jobs
        env:
          NEEDS: ${{ toJSON(needs) }}
        run: |
          # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...}
          # Fail if any job broke. In the merge queue, also fail if any never ran.
          echo "$NEEDS"
          broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS")
          if [ -n "$broke" ]; then
            echo "FAILED: $broke"
            exit 1
          fi
          if [ "${{ github.event_name }}" = "merge_group" ]; then
            absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS")
            if [ -n "$absent" ]; then
              echo "DID NOT RUN IN MERGE QUEUE: $absent"
              exit 1
            fi
          fi

  test-embeddable-posix:
    name: Test Embeddable (${{ matrix.label }})
    runs-on: ${{ matrix.os }}
    needs: build-lemonade-embeddable-linux
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        include:
          - label: Linux
            os: ubuntu-latest
            platform: ubuntu-x64
            artifact: lemonade-embeddable-linux-x86_64
          - label: Linux ARM64
            os: ubuntu-24.04-arm
            platform: ubuntu-arm64
            artifact: lemonade-embeddable-linux-arm64
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-linux.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Download embeddable archive
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.artifact }}
          path: .

      - name: Smoke test embeddable archive
        uses: ./.github/actions/smoke-test-embeddable
        with:
          version: ${{ env.LEMONADE_VERSION }}
          platform: ${{ matrix.platform }}
          label: ${{ matrix.label }}

  test-embeddable-macos:
    name: Test Embeddable (macOS)
    # Own job rather than a leg of test-embeddable-posix so that PR pushes can
    # skip it at job level. Build Embeddable Lemonade (macOS) still runs on every
    # PR as the AppleClang compile canary; this job re-tests the packaged archive
    # on a clean runner, which is merge-queue work.
    # Label a PR `ci:macos` to opt it back in.
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:macos')
    runs-on: macos-latest
    needs: build-lemonade-embeddable-macos
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-macos.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Download embeddable archive
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-macos
          path: .

      - name: Smoke test embeddable archive
        uses: ./.github/actions/smoke-test-embeddable
        with:
          version: ${{ env.LEMONADE_VERSION }}
          platform: macos-arm64
          label: macOS

  test-embeddable-windows:
    name: Test Embeddable (Windows)
    runs-on: windows-latest
    needs: build-lemonade-embeddable-windows
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-embeddable-windows.outputs.version }}
    steps:
      - name: Download embeddable archive
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-windows
          path: .

      - name: Validate archive structure
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          $archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"
          Expand-Archive -Path "$archiveDir.zip" -DestinationPath .

          # Verify expected files
          $expected = @(
              "$archiveDir\lemond.exe",
              "$archiveDir\lemonade.exe",
              "$archiveDir\LICENSE",
              "$archiveDir\resources\server_models.json",
              "$archiveDir\resources\backend_versions.json",
              "$archiveDir\resources\defaults.json",
              "$archiveDir\resources\toolDefinitions.json"
          )
          $failures = @()
          foreach ($f in $expected) {
              if (-not (Test-Path $f)) {
                  Write-Host "ERROR: $f not found!" -ForegroundColor Red
                  $failures += $f
              }
          }
          if ($failures.Count -gt 0) { exit 1 }
          Write-Host "All expected files present" -ForegroundColor Green

          # Verify NO forbidden files
          $forbidden = @(
              "$archiveDir\LemonadeServer.exe",
              "$archiveDir\lemonade-app.exe",
              "$archiveDir\lemonade-tray.exe",
              "$archiveDir\resources\web-app"
          )
          foreach ($f in $forbidden) {
              if (Test-Path $f) {
                  Write-Host "ERROR: Forbidden file found: $f" -ForegroundColor Red
                  exit 1
              }
          }
          Write-Host "No forbidden files found" -ForegroundColor Green

      - name: Test version commands
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"
          & "$archiveDir\lemond.exe" --version
          & "$archiveDir\lemonade.exe" --version

      - name: Test lemond startup and health check
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"
          $archiveDir = "lemonade-embeddable-$($env:LEMONADE_VERSION)-windows-x64"

          Push-Location $archiveDir
          $proc = Start-Process -FilePath ".\lemond.exe" -ArgumentList ".\" -PassThru -NoNewWindow

          # Wait for server to become healthy
          $healthy = $false
          for ($i = 0; $i -lt 15; $i++) {
            try {
              $response = Invoke-WebRequest -Uri "http://localhost:13305/api/v1/health" -UseBasicParsing -TimeoutSec 2
              if ($response.StatusCode -eq 200) {
                Write-Host "Server is healthy!" -ForegroundColor Green
                Write-Host "Health response: $($response.Content)"
                $healthy = $true
                break
              }
            } catch {}
            Start-Sleep -Seconds 2
          }

          if (-not $healthy) {
            Write-Host "ERROR: Server failed to start within 30 seconds" -ForegroundColor Red
            Stop-Process $proc -Force -ErrorAction SilentlyContinue
            Pop-Location
            exit 1
          }

          Stop-Process $proc -Force -ErrorAction SilentlyContinue
          Pop-Location
          Write-Host "Embeddable Windows smoke test PASSED!" -ForegroundColor Green

      - name: Cleanup processes (Windows)
        if: always()
        shell: PowerShell
        run: |
          Stop-Process -Name lemond -Force -ErrorAction SilentlyContinue
          Stop-Process -Name lemonade -Force -ErrorAction SilentlyContinue

  test-dmg-inference:
    name: Test .dmg - macOS inference
    runs-on: macos-latest
    needs: build-lemonade-macos-dmg
    # Inference is the only macOS coverage no other job provides, so it has to
    # run where merges are gated.
    # Label a PR `ci:macos` to opt it back in.
    if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:macos')) }}
    timeout-minutes: 45
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
      # Covers the install action's manual fallback, which runs in this shell.
      # The launchd path is covered by the launchctl setenv step below.
      GGML_METAL_NO_RESIDENCY: "1"
    steps:
      - uses: actions/checkout@v5

      - name: Set HF_HOME environment variable
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      # Must precede the install: the .pkg postflight starts lemond via launchd,
      # which passes on the launchd domain environment rather than this shell's.
      # Killing that server and starting our own instead loses a race for
      # 127.0.0.1:13305 often enough to be unusable — lemond then binds IPv6
      # only and every test that uses a literal IPv4 address fails.
      - name: Put the Metal setting in the launchd environment
        shell: bash
        run: sudo launchctl setenv GGML_METAL_NO_RESIDENCY 1

      - name: Download .pkg package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Install Lemonade Server (.pkg)
        id: install-pkg
        uses: ./.github/actions/install-lemonade-server-dmg
        with:
          version: ${{ env.LEMONADE_VERSION }}
          download-artifact: 'false'

      - name: Verify the server picked up the Metal setting
        shell: bash
        env:
          BIN_PATH: ${{ steps.install-pkg.outputs.bin-path }}
        run: |
          set -e
          echo "CLI_BINARY=$BIN_PATH/lemonade" >> "$GITHUB_ENV"

          PID=$(sudo lsof -t -nP -iTCP:13305 -sTCP:LISTEN 2>/dev/null | head -1)
          if [ -z "$PID" ]; then
            echo "ERROR: nothing is listening on port 13305"
            exit 1
          fi
          # Assert rather than assume: this job exists to exercise Metal, and a
          # server without the setting would quietly test something else.
          if ! sudo ps eww -p "$PID" | grep -q GGML_METAL_NO_RESIDENCY; then
            echo "ERROR: lemond (PID $PID) is missing GGML_METAL_NO_RESIDENCY"
            sudo ps eww -p "$PID" || true
            exit 1
          fi
          echo "lemond (PID $PID) has GGML_METAL_NO_RESIDENCY"

      - name: Setup Python and virtual environment
        id: setup
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      # All inference tests run in one job against the already-started server so
      # they share a single setup and model cache (issue #2103). Each step runs
      # even if an earlier test failed; the job fails if any failed.
      - name: Test llamacpp-metal
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        run: .venv/bin/python test/server_llm.py --wrapped-server llamacpp --backend metal --cli-binary "$CLI_BINARY"

      - name: Test whisper-metal
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        run: .venv/bin/python test/server_whisper.py --wrapped-server whispercpp --backend metal --cli-binary "$CLI_BINARY"

      # Temporarily disabled — sd-cpp metal test is being skipped
      # - name: Test sd-cpp-metal
      #   if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
      #   env:
      #     LEMONADE_TEST_SD_MODEL: SD-Turbo-GGUF
      #   run: .venv/bin/python test/server_sd.py --wrapped-server sd-cpp --backend metal --cli-binary "$CLI_BINARY"

      - name: Test moonshine
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        run: .venv/bin/python test/server_moonshine.py --wrapped-server moonshine --backend cpu --cli-binary "$CLI_BINARY"

      - name: Test kokoro-metal
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        run: .venv/bin/python test/server_tts.py --cli-binary "$CLI_BINARY"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-dmg-inference

  # ========================================================================
  # CLI AND ENDPOINTS TESTS - Run on GitHub-hosted runners (no GPU needed)
  # ========================================================================

  test-cli-endpoints-linux:
    name: Test CLI/Endpoints (ubuntu-latest)
    runs-on: ubuntu-latest
    needs: build-lemonade-deb
    timeout-minutes: 25
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      HF_HOME: ${{ github.workspace }}/hf-cache
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Install pip and test dependencies
        shell: bash
        run: |
          set -e
          # Install pip if not already available
          if ! command -v pip3 &> /dev/null; then
            apt-get update
            apt-get install -y python3-pip python3-venv
          fi
          # Create venv and install test dependencies
          python3 -m venv .venv
          .venv/bin/pip install --upgrade pip
          .venv/bin/pip install -r test/requirements.txt

      - name: Prepare HF cache
        id: setup
        shell: bash
        run: |
          set -e
          # HF_HOME is job-level env so install-lemonade-deb and the lemond
          # process it starts inherit the same cache path.
          mkdir -p "$HF_HOME"
          echo "HF_HOME=$HF_HOME"

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}

      # All test scripts run in one job so they share a single setup and model
      # cache (issue #2103). Each step runs even if an earlier test failed
      # (matching the old fail-fast: false matrix); the job fails if any failed.
      - name: Test cli
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu cli"
          .venv/bin/python test/server_cli2.py --cli-binary lemonade

      - name: Test endpoints
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu endpoints"
          .venv/bin/python test/server_endpoints.py --cli-binary lemonade
          echo "Running WebSocket idle test..."
          .venv/bin/python test/test_websocket_idle.py
          echo "WebSocket idle test PASSED!"
          echo "Running WebSocket auth tests..."
          .venv/bin/python test/server_websocket_auth.py
          echo "WebSocket auth tests PASSED!"

      - name: Test router
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu router"
          .venv/bin/python test/server_router.py --cli-binary lemonade

      # The full smoke test can be added to a different GPU equipped pipeline
      # as well if needed, but it would significantly add to the pipeline runtime.
      - name: Test mcp
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu mcp"
          .venv/bin/python test/server_mcp.py --cli-binary lemonade
      - name: Test web-app path traversal
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu webapp"
          .venv/bin/python test/server_webapp.py --cli-binary lemonade

      - name: Test model name normalization
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu model-normalization"
          .venv/bin/python test/test_model_name_normalization.py --cli-binary lemonade

      - name: Test ollama
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        env:
          # No persistent HF cache here, so prefer the 2 GB GGUF build.
          LEMONADE_TEST_SD_MODEL: SD-Turbo-GGUF
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu ollama"
          .venv/bin/python test/test_ollama.py --cli-binary lemonade

      - name: Test streaming-errors
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu streaming-errors"
          .venv/bin/python test/server_streaming_errors.py --cli-binary lemonade

      - name: Test jobs
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu jobs"
          .venv/bin/python test/server_jobs.py --lemond-binary lemond

      # Must run last: its tearDown stops lemond via /internal/shutdown to
      # launch its own instance, and nothing after it can rely on the server.
      - name: Test llamacpp-system
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu llamacpp-system"
          .venv/bin/python test/test_llamacpp_system_backend.py --cli-binary lemonade

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-ubuntu-latest

  test-cli-endpoints-linux-arm64:
    name: Test ${{ matrix.test_type }} (ubuntu-24.04-arm)
    runs-on: ubuntu-24.04-arm
    needs: build-lemonade-linux-arm64
    strategy:
      fail-fast: ${{ github.event_name == 'merge_group' }}
      matrix:
        # llamacpp-system and env-vars omitted: no system llama-server or .deb on
        # GitHub-hosted ARM64 runners; Vulkan tests omitted: no GPU available.
        test_type: [cli, endpoints, ollama, streaming-errors]
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      - uses: actions/checkout@v5

      - name: Download build artifacts
        uses: actions/download-artifact@v7
        with:
          name: lemonade-linux-arm64-build
          path: build

      - name: Make binaries executable
        run: chmod +x build/lemond build/lemonade

      - name: Set HF_HOME
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      - name: Start lemond server
        run: |
          mkdir -p "$RUNNER_TEMP/xdg-runtime"
          chmod 700 "$RUNNER_TEMP/xdg-runtime"
          echo "XDG_RUNTIME_DIR=$RUNNER_TEMP/xdg-runtime" >> "$GITHUB_ENV"
          export XDG_RUNTIME_DIR="$RUNNER_TEMP/xdg-runtime"
          ./build/lemond > "$RUNNER_TEMP/lemond.log" 2>&1 &
          LEMOND_PID=$!
          disown "$LEMOND_PID"
          for i in $(seq 1 30); do
            if curl -sf http://localhost:13305/live > /dev/null 2>&1; then
              echo "Server is running and healthy (PID $LEMOND_PID)"
              exit 0
            fi
            echo "Waiting for server... ($i/30)"
            sleep 2
          done
          echo "ERROR: Server did not start within 60 seconds"
          cat "$RUNNER_TEMP/lemond.log" 2>/dev/null || true
          exit 1

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run tests
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          VENV_PYTHON=.venv/bin/python
          CLI_BINARY=./build/lemonade

          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "arm64 ${{ matrix.test_type }}"

          if [ "${{ matrix.test_type }}" = "cli" ]; then
            echo "Running CLI tests..."
            $VENV_PYTHON test/server_cli2.py --cli-binary "$CLI_BINARY"
          elif [ "${{ matrix.test_type }}" = "endpoints" ]; then
            echo "Running endpoint tests..."
            $VENV_PYTHON test/server_endpoints.py --cli-binary "$CLI_BINARY"
            echo "Running WebSocket idle test..."
            $VENV_PYTHON test/test_websocket_idle.py
            echo "WebSocket idle test PASSED!"
            echo "Running WebSocket auth tests..."
            $VENV_PYTHON test/server_websocket_auth.py
            echo "WebSocket auth tests PASSED!"
          elif [ "${{ matrix.test_type }}" = "ollama" ]; then
            echo "Running Ollama API tests..."
            $VENV_PYTHON test/test_ollama.py --cli-binary "$CLI_BINARY"
          elif [ "${{ matrix.test_type }}" = "streaming-errors" ]; then
            echo "Running streaming error termination tests..."
            $VENV_PYTHON test/server_streaming_errors.py --cli-binary "$CLI_BINARY"
          fi
          echo "${{ matrix.test_type }} tests PASSED!"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-arm64-${{ matrix.test_type }}

  test-cli-endpoints-windows:
    name: Test CLI/Endpoints (windows-latest)
    runs-on: windows-latest
    needs: build-lemonade-server-installer
    timeout-minutes: 30
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      # Pre-checkout cleanup is run here as a safeguard to release file locks on Windows
      # in case this job is mapped to persistent self-hosted runners labeled as 'windows-latest'.
      - name: Prepare Windows runner for long paths and stale workspace caches
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Continue"

          # 1. Kill any running/orphaned processes to release file locks
          $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer")
          foreach ($p in $patterns) {
              Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue
          }

      - uses: actions/checkout@v5

      - name: Setup & sanitize Windows CI environment
        uses: ./.github/actions/setup-windows-ci-env
        with:
          matrix-name: windows-latest

      - name: Install Lemonade Server (Windows)
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Set paths (Windows)
        shell: powershell
        run: |
          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
          echo "CLI_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade.exe" >> $Env:GITHUB_ENV

      - name: Setup Python and virtual environment
        id: setup
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      # All test scripts run in one job so they share a single setup and model
      # cache (issue #2103). Each step runs even if an earlier test failed; the
      # job fails if any failed.
      - name: Test cli
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest cli"
          $VENV_PYTHON test/server_cli2.py --cli-binary "$CLI_BINARY"

      - name: Test endpoints
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest endpoints"
          $VENV_PYTHON test/server_endpoints.py --cli-binary "$CLI_BINARY"
          echo "Running WebSocket idle test..."
          $VENV_PYTHON test/test_websocket_idle.py
          echo "WebSocket idle test PASSED!"

      - name: Test router
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest router"
          $VENV_PYTHON test/server_router.py --cli-binary "$CLI_BINARY"

      - name: Test web-app path traversal
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest webapp"
          $VENV_PYTHON test/server_webapp.py --cli-binary "$CLI_BINARY"

      - name: Test ollama
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        env:
          # No persistent HF cache here, so prefer the 2 GB GGUF build.
          LEMONADE_TEST_SD_MODEL: SD-Turbo-GGUF
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest ollama"
          $VENV_PYTHON test/test_ollama.py --cli-binary "$CLI_BINARY"

      - name: Test streaming-errors
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest streaming-errors"
          $VENV_PYTHON test/server_streaming_errors.py --cli-binary "$CLI_BINARY"

      # Must run last: its tearDown stops lemond via /internal/shutdown to
      # launch its own instance, and nothing after it can rely on the server.
      - name: Test llamacpp-system
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "windows-latest llamacpp-system"
          $VENV_PYTHON test/test_llamacpp_system_backend.py --cli-binary "$CLI_BINARY"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-windows-latest

      - name: Teardown Windows CI environment
        if: always()
        uses: ./.github/actions/teardown-windows-ci-env

  test-cli-endpoints-macos:
    name: Test CLI/Endpoints (macos-latest)
    # Skipped at job level rather than dropped from a matrix: this is a required
    # status check, and a required context that is never created leaves the PR
    # pending forever, whereas a job skipped by `if:` reports success.
    # Label a PR `ci:macos` to opt it back in.
    if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:macos')
    runs-on: macos-latest
    needs: build-lemonade-macos-dmg
    timeout-minutes: 45
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
    # These suites drive the server the .pkg postflight starts via launchd, which
    # inherits only the plist environment. Job-level server tuning such as
    # GGML_METAL_NO_RESIDENCY would not reach it, so it is not set here; Metal
    # inference is covered by test-dmg-inference, which owns its server.
    steps:
      - uses: actions/checkout@v5

      - name: Download .pkg package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Install Lemonade Server (.pkg)
        id: install-pkg
        uses: ./.github/actions/install-lemonade-server-dmg
        with:
          version: ${{ env.LEMONADE_VERSION }}
          download-artifact: 'false'

      - name: Set HF_HOME environment variable
        shell: bash
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      - name: Stop the tray agent started by the .pkg
        # postinst-full-mac bootstraps com.lemonade.tray into the GUI domain and
        # the tray is single-instance, so test_tray_https.py cannot start its own
        # while this one runs. KeepAlive is false, so it stays down.
        shell: bash
        run: |
          set -e
          launchctl bootout "gui/$(id -u)/com.lemonade.tray" 2>/dev/null || true
          pkill -f lemonade-tray 2>/dev/null || true
          sudo pkill -f lemonade-tray 2>/dev/null || true
          for i in $(seq 1 15); do
            pgrep -f lemonade-tray > /dev/null 2>&1 || break
            sleep 1
          done

      - name: Set environment
        shell: bash
        run: |
          set -e
          echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
          echo "CLI_BINARY=${{ steps.install-pkg.outputs.bin-path }}/lemonade" >> $GITHUB_ENV
          echo "LEMOND_BINARY=${{ steps.install-pkg.outputs.bin-path }}/lemond" >> $GITHUB_ENV

      - name: Setup Python and virtual environment
        id: setup
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      # All test scripts run in one job so they share a single setup and model
      # cache (issue #2103). Each step runs even if an earlier test failed; the
      # job fails if any failed.
      - name: Test cli
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest cli"
          $VENV_PYTHON test/server_cli2.py --cli-binary "$CLI_BINARY"

      - name: Test endpoints
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest endpoints"
          $VENV_PYTHON test/server_endpoints.py --cli-binary "$CLI_BINARY"
          $VENV_PYTHON test/test_websocket_idle.py
          $VENV_PYTHON test/server_websocket_auth.py
          $VENV_PYTHON test/test_tray_https.py
          $VENV_PYTHON test/server_jobs.py --lemond-binary "$LEMOND_BINARY"

      - name: Test router
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest router"
          $VENV_PYTHON test/server_router.py --cli-binary "$CLI_BINARY"

      - name: Test web-app path traversal
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest webapp"
          $VENV_PYTHON test/server_webapp.py --cli-binary "$CLI_BINARY"

      - name: Test model name normalization
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest model-normalization"
          $VENV_PYTHON test/test_model_name_normalization.py --cli-binary "$CLI_BINARY"

      - name: Test ollama
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest ollama"
          $VENV_PYTHON test/test_ollama.py --cli-binary "$CLI_BINARY"

      - name: Test streaming-errors
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest streaming-errors"
          $VENV_PYTHON test/server_streaming_errors.py --cli-binary "$CLI_BINARY"

      # Must run last: its tearDown stops lemond via /internal/shutdown to
      # launch its own instance, and nothing after it can rely on the server.
      - name: Test llamacpp-system
        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
        shell: bash
        run: |
          $VENV_PYTHON -m test.utils.reset_server_state --best-effort --label "macos-latest llamacpp-system"
          $VENV_PYTHON test/test_llamacpp_system_backend.py --cli-binary "$CLI_BINARY"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-macos-latest

  # ========================================================================
  # API KEY TESTS - Separate job with LEMONADE_API_KEY env var
  # ========================================================================

  test-cli-apikey-linux:
    name: Test API Key (ubuntu-latest)
    runs-on: ubuntu-latest
    needs: build-lemonade-deb
    container:
      image: ghcr.io/lemonade-sdk/lemonade/build-environment:ubuntu24.04
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      LEMONADE_API_KEY: "test-api-key-12345"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v5

      - name: Configure git safe directory
        run: git config --global --add safe.directory $(pwd)

      - name: Install pip and test dependencies
        shell: bash
        run: |
          set -e
          if ! command -v pip3 &> /dev/null; then
            apt-get update
            apt-get install -y python3-pip python3-venv
          fi
          python3 -m venv .venv
          .venv/bin/pip install --upgrade pip
          .venv/bin/pip install -r test/requirements.txt

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Verify API key enforcement
        shell: bash
        run: |
          set -e
          PASS=0
          FAIL=0

          # 1. No API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: No API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: No API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 2. Wrong API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-key" http://127.0.0.1:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: Wrong API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Wrong API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 3. Correct API key → must get 200
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $LEMONADE_API_KEY" http://127.0.0.1:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "200" ]; then
            echo "PASS: Correct API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Correct API key → $HTTP_CODE (expected 200)"
            FAIL=$((FAIL+1))
          fi

          echo ""
          echo "Results: $PASS passed, $FAIL failed"
          if [ "$FAIL" -gt 0 ]; then
            echo "ERROR: API key enforcement tests failed"
            exit 1
          fi

      - name: Run CLI tests with API key
        shell: bash
        run: |
          set -e
          # Keep this job focused on API-key propagation. The full CLI suite
          # runs elsewhere and includes model-pull/delete coverage unrelated to auth.
          .venv/bin/python test/server_cli_apikey.py --cli-binary lemonade

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-apikey-ubuntu-latest

  test-cli-apikey:
    name: Test API Key (windows-latest)
    runs-on: windows-latest
    needs:
      - build-lemonade-server-installer
    env:
      LEMONADE_CI_MODE: "True"
      HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }}
      LEMONADE_API_KEY: "test-api-key-12345"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      # Pre-checkout cleanup is run here as a safeguard to release file locks on Windows
      # in case this job is mapped to persistent self-hosted runners labeled as 'windows-latest'.
      - name: Prepare Windows runner for long paths and stale workspace caches
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Continue"

          # 1. Kill any running/orphaned processes to release file locks
          $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer")
          foreach ($p in $patterns) {
              Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue
          }


      - uses: actions/checkout@v5

      - name: Setup & sanitize Windows CI environment
        uses: ./.github/actions/setup-windows-ci-env
        with:
          matrix-name: apikey-windows-latest

      - name: Install Lemonade Server (Windows)
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Set paths (Windows)
        shell: powershell
        run: |
          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
          echo "CLI_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade.exe" >> $Env:GITHUB_ENV

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Verify API key enforcement
        shell: bash
        run: |
          set -e
          PASS=0
          FAIL=0

          # 1. No API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: No API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: No API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 2. Wrong API key → must get 401
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer wrong-key" http://localhost:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "401" ]; then
            echo "PASS: Wrong API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Wrong API key → $HTTP_CODE (expected 401)"
            FAIL=$((FAIL+1))
          fi

          # 3. Correct API key → must get 200
          HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $LEMONADE_API_KEY" http://localhost:13305/api/v1/health 2>/dev/null || echo "000")
          if [ "$HTTP_CODE" = "200" ]; then
            echo "PASS: Correct API key → $HTTP_CODE"
            PASS=$((PASS+1))
          else
            echo "FAIL: Correct API key → $HTTP_CODE (expected 200)"
            FAIL=$((FAIL+1))
          fi

          echo ""
          echo "Results: $PASS passed, $FAIL failed"
          if [ "$FAIL" -gt 0 ]; then
            echo "ERROR: API key enforcement tests failed"
            exit 1
          fi

      - name: Run CLI tests with API key
        shell: bash
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          # Keep this job focused on API-key propagation. The full CLI suite
          # runs elsewhere and includes model-pull/delete coverage unrelated to auth.
          "$VENV_PYTHON" test/server_cli_apikey.py --cli-binary "$CLI_BINARY"

      - name: Capture and upload server logs
        if: always()
        uses: ./.github/actions/capture-server-logs
        with:
          artifact-name: server-logs-apikey-windows-latest

      - name: Teardown Windows CI environment
        if: always()
        uses: ./.github/actions/teardown-windows-ci-env

  # ========================================================================
  # RELEASE JOB - Add artifacts to GitHub release
  # ========================================================================

  release:
    name: Create GitHub Release
    runs-on: ubuntu-latest
    needs:
      - sign-msi-installers
      - build-lemonade-rpm
      - build-lemonade-debian13
      - test-lemonade-debian13-smoke
      - build-lemonade-macos-dmg
      - build-lemonade-embeddable-linux
      - build-lemonade-embeddable-windows
      - build-lemonade-embeddable-macos
      - test-cli-endpoints-windows
      - test-cli-endpoints-macos
      - test-rpm-package
      - test-embeddable-posix
      - test-embeddable-macos
      - test-embeddable-windows
    if: startsWith(github.ref, 'refs/tags/v')
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
    steps:
      - name: Checkout for release notes action
        uses: actions/checkout@v5
        with:
          sparse-checkout: .github

      - name: Download Signed Lemonade Server Installer (Windows)
        uses: actions/download-artifact@v7
        with:
          name: Lemonade_Server_MSI_Signed
          path: .

      - name: Download Lemonade .rpm Packages (x86_64)
        uses: actions/download-artifact@v7
        with:
          pattern: lemonade-rpm-fedora-*-x86_64
          path: .
          merge-multiple: true

      - name: Download Lemonade .rpm Packages (ARM64)
        uses: actions/download-artifact@v7
        with:
          pattern: lemonade-rpm-fedora-*-arm64
          path: .
          merge-multiple: true

      - name: Download Lemonade .deb (Debian 13, X86_64)
        uses: actions/download-artifact@v7
        with:
          name: lemonade-debian13-x86_64
          path: .

      - name: Download Lemonade .deb (Debian 13, ARM64)
        uses: actions/download-artifact@v7
        with:
          name: lemonade-debian13-arm64
          path: .

      - name: Download Lemonade macOS .pkg Package
        uses: actions/download-artifact@v7
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Download Lemonade Embeddable (Linux, X86_64)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-linux-x86_64
          path: .

      - name: Download Lemonade Embeddable (Linux, ARM64)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-linux-arm64
          path: .

      - name: Download Lemonade Embeddable (Windows)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-windows
          path: .

      - name: Download Lemonade Embeddable (macOS)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-embeddable-macos
          path: .

      - name: Verify release artifacts
        run: |
          echo "Release artifacts:"
          ls -lh lemonade-server-minimal.msi
          ls -lh lemonade.msi
          ls -lh lemonade-server-*.x86_64.rpm
          ls -lh lemonade-server-*.aarch64.rpm
          ls -lh lemonade-server_${LEMONADE_VERSION}-debian13_amd64.deb
          ls -lh lemonade-server_${LEMONADE_VERSION}-debian13_arm64.deb
          ls -lh *.pkg
          ls -lh lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-x64.tar.gz
          ls -lh lemonade-embeddable-${LEMONADE_VERSION}-ubuntu-arm64.tar.gz
          ls -lh lemonade-embeddable-${LEMONADE_VERSION}-windows-x64.zip
          ls -lh lemonade-embeddable-${LEMONADE_VERSION}-macos-arm64.tar.gz

      - name: Generate release notes
        id: release-notes
        uses: ./.github/actions/generate-release-notes
        with:
          version: ${{ env.LEMONADE_VERSION }}
          repo: ${{ github.repository }}
          github_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Create Release
        uses: softprops/action-gh-release@v2
        with:
          name: ${{ github.ref_name }}
          body_path: ${{ steps.release-notes.outputs.release_notes_file }}
          files: |
            lemonade-server-minimal.msi
            lemonade.msi
            lemonade-server-*.x86_64.rpm
            lemonade-server-*.aarch64.rpm
            lemonade-server_${{ env.LEMONADE_VERSION }}-debian13_amd64.deb
            lemonade-server_${{ env.LEMONADE_VERSION }}-debian13_arm64.deb
            *.pkg
            lemonade-embeddable-${{ env.LEMONADE_VERSION }}-ubuntu-x64.tar.gz
            lemonade-embeddable-${{ env.LEMONADE_VERSION }}-ubuntu-arm64.tar.gz
            lemonade-embeddable-${{ env.LEMONADE_VERSION }}-windows-x64.zip
            lemonade-embeddable-${{ env.LEMONADE_VERSION }}-macos-arm64.tar.gz
          fail_on_unmatched_files: true