Backend Battle Nightly Regression #1123
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate New stable-diffusion.cpp Release | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| release: | |
| description: "Optional upstream leejet tag, e.g. master-714-db48014. Empty = latest." | |
| required: false | |
| type: string | |
| cuda_release: | |
| description: "Optional lemonade-sdk CUDA tag, e.g. master-714-db48014. Empty = latest CUDA release." | |
| required: false | |
| type: string | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| merge_group: | |
| schedule: | |
| # Sunday at 17:20 UTC, staggered after the llama.cpp auto-update workflow. | |
| - cron: "20 17 * * 0" | |
| permissions: | |
| contents: read | |
| env: | |
| LEMONADE_DISABLE_SYSTEMD_JOURNAL: "1" | |
| LEMONADE_CI_MODE: "True" | |
| PYTHONIOENCODING: utf-8 | |
| HF_TOKEN: ${{ secrets.HUGGINGFACE_ACCESS_TOKEN }} | |
| SDCPP_REPO: leejet/stable-diffusion.cpp | |
| SDCPP_CUDA_REPO: lemonade-sdk/stable-diffusion.cpp | |
| # Non-CUDA assets are published by leejet. CUDA assets are published by the | |
| # lemonade-sdk fork and may intentionally be on a different release tag. | |
| SDCPP_BASE_UPDATE_BACKENDS: cpu,vulkan,rocm-stable,metal | |
| SDCPP_CUDA_UPDATE_BACKENDS: cuda | |
| SDCPP_VALIDATED_LABELS: cpu,vulkan,rocm-stable | |
| SDCPP_TEST_PROMPT: "A small glass of lemonade on a clean table, product photo, high detail" | |
| SDCPP_TEST_MODELS: "SD-Turbo-GGUF,Flux-2-Klein-4B" | |
| SDCPP_TEST_SIZES: "512x256,1024x1024" | |
| SDCPP_TEST_SEED: "12345" | |
| SDCPP_TEST_STEPS: "4" | |
| LEMONADE_VALIDATE_SD_TIMEOUT: "7200" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'pull_request' && 'pr' || github.run_id }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| discover-release: | |
| name: Discover stable-diffusion.cpp releases | |
| # Label a PR `ci:upgrades` to opt it back in. | |
| if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:upgrades') | |
| runs-on: ubuntu-latest | |
| outputs: | |
| base_release: ${{ steps.release.outputs.base_release }} | |
| base_short_release: ${{ steps.release.outputs.base_short_release }} | |
| cuda_release: ${{ steps.release.outputs.cuda_release }} | |
| cuda_short_release: ${{ steps.release.outputs.cuda_short_release }} | |
| is_update_run: ${{ steps.release.outputs.is_update_run }} | |
| base_update_backends: ${{ steps.assets.outputs.base_update_backends }} | |
| cuda_update_backends: ${{ steps.assets.outputs.cuda_update_backends }} | |
| update_backends: ${{ steps.assets.outputs.update_backends }} | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Resolve release tags | |
| id: release | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| INPUT_RELEASE: ${{ github.event_name == 'workflow_dispatch' && inputs.release || '' }} | |
| INPUT_CUDA_RELEASE: ${{ github.event_name == 'workflow_dispatch' && inputs.cuda_release || '' }} | |
| run: | | |
| set -euo pipefail | |
| # Only the auto-update runs resolve upstream releases. Everywhere else | |
| # the build validates the pins committed in backend_versions.json, so a | |
| # bad upstream release can't fail a merge or hide a broken pin. | |
| UPDATE_RUN=false | |
| if [ "${{ github.event_name }}" = "schedule" ] || [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| UPDATE_RUN=true | |
| fi | |
| if [ "$UPDATE_RUN" = "false" ]; then | |
| read -r BASE_RELEASE CUDA_RELEASE < <(python3 - <<'PY' | |
| import json | |
| with open('src/cpp/resources/backend_versions.json', encoding='utf-8') as f: | |
| data = json.load(f) | |
| print(data['sd-cpp']['cpu'], data['sd-cpp']['cuda']) | |
| PY | |
| ) | |
| else | |
| if [ -n "$INPUT_RELEASE" ]; then | |
| BASE_RELEASE="$INPUT_RELEASE" | |
| else | |
| BASE_RELEASE=$(gh api repos/${SDCPP_REPO}/releases/latest --jq '.tag_name') | |
| fi | |
| if [ -n "$INPUT_CUDA_RELEASE" ]; then | |
| CUDA_RELEASE="$INPUT_CUDA_RELEASE" | |
| else | |
| CUDA_RELEASE=$(gh api repos/${SDCPP_CUDA_REPO}/releases/latest --jq '.tag_name') | |
| fi | |
| fi | |
| python3 - "$BASE_RELEASE" "$CUDA_RELEASE" <<'PY' > release_outputs.env | |
| import re | |
| import sys | |
| release_re = re.compile(r"^master-[0-9]+-[0-9a-f]{7,40}$") | |
| def short_release(value: str) -> str: | |
| parts = value.split('-', 2) | |
| if len(parts) != 3: | |
| raise SystemExit(f"Invalid stable-diffusion.cpp release tag: {value}") | |
| return f"{parts[0]}-{parts[2]}" | |
| base_release, cuda_release = sys.argv[1], sys.argv[2] | |
| for label, value in (("base", base_release), ("cuda", cuda_release)): | |
| if not release_re.match(value): | |
| raise SystemExit( | |
| f"Invalid {label} stable-diffusion.cpp release tag: {value}. " | |
| "Expected format like master-714-db48014." | |
| ) | |
| print(f"base_release={base_release}") | |
| print(f"base_short_release={short_release(base_release)}") | |
| print(f"cuda_release={cuda_release}") | |
| print(f"cuda_short_release={short_release(cuda_release)}") | |
| PY | |
| cat release_outputs.env >> "$GITHUB_OUTPUT" | |
| echo "is_update_run=$UPDATE_RUN" >> "$GITHUB_OUTPUT" | |
| echo "Using base stable-diffusion.cpp release: ${BASE_RELEASE}" | |
| echo "Using CUDA stable-diffusion.cpp release: ${CUDA_RELEASE}" | |
| - name: Verify base release assets | |
| id: assets | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BASE_RELEASE: ${{ steps.release.outputs.base_release }} | |
| BASE_SHORT_RELEASE: ${{ steps.release.outputs.base_short_release }} | |
| run: | | |
| set -euo pipefail | |
| echo "Base assets from ${SDCPP_REPO}@${BASE_RELEASE}:" | |
| gh api repos/${SDCPP_REPO}/releases/tags/${BASE_RELEASE} --jq '.assets[].name' | sort > assets.txt | |
| cat assets.txt | |
| python3 <<'PY' | |
| import os | |
| import re | |
| import sys | |
| from pathlib import Path | |
| base_short = os.environ['BASE_SHORT_RELEASE'] | |
| base_assets = set(Path('assets.txt').read_text(encoding='utf-8').splitlines()) | |
| checks = { | |
| 'cpu': [ | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-win-avx2-x64\.zip$'), | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-Linux-Ubuntu-24\.04-x86_64\.zip$'), | |
| ], | |
| 'vulkan': [ | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-win-vulkan-x64\.zip$'), | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-Linux-Ubuntu-24\.04-x86_64-vulkan\.zip$'), | |
| ], | |
| 'rocm-stable': [ | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-win-rocm-[0-9][^/]*-x64\.zip$'), | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-Linux-Ubuntu-24\.04-x86_64-rocm-[0-9][^/]*\.zip$'), | |
| ], | |
| 'metal': [ | |
| (base_assets, rf'^sd-{re.escape(base_short)}-bin-Darwin-macOS-[^/]+-arm64\.zip$'), | |
| ], | |
| } | |
| missing = [] | |
| for backend, patterns in checks.items(): | |
| for assets, pattern in patterns: | |
| if not any(re.match(pattern, asset) for asset in assets): | |
| missing.append(f"{backend}: {pattern}") | |
| if missing: | |
| print('Missing expected release assets:', file=sys.stderr) | |
| for item in missing: | |
| print(f' - {item}', file=sys.stderr) | |
| sys.exit(1) | |
| PY | |
| echo "base_update_backends=${SDCPP_BASE_UPDATE_BACKENDS}" >> "$GITHUB_OUTPUT" | |
| echo "cuda_update_backends=${SDCPP_CUDA_UPDATE_BACKENDS}" >> "$GITHUB_OUTPUT" | |
| echo "update_backends=${SDCPP_BASE_UPDATE_BACKENDS},${SDCPP_CUDA_UPDATE_BACKENDS}" >> "$GITHUB_OUTPUT" | |
| echo "Will update base sd-cpp pins: ${SDCPP_BASE_UPDATE_BACKENDS} -> ${BASE_RELEASE}" | |
| echo "Will update CUDA sd-cpp pins later after validation: ${SDCPP_CUDA_UPDATE_BACKENDS} -> ${{ steps.release.outputs.cuda_release }}" | |
| build: | |
| name: Build Lemonade with sd-cpp pins | |
| needs: discover-release | |
| # Label a PR `ci:upgrades` to opt it back in. | |
| if: (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:upgrades')) && always() && !failure() && !cancelled() | |
| runs-on: windows-latest | |
| steps: | |
| - name: Prepare Windows runner for long paths and stale workspace caches | |
| if: runner.os == 'Windows' | |
| shell: PowerShell | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| # 1. Kill any running/orphaned processes to release file locks | |
| $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer") | |
| foreach ($p in $patterns) { | |
| Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue | |
| } | |
| - uses: actions/checkout@v5 | |
| with: | |
| clean: true | |
| fetch-depth: 0 | |
| - name: Update backend_versions.json for validation | |
| if: needs.discover-release.outputs.is_update_run == 'true' | |
| shell: PowerShell | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| python .github/scripts/update_sdcpp_versions.py ` | |
| --release "${{ needs.discover-release.outputs.base_release }}" ` | |
| --backends "${{ needs.discover-release.outputs.base_update_backends }}" | |
| python .github/scripts/update_sdcpp_versions.py ` | |
| --release "${{ needs.discover-release.outputs.cuda_release }}" ` | |
| --backends "${{ needs.discover-release.outputs.cuda_update_backends }}" | |
| - name: Build lemond and the CLI | |
| shell: PowerShell | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| Write-Host "Building Lemonade server binaries..." -ForegroundColor Cyan | |
| if (Test-Path "build") { Remove-Item -Recurse -Force "build" } | |
| # BUILD_WEB_APP=OFF skips the configure-time CONFIGURE_DEPENDS glob | |
| # over src/app; the web-app target is not in this build graph anyway. | |
| cmake --preset vs18 -DBUILD_WEB_APP=OFF | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| # /m:2 goes straight to MSBuild rather than through --parallel, which | |
| # makes CMake set CL_MPCount=1 and cancel the /MP in CMakeLists.txt. | |
| # Capped at 2 because /MP already spawns one cl.exe per core. | |
| cmake --build build --config Release --target lemond lemonade -- /m:2 | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| if (-not (Test-Path "build\Release\lemond.exe")) { | |
| Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red | |
| exit 1 | |
| } | |
| if (-not (Test-Path "build\Release\lemonade.exe")) { | |
| Write-Host "ERROR: lemonade.exe not found!" -ForegroundColor Red | |
| exit 1 | |
| } | |
| Write-Host "Build successful!" -ForegroundColor Green | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sdcpp-build | |
| path: | | |
| build/Release/ | |
| build/resources/ | |
| retention-days: 7 | |
| validate: | |
| name: Validate ${{ matrix.label }} | |
| needs: [discover-release, build] | |
| if: always() && needs.build.result == 'success' | |
| runs-on: ${{ matrix.runner }} | |
| timeout-minutes: 420 | |
| strategy: | |
| fail-fast: ${{ github.event_name == 'merge_group' }} | |
| matrix: | |
| # SDCPP_TEST_MODELS peaks at ~18 GB resident, so these legs never needed | |
| # the 128gb runner; the sd-cpp suite in cpp_server_build_test_release.yml | |
| # already runs on hosts with no size label at all. | |
| include: | |
| - label: cpu | |
| backend: cpu | |
| channel: "" | |
| runner: [self-hosted, Windows, stx-halo, vulkan, lemon-prod] | |
| - label: vulkan | |
| backend: vulkan | |
| channel: "" | |
| runner: [self-hosted, Windows, stx-halo, vulkan, lemon-prod] | |
| - label: rocm-stable | |
| backend: rocm | |
| channel: stable | |
| runner: [self-hosted, Windows, stx-halo, rocm, lemon-prod] | |
| # CUDA is pinned by the updater and its assets are required by the | |
| # verify-cuda-assets job after this validation matrix completes. | |
| # Validation stays disabled until a matching runner exists. | |
| # - label: cuda | |
| # backend: cuda | |
| # channel: "" | |
| # runner: [self-hosted, Windows, 128gb, cuda] | |
| steps: | |
| - name: Prepare Windows runner for long paths and stale workspace caches | |
| if: runner.os == 'Windows' | |
| shell: PowerShell | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| # 1. Kill any running/orphaned processes to release file locks | |
| $patterns = @("lemonade", "lemond", "llama-server", "llama", "flm", "ort-server", "moonshine-server", "wscript", "LemonadeServer") | |
| foreach ($p in $patterns) { | |
| Get-Process | Where-Object { $_.ProcessName -like "*$p*" } | Stop-Process -Force -ErrorAction SilentlyContinue | |
| } | |
| - uses: actions/checkout@v5 | |
| with: | |
| clean: true | |
| - name: Cleanup processes | |
| uses: ./.github/actions/cleanup-processes-windows | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: sdcpp-build | |
| path: build | |
| - name: Verify binaries | |
| shell: PowerShell | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| $lemondExe = "build\Release\lemond.exe" | |
| $lemonadeExe = "build\Release\lemonade.exe" | |
| if (-not (Test-Path $lemondExe)) { | |
| Write-Host "ERROR: lemond.exe not found!" -ForegroundColor Red | |
| Get-ChildItem -Recurse build | Select-Object FullName | |
| exit 1 | |
| } | |
| if (-not (Test-Path $lemonadeExe)) { | |
| Write-Host "ERROR: lemonade.exe not found!" -ForegroundColor Red | |
| Get-ChildItem -Recurse build | Select-Object FullName | |
| exit 1 | |
| } | |
| & $lemondExe --version | |
| & $lemonadeExe --version | |
| Write-Host "Binaries verified!" -ForegroundColor Green | |
| - name: Setup Python and virtual environment | |
| uses: ./.github/actions/setup-venv | |
| with: | |
| venv-name: '.venv' | |
| python-version: '3.10' | |
| requirements-file: 'test/requirements.txt' | |
| - name: Run validation with lemond | |
| shell: PowerShell | |
| env: | |
| HF_TOKEN: ${{ env.HF_TOKEN }} | |
| LEMONADE_VALIDATE_SD_PROMPT: ${{ env.SDCPP_TEST_PROMPT }} | |
| LEMONADE_VALIDATE_SD_STEPS: ${{ env.SDCPP_TEST_STEPS }} | |
| LEMONADE_VALIDATE_SD_TIMEOUT: ${{ env.LEMONADE_VALIDATE_SD_TIMEOUT }} | |
| run: | | |
| $ErrorActionPreference = "Stop" | |
| $lemondExe = (Resolve-Path "build\Release\lemond.exe").Path | |
| $backend = "${{ matrix.backend }}" | |
| $channel = "${{ matrix.channel }}" | |
| $label = "${{ matrix.label }}" | |
| $logsDir = "server-logs-$label" | |
| $cacheDir = Join-Path $PWD "ci-cache-$label" | |
| $venvPython = ".\.venv\Scripts\python.exe" | |
| New-Item -ItemType Directory -Force -Path $logsDir | Out-Null | |
| New-Item -ItemType Directory -Force -Path $cacheDir | Out-Null | |
| $stdoutLog = Join-Path $PWD "$logsDir\lemond.stdout.log" | |
| $stderrLog = Join-Path $PWD "$logsDir\lemond.stderr.log" | |
| $proc = Start-Process ` | |
| -FilePath $lemondExe ` | |
| -ArgumentList @($cacheDir, "--port", "13305", "--host", "127.0.0.1") ` | |
| -RedirectStandardOutput $stdoutLog ` | |
| -RedirectStandardError $stderrLog ` | |
| -PassThru | |
| Write-Host "Started lemond PID $($proc.Id)" -ForegroundColor Cyan | |
| try { | |
| $validationArgs = @( | |
| "test/validate_sdcpp.py", | |
| "--backend", $backend, | |
| "--seed", "${{ env.SDCPP_TEST_SEED }}", | |
| "--warmup", | |
| "--output", "sdcpp_validation_$label.json", | |
| "--images-dir", "sdcpp-images-$label" | |
| ) | |
| foreach ($model in "${{ env.SDCPP_TEST_MODELS }}".Split(',')) { | |
| $model = $model.Trim() | |
| if ($model) { | |
| $validationArgs += "--model" | |
| $validationArgs += $model | |
| } | |
| } | |
| foreach ($size in "${{ env.SDCPP_TEST_SIZES }}".Split(',')) { | |
| $size = $size.Trim() | |
| if ($size) { | |
| $validationArgs += "--size" | |
| $validationArgs += $size | |
| } | |
| } | |
| if ($channel) { | |
| $validationArgs += "--channel" | |
| $validationArgs += $channel | |
| } | |
| & $venvPython @validationArgs | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| } finally { | |
| try { | |
| Invoke-WebRequest -Uri "http://127.0.0.1:13305/internal/shutdown" ` | |
| -Method POST -TimeoutSec 10 | Out-Null | |
| Start-Sleep -Seconds 2 | |
| } catch { | |
| Write-Host "lemond shutdown not reachable; relying on cleanup step." -ForegroundColor Yellow | |
| } | |
| } | |
| - name: Upload validation evidence | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sdcpp-validation-${{ matrix.label }} | |
| path: | | |
| sdcpp_validation_${{ matrix.label }}.json | |
| sdcpp-images-${{ matrix.label }}/ | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| - name: Upload server logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: sdcpp-server-logs-${{ matrix.label }} | |
| path: server-logs-${{ matrix.label }}/ | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| - name: Cleanup | |
| if: always() | |
| uses: ./.github/actions/cleanup-processes-windows | |
| verify-cuda-assets: | |
| name: Verify required CUDA assets | |
| needs: [discover-release, validate] | |
| if: always() && needs.discover-release.result == 'success' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Verify CUDA release assets | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| CUDA_RELEASE: ${{ needs.discover-release.outputs.cuda_release }} | |
| CUDA_SHORT_RELEASE: ${{ needs.discover-release.outputs.cuda_short_release }} | |
| run: | | |
| set -euo pipefail | |
| echo "CUDA asset validation intentionally runs after the image validation matrix." | |
| echo "If CUDA assets are missing, this job hard-fails after CPU/Vulkan/ROCm evidence is available." | |
| echo "CUDA assets from ${SDCPP_CUDA_REPO}@${CUDA_RELEASE}:" | |
| gh api repos/${SDCPP_CUDA_REPO}/releases/tags/${CUDA_RELEASE} --jq '.assets[].name' | sort > cuda-assets.txt | |
| cat cuda-assets.txt | |
| python3 <<'PY' | |
| import os | |
| import re | |
| import sys | |
| from pathlib import Path | |
| cuda_short = os.environ['CUDA_SHORT_RELEASE'] | |
| cuda_assets = set(Path('cuda-assets.txt').read_text(encoding='utf-8').splitlines()) | |
| patterns = [ | |
| rf'^sd-{re.escape(cuda_short)}-windows-cuda-sm_[0-9]+-x64\.zip$', | |
| rf'^sd-{re.escape(cuda_short)}-ubuntu-cuda-sm_[0-9]+-x64\.tar\.xz$', | |
| ] | |
| missing = [] | |
| for pattern in patterns: | |
| if not any(re.match(pattern, asset) for asset in cuda_assets): | |
| missing.append(pattern) | |
| if missing: | |
| print('Missing expected CUDA release assets:', file=sys.stderr) | |
| for pattern in missing: | |
| print(f' - {pattern}', file=sys.stderr) | |
| sys.exit(1) | |
| PY | |
| create-pr: | |
| name: Create update PR | |
| needs: [discover-release, validate, verify-cuda-assets] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| if: >- | |
| (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && | |
| needs.validate.result == 'success' && | |
| needs.verify-cuda-assets.result == 'success' | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - name: Download validation evidence | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: sdcpp-validation-* | |
| merge-multiple: true | |
| - name: Update backend_versions.json | |
| run: | | |
| python3 .github/scripts/update_sdcpp_versions.py \ | |
| --release "${{ needs.discover-release.outputs.base_release }}" \ | |
| --backends "${{ needs.discover-release.outputs.base_update_backends }}" | |
| python3 .github/scripts/update_sdcpp_versions.py \ | |
| --release "${{ needs.discover-release.outputs.cuda_release }}" \ | |
| --backends "${{ needs.discover-release.outputs.cuda_update_backends }}" | |
| - name: Check update diff | |
| id: update-diff | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| if git diff --quiet src/cpp/resources/backend_versions.json; then | |
| echo "backend_versions.json is unchanged; nothing to update." | |
| echo "has_update=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "has_update=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Normalize validation evidence and generate PR body | |
| if: steps.update-diff.outputs.has_update == 'true' | |
| shell: bash | |
| env: | |
| BASE_RELEASE: ${{ needs.discover-release.outputs.base_release }} | |
| CUDA_RELEASE: ${{ needs.discover-release.outputs.cuda_release }} | |
| BASE_UPDATE_BACKENDS: ${{ needs.discover-release.outputs.base_update_backends }} | |
| CUDA_UPDATE_BACKENDS: ${{ needs.discover-release.outputs.cuda_update_backends }} | |
| run: | | |
| set -euo pipefail | |
| UPDATE_BRANCH="auto/sdcpp-update-${BASE_RELEASE}-cuda-${CUDA_RELEASE}" | |
| EVIDENCE_BRANCH="auto/sdcpp-validation-images" | |
| EVIDENCE_ROOT=".github/sdcpp-validation" | |
| EVIDENCE_NAME="${BASE_RELEASE}-cuda-${CUDA_RELEASE}" | |
| EVIDENCE_DIR="${EVIDENCE_ROOT}/${EVIDENCE_NAME}" | |
| # The shared evidence branch keeps the newest three validation results. | |
| # Older historical PR image links intentionally expire after pruning. | |
| rm -rf "$EVIDENCE_ROOT" | |
| python3 .github/scripts/render_sdcpp_pr_body.py \ | |
| --base-release "$BASE_RELEASE" \ | |
| --cuda-release "$CUDA_RELEASE" \ | |
| --base-update-backends "$BASE_UPDATE_BACKENDS" \ | |
| --cuda-update-backends "$CUDA_UPDATE_BACKENDS" \ | |
| --validated-labels "${SDCPP_VALIDATED_LABELS}" \ | |
| --repository "${GITHUB_REPOSITORY}" \ | |
| --image-ref "$EVIDENCE_BRANCH" \ | |
| --evidence-dir "$EVIDENCE_DIR" \ | |
| --models "${SDCPP_TEST_MODELS}" \ | |
| --sizes "${SDCPP_TEST_SIZES}" \ | |
| --output pr_body.md | |
| { | |
| echo "UPDATE_BRANCH=$UPDATE_BRANCH" | |
| echo "EVIDENCE_BRANCH=$EVIDENCE_BRANCH" | |
| echo "EVIDENCE_ROOT=$EVIDENCE_ROOT" | |
| echo "EVIDENCE_NAME=$EVIDENCE_NAME" | |
| } >> "$GITHUB_ENV" | |
| - name: Publish validation images | |
| if: steps.update-diff.outputs.has_update == 'true' | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BASE_RELEASE: ${{ needs.discover-release.outputs.base_release }} | |
| CUDA_RELEASE: ${{ needs.discover-release.outputs.cuda_release }} | |
| run: | | |
| set -euo pipefail | |
| if [ ! -d "$EVIDENCE_ROOT/$EVIDENCE_NAME" ]; then | |
| echo "Expected validation evidence directory is missing: $EVIDENCE_ROOT/$EVIDENCE_NAME" >&2 | |
| exit 1 | |
| fi | |
| PUBLISH_DIR="$(mktemp -d)" | |
| SNAPSHOT_DIR="$(mktemp -d)" | |
| CURRENT_EVIDENCE="$(mktemp -d)" | |
| cp -R "$EVIDENCE_ROOT/$EVIDENCE_NAME" "$CURRENT_EVIDENCE/$EVIDENCE_NAME" | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| git init "$PUBLISH_DIR" | |
| cd "$PUBLISH_DIR" | |
| git remote add origin "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| REMOTE_SHA=$(git ls-remote origin "refs/heads/$EVIDENCE_BRANCH" | awk '{print $1}') | |
| if [ -n "$REMOTE_SHA" ]; then | |
| git fetch --depth 1 origin "refs/heads/$EVIDENCE_BRANCH" | |
| git checkout --detach FETCH_HEAD | |
| if [ -d .github/sdcpp-validation ]; then | |
| cp -R .github/sdcpp-validation "$SNAPSHOT_DIR/sdcpp-validation" | |
| fi | |
| fi | |
| mkdir -p "$SNAPSHOT_DIR/sdcpp-validation" | |
| RESULT_DIR="$SNAPSHOT_DIR/sdcpp-validation/$EVIDENCE_NAME" | |
| # Preserve the original publication time when a workflow reruns for the | |
| # same release combination, so retention order remains deterministic. | |
| if [ -f "$RESULT_DIR/.published-at" ]; then | |
| PUBLISHED_AT=$(cat "$RESULT_DIR/.published-at") | |
| else | |
| PUBLISHED_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| fi | |
| rm -rf "$RESULT_DIR" | |
| cp -R "$CURRENT_EVIDENCE/$EVIDENCE_NAME" "$RESULT_DIR" | |
| printf '%s\n' "$PUBLISHED_AT" > "$RESULT_DIR/.published-at" | |
| python3 - "$SNAPSHOT_DIR/sdcpp-validation" <<'PY' | |
| import shutil | |
| import sys | |
| from pathlib import Path | |
| root = Path(sys.argv[1]) | |
| entries = [] | |
| for directory in root.iterdir(): | |
| if not directory.is_dir(): | |
| continue | |
| marker = directory / ".published-at" | |
| timestamp = marker.read_text(encoding="utf-8").strip() if marker.exists() else "" | |
| entries.append((timestamp, directory.name, directory)) | |
| entries.sort(reverse=True) | |
| for _, _, directory in entries[3:]: | |
| print(f"Pruning old validation evidence: {directory.name}") | |
| shutil.rmtree(directory) | |
| PY | |
| git checkout --orphan evidence-publish | |
| git rm -rf . >/dev/null 2>&1 || true | |
| find . -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + | |
| mkdir -p .github | |
| cp -R "$SNAPSHOT_DIR/sdcpp-validation" .github/sdcpp-validation | |
| git add .github/sdcpp-validation | |
| git commit -m "Publish stable-diffusion.cpp validation images for ${BASE_RELEASE} and CUDA ${CUDA_RELEASE}" | |
| if [ -n "$REMOTE_SHA" ]; then | |
| git push \ | |
| --force-with-lease="refs/heads/$EVIDENCE_BRANCH:$REMOTE_SHA" \ | |
| origin "HEAD:refs/heads/$EVIDENCE_BRANCH" | |
| else | |
| git push origin "HEAD:refs/heads/$EVIDENCE_BRANCH" | |
| fi | |
| - name: Create or update Pull Request | |
| if: steps.update-diff.outputs.has_update == 'true' | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BASE_RELEASE: ${{ needs.discover-release.outputs.base_release }} | |
| CUDA_RELEASE: ${{ needs.discover-release.outputs.cuda_release }} | |
| run: | | |
| set -euo pipefail | |
| BRANCH="$UPDATE_BRANCH" | |
| TITLE="Update stable-diffusion.cpp to ${BASE_RELEASE} and CUDA ${CUDA_RELEASE}" | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git checkout -B "$BRANCH" | |
| git add src/cpp/resources/backend_versions.json | |
| git commit -m "Update stable-diffusion.cpp pins to ${BASE_RELEASE} and CUDA ${CUDA_RELEASE}" | |
| REMOTE_SHA=$(git ls-remote origin "refs/heads/$BRANCH" | awk '{print $1}') | |
| if [ -n "$REMOTE_SHA" ]; then | |
| git push --force-with-lease="refs/heads/$BRANCH:$REMOTE_SHA" origin "$BRANCH" | |
| else | |
| git push origin "$BRANCH" | |
| fi | |
| EXISTING_PR=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || true) | |
| if [ -n "$EXISTING_PR" ]; then | |
| gh pr edit "$EXISTING_PR" \ | |
| --title "$TITLE" \ | |
| --body-file pr_body.md \ | |
| --base main | |
| else | |
| gh pr create \ | |
| --title "$TITLE" \ | |
| --body-file pr_body.md \ | |
| --base main \ | |
| --head "$BRANCH" | |
| fi | |
| # Gate job that ensures: 1. in the merge queue, all jobs in `needs:` ran | |
| # successfully (otherwise the merge is blocked), and 2. these jobs do not need | |
| # to run on ordinary pull request pushes. | |
| validation-gate: | |
| name: stable-diffusion.cpp validation | |
| needs: [discover-release, build, validate, verify-cuda-assets] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check gated jobs | |
| env: | |
| NEEDS: ${{ toJSON(needs) }} | |
| run: | | |
| # $NEEDS: {"job-id": {"result": "success|failure|skipped|cancelled"}, ...} | |
| # Fail if any job broke. In the merge queue, also fail if any never ran. | |
| echo "$NEEDS" | |
| broke=$(jq -r 'to_entries[]|select(.value.result=="failure" or .value.result=="cancelled")|.key' <<<"$NEEDS") | |
| if [ -n "$broke" ]; then | |
| echo "FAILED: $broke" | |
| exit 1 | |
| fi | |
| if [ "${{ github.event_name }}" = "merge_group" ]; then | |
| absent=$(jq -r 'to_entries[]|select(.value.result!="success")|.key' <<<"$NEEDS") | |
| if [ -n "$absent" ]; then | |
| echo "DID NOT RUN IN MERGE QUEUE: $absent" | |
| exit 1 | |
| fi | |
| fi |