test #8
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| # Conformance run for the Vyges Loom sign-off engines, driven through `vyges mcp`. | |
| # deterministic — replays known-good tool calls (no LLM); this GATES. | |
| # agentic — one GitHub Models LLM must pick each tool + form its args from the | |
| # engine descriptors alone. ADVISORY (never fails the run). | |
| # report — publishes a dashboard of both to GitHub Pages. | |
| on: | |
| workflow_dispatch: # manual only — the agentic sweep uses GitHub Models quota, so never auto-run | |
| permissions: | |
| contents: read | |
| models: read | |
| pages: write | |
| id-token: write | |
| env: | |
| MODEL: openai/gpt-4.1 | |
| jobs: | |
| deterministic: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install the Vyges CLI + Loom engines | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -LsSf \ | |
| https://github.com/vyges-tools/cli/releases/latest/download/vyges-installer.sh | sh | |
| echo "$HOME/.vyges/bin" >> "$GITHUB_PATH" | |
| echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - run: vyges install loom | |
| - name: Deterministic conformance (gates) | |
| run: | | |
| python3 conformance.py cases.json --driver echo --report report-echo.json | tee out.txt | |
| { echo '## Deterministic (echo)'; echo '```'; cat out.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Render a real chip for the dashboard (best-effort) | |
| continue-on-error: true | |
| run: vyges-gds-view render fixtures/drc/edge_sensor_glue.gds --top edge_sensor_glue --png --width 380 -o chip.png | |
| - uses: actions/upload-artifact@v4 | |
| with: { name: report-echo, path: report-echo.json } | |
| - uses: actions/upload-artifact@v4 | |
| with: { name: chip, path: chip.png, if-no-files-found: ignore } | |
| agentic: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install the Vyges CLI + Loom engines | |
| run: | | |
| curl --proto '=https' --tlsv1.2 -LsSf \ | |
| https://github.com/vyges-tools/cli/releases/latest/download/vyges-installer.sh | sh | |
| echo "$HOME/.vyges/bin" >> "$GITHUB_PATH" | |
| echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" | |
| - run: vyges install loom | |
| - name: Agentic conformance (advisory) — ${{ env.MODEL }} | |
| continue-on-error: true # advisory: a model miss / rate-limit never fails CI | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.MODELS_TOKEN || github.token }} | |
| run: | | |
| python3 conformance.py cases.json --driver github --model "$MODEL" \ | |
| --report "report-agentic.json" | tee out.txt | |
| { echo "## Agentic — $MODEL"; echo '```'; cat out.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" | |
| - uses: actions/upload-artifact@v4 | |
| with: { name: report-agentic, path: report-agentic.json } | |
| report: | |
| needs: [deterministic, agentic] | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deploy.outputs.page_url }} | |
| concurrency: | |
| group: pages | |
| cancel-in-progress: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: { path: reports, merge-multiple: true } | |
| - name: Render the dashboard | |
| run: | | |
| python3 render_report.py --out site --chip reports/chip.png \ | |
| --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ | |
| reports/report-echo.json reports/report-agentic.json | |
| - uses: actions/configure-pages@v5 | |
| - uses: actions/upload-pages-artifact@v3 | |
| with: { path: site } | |
| - id: deploy | |
| uses: actions/deploy-pages@v4 |