fix(eval): harden agile skill quality #31
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Skill Eval (promptfoo) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| skill: | |
| description: "Eval config name: agile-delivery-agent | agile-story-writer | agile-story-splitter | problem-framing | sprint-goal-writer (blank = all)" | |
| required: false | |
| default: "" | |
| pull_request: | |
| paths: | |
| - ".github/skills/**" | |
| - ".github/agents/**" | |
| - "evals/**" | |
| - "package.json" | |
| - "package-lock.json" | |
| - "scripts/**" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| eval: | |
| name: promptfoo eval | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "24.15.0" | |
| cache: npm | |
| - name: Install dependencies | |
| run: npm ci | |
| # Guard: skip (with a visible warning) when no API key is configured. | |
| # This keeps forks and first-time contributors unblocked without silently | |
| # passing on mock data. Supports any provider (Anthropic, OpenAI, Google, OpenRouter). | |
| - name: Check API key | |
| id: key_check | |
| run: | | |
| if [ -n "$EVAL_MODEL_CONFIGURED" ]; then | |
| eval_model="$EVAL_MODEL_CONFIGURED" | |
| elif [ -n "$ANTHROPIC_API_KEY" ]; then | |
| eval_model="anthropic:claude-3-7-sonnet-20250219" | |
| elif [ -n "$OPENAI_API_KEY" ]; then | |
| eval_model="openai:gpt-4o" | |
| elif [ -n "$GOOGLE_API_KEY" ]; then | |
| eval_model="google:gemini-2.0-flash" | |
| elif [ -n "$OPENROUTER_API_KEY" ]; then | |
| eval_model="openrouter:mistralai/mistral-large" | |
| else | |
| eval_model="" | |
| fi | |
| if [ -z "$ANTHROPIC_API_KEY" ] && [ -z "$OPENAI_API_KEY" ] && [ -z "$GOOGLE_API_KEY" ] && [ -z "$OPENROUTER_API_KEY" ]; then | |
| echo "has_key=false" >> "$GITHUB_OUTPUT" | |
| echo "::warning::No API key secret configured — eval skipped. \ | |
| Add one of: ANTHROPIC_API_KEY, OPENAI_API_KEY, GOOGLE_API_KEY, or OPENROUTER_API_KEY \ | |
| in Settings → Secrets → Actions to enable live CI evals." | |
| else | |
| echo "has_key=true" >> "$GITHUB_OUTPUT" | |
| echo "eval_model=$eval_model" >> "$GITHUB_OUTPUT" | |
| echo "Using EVAL_MODEL=$eval_model" | |
| fi | |
| env: | |
| EVAL_MODEL_CONFIGURED: ${{ secrets.EVAL_MODEL }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| - name: Run evals — specific skill | |
| if: steps.key_check.outputs.has_key == 'true' && github.event_name == 'workflow_dispatch' && inputs.skill != '' | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| EVAL_MODEL: ${{ steps.key_check.outputs.eval_model }} | |
| run: | | |
| mkdir -p promptfoo-results | |
| npm exec -- promptfoo eval \ | |
| --config "evals/${{ github.event.inputs.skill }}.yaml" \ | |
| --output "promptfoo-results/${{ github.event.inputs.skill }}.json" | |
| - name: Run evals — all skills | |
| if: steps.key_check.outputs.has_key == 'true' && (github.event_name != 'workflow_dispatch' || inputs.skill == '') | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| EVAL_MODEL: ${{ steps.key_check.outputs.eval_model }} | |
| run: | | |
| EXIT=0 | |
| mkdir -p promptfoo-results | |
| for config in evals/agile-delivery-agent.yaml \ | |
| evals/agile-story-writer.yaml \ | |
| evals/agile-story-splitter.yaml \ | |
| evals/problem-framing.yaml \ | |
| evals/sprint-goal-writer.yaml; do | |
| echo "──────────────────────────────────────────" | |
| echo "Evaluating: $config" | |
| echo "──────────────────────────────────────────" | |
| name="$(basename "$config" .yaml)" | |
| npm exec -- promptfoo eval \ | |
| --config "$config" \ | |
| --output "promptfoo-results/${name}.json" || EXIT=1 | |
| done | |
| exit $EXIT | |
| - name: Report eval costs | |
| if: always() && steps.key_check.outputs.has_key == 'true' | |
| run: node scripts/report-eval-costs.mjs promptfoo-results | |
| - name: Upload eval artifacts | |
| if: always() && steps.key_check.outputs.has_key == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: promptfoo-results | |
| path: promptfoo-results/ | |
| if-no-files-found: ignore |