|
| 1 | +name: kolm quality gate |
| 2 | +description: One-line CI gate for a .kolm artifact. Verifies signature + receipt chain, blocks on K-Score regression vs a baseline, and posts a PR comment when the gate fails. Wraps `kolm verify` + the K-Score regression check. |
| 3 | +author: kolm.ai |
| 4 | + |
| 5 | +branding: |
| 6 | + icon: shield |
| 7 | + color: gray-dark |
| 8 | + |
| 9 | +inputs: |
| 10 | + artifact: |
| 11 | + description: Path to the .kolm artifact to gate (relative to workspace root). |
| 12 | + required: true |
| 13 | + threshold: |
| 14 | + description: Minimum K-Score required to pass (0.0 - 1.0). Default 0.85. |
| 15 | + required: false |
| 16 | + default: "0.85" |
| 17 | + baseline: |
| 18 | + description: Optional path to a baseline .kolm artifact (e.g. the artifact pinned on main). When set, the action fails if the candidate's K-Score is more than `max-regression` below the baseline. |
| 19 | + required: false |
| 20 | + default: "" |
| 21 | + max-regression: |
| 22 | + description: Maximum allowed K-Score regression vs baseline, as a fraction (0.02 = 2 percentage points). Only applied when `baseline` is set. |
| 23 | + required: false |
| 24 | + default: "0.01" |
| 25 | + namespace: |
| 26 | + description: Optional namespace label written into the receipt envelope when the gate publishes a report receipt (for the PR-comment audit trail). |
| 27 | + required: false |
| 28 | + default: "" |
| 29 | + kolm-api-key: |
| 30 | + description: Optional KOLM_API_KEY. Only needed when the gate calls api.kolm.ai for tenant-scoped verification (the default offline verify path does not need it). |
| 31 | + required: false |
| 32 | + default: "" |
| 33 | + comment-on-pr: |
| 34 | + description: Post a comment on the PR with the K-Score result and any regression details. Requires `pull-requests:write` on the calling workflow. |
| 35 | + required: false |
| 36 | + default: "true" |
| 37 | + github-token: |
| 38 | + description: GITHUB_TOKEN with `pull-requests:write` scope. Required only when `comment-on-pr` is true. |
| 39 | + required: false |
| 40 | + default: ${{ github.token }} |
| 41 | + |
| 42 | +outputs: |
| 43 | + k-score: |
| 44 | + description: K-Score of the candidate artifact (0.0 - 1.0). |
| 45 | + value: ${{ steps.gate.outputs.k_score }} |
| 46 | + baseline-k-score: |
| 47 | + description: K-Score of the baseline artifact (empty when no baseline supplied). |
| 48 | + value: ${{ steps.gate.outputs.baseline_k_score }} |
| 49 | + delta: |
| 50 | + description: Candidate minus baseline (positive = improvement, negative = regression). Empty when no baseline. |
| 51 | + value: ${{ steps.gate.outputs.delta }} |
| 52 | + passed: |
| 53 | + description: '"true" or "false" — whether the gate passed.' |
| 54 | + value: ${{ steps.gate.outputs.passed }} |
| 55 | + receipt-id: |
| 56 | + description: Receipt id for the verification call (empty on offline-only verify). |
| 57 | + value: ${{ steps.gate.outputs.receipt_id }} |
| 58 | + |
| 59 | +runs: |
| 60 | + using: composite |
| 61 | + steps: |
| 62 | + - name: install kolm CLI |
| 63 | + shell: bash |
| 64 | + run: | |
| 65 | + if ! command -v kolm >/dev/null 2>&1; then |
| 66 | + npm i -g github:kolm-ai/kolmogorov-stack |
| 67 | + else |
| 68 | + echo "kolm CLI on PATH ($(kolm --version 2>/dev/null || echo unknown))" |
| 69 | + fi |
| 70 | +
|
| 71 | + - name: gate |
| 72 | + id: gate |
| 73 | + shell: bash |
| 74 | + env: |
| 75 | + KOLM_GATE_ARTIFACT: ${{ inputs.artifact }} |
| 76 | + KOLM_GATE_THRESHOLD: ${{ inputs.threshold }} |
| 77 | + KOLM_GATE_BASELINE: ${{ inputs.baseline }} |
| 78 | + KOLM_GATE_MAX_REGRESSION: ${{ inputs.max-regression }} |
| 79 | + KOLM_GATE_NAMESPACE: ${{ inputs.namespace }} |
| 80 | + KOLM_API_KEY: ${{ inputs.kolm-api-key }} |
| 81 | + run: | |
| 82 | + set -e |
| 83 | + if [ ! -f "$KOLM_GATE_ARTIFACT" ]; then |
| 84 | + echo "::error::artifact not found: $KOLM_GATE_ARTIFACT" |
| 85 | + exit 1 |
| 86 | + fi |
| 87 | + cand=$(kolm verify "$KOLM_GATE_ARTIFACT" --json) |
| 88 | + echo "candidate verify:" |
| 89 | + echo "$cand" |
| 90 | + cand_ok=$(echo "$cand" | jq -r '.ok') |
| 91 | + cand_score=$(echo "$cand" | jq -r '.k_score // .manifest.k_score // empty') |
| 92 | + cand_receipt=$(echo "$cand" | jq -r '.receipt_id // empty') |
| 93 | + if [ "$cand_ok" != "true" ]; then |
| 94 | + echo "::error::candidate artifact failed verification" |
| 95 | + exit 1 |
| 96 | + fi |
| 97 | + if [ -z "$cand_score" ] || [ "$cand_score" = "null" ]; then |
| 98 | + echo "::error::candidate artifact has no k_score in manifest — cannot gate" |
| 99 | + exit 1 |
| 100 | + fi |
| 101 | + echo "k_score=$cand_score" >> "$GITHUB_OUTPUT" |
| 102 | + [ -n "$cand_receipt" ] && echo "receipt_id=$cand_receipt" >> "$GITHUB_OUTPUT" || true |
| 103 | +
|
| 104 | + threshold="$KOLM_GATE_THRESHOLD" |
| 105 | + below_thresh=$(awk -v a="$cand_score" -v b="$threshold" 'BEGIN{ print (a+0 < b+0) ? 1 : 0 }') |
| 106 | + if [ "$below_thresh" = "1" ]; then |
| 107 | + echo "::error::K-Score $cand_score is below threshold $threshold" |
| 108 | + echo "passed=false" >> "$GITHUB_OUTPUT" |
| 109 | + echo "GATE_REASON=below_threshold" >> "$GITHUB_ENV" |
| 110 | + echo "GATE_CAND_SCORE=$cand_score" >> "$GITHUB_ENV" |
| 111 | + echo "GATE_THRESHOLD=$threshold" >> "$GITHUB_ENV" |
| 112 | + exit 1 |
| 113 | + fi |
| 114 | +
|
| 115 | + if [ -n "$KOLM_GATE_BASELINE" ]; then |
| 116 | + if [ ! -f "$KOLM_GATE_BASELINE" ]; then |
| 117 | + echo "::error::baseline artifact not found: $KOLM_GATE_BASELINE" |
| 118 | + exit 1 |
| 119 | + fi |
| 120 | + base=$(kolm verify "$KOLM_GATE_BASELINE" --json) |
| 121 | + base_ok=$(echo "$base" | jq -r '.ok') |
| 122 | + base_score=$(echo "$base" | jq -r '.k_score // .manifest.k_score // empty') |
| 123 | + if [ "$base_ok" != "true" ]; then |
| 124 | + echo "::error::baseline artifact failed verification" |
| 125 | + exit 1 |
| 126 | + fi |
| 127 | + if [ -z "$base_score" ] || [ "$base_score" = "null" ]; then |
| 128 | + echo "::error::baseline artifact has no k_score in manifest" |
| 129 | + exit 1 |
| 130 | + fi |
| 131 | + delta=$(awk -v a="$cand_score" -v b="$base_score" 'BEGIN{ printf "%.6f", (a+0) - (b+0) }') |
| 132 | + echo "baseline_k_score=$base_score" >> "$GITHUB_OUTPUT" |
| 133 | + echo "delta=$delta" >> "$GITHUB_OUTPUT" |
| 134 | + regressed=$(awk -v d="$delta" -v m="$KOLM_GATE_MAX_REGRESSION" 'BEGIN{ print (d+0 < -1 * (m+0)) ? 1 : 0 }') |
| 135 | + if [ "$regressed" = "1" ]; then |
| 136 | + echo "::error::K-Score regressed by $delta vs baseline (max allowed: -$KOLM_GATE_MAX_REGRESSION)" |
| 137 | + echo "passed=false" >> "$GITHUB_OUTPUT" |
| 138 | + echo "GATE_REASON=regression_vs_baseline" >> "$GITHUB_ENV" |
| 139 | + echo "GATE_CAND_SCORE=$cand_score" >> "$GITHUB_ENV" |
| 140 | + echo "GATE_BASE_SCORE=$base_score" >> "$GITHUB_ENV" |
| 141 | + echo "GATE_DELTA=$delta" >> "$GITHUB_ENV" |
| 142 | + exit 1 |
| 143 | + fi |
| 144 | + fi |
| 145 | + echo "passed=true" >> "$GITHUB_OUTPUT" |
| 146 | + echo "K-Score gate PASSED — candidate=$cand_score threshold=$threshold ${KOLM_GATE_BASELINE:+baseline=$base_score delta=$delta}" |
| 147 | +
|
| 148 | + - name: comment on PR (regression) |
| 149 | + if: failure() && inputs.comment-on-pr == 'true' && github.event_name == 'pull_request' |
| 150 | + uses: actions/github-script@v7 |
| 151 | + env: |
| 152 | + GATE_REASON: ${{ env.GATE_REASON }} |
| 153 | + GATE_CAND_SCORE: ${{ env.GATE_CAND_SCORE }} |
| 154 | + GATE_BASE_SCORE: ${{ env.GATE_BASE_SCORE }} |
| 155 | + GATE_THRESHOLD: ${{ env.GATE_THRESHOLD }} |
| 156 | + GATE_DELTA: ${{ env.GATE_DELTA }} |
| 157 | + GATE_ARTIFACT: ${{ inputs.artifact }} |
| 158 | + GATE_NAMESPACE: ${{ inputs.namespace }} |
| 159 | + with: |
| 160 | + github-token: ${{ inputs.github-token }} |
| 161 | + script: | |
| 162 | + const reason = process.env.GATE_REASON || 'unknown'; |
| 163 | + const cand = process.env.GATE_CAND_SCORE || 'unknown'; |
| 164 | + const base = process.env.GATE_BASE_SCORE || ''; |
| 165 | + const thresh = process.env.GATE_THRESHOLD || ''; |
| 166 | + const delta = process.env.GATE_DELTA || ''; |
| 167 | + const artifact = process.env.GATE_ARTIFACT || ''; |
| 168 | + const namespace = process.env.GATE_NAMESPACE || ''; |
| 169 | + let body = `### kolm quality gate FAILED\n\n`; |
| 170 | + body += `**Artifact:** \`${artifact}\`\n`; |
| 171 | + if (namespace) body += `**Namespace:** \`${namespace}\`\n`; |
| 172 | + body += `\n`; |
| 173 | + if (reason === 'below_threshold') { |
| 174 | + body += `**Reason:** K-Score \`${cand}\` is below threshold \`${thresh}\`.\n\n`; |
| 175 | + body += `Run \`kolm verify ${artifact}\` locally to reproduce, then \`kolm compile --refit\` to retrain on the latest captured corpus.`; |
| 176 | + } else if (reason === 'regression_vs_baseline') { |
| 177 | + body += `**Reason:** K-Score regressed by \`${delta}\` vs baseline.\n\n`; |
| 178 | + body += `| | K-Score |\n|---|---|\n| candidate | \`${cand}\` |\n| baseline | \`${base}\` |\n| delta | \`${delta}\` |\n\n`; |
| 179 | + body += `If this regression is expected (broader eval set, capture drift, intentional smaller student), bump the artifact version and adjust the gate inputs.`; |
| 180 | + } else { |
| 181 | + body += `**Reason:** \`${reason}\`. Check the workflow logs for the kolm CLI output.`; |
| 182 | + } |
| 183 | + body += `\n\n---\n[kolm quality gate docs](https://kolm.ai/docs/ci/quality-gate)`; |
| 184 | + await github.rest.issues.createComment({ |
| 185 | + issue_number: context.issue.number, |
| 186 | + owner: context.repo.owner, |
| 187 | + repo: context.repo.repo, |
| 188 | + body, |
| 189 | + }); |
0 commit comments