fix: make eval_pipeline scripts pass pre-commit after rebase onto main

Rebase cl_agentic_grader onto upstream/main exposed skills/eval_pipeline
files that failed black/isort/pylint. Apply formatting and resolve pylint
warnings so the PR merge commit passes the CI grep-on-Failed check.

Author: chr6192

skills/eval_pipeline/01-eval-design/scripts/coverage_check.py

@@ -20,6 +20,7 @@
 
 EXIT: 0 if coverage is adequate, 2 if thin cells found, 1 on usage error.
 """
+# pylint: disable=missing-function-docstring
 from __future__ import annotations
 
 import argparse
@@ -101,14 +102,16 @@ def _self_test() -> None:
     data += [{"metadata": {"dimension": "order_accuracy", "difficulty": "easy"}}] * 12
     data += [{"metadata": {"dimension": "order_accuracy", "difficulty": "boundary"}}] * 11
     data += [{"metadata": {"dimension": "order_accuracy", "difficulty": "adversarial"}}] * 10
-    data += [{"metadata": {"dimension": "tone", "difficulty": "easy"}}] * 3   # thin dimension
+    data += [{"metadata": {"dimension": "tone", "difficulty": "easy"}}] * 3  # thin dimension
     r = analyze(data)
     assert "tone" in r["thin_dimensions"], r["thin_dimensions"]
     assert r["verdict"] == "thin_coverage"
     # A well-covered single dimension.
-    good = ([{"metadata": {"dimension": "d", "difficulty": "easy"}}] * 11
-            + [{"metadata": {"dimension": "d", "difficulty": "boundary"}}] * 10
-            + [{"metadata": {"dimension": "d", "difficulty": "adversarial"}}] * 10)
+    good = (
+        [{"metadata": {"dimension": "d", "difficulty": "easy"}}] * 11
+        + [{"metadata": {"dimension": "d", "difficulty": "boundary"}}] * 10
+        + [{"metadata": {"dimension": "d", "difficulty": "adversarial"}}] * 10
+    )
     rg = analyze(good)
     assert rg["verdict"] == "adequate", rg
     print("self-test OK")

skills/eval_pipeline/03-align-human/scripts/calibration.py

@@ -29,6 +29,7 @@
 
 EXIT CODE: 0 if calibrated, 2 if not_calibrated/insufficient_evidence, 1 on usage error.
 """
+# pylint: disable=missing-function-docstring
 from __future__ import annotations
 
 import argparse
@@ -86,12 +87,14 @@ def load_pairs(
     out: list[dict[str, str]] = []
     if pairs is not None:
         for row in _read_jsonl(pairs):
-            out.append({
-                "id": str(row.get("id", len(out))),
-                "judge": _norm(row["judge"]),
-                "human": _norm(row["human"]),
-                "stratum": str(row.get(stratum_key, "")) or "all",
-            })
+            out.append(
+                {
+                    "id": str(row.get("id", len(out))),
+                    "judge": _norm(row["judge"]),
+                    "human": _norm(row["human"]),
+                    "stratum": str(row.get(stratum_key, "")) or "all",
+                }
+            )
         return out
 
     if verdicts is None or labels is None:
@@ -101,12 +104,14 @@ def load_pairs(
         rid = str(row["id"])
         if rid not in label_by_id:
             continue
-        out.append({
-            "id": rid,
-            "judge": _norm(row.get("verdict", row.get("judge"))),
-            "human": _norm(label_by_id[rid]),
-            "stratum": str(row.get(stratum_key, "")) or "all",
-        })
+        out.append(
+            {
+                "id": rid,
+                "judge": _norm(row.get("verdict", row.get("judge"))),
+                "human": _norm(label_by_id[rid]),
+                "stratum": str(row.get(stratum_key, "")) or "all",
+            }
+        )
     return out
 
 
@@ -133,7 +138,7 @@ def _safe_div(num: float, den: float) -> float:
 def core_metrics(pairs: list[dict[str, str]], positive: str = "fail") -> dict[str, float]:
     c = confusion(pairs, positive)
     tp, fp, tn, fn = c["tp"], c["fp"], c["tn"], c["fn"]
-    tpr = _safe_div(tp, tp + fn)            # recall on the positive (caught failures)
+    tpr = _safe_div(tp, tp + fn)  # recall on the positive (caught failures)
     tnr = _safe_div(tn, tn + fp)
     precision = _safe_div(tp, tp + fp)
     f1 = _safe_div(2 * precision * tpr, precision + tpr)
@@ -216,8 +221,9 @@ def per_stratum(pairs: list[dict[str, str]], positive: str = "fail") -> dict[str
     return out
 
 
-def calibration_gate(pairs: list[dict[str, str]], metrics: dict[str, float], kappa: float,
-                     positive: str = "fail") -> dict[str, Any]:
+def calibration_gate(
+    pairs: list[dict[str, str]], metrics: dict[str, float], kappa: float, positive: str = "fail"
+) -> dict[str, Any]:
     """Apply the SKILL.md hard gate. Returns verdict + blocking reasons."""
     n = len(pairs)
     n_pos = sum(1 for p in pairs if p["human"] == positive)
@@ -228,20 +234,29 @@ def calibration_gate(pairs: list[dict[str, str]], metrics: dict[str, float], kap
     if min(n_pos, n_neg) < N_PER_CLASS_MIN:
         reasons.append(f"min class count {min(n_pos, n_neg)} (need >= {N_PER_CLASS_MIN} per class)")
     if reasons:
-        return {"verdict": "insufficient_evidence", "blocking": reasons,
-                "next": f"collect more labels: aim for >= {N_TOTAL_MIN} total, >= {N_PER_CLASS_MIN} per class"}
+        return {
+            "verdict": "insufficient_evidence",
+            "blocking": reasons,
+            "next": f"collect more labels: aim for >= {N_TOTAL_MIN} total, >= {N_PER_CLASS_MIN} per class",
+        }
     not_ready = []
     if metrics["tpr"] < TPR_MIN:
         not_ready.append(f"TPR {metrics['tpr']:.2f} < {TPR_MIN}")
     if metrics["tnr"] < TNR_MIN:
         not_ready.append(f"TNR {metrics['tnr']:.2f} < {TNR_MIN}")
     if not_ready:
-        return {"verdict": "not_calibrated", "blocking": not_ready,
-                "next": "refine the judge prompt (add borderline few-shot) then re-measure"}
+        return {
+            "verdict": "not_calibrated",
+            "blocking": not_ready,
+            "next": "refine the judge prompt (add borderline few-shot) then re-measure",
+        }
     phase = 2 if kappa >= KAPPA_PHASE2_MIN else 1
-    return {"verdict": "calibrated", "blocking": [],
-            "human_reduction_phase": phase,
-            "next": "phase 2 (assisted) ok" if phase == 2 else "calibrated but kappa<0.6: keep humans primary"}
+    return {
+        "verdict": "calibrated",
+        "blocking": [],
+        "human_reduction_phase": phase,
+        "next": "phase 2 (assisted) ok" if phase == 2 else "calibrated but kappa<0.6: keep humans primary",
+    }
 
 
 def analyze(pairs: list[dict[str, str]], positive: str = "fail", n_iter: int = 1000) -> dict[str, Any]:
@@ -279,8 +294,11 @@ def render(report: dict[str, Any]) -> str:
         f"  kappa={report['kappa']:.2f}  Gwet's AC1={report['gwet_ac1']:.2f}"
         + ("  [gap>0.15: class imbalance — trust AC1]" if report["kappa_ac1_gap"] > 0.15 else ""),
         f"  bias={report['bias']:+.2f} "
-        + ("(stricter than humans)" if report["bias"] > 0.1 else
-           "(more lenient than humans)" if report["bias"] < -0.1 else "(no significant bias)"),
+        + (
+            "(stricter than humans)"
+            if report["bias"] > 0.1
+            else "(more lenient than humans)" if report["bias"] < -0.1 else "(no significant bias)"
+        ),
     ]
     if report["per_stratum"]:
         lines.append("  per-stratum:")
@@ -327,8 +345,12 @@ def main(argv: list[str] | None = None) -> int:
     ap.add_argument("--pairs", type=Path, help="JSONL with id,judge,human[,stratum].")
     ap.add_argument("--verdicts", type=Path, help="JSONL with id,verdict[,stratum].")
     ap.add_argument("--labels", type=Path, help="JSONL with id,label.")
-    ap.add_argument("--positive", default="fail", choices=["fail", "pass"],
-                    help="Positive class for TPR (default: fail = caught problems).")
+    ap.add_argument(
+        "--positive",
+        default="fail",
+        choices=["fail", "pass"],
+        help="Positive class for TPR (default: fail = caught problems).",
+    )
     ap.add_argument("--stratum-key", default="stratum", help="Field name for stratum (default: stratum).")
     ap.add_argument("--n-iter", type=int, default=1000, help="Bootstrap iterations.")
     ap.add_argument("--json", action="store_true", help="Emit JSON instead of a text report.")

skills/eval_pipeline/05-rag-eval/scripts/rag_diagnostic.py

@@ -23,6 +23,7 @@
 
 EXIT: 0 always (diagnostic), 1 on usage error.
 """
+# pylint: disable=missing-function-docstring
 from __future__ import annotations
 
 import argparse
@@ -59,21 +60,26 @@ def _retrieval_good(row: dict[str, Any], threshold: float) -> bool | None:
     return None
 
 
-def analyze(traces: list[dict[str, Any]], faithful_threshold: float = 4.0,
-            recall_threshold: float = 0.5) -> dict[str, Any]:
+def analyze(
+    traces: list[dict[str, Any]], faithful_threshold: float = 4.0, recall_threshold: float = 0.5
+) -> dict[str, Any]:
     n = len(traces)
     faith = [_faithful(t, faithful_threshold) for t in traces]
     retr = [_retrieval_good(t, recall_threshold) for t in traces]
     have_retrieval = all(r is not None for r in retr)
 
-    gen = {"faithful": round(sum(faith) / n, 3) if n else 0.0,
-           "hallucinating": round(sum(1 for f in faith if not f) / n, 3) if n else 0.0}
+    gen = {
+        "faithful": round(sum(faith) / n, 3) if n else 0.0,
+        "hallucinating": round(sum(1 for f in faith if not f) / n, 3) if n else 0.0,
+    }
 
     result: dict[str, Any] = {"n": n, "generation": gen, "has_retrieval_signal": have_retrieval}
     if not have_retrieval:
         result["primary_issue"] = (
-            "generation: {:.0%} of answers hallucinate".format(gen["hallucinating"])
-            if gen["hallucinating"] > 0.1 else "generation looks healthy; add a retrieval signal to localize further")
+            f"generation: {gen['hallucinating']:.0%} of answers hallucinate"
+            if gen["hallucinating"] > 0.1
+            else "generation looks healthy; add a retrieval signal to localize further"
+        )
         return result
 
     # 2x2 matrix
@@ -84,21 +90,27 @@ def analyze(traces: list[dict[str, Any]], faithful_threshold: float = 4.0,
     matrix = {k: round(v / n, 3) for k, v in cells.items()} if n else cells
     # Primary issue heuristic
     if matrix["good_hallucinating"] >= matrix["poor_hallucinating"] and matrix["good_hallucinating"] > 0.1:
-        primary = (f"generation: {matrix['good_hallucinating']:.0%} hallucinate DESPITE good retrieval "
-                   "-> fix the generation prompt/model, not retrieval")
+        primary = (
+            f"generation: {matrix['good_hallucinating']:.0%} hallucinate DESPITE good retrieval "
+            "-> fix the generation prompt/model, not retrieval"
+        )
     elif matrix["poor_hallucinating"] > 0.15:
-        primary = (f"retrieval: {matrix['poor_hallucinating']:.0%} have poor retrieval AND hallucinate "
-                   "-> fix retrieval (chunking/embeddings) first")
+        primary = (
+            f"retrieval: {matrix['poor_hallucinating']:.0%} have poor retrieval AND hallucinate "
+            "-> fix retrieval (chunking/embeddings) first"
+        )
     else:
         primary = "system largely healthy"
     result.update({"matrix": matrix, "primary_issue": primary})
     return result
 
 
 def render(report: dict[str, Any]) -> str:
-    lines = [f"RAG diagnostic  (n={report['n']})",
-             f"  generation: faithful={report['generation']['faithful']:.0%}  "
-             f"hallucinating={report['generation']['hallucinating']:.0%}"]
+    lines = [
+        f"RAG diagnostic  (n={report['n']})",
+        f"  generation: faithful={report['generation']['faithful']:.0%}  "
+        f"hallucinating={report['generation']['hallucinating']:.0%}",
+    ]
     if report["has_retrieval_signal"]:
         m = report["matrix"]
         lines += [

skills/eval_pipeline/06-prompt-regression/scripts/pairwise.py

@@ -18,6 +18,7 @@
 
 EXIT CODE: 0 if candidate is BETTER, 2 otherwise (worse/tied/inconclusive), 1 on usage error.
 """
+# pylint: disable=missing-function-docstring
 from __future__ import annotations
 
 import argparse
@@ -27,8 +28,8 @@
 from pathlib import Path
 from typing import Any
 
-MIN_SAMPLES = 10          # below this, CI is too wide to conclude (see SKILL.md)
-TIE_CI_WIDTH = 0.30       # CI brackets 0.5 and is narrower than this => genuine tie
+MIN_SAMPLES = 10  # below this, CI is too wide to conclude (see SKILL.md)
+TIE_CI_WIDTH = 0.30  # CI brackets 0.5 and is narrower than this => genuine tie
 
 try:
     import numpy as _np
@@ -94,8 +95,14 @@ def verdict_for(wins: list[str], candidate: str, baseline: str, n_iter: int = 10
             verdict = "TIED (CI brackets 0.5, narrow)"
         else:
             verdict = "INCONCLUSIVE (CI too wide — need more samples)"
-    return {"n": n, "candidate_win_rate": round(cand, 3), "baseline_win_rate": round(base, 3),
-            "tie_rate": round(tie, 3), "candidate_ci95": [round(ci[0], 3), round(ci[1], 3)], "verdict": verdict}
+    return {
+        "n": n,
+        "candidate_win_rate": round(cand, 3),
+        "baseline_win_rate": round(base, 3),
+        "tie_rate": round(tie, 3),
+        "candidate_ci95": [round(ci[0], 3), round(ci[1], 3)],
+        "verdict": verdict,
+    }
 
 
 def analyze(comparisons: list[dict[str, Any]], candidate: str, baseline: str, n_iter: int = 1000) -> dict[str, Any]:
@@ -108,22 +115,42 @@ def analyze(comparisons: list[dict[str, Any]], candidate: str, baseline: str, n_
 
 
 def render(report: dict[str, Any]) -> str:
-    lines = [f"Pairwise: candidate='{report['candidate']}' vs baseline='{report['baseline']}'",
-             f"{'Dimension':<16}{'Cand':>7}{'Base':>7}{'Tie':>7}  {'95% CI':>14}  Verdict",
-             "-" * 78]
+    lines = [
+        f"Pairwise: candidate='{report['candidate']}' vs baseline='{report['baseline']}'",
+        f"{'Dimension':<16}{'Cand':>7}{'Base':>7}{'Tie':>7}  {'95% CI':>14}  Verdict",
+        "-" * 78,
+    ]
     for d, v in report["by_dimension"].items():
         ci = f"[{v['candidate_ci95'][0]:.2f},{v['candidate_ci95'][1]:.2f}]"
-        lines.append(f"{d:<16}{v['candidate_win_rate']:>7.0%}{v['baseline_win_rate']:>7.0%}"
-                     f"{v['tie_rate']:>7.0%}  {ci:>14}  {v['verdict']}")
+        lines.append(
+            f"{d:<16}{v['candidate_win_rate']:>7.0%}{v['baseline_win_rate']:>7.0%}"
+            f"{v['tie_rate']:>7.0%}  {ci:>14}  {v['verdict']}"
+        )
     return "\n".join(lines)
 
 
 def _self_test() -> None:
     comps = []
     # Candidate clearly better on relevance: wins both orders for 15 queries (30 rows).
     for i in range(15):
-        comps.append({"id": f"q{i}", "model_a": "baseline", "model_b": "candidate", "score": 0.0, "dimension": "relevance"})
-        comps.append({"id": f"q{i}", "model_a": "candidate", "model_b": "baseline", "score": 1.0, "dimension": "relevance"})
+        comps.append(
+            {
+                "id": f"q{i}",
+                "model_a": "baseline",
+                "model_b": "candidate",
+                "score": 0.0,
+                "dimension": "relevance",
+            }
+        )
+        comps.append(
+            {
+                "id": f"q{i}",
+                "model_a": "candidate",
+                "model_b": "baseline",
+                "score": 1.0,
+                "dimension": "relevance",
+            }
+        )
     w = winners(comps)
     assert all(x == "candidate" for x in w), w[:5]
     r = analyze(comps, "candidate", "baseline", n_iter=200)

skills/eval_pipeline/07-redteam/scripts/asr_report.py

@@ -21,6 +21,7 @@
 
 EXIT: 0 always for reporting (this is diagnostic, not a gate); 1 on usage error.
 """
+# pylint: disable=missing-function-docstring
 from __future__ import annotations
 
 import argparse
@@ -42,7 +43,7 @@ def _read_jsonl(path: Path) -> list[dict[str, Any]]:
     return rows
 
 
-def _violated(row: dict[str, Any], score_scale: float, harm_threshold: float) -> bool:
+def _violated(row: dict[str, Any], _score_scale: float, harm_threshold: float) -> bool:
     """Determine if an attack succeeded (produced a violation)."""
     if "violated" in row:
         return bool(row["violated"])
@@ -70,8 +71,10 @@ def asr(attacks: list[dict[str, Any]], score_scale: float = 1.0, harm_threshold:
             "asr": round(v / len(rows), 3) if rows else 0.0,
             "n": len(rows),
             "by_vector": {
-                vec: {"asr": round(sum(1 for r in vr if _violated(r, score_scale, harm_threshold)) / len(vr), 3),
-                      "n": len(vr)}
+                vec: {
+                    "asr": round(sum(1 for r in vr if _violated(r, score_scale, harm_threshold)) / len(vr), 3),
+                    "n": len(vr),
+                }
                 for vec, vr in sorted(vec_rows.items())
             },
         }
@@ -87,12 +90,19 @@ def over_refusal(rows: list[dict[str, Any]]) -> dict[str, Any]:
     for cat, crows in cat_rows.items():
         ref = sum(1 for r in crows if r.get("refused"))
         by_cat[cat] = {"over_refusal_rate": round(ref / len(crows), 3) if crows else 0.0, "n": len(crows)}
-    return {"overall_over_refusal": round(total_r / len(rows), 3) if rows else 0.0,
-            "n_benign": len(rows), "by_category": by_cat}
-
-
-def analyze(attacks: list[dict[str, Any]], over_refusals: list[dict[str, Any]] | None,
-            score_scale: float = 1.0, harm_threshold: float = 1.0) -> dict[str, Any]:
+    return {
+        "overall_over_refusal": round(total_r / len(rows), 3) if rows else 0.0,
+        "n_benign": len(rows),
+        "by_category": by_cat,
+    }
+
+
+def analyze(
+    attacks: list[dict[str, Any]],
+    over_refusals: list[dict[str, Any]] | None,
+    score_scale: float = 1.0,
+    harm_threshold: float = 1.0,
+) -> dict[str, Any]:
     out: dict[str, Any] = {"attack": asr(attacks, score_scale, harm_threshold)}
     out["over_refusal"] = over_refusal(over_refusals) if over_refusals else None
     return out
@@ -106,8 +116,10 @@ def render(report: dict[str, Any]) -> str:
         lines.append(f"  {cat:<22} ASR={v['asr']:.0%} (n={v['n']})   [{vecs}]")
     o = report["over_refusal"]
     if o is None:
-        lines.append("Over-Refusal: NOT MEASURED — ASR alone is misleading "
-                     "(a system that refuses everything shows ASR=0%). Add a benign over-refusal set.")
+        lines.append(
+            "Over-Refusal: NOT MEASURED — ASR alone is misleading "
+            "(a system that refuses everything shows ASR=0%). Add a benign over-refusal set."
+        )
     else:
         lines.append(f"Over-Refusal Rate  (overall={o['overall_over_refusal']:.0%}, n={o['n_benign']})")
         for cat, v in sorted(o["by_category"].items()):