Count non-binding budgets correctly in published results (#14)

Author: FelixTJDietrich

site/attempt-outcome.ts

@@ -0,0 +1,63 @@
+import type { AttemptAnalysisRow } from "../src/export/analysis.ts";
+
+export type PublicAttemptOutcome =
+  | "accepted"
+  | "quality_failed"
+  | "abstained"
+  | "generation_failed"
+  | "budget_exceeded"
+  | "budget_unverifiable"
+  | "infrastructure_failed"
+  | "not_started";
+
+export interface PublicAttemptClassificationInput {
+  generation_state: string;
+  generation_outcome: string | null;
+  evaluation_status: string | null;
+  evaluator_strict_success: boolean | null;
+  strict_success: boolean | null;
+  generation_budget_status: AttemptAnalysisRow["generation_budget_status"];
+}
+
+export function classifyPublicOutcome(
+  attempt: PublicAttemptClassificationInput,
+): PublicAttemptOutcome {
+  if (attempt.generation_state === "planned") {
+    return "not_started";
+  }
+  const generated =
+    attempt.generation_state === "completed" && attempt.generation_outcome === "succeeded";
+  if (attempt.generation_state === "completed" && attempt.generation_outcome === "abstained") {
+    return "abstained";
+  }
+  if (attempt.generation_outcome === "infra_failed") {
+    return "infrastructure_failed";
+  }
+  if (attempt.generation_state === "completed" && attempt.generation_outcome !== "succeeded") {
+    return "generation_failed";
+  }
+  if (generated && attempt.generation_budget_status === "exceeded") {
+    return "budget_exceeded";
+  }
+  if (generated && attempt.generation_budget_status === "unverifiable") {
+    return "budget_unverifiable";
+  }
+  const evaluatorStrictSuccess = attempt.evaluator_strict_success ?? attempt.strict_success;
+  if (
+    generated &&
+    (attempt.generation_budget_status === "compliant" ||
+      attempt.generation_budget_status === "non_binding") &&
+    attempt.evaluation_status === "succeeded" &&
+    evaluatorStrictSuccess === true
+  ) {
+    return "accepted";
+  }
+  if (
+    generated &&
+    attempt.evaluation_status === "quality_failed" &&
+    evaluatorStrictSuccess === false
+  ) {
+    return "quality_failed";
+  }
+  return "infrastructure_failed";
+}

site/publish.ts

@@ -4,25 +4,24 @@ import { canonicalJson, sha256 } from "../src/core/canonical.ts";
 import { toCsv, toJsonLines } from "../src/export/serialize.ts";
 import { verifyRelease } from "../src/export/verify.ts";
 import { assignApproachColors } from "./approach-colors.ts";
+import {
+  classifyPublicOutcome,
+  type PublicAttemptClassificationInput,
+  type PublicAttemptOutcome,
+} from "./attempt-outcome.ts";
 import { buildStaticSite } from "./build.ts";
 import {
   type FormalReleaseStatus,
   formalReleaseDesignationSchema,
   publicReleaseSchema,
 } from "./contracts.ts";
 
-interface FormalAttempt {
+interface FormalAttempt extends PublicAttemptClassificationInput {
   attempt_id: string;
   case_id: string;
   system_id: string;
   replicate: number;
-  generation_state: string;
-  generation_outcome: string | null;
   generation_error: string | null;
-  evaluation_status: string | null;
-  evaluator_strict_success: boolean | null;
-  strict_success: boolean | null;
-  generation_budget_status: "compliant" | "exceeded" | "unverifiable" | null;
   evaluation_failure: string | null;
   generation_duration_ms: number | null;
   cost_amount: number | null;
@@ -35,15 +34,7 @@ interface PublicAttempt {
   system_id: string;
   replicate: number;
   lifecycle: string;
-  outcome:
-    | "accepted"
-    | "quality_failed"
-    | "abstained"
-    | "generation_failed"
-    | "budget_exceeded"
-    | "budget_unverifiable"
-    | "infrastructure_failed"
-    | "not_started";
+  outcome: PublicAttemptOutcome;
   strict_accepted: boolean | null;
   evaluator_strict_accepted?: boolean | null;
   generation_completed: boolean;
@@ -96,46 +87,6 @@ async function readJson<T>(path: string): Promise<T> {
   return JSON.parse(await readFile(path, "utf8")) as T;
 }
 
-function disposition(attempt: FormalAttempt): PublicAttempt["outcome"] {
-  if (attempt.generation_state === "planned") {
-    return "not_started";
-  }
-  const generated =
-    attempt.generation_state === "completed" && attempt.generation_outcome === "succeeded";
-  if (attempt.generation_state === "completed" && attempt.generation_outcome === "abstained") {
-    return "abstained";
-  }
-  if (attempt.generation_outcome === "infra_failed") {
-    return "infrastructure_failed";
-  }
-  if (attempt.generation_state === "completed" && attempt.generation_outcome !== "succeeded") {
-    return "generation_failed";
-  }
-  if (generated && attempt.generation_budget_status === "exceeded") {
-    return "budget_exceeded";
-  }
-  if (generated && attempt.generation_budget_status === "unverifiable") {
-    return "budget_unverifiable";
-  }
-  const evaluatorStrictSuccess = attempt.evaluator_strict_success ?? attempt.strict_success;
-  if (
-    generated &&
-    attempt.generation_budget_status === "compliant" &&
-    attempt.evaluation_status === "succeeded" &&
-    evaluatorStrictSuccess === true
-  ) {
-    return "accepted";
-  }
-  if (
-    generated &&
-    attempt.evaluation_status === "quality_failed" &&
-    evaluatorStrictSuccess === false
-  ) {
-    return "quality_failed";
-  }
-  return "infrastructure_failed";
-}
-
 function strictValue(outcome: PublicAttempt["outcome"]): boolean | null {
   if (outcome === "accepted") {
     return true;
@@ -235,7 +186,7 @@ export async function publishSite(options: {
     await readFile(join(releaseDirectory, "data", "attempts.jsonl"), "utf8"),
   );
   const publicAttempts: PublicAttempt[] = attempts.map((attempt) => {
-    const outcome = disposition(attempt);
+    const outcome = classifyPublicOutcome(attempt);
     const costUsd =
       attempt.generation_state === "planned"
         ? 0

tests/site-publish.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, test } from "bun:test";
+import {
+  classifyPublicOutcome,
+  type PublicAttemptClassificationInput,
+  type PublicAttemptOutcome,
+} from "../site/attempt-outcome.ts";
+
+const successfulAttempt: PublicAttemptClassificationInput = {
+  generation_state: "completed",
+  generation_outcome: "succeeded",
+  evaluation_status: "succeeded",
+  evaluator_strict_success: true,
+  strict_success: true,
+  generation_budget_status: "compliant",
+};
+
+describe("public attempt classification", () => {
+  test.each(["compliant", "non_binding"] as const)(
+    "accepts a successful evaluation with a %s generation budget",
+    (generationBudgetStatus) => {
+      expect(
+        classifyPublicOutcome({
+          ...successfulAttempt,
+          generation_budget_status: generationBudgetStatus,
+        }),
+      ).toBe("accepted");
+    },
+  );
+
+  test("falls back to the release strict-success field", () => {
+    expect(classifyPublicOutcome({ ...successfulAttempt, evaluator_strict_success: null })).toBe(
+      "accepted",
+    );
+  });
+
+  test.each([
+    [{ generation_state: "planned" }, "not_started"],
+    [{ generation_outcome: "abstained" }, "abstained"],
+    [{ generation_outcome: "infra_failed" }, "infrastructure_failed"],
+    [{ generation_outcome: "failed" }, "generation_failed"],
+    [{ generation_budget_status: "exceeded" }, "budget_exceeded"],
+    [{ generation_budget_status: "unverifiable" }, "budget_unverifiable"],
+    [
+      {
+        evaluation_status: "quality_failed",
+        evaluator_strict_success: false,
+        strict_success: false,
+      },
+      "quality_failed",
+    ],
+    [
+      {
+        evaluation_status: null,
+        evaluator_strict_success: null,
+        strict_success: null,
+      },
+      "infrastructure_failed",
+    ],
+  ] satisfies Array<[Partial<PublicAttemptClassificationInput>, PublicAttemptOutcome]>)(
+    "classifies %j as %s",
+    (overrides, expected) => {
+      expect(classifyPublicOutcome({ ...successfulAttempt, ...overrides })).toBe(expected);
+    },
+  );
+});