Revert Claude Code eval runner variants (#182)

* Revert "Add claude-code-convex variant (plugin cell) (#180)"

This reverts commit f51d0ef.

* Revert "Add claude-code apiKind (agentic, vanilla cell) (#178)"

This reverts commit 09e053c.

* Remove stale Claude Code workflow override

Author: mikecann

.github/workflows/manual_evals.yml

@@ -29,7 +29,6 @@ jobs:
       ENVIRONMENT: ci
       OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
       CURSOR_API_KEY: ${{ secrets.CURSOR_API_KEY }}
-      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
 
     steps:
       - uses: actions/checkout@v4
@@ -45,15 +44,6 @@ jobs:
       - name: Install dependencies
         run: bun install
 
-      - name: Install Claude Code CLI and Convex plugin
-        if: contains(inputs.models, 'claude-code-convex')
-        run: |
-          npm install -g @anthropic-ai/claude-code
-          claude --version
-          claude plugin marketplace add anthropics/claude-plugins-official
-          claude plugin install convex@claude-plugins-official
-          ls -la "$HOME/.claude/plugins" || true
-
       - name: Create temp directory
         run: mkdir -p /tmp/convex-evals
 
@@ -71,24 +61,11 @@ jobs:
       - name: Run evaluations
         env:
           OUTPUT_TEMPDIR: /tmp/convex-evals
-          OPENROUTER_CONCURRENCY: ${{ contains(inputs.models, 'claude-code') && '2' || '4' }}
         run: bun run runner/index.ts
 
       - name: Run evaluations (no_guidelines experiment)
         if: ${{ inputs.run_no_guidelines == 'true' }}
         env:
           OUTPUT_TEMPDIR: /tmp/convex-evals-no-guidelines
           EVALS_EXPERIMENT: no_guidelines
-          OPENROUTER_CONCURRENCY: ${{ contains(inputs.models, 'claude-code') && '2' || '4' }}
         run: bun run runner/index.ts
-
-      - name: Upload eval outputs
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: eval-outputs-${{ github.run_id }}
-          path: |
-            /tmp/convex-evals
-            /tmp/convex-evals-no-guidelines
-          if-no-files-found: ignore
-          retention-days: 14

bun.lock

@@ -24,7 +24,6 @@
     "@ai-sdk/openai-compatible": "^2.0.27",
     "@ai-sdk/togetherai": "^2.0.30",
     "@ai-sdk/xai": "^3.0.47",
-    "@anthropic-ai/claude-agent-sdk": "^0.3.154",
     "@cursor/sdk": "1.0.13",
     "@inquirer/prompts": "^8.0.0",
     "@types/bun": "^1.3.8",

package.json

@@ -25,7 +25,6 @@ import {
   type ResolvedModel,
   resolveModelDefaults,
   OPENROUTER_API_KEY_VAR,
-  ANTHROPIC_API_KEY_VAR,
   DEFAULT_MAX_CONCURRENCY,
 } from "./models/index.js";
 import {
@@ -286,9 +285,7 @@ export async function runEvalsForModel(
         model.name,
         modelDisplayName,
         provider,
-        model.apiKind === "cursor-sdk" || model.apiKind === "claude-code"
-          ? "chat"
-          : model.apiKind,
+        model.apiKind === "cursor-sdk" ? "chat" : model.apiKind,
         metadata?.openRouterFirstSeenAt,
       );
       if (modelId) {
@@ -318,11 +315,7 @@ export async function runEvalsForModel(
     let modelApiKey: string | null = null;
     if (executionMode === "generate") {
       const apiKeyVar =
-        model.apiKind === "cursor-sdk"
-          ? CURSOR_API_KEY_VAR
-          : model.apiKind === "claude-code"
-            ? ANTHROPIC_API_KEY_VAR
-            : OPENROUTER_API_KEY_VAR;
+        model.apiKind === "cursor-sdk" ? CURSOR_API_KEY_VAR : OPENROUTER_API_KEY_VAR;
       const apiKey = process.env[apiKeyVar];
       if (!apiKey) {
         console.error(`${apiKeyVar} is not set`);
@@ -332,7 +325,7 @@ export async function runEvalsForModel(
     }
 
     if (executionMode === "generate" && modelApiKey) {
-      if (model.apiKind !== "cursor-sdk" && model.apiKind !== "claude-code") {
+      if (model.apiKind !== "cursor-sdk") {
         logInfo(`[preflight] Checking endpoint availability for ${model.name}...`);
         try {
           await preflightOpenRouterEndpoint(model, modelApiKey);

runner/index.ts

@@ -32,8 +32,6 @@ describe("ALL_MODELS", () => {
     expect(ALL_MODELS).toContain("anthropic/claude-opus-4.6");
     expect(ALL_MODELS).toContain("anthropic/claude-opus-4.7");
     expect(ALL_MODELS).toContain("anthropic/claude-opus-4.8");
-    expect(ALL_MODELS).toContain("claude-code/opus-4.7");
-    expect(ALL_MODELS).toContain("claude-code-convex/opus-4.7");
     expect(ALL_MODELS).toContain("deepseek/deepseek-v4-pro");
     expect(ALL_MODELS).toContain("moonshotai/kimi-k2.6");
     expect(ALL_MODELS).toContain("google/gemini-2.5-flash");

runner/models.test.ts

@@ -5,7 +5,6 @@
 
 export const OPENROUTER_API_KEY_VAR = "OPENROUTER_API_KEY";
 export const OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1";
-export const ANTHROPIC_API_KEY_VAR = "ANTHROPIC_API_KEY";
 export const DEFAULT_MAX_CONCURRENCY = parseInt(
   process.env.OPENROUTER_CONCURRENCY ?? "4",
   10,
@@ -16,7 +15,7 @@ export interface ResolvedModel {
   runnableName: string;
   formattedName: string;
   baseURL: string;
-  apiKind: "chat" | "responses" | "cursor-sdk" | "claude-code";
+  apiKind: "chat" | "responses" | "cursor-sdk";
 }
 
 export function resolveModelDefaults(name: string): ResolvedModel {
@@ -41,8 +40,6 @@ export const ALL_MODELS: string[] = [
   "anthropic/claude-opus-4.6",
   "anthropic/claude-opus-4.7",
   "anthropic/claude-opus-4.8",
-  "claude-code/opus-4.7",
-  "claude-code-convex/opus-4.7",
   "openai/o4-mini",
   "openai/gpt-4.1",
   "openai/gpt-5.1",