From 869a629a8a843ab54382245deccaa72b9118a62e Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Wed, 20 May 2026 12:09:36 -0700 Subject: [PATCH 1/7] When --warmups=0, use a new browser for each task --- packages/web/benchmark/src/index.ts | 33 +++++-- packages/web/benchmark/src/stats.ts | 28 ++++++ packages/web/benchmark/src/types.ts | 2 + .../web/scripts/lib/run-benchmark-page.ts | 94 +++++++++++++++++-- 4 files changed, 144 insertions(+), 13 deletions(-) create mode 100644 packages/web/benchmark/src/stats.ts diff --git a/packages/web/benchmark/src/index.ts b/packages/web/benchmark/src/index.ts index 02d79893e..f6144a06c 100644 --- a/packages/web/benchmark/src/index.ts +++ b/packages/web/benchmark/src/index.ts @@ -53,7 +53,8 @@ async function benchmarkSource( service: DatabaseServiceWebWorker, sourceNames: string[], iterations: number, - warmupRounds: number + warmupRounds: number, + tasks: typeof BENCHMARK_TASKS ): Promise { const { annotationSvc, fileSvc } = createServices(service, sourceNames); @@ -64,17 +65,17 @@ async function benchmarkSource( // of every task reflect cold-start overhead rather than steady-state cost. setStatus(`Warming up ${sourceNames.join(", ")} (${warmupRounds} rounds)...`); for (let w = 0; w < warmupRounds; w++) { - for (const task of BENCHMARK_TASKS) { + for (const task of tasks) { service.clearTimings(); await task.run(annotationSvc, fileSvc); } } - const timingsMap = new Map(BENCHMARK_TASKS.map(({ name }) => [name, []])); + const timingsMap = new Map(tasks.map(({ name }) => [name, []])); for (let i = 0; i < iterations; i++) { setStatus(`Timing ${sourceNames.join(", ")} — iteration ${i + 1}/${iterations}...`); - for (const task of shuffle(BENCHMARK_TASKS)) { + for (const task of shuffle(tasks)) { if (task.resetAnnotationCache) { for (const sourceName of sourceNames) { service.clearAnnotationCache(sourceName); @@ -94,7 +95,7 @@ async function benchmarkSource( } } - return BENCHMARK_TASKS.map(({ name }) => { + return tasks.map(({ name }) => { const timings = [...(timingsMap.get(name) ?? [])].sort((a, b) => a - b); return { name, @@ -113,6 +114,16 @@ async function main() { } const iterations = config.iterations ?? DEFAULT_ITERATIONS; const warmupRounds = config.warmupRounds ?? DEFAULT_WARMUP_ROUNDS; + const taskFilter = config.taskFilter; + + // When a taskFilter is provided, only run the requested tasks. + if (taskFilter) { + const validNames = new Set(BENCHMARK_TASKS.map((t) => t.name)); + const invalid = taskFilter.filter((n) => !validNames.has(n)); + if (invalid.length) { + throw new Error(`Unknown task(s) in taskFilter: ${invalid.join(", ")}`); + } + } setStatus("Initializing DuckDB-WASM..."); const initStart = performance.now(); @@ -149,6 +160,10 @@ async function main() { await service.execute('DROP VIEW IF EXISTS "__bff_warmup__"'); } + const activeTasks = taskFilter + ? BENCHMARK_TASKS.filter((t) => taskFilter.includes(t.name)) + : BENCHMARK_TASKS; + const sourceResults: SourceResult[] = []; for (const sources of config.testCases) { @@ -171,7 +186,13 @@ async function main() { const registrationMs = performance.now() - regStart; const labels = sources.map((source) => source.label); - const queries = await benchmarkSource(service, labels, iterations, warmupRounds); + const queries = await benchmarkSource( + service, + labels, + iterations, + warmupRounds, + activeTasks + ); sourceResults.push({ labels, registrationMs, queries }); for (const source of sources) { diff --git a/packages/web/benchmark/src/stats.ts b/packages/web/benchmark/src/stats.ts new file mode 100644 index 000000000..83c0e3d32 --- /dev/null +++ b/packages/web/benchmark/src/stats.ts @@ -0,0 +1,28 @@ +import { QueryResult } from "./types"; + +export const DEFAULT_ITERATIONS = 5; +export const DEFAULT_WARMUP_ROUNDS = 1; + +// Nearest-rank percentile over a pre-sorted array. Used to report p50 and p95 +// across timed iterations — p95 surfaces occasional slow outliers (GC pauses, +// DuckDB cache misses) that the median would hide. +export function percentile(sorted: number[], p: number): number { + const idx = Math.ceil((p / 100) * sorted.length) - 1; + return sorted[Math.max(0, idx)]; +} + +/** + * Build a QueryResult from raw timing samples: sorts them and computes + * p50/p95/p99. Used by both the in-browser benchmark engine and the + * Playwright runner's cold-start aggregation path. + */ +export function buildQueryResult(name: string, rawTimings: number[]): QueryResult { + const timings = [...rawTimings].sort((a, b) => a - b); + return { + name, + timings, + p50: percentile(timings, 50), + p95: percentile(timings, 95), + p99: percentile(timings, 99), + }; +} diff --git a/packages/web/benchmark/src/types.ts b/packages/web/benchmark/src/types.ts index 3f1cb6f25..358dd375b 100644 --- a/packages/web/benchmark/src/types.ts +++ b/packages/web/benchmark/src/types.ts @@ -15,6 +15,8 @@ export interface BenchmarkConfig { testCases: TestCase[]; iterations?: number; warmupRounds?: number; + /** When set, only tasks whose name appears in this list will run. */ + taskFilter?: string[]; } export interface QueryResult { diff --git a/packages/web/scripts/lib/run-benchmark-page.ts b/packages/web/scripts/lib/run-benchmark-page.ts index 783a7a459..c800227ef 100644 --- a/packages/web/scripts/lib/run-benchmark-page.ts +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -23,7 +23,8 @@ import http from "http"; import fs from "fs"; import path from "path"; import { execSync } from "child_process"; -import { BenchmarkResults, TestCase } from "../../benchmark/src/types"; +import { BenchmarkResults, QueryResult, SourceResult, TestCase } from "../../benchmark/src/types"; +import { BENCHMARK_TASKS } from "../../benchmark/src/tasks"; const DIST_DIR = path.join(__dirname, "..", "..", "benchmark", "dist"); const FIXTURES_DIR = path.join(__dirname, "..", "..", "fixtures"); @@ -154,7 +155,7 @@ export async function runBenchmarkPage({ iterations?: number; warmupRounds?: number; channel?: string; -}): Promise { +}) { if (!skipBuild) buildBenchmark(); if (!fs.existsSync(path.join(DIST_DIR, "index.html"))) { @@ -164,6 +165,77 @@ export async function runBenchmarkPage({ } const server = await startServer(); + + try { + // When warmups=0, each task gets its own browser for cold-start isolation. + // Otherwise all tasks share a single browser instance. + const allTaskNames = BENCHMARK_TASKS.map((t) => t.name); + const taskBatches: string[][] = + warmupRounds === 0 ? allTaskNames.map((name) => [name]) : [allTaskNames]; + + if (warmupRounds === 0) { + console.log( + `[playwright] warmupRounds=0: running ${allTaskNames.length} task(s) in separate browser instances` + ); + } + + let initTimeMs = 0; + const sourceResults: SourceResult[] = []; + + for (const testCase of testCases) { + const queries: QueryResult[] = []; + let registrationMs = 0; + + for (const batch of taskBatches) { + console.log( + `[playwright] Launching browser for task(s): ${batch.join(", ")} ` + + `(${testCase.map((s) => s.label).join(", ")})` + ); + const run = await runSingleBenchmark({ + testCase, + iterations, + warmupRounds, + channel, + taskFilter: batch, + }); + + if (initTimeMs === 0) initTimeMs = run.initTimeMs; + if (registrationMs === 0) registrationMs = run.registrationMs; + queries.push(...run.queries); + } + + sourceResults.push({ + labels: testCase.map((testCase) => testCase.label), + registrationMs, + queries, + }); + } + + return { + timestamp: new Date().toISOString(), + commit: "unknown", + branch: "unknown", + initTimeMs, + results: sourceResults, + } as BenchmarkResults; + } finally { + await new Promise((res) => server.close(res)); + } +} + +async function runSingleBenchmark({ + testCase, + iterations, + warmupRounds, + channel, + taskFilter, +}: { + testCase: TestCase; + iterations?: number; + warmupRounds?: number; + channel?: string; + taskFilter: string[]; +}): Promise<{ initTimeMs: number; registrationMs: number; queries: QueryResult[] }> { const browser = await chromium.launch({ channel, headless: true, @@ -182,13 +254,14 @@ export async function runBenchmarkPage({ // synchronously on startup — no callback handshake needed. await page.addInitScript({ content: `window.__benchmarkConfig = ${JSON.stringify({ - testCases, + testCases: [testCase], iterations, warmupRounds, + taskFilter, })};`, }); - console.log(`[playwright] Starting benchmark (${testCases.length} test case(s))...`); + console.log(`[playwright] Starting benchmark...`); await page.goto(`http://localhost:${PORT}/`, { waitUntil: "domcontentloaded" }); // Wait for the benchmark to signal it's ready for file injection @@ -201,7 +274,7 @@ export async function runBenchmarkPage({ // which is identical to how the real app loads files via the file picker — // no HTTP range-request overhead, so DuckDB sort performance matches real-user timing. const loaded = new Set(); - for (const source of testCases.flat()) { + for (const source of testCase) { if (source.label in loaded) continue; // Don't add duplicate sources const localMatch = source.url.match( new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`) @@ -251,10 +324,17 @@ export async function runBenchmarkPage({ const error = await page.evaluate(() => window.__benchmarkError ?? null); if (error) throw new Error(`Benchmark failed in browser: ${error}`); - return await page.evaluate(() => window.__benchmarkResults); + const benchmarkResults: BenchmarkResults = await page.evaluate( + () => window.__benchmarkResults + ); + const result = benchmarkResults.results[0]; + return { + initTimeMs: benchmarkResults.initTimeMs, + registrationMs: result.registrationMs, + queries: result.queries, + }; } finally { await browser.close(); - await new Promise((res) => server.close(res)); } } From 5b15db0da595d541feaf32f2c2301ff063a2c865 Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Wed, 20 May 2026 10:19:56 -0700 Subject: [PATCH 2/7] New browser per iteration, not just per task --- packages/web/benchmark/src/index.ts | 23 +----- .../web/scripts/lib/run-benchmark-page.ts | 78 +++++++++++++------ 2 files changed, 57 insertions(+), 44 deletions(-) diff --git a/packages/web/benchmark/src/index.ts b/packages/web/benchmark/src/index.ts index f6144a06c..b98b21585 100644 --- a/packages/web/benchmark/src/index.ts +++ b/packages/web/benchmark/src/index.ts @@ -1,10 +1,8 @@ import { BENCHMARK_TASKS, createServices } from "./tasks"; import { BenchmarkConfig, BenchmarkResults, QueryResult, SourceResult } from "./types"; +import { DEFAULT_ITERATIONS, DEFAULT_WARMUP_ROUNDS, buildQueryResult } from "./stats"; import DatabaseServiceWebWorker from "../../src/services/DatabaseServiceWeb/duckdb-worker.worker"; -const DEFAULT_ITERATIONS = 5; -const DEFAULT_WARMUP_ROUNDS = 1; - // Updates the #status element in the benchmark HTML page and mirrors to console. // The page can run headlessly in CI (Playwright), so the console log is the // only visible progress signal when there is no browser UI to observe. @@ -14,14 +12,6 @@ function setStatus(msg: string) { console.log("[benchmark]", msg); } -// Nearest-rank percentile over a pre-sorted array. Used to report p50 and p95 -// across timed iterations — p95 surfaces occasional slow outliers (GC pauses, -// DuckDB cache misses) that the median would hide. -function percentile(sorted: number[], p: number): number { - const idx = Math.ceil((p / 100) * sorted.length) - 1; - return sorted[Math.max(0, idx)]; -} - // Fisher-Yates shuffle — randomizes task order each timed iteration so that a // consistently slow task doesn't inflate the times of everything that follows it // (DuckDB buffer pool and OS page cache warm up over repeated runs). @@ -95,16 +85,7 @@ async function benchmarkSource( } } - return tasks.map(({ name }) => { - const timings = [...(timingsMap.get(name) ?? [])].sort((a, b) => a - b); - return { - name, - timings, - p50: percentile(timings, 50), - p95: percentile(timings, 95), - p99: percentile(timings, 99), - }; - }); + return tasks.map(({ name }) => buildQueryResult(name, timingsMap.get(name) ?? [])); } async function main() { diff --git a/packages/web/scripts/lib/run-benchmark-page.ts b/packages/web/scripts/lib/run-benchmark-page.ts index c800227ef..27e6c6c25 100644 --- a/packages/web/scripts/lib/run-benchmark-page.ts +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -25,7 +25,7 @@ import path from "path"; import { execSync } from "child_process"; import { BenchmarkResults, QueryResult, SourceResult, TestCase } from "../../benchmark/src/types"; import { BENCHMARK_TASKS } from "../../benchmark/src/tasks"; - +import { DEFAULT_ITERATIONS, buildQueryResult } from "../../benchmark/src/stats"; const DIST_DIR = path.join(__dirname, "..", "..", "benchmark", "dist"); const FIXTURES_DIR = path.join(__dirname, "..", "..", "fixtures"); const PORT = 18765; @@ -167,45 +167,77 @@ export async function runBenchmarkPage({ const server = await startServer(); try { - // When warmups=0, each task gets its own browser for cold-start isolation. - // Otherwise all tasks share a single browser instance. - const allTaskNames = BENCHMARK_TASKS.map((t) => t.name); - const taskBatches: string[][] = - warmupRounds === 0 ? allTaskNames.map((name) => [name]) : [allTaskNames]; - - if (warmupRounds === 0) { - console.log( - `[playwright] warmupRounds=0: running ${allTaskNames.length} task(s) in separate browser instances` - ); - } + const allTaskNames = BENCHMARK_TASKS.map((task) => task.name); + const iterationCount = iterations ?? DEFAULT_ITERATIONS; let initTimeMs = 0; const sourceResults: SourceResult[] = []; for (const testCase of testCases) { - const queries: QueryResult[] = []; let registrationMs = 0; + let queries: QueryResult[]; - for (const batch of taskBatches) { + if (warmupRounds === 0) { + // Each (task, iteration) pair gets a fresh browser so every + // measurement is a cold start. console.log( - `[playwright] Launching browser for task(s): ${batch.join(", ")} ` + - `(${testCase.map((s) => s.label).join(", ")})` + `[playwright] warmupRounds=0: running ${allTaskNames.length} task(s) × ` + + `${iterationCount} iteration(s) in separate browser instances` + ); + + const timingsMap = new Map( + allTaskNames.map((name) => [name, []]) + ); + + for (const taskName of allTaskNames) { + for (let i = 0; i < iterationCount; i++) { + console.log( + `[playwright] Launching browser for "${taskName}" ` + + `iteration ${i + 1}/${iterationCount} ` + + `(${testCase.map((source) => source.label).join(", ")})` + ); + const run = await runSingleBenchmark({ + testCase, + iterations: 1, + warmupRounds: 0, + channel, + taskFilter: [taskName], + }); + + if (initTimeMs === 0) initTimeMs = run.initTimeMs; + if (registrationMs === 0) registrationMs = run.registrationMs; + + const timing = run.queries[0]?.timings[0]; + if (timing !== undefined) { + timingsMap.get(taskName)!.push(timing); + } + } + } + + queries = allTaskNames.map((name) => + buildQueryResult(name, timingsMap.get(name) ?? []) + ); + } else { + // Warmups > 0: all tasks share a single browser instance. + console.log( + `[playwright] Launching browser for task(s): ${allTaskNames.join(", ")} ` + + `(${testCase.map((source) => source.label).join(", ")})` ); const run = await runSingleBenchmark({ testCase, - iterations, + iterations: iterationCount, warmupRounds, channel, - taskFilter: batch, + taskFilter: allTaskNames, }); - if (initTimeMs === 0) initTimeMs = run.initTimeMs; - if (registrationMs === 0) registrationMs = run.registrationMs; - queries.push(...run.queries); + initTimeMs = initTimeMs || run.initTimeMs; + registrationMs = run.registrationMs; + queries = run.queries; } sourceResults.push({ - labels: testCase.map((testCase) => testCase.label), + labels: testCase.map((source) => source.label), registrationMs, queries, }); @@ -231,7 +263,7 @@ async function runSingleBenchmark({ taskFilter, }: { testCase: TestCase; - iterations?: number; + iterations: number; warmupRounds?: number; channel?: string; taskFilter: string[]; From d07858f731b5efcd01fed192fb8a40cbdd5b7a0d Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Thu, 21 May 2026 12:10:33 -0700 Subject: [PATCH 3/7] Remove verbose LLM comments --- packages/web/benchmark/src/stats.ts | 8 -------- packages/web/benchmark/src/types.ts | 1 - 2 files changed, 9 deletions(-) diff --git a/packages/web/benchmark/src/stats.ts b/packages/web/benchmark/src/stats.ts index 83c0e3d32..245911051 100644 --- a/packages/web/benchmark/src/stats.ts +++ b/packages/web/benchmark/src/stats.ts @@ -3,19 +3,11 @@ import { QueryResult } from "./types"; export const DEFAULT_ITERATIONS = 5; export const DEFAULT_WARMUP_ROUNDS = 1; -// Nearest-rank percentile over a pre-sorted array. Used to report p50 and p95 -// across timed iterations — p95 surfaces occasional slow outliers (GC pauses, -// DuckDB cache misses) that the median would hide. export function percentile(sorted: number[], p: number): number { const idx = Math.ceil((p / 100) * sorted.length) - 1; return sorted[Math.max(0, idx)]; } -/** - * Build a QueryResult from raw timing samples: sorts them and computes - * p50/p95/p99. Used by both the in-browser benchmark engine and the - * Playwright runner's cold-start aggregation path. - */ export function buildQueryResult(name: string, rawTimings: number[]): QueryResult { const timings = [...rawTimings].sort((a, b) => a - b); return { diff --git a/packages/web/benchmark/src/types.ts b/packages/web/benchmark/src/types.ts index 358dd375b..3208d4dd0 100644 --- a/packages/web/benchmark/src/types.ts +++ b/packages/web/benchmark/src/types.ts @@ -15,7 +15,6 @@ export interface BenchmarkConfig { testCases: TestCase[]; iterations?: number; warmupRounds?: number; - /** When set, only tasks whose name appears in this list will run. */ taskFilter?: string[]; } From efc248343e570d2b2a1a7d86372e32aac395288e Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Thu, 28 May 2026 13:56:35 -0700 Subject: [PATCH 4/7] Restore type of runBenchmarkPage --- packages/web/scripts/lib/run-benchmark-page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/scripts/lib/run-benchmark-page.ts b/packages/web/scripts/lib/run-benchmark-page.ts index c7c47c7c3..de11d2c3a 100644 --- a/packages/web/scripts/lib/run-benchmark-page.ts +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -158,7 +158,7 @@ export async function runBenchmarkPage({ iterations?: number; warmupRounds?: number; channel?: string; -}) { +}): Promise { if (!skipBuild) buildBenchmark(); if (!fs.existsSync(path.join(DIST_DIR, "index.html"))) { From 4a857ce2f3e37064ea3b4ab44ebbf57743cfa84a Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Thu, 28 May 2026 14:43:49 -0700 Subject: [PATCH 5/7] When warmups>0, continue using one browser for all test cases --- .../web/scripts/lib/run-benchmark-page.ts | 235 +++++++++--------- 1 file changed, 115 insertions(+), 120 deletions(-) diff --git a/packages/web/scripts/lib/run-benchmark-page.ts b/packages/web/scripts/lib/run-benchmark-page.ts index de11d2c3a..085639e45 100644 --- a/packages/web/scripts/lib/run-benchmark-page.ts +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -22,13 +22,15 @@ declare global { } import { chromium } from "playwright"; +import type { Page } from "playwright"; import http from "http"; import fs from "fs"; import path from "path"; import { execSync } from "child_process"; -import { BenchmarkResults, QueryResult, SourceResult, TestCase } from "../../benchmark/src/types"; +import { BenchmarkResults, SourceResult, TestCase } from "../../benchmark/src/types"; import { BENCHMARK_TASKS } from "../../benchmark/src/tasks"; import { DEFAULT_ITERATIONS, buildQueryResult } from "../../benchmark/src/stats"; + const DIST_DIR = path.join(__dirname, "..", "..", "benchmark", "dist"); const FIXTURES_DIR = path.join(__dirname, "..", "..", "fixtures"); const PORT = 18765; @@ -170,107 +172,106 @@ export async function runBenchmarkPage({ const server = await startServer(); try { - const allTaskNames = BENCHMARK_TASKS.map((task) => task.name); - const iterationCount = iterations ?? DEFAULT_ITERATIONS; - - let initTimeMs = 0; - const sourceResults: SourceResult[] = []; - - for (const testCase of testCases) { - let registrationMs = 0; - let queries: QueryResult[]; + if (warmupRounds === 0) { + return await runColdStartBenchmarks({ testCases, iterations, channel }); + } else { + // Warmups > 0: all test cases share a single browser instance. + return await runInBrowser({ testCases, iterations, warmupRounds, channel }); + } + } finally { + await new Promise((res) => server.close(res)); + } +} - if (warmupRounds === 0) { - // Each (task, iteration) pair gets a fresh browser so every - // measurement is a cold start. - console.log( - `[playwright] warmupRounds=0: running ${allTaskNames.length} task(s) × ` + - `${iterationCount} iteration(s) in separate browser instances` - ); +/** + * warmupRounds=0 path: each (testCase, task, iteration) triple gets a fresh + * browser so every measurement is a cold start. + */ +async function runColdStartBenchmarks({ + testCases, + iterations, + channel, +}: { + testCases: TestCase[]; + iterations?: number; + channel?: string; +}): Promise { + const allTaskNames = BENCHMARK_TASKS.map((task) => task.name); + const iterationCount = iterations ?? DEFAULT_ITERATIONS; - const timingsMap = new Map( - allTaskNames.map((name) => [name, []]) - ); + let initTimeMs = 0; + const sourceResults: SourceResult[] = []; - for (const taskName of allTaskNames) { - for (let i = 0; i < iterationCount; i++) { - console.log( - `[playwright] Launching browser for "${taskName}" ` + - `iteration ${i + 1}/${iterationCount} ` + - `(${testCase.map((source) => source.label).join(", ")})` - ); - const run = await runSingleBenchmark({ - testCase, - iterations: 1, - warmupRounds: 0, - channel, - taskFilter: [taskName], - }); + for (const testCase of testCases) { + let registrationMs = 0; - if (initTimeMs === 0) initTimeMs = run.initTimeMs; - if (registrationMs === 0) registrationMs = run.registrationMs; + console.log( + `[playwright] warmupRounds=0: running ${allTaskNames.length} task(s) × ` + + `${iterationCount} iteration(s) in separate browser instances` + ); - const timing = run.queries[0]?.timings[0]; - if (timing !== undefined) { - timingsMap.get(taskName)!.push(timing); - } - } - } + const timingsMap = new Map(allTaskNames.map((name) => [name, []])); - queries = allTaskNames.map((name) => - buildQueryResult(name, timingsMap.get(name) ?? []) - ); - } else { - // Warmups > 0: all tasks share a single browser instance. + for (const taskName of allTaskNames) { + for (let i = 0; i < iterationCount; i++) { console.log( - `[playwright] Launching browser for task(s): ${allTaskNames.join(", ")} ` + + `[playwright] Launching browser for "${taskName}" ` + + `iteration ${i + 1}/${iterationCount} ` + `(${testCase.map((source) => source.label).join(", ")})` ); - const run = await runSingleBenchmark({ - testCase, - iterations: iterationCount, - warmupRounds, + const run = await runInBrowser({ + testCases: [testCase], + iterations: 1, + warmupRounds: 0, channel, - taskFilter: allTaskNames, + taskFilter: [taskName], }); - initTimeMs = initTimeMs || run.initTimeMs; - registrationMs = run.registrationMs; - queries = run.queries; - } + if (initTimeMs === 0) initTimeMs = run.initTimeMs; + const runResult = run.results[0]; + if (registrationMs === 0) registrationMs = runResult.registrationMs; - sourceResults.push({ - labels: testCase.map((source) => source.label), - registrationMs, - queries, - }); + const timing = runResult.queries[0]?.timings[0]; + if (timing !== undefined) { + timingsMap.get(taskName)!.push(timing); + } + } } - return { - timestamp: new Date().toISOString(), - commit: "unknown", - branch: "unknown", - initTimeMs, - results: sourceResults, - } as BenchmarkResults; - } finally { - await new Promise((res) => server.close(res)); + sourceResults.push({ + labels: testCase.map((source) => source.label), + registrationMs, + queries: allTaskNames.map((name) => buildQueryResult(name, timingsMap.get(name) ?? [])), + }); } + + return { + timestamp: new Date().toISOString(), + commit: "unknown", + branch: "unknown", + initTimeMs, + results: sourceResults, + }; } -async function runSingleBenchmark({ - testCase, +/** + * Launch a single browser, run the benchmark with the given config, and return + * the raw BenchmarkResults. Both the warmup>0 path (all test cases) and the + * warmup=0 path (one test case + task filter) funnel through here. + */ +async function runInBrowser({ + testCases, iterations, warmupRounds, channel, taskFilter, }: { - testCase: TestCase; - iterations: number; + testCases: TestCase[]; + iterations?: number; warmupRounds?: number; channel?: string; - taskFilter: string[]; -}): Promise<{ initTimeMs: number; registrationMs: number; queries: QueryResult[] }> { + taskFilter?: string[]; +}): Promise { const browser = await chromium.launch({ channel, headless: true, @@ -289,14 +290,14 @@ async function runSingleBenchmark({ // synchronously on startup — no callback handshake needed. await page.addInitScript({ content: `window.__benchmarkConfig = ${JSON.stringify({ - testCases: [testCase], + testCases, iterations, warmupRounds, taskFilter, })};`, }); - console.log(`[playwright] Starting benchmark...`); + console.log(`[playwright] Starting benchmark (${testCases.length} test case(s))...`); await page.goto(`http://localhost:${PORT}/`, { waitUntil: "domcontentloaded" }); // Wait for the benchmark to signal it's ready for file injection @@ -308,38 +309,7 @@ async function runSingleBenchmark({ // The browser reads the file lazily via FileReader (BROWSER_FILEREADER protocol), // which is identical to how the real app loads files via the file picker — // no HTTP range-request overhead, so DuckDB sort performance matches real-user timing. - const loaded = new Set(); - for (const source of testCase) { - if (source.label in loaded) continue; // Don't add duplicate sources - const localMatch = source.url.match( - new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`) - ); - if (!localMatch) continue; - const fixturePath = path.join(FIXTURES_DIR, localMatch[1]); - if (!fs.existsSync(fixturePath)) continue; - - console.log(`[playwright] Injecting ${source.label} via setInputFiles...`); - const inputHandle = await page.evaluateHandle(() => { - const inp: HTMLInputElement = document.createElement("input"); - inp.type = "file"; - document.body.appendChild(inp); - return inp; - }); - await inputHandle.setInputFiles(fixturePath); - await page.evaluate((label) => { - const inputs: NodeListOf = document.querySelectorAll( - "input[type=file]" - ); - const inp = inputs[inputs.length - 1]; - window.__pendingLocalFiles = window.__pendingLocalFiles || {}; - if (!inp.files) { - throw new Error(`Injected file not found for ${label}.`); - } - window.__pendingLocalFiles[label] = inp.files[0]; - inp.remove(); - }, source.label); - loaded.add(source.label); - } + await injectFixtures(page, testCases); // Signal the benchmark to proceed with injected File objects await page.evaluate(() => { @@ -359,18 +329,43 @@ async function runSingleBenchmark({ const error = await page.evaluate(() => window.__benchmarkError ?? null); if (error) throw new Error(`Benchmark failed in browser: ${error}`); - const benchmarkResults: BenchmarkResults = await page.evaluate( - () => window.__benchmarkResults - ); - const result = benchmarkResults.results[0]; - return { - initTimeMs: benchmarkResults.initTimeMs, - registrationMs: result.registrationMs, - queries: result.queries, - }; + return await page.evaluate(() => window.__benchmarkResults); } finally { await browser.close(); } } +async function injectFixtures(page: Page, testCases: TestCase[]) { + const loaded = new Set(); + for (const source of testCases.flat()) { + if (loaded.has(source.label)) continue; + const localMatch = source.url.match(new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`)); + if (!localMatch) continue; + const fixturePath = path.join(FIXTURES_DIR, localMatch[1]); + if (!fs.existsSync(fixturePath)) continue; + + console.log(`[playwright] Injecting ${source.label} via setInputFiles...`); + const inputHandle = await page.evaluateHandle(() => { + const inp: HTMLInputElement = document.createElement("input"); + inp.type = "file"; + document.body.appendChild(inp); + return inp; + }); + await inputHandle.setInputFiles(fixturePath); + await page.evaluate((label) => { + const inputs: NodeListOf = document.querySelectorAll( + "input[type=file]" + ); + const inp = inputs[inputs.length - 1]; + window.__pendingLocalFiles = window.__pendingLocalFiles || {}; + if (!inp.files) { + throw new Error(`Injected file not found for ${label}.`); + } + window.__pendingLocalFiles[label] = inp.files[0]; + inp.remove(); + }, source.label); + loaded.add(source.label); + } +} + module.exports = { runBenchmarkPage }; From 9e1c2a5ecdbde6ca99927a070ed6a8f5a05a2136 Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Thu, 28 May 2026 15:01:50 -0700 Subject: [PATCH 6/7] Retain previous comment and formatting --- packages/web/scripts/lib/run-benchmark-page.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/web/scripts/lib/run-benchmark-page.ts b/packages/web/scripts/lib/run-benchmark-page.ts index 085639e45..b4f911b58 100644 --- a/packages/web/scripts/lib/run-benchmark-page.ts +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -338,7 +338,7 @@ async function runInBrowser({ async function injectFixtures(page: Page, testCases: TestCase[]) { const loaded = new Set(); for (const source of testCases.flat()) { - if (loaded.has(source.label)) continue; + if (loaded.has(source.label)) continue; // Don't add duplicate sources const localMatch = source.url.match(new RegExp(`^http://localhost:${PORT}/fixtures/(.+)$`)); if (!localMatch) continue; const fixturePath = path.join(FIXTURES_DIR, localMatch[1]); From 4576877042e32f15620e359f0a974a6f7b6bd1da Mon Sep 17 00:00:00 2001 From: Philip Garrison Date: Thu, 4 Jun 2026 11:58:33 -0700 Subject: [PATCH 7/7] Avoid non-null assertion --- packages/web/scripts/lib/run-benchmark-page.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/web/scripts/lib/run-benchmark-page.ts b/packages/web/scripts/lib/run-benchmark-page.ts index b4f911b58..d43a56538 100644 --- a/packages/web/scripts/lib/run-benchmark-page.ts +++ b/packages/web/scripts/lib/run-benchmark-page.ts @@ -233,7 +233,9 @@ async function runColdStartBenchmarks({ const timing = runResult.queries[0]?.timings[0]; if (timing !== undefined) { - timingsMap.get(taskName)!.push(timing); + const timings = timingsMap.get(taskName); + if (!timings) throw Error(`${taskName} not in timingsMap!`); + timings.push(timing); } } }