Add engine-integration tests (parser unit + analysis e2e)

Unit-test parseAnalysis against the golden transcripts: winrate normalization for both dialects, scoreLead present/absent, non-negative visits (real lz-analyze emits visits 0 for prior-only moves), and pv parsing. Add a replay fake engine (test/engines/replayEngine.js) that streams a recorded transcript over GTP, and an e2e spec driving attach -> analyze -> SBKV written onto the node. Registers the engine-analysis Playwright project.

Author: apetresc

e2e/engine-analysis.spec.js

@@ -0,0 +1,92 @@
+const {expect} = require('@playwright/test')
+const path = require('path')
+const {test} = require('./fixtures/electron-app')
+const {
+  loadSgfAndWait,
+  attachAndWaitForEngines,
+  detachAndWait,
+} = require('./helpers')
+
+// End-to-end coverage of the analysis pipeline: attach an engine, start
+// analysis, and confirm the engine's reported win rate is written onto the
+// current node as the SBKV property.
+//
+// The engine here is test/engines/replayEngine.js, which replays a GOLDEN
+// TRANSCRIPT — real `info ...` lines recorded from KataGo by
+// scripts/engine-transcripts/capture.mjs. So this exercises the genuine
+// attach → GTP analyze → parseAnalysis → SBKV path against real engine output,
+// deterministically and without needing an engine (or GPU) at test time.
+
+const RES = path.resolve(
+  __dirname,
+  '..',
+  'test',
+  'resources',
+  'engine-transcripts',
+)
+const REPLAY_ENGINE = path.resolve(
+  __dirname,
+  '..',
+  'test',
+  'engines',
+  'replayEngine.js',
+)
+const SGF = path.join(RES, 'sgf', 'opening-19.sgf')
+const TRANSCRIPT = path.join(
+  RES,
+  'katago-1.16.4',
+  'opening-19.kata-analyze.txt',
+)
+
+test.describe('Engine Analysis Integration', () => {
+  test('replayed KataGo analysis writes SBKV onto the current node', async ({
+    page,
+  }) => {
+    await loadSgfAndWait(page, SGF)
+    await page.evaluate(() => window.__sabaki.goToEnd())
+
+    const [syncerId] = await attachAndWaitForEngines(page, [
+      {
+        name: 'ReplayEngine',
+        path: process.execPath,
+        args: `${REPLAY_ENGINE} --transcript ${TRANSCRIPT} --analyze-command kata-analyze`,
+      },
+    ])
+
+    // The engine advertises kata-analyze, so Sabaki will choose it.
+    const supportsAnalyze = await page.evaluate((id) => {
+      const s = window.__sabaki.state.attachedEngineSyncers.find(
+        (x) => x.id === id,
+      )
+      return s != null && s.commands.includes('kata-analyze')
+    }, syncerId)
+    expect(supportsAnalyze).toBe(true)
+
+    await page.evaluate((id) => window.__sabaki.startAnalysis(id), syncerId)
+
+    // Win rate from the replayed analysis should be written onto the node.
+    await page.waitForFunction(
+      () => {
+        const s = window.__sabaki
+        const tree = s.state.gameTrees[s.state.gameIndex]
+        const node = tree.get(s.state.treePosition)
+        return node != null && node.data != null && node.data.SBKV != null
+      },
+      {timeout: 15000},
+    )
+
+    const sbkv = await page.evaluate(() => {
+      const s = window.__sabaki
+      const tree = s.state.gameTrees[s.state.gameIndex]
+      return tree.get(s.state.treePosition).data.SBKV[0]
+    })
+
+    const winrate = parseFloat(sbkv)
+    expect(Number.isFinite(winrate)).toBe(true)
+    expect(winrate).toBeGreaterThanOrEqual(0)
+    expect(winrate).toBeLessThanOrEqual(100)
+
+    await page.evaluate(() => window.__sabaki.stopAnalysis())
+    await detachAndWait(page, [syncerId])
+  })
+})

playwright.config.js

@@ -14,6 +14,11 @@ module.exports = defineConfig({
       dependencies: ['smoke'],
     },
     {name: 'engine', testMatch: /engine\.spec\.js/, dependencies: ['smoke']},
+    {
+      name: 'engine-analysis',
+      testMatch: /engine-analysis\.spec\.js/,
+      dependencies: ['smoke'],
+    },
     {
       name: 'move-numbers',
       testMatch: /move-numbers\.spec\.js/,

test/analysisTests.js

@@ -0,0 +1,159 @@
+import assert from 'assert'
+import {readFileSync, existsSync} from 'fs'
+import {join} from 'path'
+import {fromDimensions as newBoard} from '@sabaki/go-board'
+
+import {parseAnalysis} from '../src/modules/analysis.js'
+
+// These tests verify the GTP analysis parser (the SBKV / scoreLead extraction
+// path) against GOLDEN TRANSCRIPTS  (real `info ...` lines recorded from real
+// engines by scripts/engine-transcripts/capture.mjs). The fixtures are replayed
+// here so we test the exact output dialects engines emit, deterministically and
+// without an engine at test time.
+//
+// Regenerate the fixtures with `npm run gen:engine-transcripts` after adding an engine
+// version or SGF to scripts/engine-transcripts/engines.config.mjs.
+
+const RES = join(__dirname, 'resources', 'engine-transcripts')
+const manifestPath = join(RES, 'manifest.json')
+
+// The last `info` line of a transcript is the most-settled analysis update —
+// the one whose values ultimately get written to the node as SBKV/SBKS.
+function lastInfoLine(file) {
+  let lines = readFileSync(join(RES, file), 'utf8')
+    .split('\n')
+    .filter((l) => l.startsWith('info '))
+  return lines[lines.length - 1]
+}
+
+if (!existsSync(manifestPath)) {
+  describe('parseAnalysis (golden engine transcripts)', () => {
+    it('requires generated transcripts — run `npm run gen:engine-transcripts`', () => {
+      assert.fail(`missing ${manifestPath}`)
+    })
+  })
+} else {
+  const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'))
+
+  describe('parseAnalysis (golden engine transcripts)', () => {
+    it('has at least one captured transcript', () => {
+      assert(manifest.cells.length > 0, 'no transcript cells in manifest')
+    })
+
+    for (const cell of manifest.cells) {
+      describe(`${cell.engineId} · ${cell.sgf} · ${cell.command}`, () => {
+        const board = newBoard(cell.boardSize, cell.boardSize)
+        const variations = parseAnalysis(lastInfoLine(cell.file), board)
+
+        it('parses at least one variation', () => {
+          assert(Array.isArray(variations) && variations.length > 0)
+        })
+
+        it('extracts winrate as a percentage in [0, 100] (the SBKV domain)', () => {
+          for (const v of variations) {
+            assert(
+              typeof v.winrate === 'number' && isFinite(v.winrate),
+              `non-finite winrate: ${v.winrate}`,
+            )
+            assert(
+              v.winrate >= 0 && v.winrate <= 100,
+              `winrate out of range: ${v.winrate}`,
+            )
+          }
+        })
+
+        it('extracts visits as non-negative integers', () => {
+          // Engines legitimately report moves with `visits 0` (evaluated by
+          // policy prior only, no playouts), as observed in real lz-analyze
+          // output, so the invariant is non-negative, not strictly positive.
+          for (const v of variations) {
+            assert(
+              Number.isInteger(v.visits) && v.visits >= 0,
+              `bad visits: ${v.visits}`,
+            )
+          }
+        })
+
+        it('extracts a legal top-move vertex and a pv move list', () => {
+          for (const v of variations) {
+            assert(
+              v.vertex == null ||
+                (Array.isArray(v.vertex) && v.vertex.length === 2),
+              `bad vertex: ${JSON.stringify(v.vertex)}`,
+            )
+            assert(Array.isArray(v.moves), 'pv moves should be an array')
+          }
+        })
+
+        if (cell.command === 'kata-analyze') {
+          it('extracts scoreLead as a finite number (KataGo dialect → SBKS)', () => {
+            for (const v of variations) {
+              assert(
+                typeof v.scoreLead === 'number' && isFinite(v.scoreLead),
+                `expected numeric scoreLead, got: ${v.scoreLead}`,
+              )
+            }
+          })
+        } else if (cell.command === 'lz-analyze') {
+          it('reports scoreLead as null (Leela-Zero dialect has no score)', () => {
+            for (const v of variations) {
+              assert.strictEqual(
+                v.scoreLead,
+                null,
+                `expected null scoreLead, got: ${v.scoreLead}`,
+              )
+            }
+          })
+        }
+      })
+    }
+  })
+}
+
+// Focused unit cases for the parser's own normalization logic. These use minimal
+// hand-written lines (not engine-format assumptions) to pin specific branches
+// that a captured snapshot may not deterministically contain.
+describe('parseAnalysis (parser logic)', () => {
+  const board = newBoard(19, 19)
+
+  it('returns [] when the line carries no info segments', () => {
+    assert.deepStrictEqual(parseAnalysis('= ', board), [])
+  })
+
+  it('normalizes integer (Leela-Zero) winrate ten-thousandths to a percentage', () => {
+    let [v] = parseAnalysis(
+      'info move Q16 visits 10 winrate 5213 pv Q16 D4',
+      board,
+    )
+    assert(Math.abs(v.winrate - 52.13) < 1e-9, `got ${v.winrate}`)
+    assert.strictEqual(v.scoreLead, null)
+  })
+
+  it('normalizes float (KataGo) winrate and reads scoreLead', () => {
+    let [v] = parseAnalysis(
+      'info move Q16 visits 10 winrate 0.5213 scoreLead 1.5 pv Q16 D4',
+      board,
+    )
+    assert(Math.abs(v.winrate - 52.13) < 1e-9, `got ${v.winrate}`)
+    assert(Math.abs(v.scoreLead - 1.5) < 1e-9, `got ${v.scoreLead}`)
+  })
+
+  it('parses multiple variations from one line', () => {
+    let vs = parseAnalysis(
+      'info move Q16 visits 20 winrate 0.55 scoreLead 2 pv Q16 D4 ' +
+        'info move D4 visits 10 winrate 0.45 scoreLead -1 pv D4 Q16',
+      board,
+    )
+    assert.strictEqual(vs.length, 2)
+    assert(Math.abs(vs[0].winrate - 55) < 1e-9)
+    assert(Math.abs(vs[1].scoreLead - -1) < 1e-9)
+  })
+
+  it('truncates the principal variation at a pass', () => {
+    let [v] = parseAnalysis(
+      'info move Q16 visits 10 winrate 0.5 pv Q16 pass D4',
+      board,
+    )
+    assert.strictEqual(v.moves.length, 1)
+  })
+})

test/engines/replayEngine.js

@@ -0,0 +1,128 @@
+// A fake GTP engine that REPLAYS a recorded analysis transcript.
+//
+// Used by the engine-analysis e2e test to drive Sabaki's real
+// analyze → parseAnalysis → SBKV pipeline deterministically, using genuine
+// engine `info ...` lines captured by scripts/engine-transcripts/capture.mjs —
+// no live engine (or GPU) needed at test time.
+//
+// It speaks GTP directly (rather than via @sabaki/gtp's Engine helper) because
+// the analyze protocol streams `info` lines while `stop` must be handled
+// concurrently — the helper processes commands strictly sequentially and would
+// deadlock. Response framing matches what KataGo emits:
+//   - normal command:  "= <content>\n\n"
+//   - analyze command: "=\n" then streamed "info ...\n" lines, with the blank
+//     line terminator withheld until `stop` arrives.
+//
+// Usage (args, parsed loosely):
+//   node replayEngine.js --transcript <path> [--analyze-command kata-analyze]
+
+const {createInterface} = require('readline')
+const {readFileSync} = require('fs')
+
+const args = process.argv.slice(2)
+const opt = (name, fallback) => {
+  let i = args.indexOf(name)
+  return i >= 0 && i + 1 < args.length ? args[i + 1] : fallback
+}
+
+const transcriptPath = opt('--transcript')
+const analyzeCommand = opt('--analyze-command', 'kata-analyze')
+
+const infoLines = transcriptPath
+  ? readFileSync(transcriptPath, 'utf8')
+      .split('\n')
+      .filter((l) => l.startsWith('info '))
+  : []
+
+const baseCommands = [
+  'protocol_version',
+  'name',
+  'version',
+  'known_command',
+  'list_commands',
+  'quit',
+  'boardsize',
+  'clear_board',
+  'komi',
+  'play',
+  'undo',
+  'genmove',
+  'stop',
+]
+const supported = [...new Set([...baseCommands, analyzeCommand])]
+
+const out = (s) => process.stdout.write(s)
+const ok = (content = '') => out(`= ${content}\n\n`)
+const err = (msg) => out(`? ${msg}\n\n`)
+
+let streamTimer = null
+let streamIndex = 0
+
+function startStreaming() {
+  if (infoLines.length === 0) {
+    // No recorded lines: still open a valid (empty) analyze response.
+    out('=\n')
+    return
+  }
+  // Open the analyze response, then emit one recorded update per tick, cycling,
+  // until `stop` closes it.
+  out('=\n')
+  let tick = () => {
+    out(infoLines[streamIndex % infoLines.length] + '\n')
+    streamIndex++
+  }
+  tick()
+  streamTimer = setInterval(tick, 100)
+}
+
+function stopStreaming() {
+  if (streamTimer != null) {
+    clearInterval(streamTimer)
+    streamTimer = null
+  }
+  // Blank line terminates the still-open analyze response.
+  out('\n')
+}
+
+createInterface({input: process.stdin}).on('line', (raw) => {
+  let line = raw.replace(/#.*$/, '').trim()
+  if (line === '') return
+
+  // Strip an optional leading command id (GTP allows it; our controller omits).
+  let parts = line.split(/\s+/)
+  if (/^\d+$/.test(parts[0])) parts.shift()
+  let [name, ...rest] = parts
+
+  if (name === analyzeCommand) {
+    startStreaming()
+    return
+  }
+
+  if (name === 'stop') {
+    if (streamTimer != null || infoLines.length > 0) stopStreaming()
+    ok()
+    return
+  }
+
+  switch (name) {
+    case 'protocol_version':
+      return ok('2')
+    case 'name':
+      return ok('Replay Engine')
+    case 'version':
+      return ok('1.0')
+    case 'list_commands':
+      return ok(supported.join('\n'))
+    case 'known_command':
+      return ok(supported.includes(rest[0]) ? 'true' : 'false')
+    case 'genmove':
+      return ok('pass')
+    case 'quit':
+      ok()
+      return process.exit(0)
+    default:
+      // Be lenient about board-setup commands (boardsize, clear_board, komi,
+      // play, undo, handicap, …) so position sync never fails.
+      return ok()
+  }
+})