Nightly corpus #11
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Nightly real-document regression matrix (docs/superpowers/plans/2026-08-15-v9-test-coverage-strategy.md, | |
| # section 2 tier 2 / section 6). Pulls a public corpus of real Office files | |
| # (never committed here), drives the real editor through open -> edit -> save | |
| # for each, and publishes the findings table. Red is a signal, not a gate. | |
| name: Nightly corpus | |
| on: | |
| schedule: | |
| - cron: "17 19 * * *" # 03:17 Asia/Shanghai | |
| workflow_dispatch: | |
| inputs: | |
| limit: | |
| description: "Max files to run (CORPUS_LIMIT)" | |
| default: "300" | |
| filter: | |
| description: "Include regex on file paths (CORPUS_FILTER)" | |
| default: "" | |
| visual: | |
| description: "Pixel-diff original vs re-opened save (CORPUS_VISUAL, empty to skip)" | |
| default: "1" | |
| jobs: | |
| corpus: | |
| name: Real-document matrix (Apache POI test-data) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 300 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Set up the toolchain | |
| uses: ./.github/actions/setup | |
| with: | |
| browsers: chromium | |
| # Apache POI's test-data tree (Apache-2.0) is the largest curated pile of | |
| # real-world .doc/.docx/.xls/.xlsx/.ppt/.pptx around, including many | |
| # regression files from real bug reports. Sparse + blobless clone keeps | |
| # it to the three directories we need. | |
| - name: Fetch public corpus (Apache POI test-data) | |
| run: | | |
| git clone --depth 1 --filter=blob:none --sparse https://github.com/apache/poi.git corpus-src | |
| git -C corpus-src sparse-checkout set test-data/spreadsheet test-data/document test-data/slideshow | |
| mkdir -p corpus | |
| cp -r corpus-src/test-data/spreadsheet corpus/spreadsheet | |
| cp -r corpus-src/test-data/document corpus/document | |
| cp -r corpus-src/test-data/slideshow corpus/slideshow | |
| echo "corpus files: $(find corpus -type f | wc -l)" | |
| - name: Run corpus matrix | |
| env: | |
| CORPUS_DIR: ${{ github.workspace }}/corpus | |
| CORPUS_LIMIT: ${{ github.event.inputs.limit || '300' }} | |
| CORPUS_FILTER: ${{ github.event.inputs.filter || '' }} | |
| CORPUS_VISUAL: ${{ github.event.inputs.visual || '1' }} | |
| # Expected-to-fail inputs: encrypted/password files, deliberately | |
| # truncated/corrupt regression samples, macro-only containers, and | |
| # POI's fuzzer output (clusterfuzz-testcase-*, *Fuzzer*, Fuzzed.doc, | |
| # poi-fuzz.xls, crash-<sha1>.*) -- byte soup kept precisely because | |
| # it broke a parser, so "this editor will not open it" is not a | |
| # finding. 15 of the 20 red rows on 2026-08-21 were these. The terms | |
| # are narrow on purpose: a bare `crash` would also drop | |
| # 51921-Word-Crash067.doc, which is a real document from a bug report. | |
| CORPUS_EXCLUDE: "password|protect|encrypt|corrupt|truncat|broken|invalid|damaged|clusterfuzz|fuzzer|fuzzed|poi-fuzz|crash-[0-9a-f]{6}|\\.xlsm$|\\.xlsb$|\\.docm$|\\.pptm$" | |
| run: pnpm exec playwright test test/e2e/corpus.spec.ts --workers=2 --reporter=list | |
| - name: Summarize | |
| if: always() | |
| run: node bin/corpus-report.mjs test-results | |
| - name: Upload corpus report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: corpus-report | |
| path: | | |
| test-results/corpus-report.json | |
| playwright-report/ | |
| retention-days: 14 | |
| browsers: | |
| name: Cross-browser (WebKit + Firefox) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Set up the toolchain | |
| uses: ./.github/actions/setup | |
| with: | |
| browsers: webkit firefox | |
| # The same PR-time suites (minus the opt-in sweeps) on the two engines | |
| # the gate does not cover. Same-realm/L0 rules apply unchanged. | |
| - name: Run E2E on WebKit and Firefox | |
| run: pnpm exec playwright test -c playwright.browsers.config.ts --grep-invert "api surface|corpus" --reporter=list | |
| - name: Upload report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: playwright-report-browsers | |
| path: playwright-report-browsers/ | |
| retention-days: 14 | |
| budgets: | |
| name: Slow-network budgets | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Set up the toolchain | |
| uses: ./.github/actions/setup | |
| with: | |
| browsers: chromium | |
| # Roadmap direction 9 (L3): flag when the ecosystem packages on npm have | |
| # moved past what this repo pins. Informational -- the bump itself is a | |
| # PR (and for ranui usually a release from chaxus/ran first). | |
| - name: ranui / ranuts upstream check | |
| run: | | |
| for pkg in ranui ranuts; do | |
| PINNED=$(node -p "require('./package.json').dependencies['$pkg']") | |
| LATEST=$(npm view "$pkg" version 2>/dev/null || echo unknown) | |
| if [ "$PINNED" = "$LATEST" ]; then | |
| echo "✅ $pkg pinned $PINNED == npm latest" | tee -a "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "⚠️ $pkg pinned $PINNED, npm latest $LATEST -- consider a bump PR (vendored IIFEs re-sync on build; ranui-vendor-sync.test.ts guards drift)" | tee -a "$GITHUB_STEP_SUMMARY" | |
| fi | |
| done | |
| # Strategy section 9.1 layer 2: click every visible toolbar button of | |
| # each editor once; L0 + "still saves" is the oracle. Found guard 8's | |
| # comment/selection crash. | |
| - name: UI crawl | |
| env: | |
| UI_CRAWL: "1" | |
| run: pnpm exec playwright test test/e2e/ui-crawl.spec.ts --reporter=list | |
| - name: Upload UI crawl reports | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: ui-crawl-reports | |
| path: test-results/ui-crawl-*.json | |
| retention-days: 14 | |
| # Cold open + first save under CDP network throttling (L4): the whole | |
| # path must stay inside the save request's allowance. | |
| - name: Throttled-network budget | |
| env: | |
| SLOW_NET: "1" | |
| run: pnpm exec playwright test test/e2e/slow-network.spec.ts --reporter=list |