Nightly E2E #164
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly E2E | |
| on: | |
| schedule: | |
| # Every day at midnight UTC | |
| - cron: "0 0 * * *" | |
| # Allow manual trigger for debugging | |
| workflow_dispatch: | |
| inputs: | |
| run_deployed: | |
| description: "Also run the DEPLOYED recovery suites (real, billable Workers)" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Deterministic Playwright suite. Drives only mock agents against an AI-free | |
| # worker, so it boots offline/instantly and needs no Cloudflare credentials. | |
| # This is the gating, always-green ai-chat e2e signal. (The Workers-AI specs | |
| # run in the separate `e2e-ai-chat-llm` job below, which is bounded by a | |
| # globalTimeout so a flaky edge connection can never cancel a job at the cap.) | |
| e2e-ai-chat: | |
| name: "E2E: ai-chat (Playwright, deterministic)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| - name: Get Playwright version | |
| id: playwright-version | |
| run: echo "version=$(node -p 'require(\"playwright/package.json\").version')" >> $GITHUB_OUTPUT | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v5 | |
| id: playwright-cache | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }} | |
| - name: Install Playwright browsers | |
| run: pnpm exec playwright install --with-deps chromium | |
| - name: Run ai-chat deterministic e2e tests | |
| run: pnpm run test:e2e:playwright | |
| working-directory: packages/ai-chat | |
| # Workers-AI Playwright suite. Needs the remote `ai` binding and a healthy | |
| # Workers AI edge connection. Real-model latency/flakiness is bounded by a | |
| # hard globalTimeout in playwright.llm.config.ts so this can never silently | |
| # run to the job's 30-minute cancel. | |
| e2e-ai-chat-llm: | |
| name: "E2E: ai-chat (Playwright, Workers AI)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| - name: Get Playwright version | |
| id: playwright-version | |
| run: echo "version=$(node -p 'require(\"playwright/package.json\").version')" >> $GITHUB_OUTPUT | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v5 | |
| id: playwright-cache | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }} | |
| - name: Install Playwright browsers | |
| run: pnpm exec playwright install --with-deps chromium | |
| - name: Run ai-chat Workers AI e2e tests | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e:playwright:llm | |
| working-directory: packages/ai-chat | |
| e2e-think: | |
| name: "E2E: think (chat recovery)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| - name: Run think e2e tests | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e | |
| working-directory: packages/think | |
| e2e-ai-chat-recovery: | |
| name: "E2E: ai-chat (SIGKILL recovery)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| # Real `wrangler dev` + mid-stream SIGKILL suite that proves AIChatAgent's | |
| # convergence onto the shared `agents/chat` recovery engine survives a real | |
| # isolate crash. Distinct from the `e2e-ai-chat` Playwright job above, which | |
| # exercises the client-side resume protocol (disconnect/reconnect), not | |
| # process death. (Excludes the env-gated `deployed-recovery` suite.) | |
| - name: Run ai-chat SIGKILL recovery e2e tests | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e | |
| working-directory: packages/ai-chat | |
| e2e-agents: | |
| name: "E2E: agents (fiber eviction)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| # Core `runFiber` SIGKILL recovery primitives (root/sub-agent, concurrency, | |
| # scan-deadline yield, poison-row aging/backoff, facet multipass). Excluded | |
| # from the default unit `test` target because it spawns real wrangler dev | |
| # processes; runs here nightly instead. | |
| - name: Run agents fiber-recovery e2e tests | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e | |
| working-directory: packages/agents | |
| e2e-engine-genericity: | |
| name: "E2E: shared-engine genericity (pi + tanstack)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| # Genericity proofs that the shared `agents/chat` recovery engine is not | |
| # AI-SDK-specific: a non-`UIMessage` PiAgent and a foreign (AG-UI / TanStack | |
| # AI) tool vocabulary each recover from a real SIGKILL through the same | |
| # engine + resume handshake + codec. Both use deterministic faux models; the | |
| # real-Workers-AI tanstack leg stays gated behind RUN_WORKERS_AI_E2E. | |
| - name: Run pi-recovery genericity e2e | |
| run: pnpm run test:e2e | |
| working-directory: experimental/pi-recovery | |
| # The real-Workers-AI tanstack leg is enabled here (secrets available in | |
| # nightly); locally it stays gated behind RUN_WORKERS_AI_E2E so contributors | |
| # without an `ai` binding skip it. | |
| - name: Run tanstack-recovery genericity e2e | |
| env: | |
| RUN_WORKERS_AI_E2E: "1" | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e | |
| working-directory: experimental/tanstack-recovery | |
| # Deterministic codemode Playwright suite (Layer 4b). Drives the dynamic | |
| # Worker-loader code-generation flow (executor.spec.ts) against an AI-free | |
| # worker; offline, no Cloudflare credentials. Mirrors `e2e-ai-chat`. A | |
| # `globalTimeout` in its playwright.config.ts bounds it so a hang can never | |
| # silently run to the cap. (The Workers-AI codemode specs run in the separate | |
| # `e2e-codemode-llm` job below.) | |
| e2e-codemode: | |
| name: "E2E: codemode (Playwright)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| - name: Get Playwright version | |
| id: playwright-version | |
| run: echo "version=$(node -p 'require(\"playwright/package.json\").version')" >> $GITHUB_OUTPUT | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v5 | |
| id: playwright-cache | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }} | |
| - name: Install Playwright browsers | |
| run: pnpm exec playwright install --with-deps chromium | |
| - name: Run codemode deterministic e2e tests | |
| run: pnpm run test:e2e | |
| working-directory: packages/codemode | |
| # Workers-AI codemode Playwright suite. Needs the remote `ai` binding | |
| # (wrangler.jsonc) and a healthy Workers AI edge connection, so it gets | |
| # credentials. Real-model latency/flakiness is bounded by a hard globalTimeout | |
| # in playwright.llm.config.ts so this can never silently run to the cap. | |
| e2e-codemode-llm: | |
| name: "E2E: codemode (Playwright, Workers AI)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| - name: Get Playwright version | |
| id: playwright-version | |
| run: echo "version=$(node -p 'require(\"playwright/package.json\").version')" >> $GITHUB_OUTPUT | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v5 | |
| id: playwright-cache | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }} | |
| - name: Install Playwright browsers | |
| run: pnpm exec playwright install --with-deps chromium | |
| - name: Run codemode Workers AI e2e tests | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e:llm | |
| working-directory: packages/codemode | |
| # Browser-connector e2e (Layer 3c). Spawns a real `wrangler dev` (local Browser | |
| # Rendering simulator + worker loader) and drives `BrowserConnector` over CDP. | |
| # Kept out of the default unit `test` target (it needs Chromium); runs here | |
| # nightly instead. | |
| e2e-agents-browser: | |
| name: "E2E: agents (browser connector)" | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| # Pin an exact pre-bug Node patch. Wrangler's local Browser Rendering | |
| # simulator downloads its own Chrome via @puppeteer/browsers, whose | |
| # extract-zip dep silently aborts mid-extraction on Node 24.16+ | |
| # (puppeteer/puppeteer#14957). That leaves a partial Chrome cache, which | |
| # Wrangler then "clears and retries" on every test — a re-corrupting loop | |
| # that, with retry:3 + a 120s test timeout, ran this job to the 30m cancel. | |
| - uses: ./.github/actions/install | |
| with: | |
| node-version: 24.15.0 | |
| - run: pnpm run build | |
| - name: Get Playwright version | |
| id: playwright-version | |
| run: echo "version=$(node -p 'require(\"playwright/package.json\").version')" >> $GITHUB_OUTPUT | |
| - name: Cache Playwright browsers | |
| uses: actions/cache@v5 | |
| id: playwright-cache | |
| with: | |
| path: ~/.cache/ms-playwright | |
| key: ${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }} | |
| - name: Install Playwright browsers | |
| run: pnpm exec playwright install --with-deps chromium | |
| # Persist Wrangler's auto-downloaded Chrome (separate from the Playwright | |
| # cache above) so it's fetched once and reused, keeping the one-time | |
| # ~hundreds-of-MB download out of the timed test window. Keyed on the | |
| # lockfile so a Wrangler bump (which can change the pinned Chrome build) | |
| # busts it. | |
| - name: Cache Wrangler Chrome | |
| uses: actions/cache@v5 | |
| with: | |
| path: ~/.cache/.wrangler/chrome | |
| key: ${{ runner.os }}-wrangler-chrome-${{ hashFiles('pnpm-lock.yaml') }} | |
| # Wall-clock guard (20m, below the 30m job cap) mirroring the Playwright | |
| # suites' globalTimeout: a future environmental stall fails WITH logs | |
| # instead of being silently cancelled at the cap. The suite's own bail:1 | |
| # already turns a real failure into a fast, reported one. | |
| - name: Run agents browser-connector e2e tests | |
| env: | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: timeout --kill-after=30s 20m pnpm run test:browser | |
| working-directory: packages/agents | |
| # ── DEPLOYED (Layer-5) recovery suites ────────────────────────────────────── | |
| # These deploy REAL, billable Workers and drive recovery on Cloudflare's edge | |
| # (not local workerd). They run on the nightly SCHEDULE. For manual | |
| # `workflow_dispatch` runs they stay opt-in (set the `run_deployed` input, or | |
| # the `RUN_DEPLOYED_E2E` repo variable) so ad-hoc debugging runs don't deploy | |
| # billable Workers unintentionally. Each suite uniquely names its throwaway | |
| # Worker and always deletes it. | |
| e2e-deployed-ai-chat: | |
| name: "E2E (deployed): ai-chat recovery on real edge" | |
| if: ${{ github.event_name == 'schedule' || vars.RUN_DEPLOYED_E2E == '1' || github.event.inputs.run_deployed == 'true' }} | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| # Deploys `wrangler.deployed.jsonc`, forces a mid-turn redeploy to evict the | |
| # live DO, asserts recovery fires + the no-false-incident counterpart, then | |
| # deletes the Worker. | |
| - name: Run ai-chat DEPLOYED recovery e2e | |
| env: | |
| RUN_DEPLOYED_E2E: "1" | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e:deployed | |
| working-directory: packages/ai-chat | |
| e2e-deployed-think-probe: | |
| name: "E2E (deployed): Think recovery probe on real edge" | |
| if: ${{ github.event_name == 'schedule' || vars.RUN_DEPLOYED_E2E == '1' || github.event.inputs.run_deployed == 'true' }} | |
| timeout-minutes: 30 | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - uses: ./.github/actions/install | |
| - run: pnpm run build | |
| # Deploys the chat-recovery-probe under a throwaway name and runs the fast, | |
| # deterministic abort-driven Think scenarios (a6 HITL, a7 server-orphan, | |
| # a8 approval, idem), then deletes the Worker. The slow real-deploy-churn | |
| # scenarios (a1/a2/a4/a5/a9/rapid) stay manual (see the probe README). | |
| - name: Run Think recovery-probe DEPLOYED suite | |
| env: | |
| RUN_DEPLOYED_E2E: "1" | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| run: pnpm run test:e2e:deployed | |
| working-directory: experimental/chat-recovery-probe |