From dc3977bc273788f15f8999b132343436a998013c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Wed, 15 Jul 2026 15:24:56 +0200 Subject: [PATCH 1/3] fix(search): route highlights beta through indexed chain (#4038) --- apps/api/src/search/execute.ts | 9 ++- apps/api/src/search/highlights.test.ts | 79 +++++++++++++++++++++ apps/api/src/search/highlights.ts | 96 ++++++++++++++++++++++---- 3 files changed, 167 insertions(+), 17 deletions(-) diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts index 74c43cdc9f..0fd58c123c 100644 --- a/apps/api/src/search/execute.ts +++ b/apps/api/src/search/execute.ts @@ -13,7 +13,7 @@ import { mergeScrapedContent, calculateScrapeCredits, } from "./scrape"; -import { applySearchHighlights, highlightsEnvReady } from "./highlights"; +import { applyIndexedSearchHighlights, highlightsEnvReady } from "./highlights"; import { runSearchHighlightsShadow } from "./highlights-shadow"; import { trackSearchResults, trackSearchRequest } from "../lib/tracking"; import type { BillingMetadata } from "../services/billing/types"; @@ -253,7 +253,12 @@ export async function executeSearch( flags?.highlightsBeta === true && highlightsEnvReady(); if (shouldApplyHighlights) { - await applySearchHighlights(searchResponse, query, logger); + await applyIndexedSearchHighlights( + searchResponse, + query, + logger, + context.requestId, + ); } else { runSearchHighlightsShadow({ response: searchResponse, diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.ts index 5162e6f6a7..2f6c119765 100644 --- a/apps/api/src/search/highlights.test.ts +++ b/apps/api/src/search/highlights.test.ts @@ -45,6 +45,7 @@ import { import { config } from "../config"; import { indexGetRecent5 } from "../db/rpc"; import { + applyIndexedSearchHighlights, applySearchHighlights, highlightsEnvReady, runIndexedSearchHighlightsShadow, @@ -114,6 +115,84 @@ describe("runIndexedSearchHighlightsShadow", () => { }); }); +describe("applyIndexedSearchHighlights", () => { + it("uses the shadow indexed request path and applies web and news responses by ID", async () => { + vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue( + new Map([ + ["0", { highlights: [], markdown: "first highlight" }], + ["1", { highlights: [], markdown: "second highlight" }], + ]), + ); + const response = { + web: [{ url: "https://first.test", description: "first fallback" }], + news: [{ url: "https://second.test", snippet: "second fallback" }], + } as any; + + const result = await applyIndexedSearchHighlights( + response, + "query", + logger, + "request-1", + ); + + expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith( + "query", + [ + { + id: "0", + url: "https://first.test", + indexObject: "index:https://first.test.json", + }, + { + id: "1", + url: "https://second.test", + indexObject: "index:https://second.test.json", + }, + ], + { + logger, + logPayload: false, + requestId: "request-1", + timeoutMs: null, + onFailure: expect.any(Function), + }, + ); + expect(generateHighlightsBatch).not.toHaveBeenCalled(); + expect(response.web[0].description).toBe("first highlight"); + expect(response.news[0].snippet).toBe("second highlight"); + expect(result).toEqual({ + attempted: 2, + indexHits: 2, + replaced: 2, + succeeded: true, + }); + }); + + it("preserves provider snippets when the indexed Chain request fails", async () => { + vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(null); + const response = { + web: [{ url: "https://first.test", description: "first fallback" }], + news: [{ url: "https://second.test", snippet: "second fallback" }], + } as any; + + const result = await applyIndexedSearchHighlights( + response, + "query", + logger, + "request-1", + ); + + expect(response.web[0].description).toBe("first fallback"); + expect(response.news[0].snippet).toBe("second fallback"); + expect(result).toEqual({ + attempted: 2, + indexHits: 2, + replaced: 0, + succeeded: false, + }); + }); +}); + describe("applySearchHighlights", () => { it("enables the in-cluster service without requiring a bearer token", () => { const token = config.HIGHLIGHT_MODEL_TOKEN; diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts index d61bc0a3cd..a8448c7c1a 100644 --- a/apps/api/src/search/highlights.ts +++ b/apps/api/src/search/highlights.ts @@ -146,17 +146,42 @@ async function getIndexObjectForURL( } } +interface IndexedSearchHighlightTarget { + url: string; + apply?: (highlight: string) => void; +} + +function indexedSearchHighlightTargets( + response: SearchV2Response, +): IndexedSearchHighlightTarget[] { + const targets: IndexedSearchHighlightTarget[] = []; + for (const result of response.web ?? []) { + if (!result.url) continue; + targets.push({ + url: result.url, + apply: highlight => { + result.description = highlight; + }, + }); + } + for (const result of response.news ?? []) { + if (!result.url) continue; + targets.push({ + url: result.url, + apply: highlight => { + result.snippet = highlight; + }, + }); + } + return targets; +} + export function searchHighlightURLs(response: SearchV2Response): string[] { - return [ - ...(response.web ?? []).flatMap(result => (result.url ? [result.url] : [])), - ...(response.news ?? []).flatMap(result => - result.url ? [result.url] : [], - ), - ]; + return indexedSearchHighlightTargets(response).map(target => target.url); } -export async function runIndexedSearchHighlightsShadow( - urls: string[], +async function runIndexedSearchHighlights( + targets: IndexedSearchHighlightTarget[], query: string, logger: Logger, requestId: string, @@ -167,16 +192,16 @@ export async function runIndexedSearchHighlightsShadow( succeeded: boolean; failureReason?: HighlightFailureReason; }> { - const attempted = urls.length; + const attempted = targets.length; const resolved = await Promise.all( - urls.map(url => getIndexObjectForURL(url, logger, false)), + targets.map(target => getIndexObjectForURL(target.url, logger, false)), ); const pages: HighlightIndexedPage[] = []; resolved.forEach((indexRef, index) => { if (!indexRef) return; pages.push({ id: String(index), - url: urls[index], + url: targets[index].url, indexObject: indexRef.name, }); }); @@ -191,10 +216,15 @@ export async function runIndexedSearchHighlightsShadow( failureReason = reason; }, }); - const replaced = results - ? Array.from(results.values()).filter(result => result.markdown.trim()) - .length - : 0; + let replaced = 0; + if (results) { + for (const page of pages) { + const highlight = results.get(page.id)?.markdown; + if (!highlight?.trim()) continue; + targets[Number(page.id)].apply?.(highlight); + replaced++; + } + } return { attempted, @@ -205,6 +235,42 @@ export async function runIndexedSearchHighlightsShadow( }; } +export async function applyIndexedSearchHighlights( + response: SearchV2Response, + query: string, + logger: Logger, + requestId: string, +): ReturnType { + const start = Date.now(); + const result = await runIndexedSearchHighlights( + indexedSearchHighlightTargets(response), + query, + logger, + requestId, + ); + logger.info("Search highlights applied", { + attempted: result.attempted, + indexHits: result.indexHits, + replaced: result.replaced, + timeTakenMs: Date.now() - start, + }); + return result; +} + +export function runIndexedSearchHighlightsShadow( + urls: string[], + query: string, + logger: Logger, + requestId: string, +): ReturnType { + return runIndexedSearchHighlights( + urls.map(url => ({ url })), + query, + logger, + requestId, + ); +} + /** * For each search result: look up the URL in our index (last 30 days), and if * present, replace the provider snippet with query-relevant highlights generated From a6872c78dd55c4d28986f4bbf3a4f5c60f611704 Mon Sep 17 00:00:00 2001 From: "cursor[bot]" <206951365+cursor[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:43:28 -0700 Subject: [PATCH 2/3] fix(security): resolve pnpm audit failures (#4037) Co-authored-by: Cursor Agent Co-authored-by: Abimael Martell --- apps/api/audit-ci.jsonc | 2 +- apps/api/package.json | 2 +- apps/api/pnpm-lock.yaml | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/api/audit-ci.jsonc b/apps/api/audit-ci.jsonc index b9ca8988de..50c587521c 100644 --- a/apps/api/audit-ci.jsonc +++ b/apps/api/audit-ci.jsonc @@ -3,7 +3,7 @@ "low": true, "allowlist": [ // GHSA-v2v4-37r5-5v8g (ip-address): - // geoip-country 5.0.202607110104 still depends on ip-address 6.x; the + // geoip-country 5.0.202607150057 still depends on ip-address 6.x; the // non-affected line is a multi-major jump outside the upstream range. // Do not override across majors without compatibility approval. // Owner: team-engineering-reviewers. Tracking: create internal ticket. diff --git a/apps/api/package.json b/apps/api/package.json index 5859418916..b68ef26fb7 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -113,7 +113,7 @@ "express": "5.2.1", "express-ws": "^5.0.2", "foundationdb": "^2.0.1", - "geoip-country": "^5.0.202607110104", + "geoip-country": "^5.0.202607150057", "he": "^1.2.0", "http-cookie-agent": "^7.0.1", "ioredis": "^5.6.1", diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml index 33f704f338..30c315cf97 100644 --- a/apps/api/pnpm-lock.yaml +++ b/apps/api/pnpm-lock.yaml @@ -166,8 +166,8 @@ importers: specifier: ^2.0.1 version: 2.0.1 geoip-country: - specifier: ^5.0.202607110104 - version: 5.0.202607110104 + specifier: ^5.0.202607150057 + version: 5.0.202607150057 he: specifier: ^1.2.0 version: 1.2.0 @@ -3049,8 +3049,8 @@ packages: resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} engines: {node: '>=18'} - geoip-country@5.0.202607110104: - resolution: {integrity: sha512-Acug9yXKPmy0QnrI41i/8QUpBfO2HY82OVZMTWYMy9Iq0SRDybEDIZOnVy5VWN3z9xqYc0tpK8ugpo1TrdX6CA==} + geoip-country@5.0.202607150057: + resolution: {integrity: sha512-BsCnrL6DluGwo4CfvCys6q9P+1DJp7CxaJOCRHarkpHQmgjPDZu6n56euBRlullrcB5dwTRODNqlcNTP/Sv4zQ==} engines: {node: '>=10.20.0'} get-east-asian-width@1.3.1: @@ -6989,7 +6989,7 @@ snapshots: transitivePeerDependencies: - supports-color - geoip-country@5.0.202607110104: + geoip-country@5.0.202607150057: dependencies: async: 2.6.4 countries-list: 3.1.1 From 3a7b1a7774291f6c1a72ff3eb0ac11187766723d Mon Sep 17 00:00:00 2001 From: Abimael Martell <1450169+abimaelmartell@users.noreply.github.com> Date: Wed, 15 Jul 2026 09:44:04 -0700 Subject: [PATCH 3/3] chore(ci): upgrade support services to pnpm 11 (#4034) --- .github/workflows/test-server.yml | 10 ++++---- apps/playwright-service-ts/Dockerfile | 18 ++++++++++----- apps/playwright-service-ts/package.json | 11 ++++----- .../playwright-service-ts/pnpm-workspace.yaml | 6 +++++ apps/test-site/package.json | 23 ++++--------------- apps/test-site/pnpm-workspace.yaml | 15 ++++++++++++ 6 files changed, 46 insertions(+), 37 deletions(-) create mode 100644 apps/playwright-service-ts/pnpm-workspace.yaml create mode 100644 apps/test-site/pnpm-workspace.yaml diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml index 47953148bf..674d3dfc40 100644 --- a/.github/workflows/test-server.yml +++ b/.github/workflows/test-server.yml @@ -77,7 +77,7 @@ jobs: - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4 with: - version: 10 + version: 11.4.0 - name: Install FoundationDB client library run: | @@ -94,7 +94,7 @@ jobs: apps/test-site/pnpm-lock.yaml apps/playwright-service-ts/pnpm-lock.yaml - - run: pnpm dlx pnpm@11.4.0 fetch + - run: pnpm fetch working-directory: apps/api - run: pnpm fetch working-directory: apps/test-site @@ -115,7 +115,7 @@ jobs: - name: Build native lib if: steps.napi_restore.outputs.cache-hit != 'true' - run: pnpm dlx pnpm@11.4.0 install + run: pnpm install working-directory: apps/api/native - name: Cache native lib @@ -213,7 +213,7 @@ jobs: working-directory: ./ - name: Install API dependencies - run: pnpm dlx pnpm@11.4.0 install --frozen-lockfile --ignore-scripts + run: pnpm install --frozen-lockfile --ignore-scripts working-directory: apps/api env: npm_config_ignore_scripts: 'true' @@ -259,7 +259,7 @@ jobs: echo "docker:docker@127.0.0.1:4500" > /tmp/fdb.cluster - name: Run tests - run: pnpm dlx pnpm@11.4.0 harness pnpm test:snips + run: pnpm harness pnpm test:snips working-directory: apps/api env: npm_config_ignore_scripts: 'true' # required currently to prevent re-building cached native lib diff --git a/apps/playwright-service-ts/Dockerfile b/apps/playwright-service-ts/Dockerfile index 47217e2568..7f72c38b05 100644 --- a/apps/playwright-service-ts/Dockerfile +++ b/apps/playwright-service-ts/Dockerfile @@ -1,21 +1,27 @@ -FROM node:18-slim +FROM node:22-slim + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +ENV CI=true + +RUN corepack enable && corepack prepare pnpm@11.4.0 --activate WORKDIR /usr/src/app -COPY package*.json ./ -RUN npm install +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +RUN pnpm install --frozen-lockfile COPY . . ENV PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright # Install Playwright dependencies -RUN npx playwright install chromium --with-deps +RUN pnpm exec playwright install chromium --with-deps -RUN npm run build +RUN pnpm run build ARG PORT ENV PORT=${PORT} EXPOSE ${PORT} -CMD [ "npm", "start" ] +CMD [ "pnpm", "start" ] diff --git a/apps/playwright-service-ts/package.json b/apps/playwright-service-ts/package.json index 43e1418a93..e905ac0040 100644 --- a/apps/playwright-service-ts/package.json +++ b/apps/playwright-service-ts/package.json @@ -11,6 +11,9 @@ "keywords": [], "author": "Jeff Pereira", "license": "ISC", + "engines": { + "node": ">=22.13.0" + }, "dependencies": { "dotenv": "^16.4.5", "express": "^5.2.1", @@ -26,11 +29,5 @@ "tsx": "^4.21.0", "typescript": "^5.9.3" }, - "pnpm": { - "overrides": { - "path-to-regexp@<8.4.0": ">=8.4.0", - "qs@<6.15.2": ">=6.15.2 <7.0.0", - "esbuild@>=0.17.0 <0.28.1": "0.28.1" - } - } + "packageManager": "pnpm@11.4.0" } diff --git a/apps/playwright-service-ts/pnpm-workspace.yaml b/apps/playwright-service-ts/pnpm-workspace.yaml new file mode 100644 index 0000000000..b8185fe456 --- /dev/null +++ b/apps/playwright-service-ts/pnpm-workspace.yaml @@ -0,0 +1,6 @@ +allowBuilds: + esbuild: true +overrides: + "path-to-regexp@<8.4.0": ">=8.4.0" + "qs@<6.15.2": ">=6.15.2 <7.0.0" + "esbuild@>=0.17.0 <0.28.1": "0.28.1" diff --git a/apps/test-site/package.json b/apps/test-site/package.json index 889c3a20fd..0526cfb339 100644 --- a/apps/test-site/package.json +++ b/apps/test-site/package.json @@ -2,6 +2,9 @@ "name": "firecrawl-test-site", "type": "module", "version": "0.0.1", + "engines": { + "node": ">=22.13.0" + }, "scripts": { "dev": "astro dev", "build": "astro build", @@ -20,23 +23,5 @@ "prettier": "^3.8.1", "prettier-plugin-astro": "^0.14.1" }, - "pnpm": { - "onlyBuiltDependencies": [ - "esbuild", - "sharp" - ], - "overrides": { - "fast-xml-builder": ">=1.1.6 <2.0.0", - "fast-xml-parser": "^5.7.0", - "devalue": ">=5.8.1 <6.0.0", - "h3": ">=1.15.9", - "picomatch@<4.0.4": ">=4.0.4", - "smol-toml@<1.6.1": ">=1.6.1", - "defu": ">=6.1.5", - "vite@>=7.0.0 <7.3.5": "7.3.5", - "js-yaml@>=4.0.0 <4.2.0": "4.2.0", - "postcss@<8.5.10": ">=8.5.10 <9.0.0", - "esbuild@>=0.17.0 <0.28.1": "0.28.1" - } - } + "packageManager": "pnpm@11.4.0" } diff --git a/apps/test-site/pnpm-workspace.yaml b/apps/test-site/pnpm-workspace.yaml new file mode 100644 index 0000000000..450cbf7a92 --- /dev/null +++ b/apps/test-site/pnpm-workspace.yaml @@ -0,0 +1,15 @@ +allowBuilds: + esbuild: true + sharp: true +overrides: + fast-xml-builder: ">=1.1.6 <2.0.0" + fast-xml-parser: "^5.7.0" + devalue: ">=5.8.1 <6.0.0" + h3: ">=1.15.9" + "picomatch@<4.0.4": ">=4.0.4" + "smol-toml@<1.6.1": ">=1.6.1" + defu: ">=6.1.5" + "vite@>=7.0.0 <7.3.5": "7.3.5" + "js-yaml@>=4.0.0 <4.2.0": "4.2.0" + "postcss@<8.5.10": ">=8.5.10 <9.0.0" + "esbuild@>=0.17.0 <0.28.1": "0.28.1"