[pull] main from firecrawl:main

.github/workflows/test-server.yml

@@ -77,7 +77,7 @@ jobs:
 
       - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
         with:
-          version: 10
+          version: 11.4.0
 
       - name: Install FoundationDB client library
         run: |
@@ -94,7 +94,7 @@ jobs:
             apps/test-site/pnpm-lock.yaml
             apps/playwright-service-ts/pnpm-lock.yaml
 
-      - run: pnpm dlx pnpm@11.4.0 fetch
+      - run: pnpm fetch
         working-directory: apps/api
       - run: pnpm fetch
         working-directory: apps/test-site
@@ -115,7 +115,7 @@ jobs:
 
       - name: Build native lib
         if: steps.napi_restore.outputs.cache-hit != 'true'
-        run: pnpm dlx pnpm@11.4.0 install
+        run: pnpm install
         working-directory: apps/api/native
 
       - name: Cache native lib
@@ -213,7 +213,7 @@ jobs:
         working-directory: ./
 
       - name: Install API dependencies
-        run: pnpm dlx pnpm@11.4.0 install --frozen-lockfile --ignore-scripts
+        run: pnpm install --frozen-lockfile --ignore-scripts
         working-directory: apps/api
         env:
           npm_config_ignore_scripts: 'true'
@@ -259,7 +259,7 @@ jobs:
           echo "docker:docker@127.0.0.1:4500" > /tmp/fdb.cluster
 
       - name: Run tests
-        run: pnpm dlx pnpm@11.4.0 harness pnpm test:snips
+        run: pnpm harness pnpm test:snips
         working-directory: apps/api
         env:
           npm_config_ignore_scripts: 'true' # required currently to prevent re-building cached native lib

apps/api/audit-ci.jsonc

@@ -3,7 +3,7 @@
     "low": true,
     "allowlist": [
         // GHSA-v2v4-37r5-5v8g (ip-address):
-        // geoip-country 5.0.202607110104 still depends on ip-address 6.x; the
+        // geoip-country 5.0.202607150057 still depends on ip-address 6.x; the
         // non-affected line is a multi-major jump outside the upstream range.
         // Do not override across majors without compatibility approval.
         // Owner: team-engineering-reviewers. Tracking: create internal ticket.

apps/api/package.json

@@ -113,7 +113,7 @@
     "express": "5.2.1",
     "express-ws": "^5.0.2",
     "foundationdb": "^2.0.1",
-    "geoip-country": "^5.0.202607110104",
+    "geoip-country": "^5.0.202607150057",
     "he": "^1.2.0",
     "http-cookie-agent": "^7.0.1",
     "ioredis": "^5.6.1",

apps/api/src/search/execute.ts

@@ -13,7 +13,7 @@ import {
   mergeScrapedContent,
   calculateScrapeCredits,
 } from "./scrape";
-import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+import { applyIndexedSearchHighlights, highlightsEnvReady } from "./highlights";
 import { runSearchHighlightsShadow } from "./highlights-shadow";
 import { trackSearchResults, trackSearchRequest } from "../lib/tracking";
 import type { BillingMetadata } from "../services/billing/types";
@@ -253,7 +253,12 @@ export async function executeSearch(
     flags?.highlightsBeta === true &&
     highlightsEnvReady();
   if (shouldApplyHighlights) {
-    await applySearchHighlights(searchResponse, query, logger);
+    await applyIndexedSearchHighlights(
+      searchResponse,
+      query,
+      logger,
+      context.requestId,
+    );
   } else {
     runSearchHighlightsShadow({
       response: searchResponse,

apps/api/src/search/highlights.test.ts

@@ -45,6 +45,7 @@ import {
 import { config } from "../config";
 import { indexGetRecent5 } from "../db/rpc";
 import {
+  applyIndexedSearchHighlights,
   applySearchHighlights,
   highlightsEnvReady,
   runIndexedSearchHighlightsShadow,
@@ -114,6 +115,84 @@ describe("runIndexedSearchHighlightsShadow", () => {
   });
 });
 
+describe("applyIndexedSearchHighlights", () => {
+  it("uses the shadow indexed request path and applies web and news responses by ID", async () => {
+    vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(
+      new Map([
+        ["0", { highlights: [], markdown: "first highlight" }],
+        ["1", { highlights: [], markdown: "second highlight" }],
+      ]),
+    );
+    const response = {
+      web: [{ url: "https://first.test", description: "first fallback" }],
+      news: [{ url: "https://second.test", snippet: "second fallback" }],
+    } as any;
+
+    const result = await applyIndexedSearchHighlights(
+      response,
+      "query",
+      logger,
+      "request-1",
+    );
+
+    expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(
+      "query",
+      [
+        {
+          id: "0",
+          url: "https://first.test",
+          indexObject: "index:https://first.test.json",
+        },
+        {
+          id: "1",
+          url: "https://second.test",
+          indexObject: "index:https://second.test.json",
+        },
+      ],
+      {
+        logger,
+        logPayload: false,
+        requestId: "request-1",
+        timeoutMs: null,
+        onFailure: expect.any(Function),
+      },
+    );
+    expect(generateHighlightsBatch).not.toHaveBeenCalled();
+    expect(response.web[0].description).toBe("first highlight");
+    expect(response.news[0].snippet).toBe("second highlight");
+    expect(result).toEqual({
+      attempted: 2,
+      indexHits: 2,
+      replaced: 2,
+      succeeded: true,
+    });
+  });
+
+  it("preserves provider snippets when the indexed Chain request fails", async () => {
+    vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(null);
+    const response = {
+      web: [{ url: "https://first.test", description: "first fallback" }],
+      news: [{ url: "https://second.test", snippet: "second fallback" }],
+    } as any;
+
+    const result = await applyIndexedSearchHighlights(
+      response,
+      "query",
+      logger,
+      "request-1",
+    );
+
+    expect(response.web[0].description).toBe("first fallback");
+    expect(response.news[0].snippet).toBe("second fallback");
+    expect(result).toEqual({
+      attempted: 2,
+      indexHits: 2,
+      replaced: 0,
+      succeeded: false,
+    });
+  });
+});
+
 describe("applySearchHighlights", () => {
   it("enables the in-cluster service without requiring a bearer token", () => {
     const token = config.HIGHLIGHT_MODEL_TOKEN;

apps/api/src/search/highlights.ts

@@ -146,17 +146,42 @@ async function getIndexObjectForURL(
   }
 }
 
+interface IndexedSearchHighlightTarget {
+  url: string;
+  apply?: (highlight: string) => void;
+}
+
+function indexedSearchHighlightTargets(
+  response: SearchV2Response,
+): IndexedSearchHighlightTarget[] {
+  const targets: IndexedSearchHighlightTarget[] = [];
+  for (const result of response.web ?? []) {
+    if (!result.url) continue;
+    targets.push({
+      url: result.url,
+      apply: highlight => {
+        result.description = highlight;
+      },
+    });
+  }
+  for (const result of response.news ?? []) {
+    if (!result.url) continue;
+    targets.push({
+      url: result.url,
+      apply: highlight => {
+        result.snippet = highlight;
+      },
+    });
+  }
+  return targets;
+}
+
 export function searchHighlightURLs(response: SearchV2Response): string[] {
-  return [
-    ...(response.web ?? []).flatMap(result => (result.url ? [result.url] : [])),
-    ...(response.news ?? []).flatMap(result =>
-      result.url ? [result.url] : [],
-    ),
-  ];
+  return indexedSearchHighlightTargets(response).map(target => target.url);
 }
 
-export async function runIndexedSearchHighlightsShadow(
-  urls: string[],
+async function runIndexedSearchHighlights(
+  targets: IndexedSearchHighlightTarget[],
   query: string,
   logger: Logger,
   requestId: string,
@@ -167,16 +192,16 @@ export async function runIndexedSearchHighlightsShadow(
   succeeded: boolean;
   failureReason?: HighlightFailureReason;
 }> {
-  const attempted = urls.length;
+  const attempted = targets.length;
   const resolved = await Promise.all(
-    urls.map(url => getIndexObjectForURL(url, logger, false)),
+    targets.map(target => getIndexObjectForURL(target.url, logger, false)),
   );
   const pages: HighlightIndexedPage[] = [];
   resolved.forEach((indexRef, index) => {
     if (!indexRef) return;
     pages.push({
       id: String(index),
-      url: urls[index],
+      url: targets[index].url,
       indexObject: indexRef.name,
     });
   });
@@ -191,10 +216,15 @@ export async function runIndexedSearchHighlightsShadow(
       failureReason = reason;
     },
   });
-  const replaced = results
-    ? Array.from(results.values()).filter(result => result.markdown.trim())
-        .length
-    : 0;
+  let replaced = 0;
+  if (results) {
+    for (const page of pages) {
+      const highlight = results.get(page.id)?.markdown;
+      if (!highlight?.trim()) continue;
+      targets[Number(page.id)].apply?.(highlight);
+      replaced++;
+    }
+  }
 
   return {
     attempted,
@@ -205,6 +235,42 @@ export async function runIndexedSearchHighlightsShadow(
   };
 }
 
+export async function applyIndexedSearchHighlights(
+  response: SearchV2Response,
+  query: string,
+  logger: Logger,
+  requestId: string,
+): ReturnType<typeof runIndexedSearchHighlights> {
+  const start = Date.now();
+  const result = await runIndexedSearchHighlights(
+    indexedSearchHighlightTargets(response),
+    query,
+    logger,
+    requestId,
+  );
+  logger.info("Search highlights applied", {
+    attempted: result.attempted,
+    indexHits: result.indexHits,
+    replaced: result.replaced,
+    timeTakenMs: Date.now() - start,
+  });
+  return result;
+}
+
+export function runIndexedSearchHighlightsShadow(
+  urls: string[],
+  query: string,
+  logger: Logger,
+  requestId: string,
+): ReturnType<typeof runIndexedSearchHighlights> {
+  return runIndexedSearchHighlights(
+    urls.map(url => ({ url })),
+    query,
+    logger,
+    requestId,
+  );
+}
+
 /**
  * For each search result: look up the URL in our index (last 30 days), and if
  * present, replace the provider snippet with query-relevant highlights generated

apps/playwright-service-ts/Dockerfile

@@ -1,21 +1,27 @@
-FROM node:18-slim
+FROM node:22-slim
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+ENV CI=true
+
+RUN corepack enable && corepack prepare pnpm@11.4.0 --activate
 
 WORKDIR /usr/src/app
-COPY package*.json ./
-RUN npm install
+COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+RUN pnpm install --frozen-lockfile
 
 COPY . .
 
 ENV PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/playwright
 
 # Install Playwright dependencies
-RUN npx playwright install chromium --with-deps
+RUN pnpm exec playwright install chromium --with-deps
 
-RUN npm run build
+RUN pnpm run build
 
 ARG PORT
 ENV PORT=${PORT}
 
 EXPOSE ${PORT}
 
-CMD [ "npm", "start" ]
+CMD [ "pnpm", "start" ]