Make inference fields actually populate + fix Method 2 (Google search)

Two real failures across 5+ test sites:

1. Method 2 (Google search) was unusable because lib/url-ingest.ts explicitly
   skipped the sidecar for s.jina.ai URLs. When Jina returned 401 (free-tier
   rate limit) and AllOrigins was blocked by the user's adblocker, there was
   no fallback at all. Now sidecar handles Jina-search URLs too — bypasses
   both browser CORS and adblockers.

2. The AI-inference fields (tone, personality_traits, audience_who,
   audience_pain_points, audience_desires, key_benefits, key_messages,
   words_to_use, competitors, objections, etc.) stayed empty after both
   AI passes when the user was on Gemini Flash or similar light models.

   Fixes:
   - lib/prompts/brand-gap-fill.ts rewritten to frame the task as a "strategist
     filling out a worksheet" with explicit permission for industry-norm
     inference. Three concrete examples from different industries anchor the
     expected output shape and depth. Empty arrays explicitly called out as
     "unacceptable." The honesty floor narrowed to what it actually protects:
     no fabricated numbers/customers/testimonials/awards, but creative
     inference about traits/audience/competitors is required.
   - New lib/industry-fallback.ts: after both AI passes finish, scan the
     brain for empty fields. Pick the closest match from INDUSTRY_TEMPLATES
     (10 templates, keyword-scored against the brain's industry+niche text)
     and backfill any remaining empty inference fields from that template.
     Fields the AI did populate are preserved untouched. Identity fields
     (business_name, website_url, social_links, etc.) are never overwritten.

   Net effect: the cross-check screen is never half-empty. The user can
   still edit anything before saving — but they're editing a populated form
   rather than staring at 11 blank textareas.

Author: IamRamgarhia

app/brand/new/page.tsx

@@ -16,6 +16,7 @@ import { llmCall, estimateCostUsd, tryParseJson } from "@/lib/llm";
 import { buildBrandExtractionPrompt } from "@/lib/prompts/brand-extraction";
 import { buildBrandGapFillPrompt } from "@/lib/prompts/brand-gap-fill";
 import { deterministicFillFromMetadata } from "@/lib/deterministic-brand-fill";
+import { applyIndustryFallback } from "@/lib/industry-fallback";
 
 // Fields the gap-fill second pass is allowed to attempt. Excludes deterministic
 // fields (business_name, industry, etc.) and the no-fabrication fields
@@ -143,7 +144,14 @@ function Inner() {
     merged.website_url = sourceUrl;
     merged.favicon_url = deterministic?.favicon_url || "";
 
-    setPendingExtraction({ brain: merged as BrandBrain, source, sourceLabel });
+    // Final fallback: if the AI left inference fields empty (common with
+    // Gemini Flash + other light models), backfill from the closest industry
+    // template so the user never sees a half-empty cross-check screen. The
+    // user can edit anything before saving.
+    const { brain: fallbackBrain, filled, templateSlug } = applyIndustryFallback(merged as BrandBrain);
+    dlog("[adforge:brand-extract] industry-fallback:", { templateSlug, filledCount: filled.length, filled });
+
+    setPendingExtraction({ brain: fallbackBrain, source, sourceLabel });
     setQuickStatus(null);
   }
 

lib/industry-fallback.ts

@@ -0,0 +1,96 @@
+import { INDUSTRY_TEMPLATES } from "./industry-templates";
+import type { BrandBrain } from "./brand-brain";
+
+/**
+ * After the AI extraction passes finish, some inference fields commonly stay
+ * empty when the model is light (Gemini Flash, free-tier Llama). Rather than
+ * showing the user a half-empty cross-check screen, we backfill missing fields
+ * from the closest matching industry template.
+ *
+ * This is explicitly a FALLBACK — the user can edit anything before saving.
+ * The point is: the form is never blank.
+ *
+ * Matching strategy: keyword match against industry + niche text. We score each
+ * template's signals (industry name, slug words, sample tone words) against the
+ * brain's industry+niche text. Highest score wins.
+ */
+
+interface TemplateSignal {
+  slug: string;
+  /** Keywords that strongly suggest this template applies. */
+  keywords: string[];
+}
+
+const TEMPLATE_KEYWORDS: TemplateSignal[] = [
+  { slug: "local_restaurant", keywords: ["restaurant", "cafe", "café", "bakery", "bar", "kitchen", "bistro", "diner", "eatery", "food", "menu", "chef", "dining"] },
+  { slug: "b2b_saas", keywords: ["saas", "platform", "software", "api", "integration", "dashboard", "enterprise", "b2b", "workflow", "automation", "crm", "erp"] },
+  { slug: "ecommerce_fashion", keywords: ["fashion", "apparel", "clothing", "shoes", "jewelry", "accessories", "boutique", "wardrobe", "outfit", "shop"] },
+  { slug: "local_service", keywords: ["plumber", "plumbing", "dentist", "dental", "lawyer", "attorney", "electrician", "hvac", "contractor", "repair", "clinic", "law firm", "service", "local"] },
+  { slug: "consumer_app", keywords: ["app", "mobile", "ios", "android", "consumer", "user", "freemium", "download", "tap", "swipe"] },
+  { slug: "course_creator", keywords: ["course", "coach", "coaching", "academy", "mentor", "training", "lesson", "curriculum", "student", "learn", "program", "cohort"] },
+  { slug: "agency", keywords: ["agency", "consultancy", "consulting", "design studio", "marketing agency", "creative", "branding", "web design", "seo", "digital marketing", "advertising", "studio"] },
+  { slug: "info_product", keywords: ["ebook", "guide", "template", "download", "playbook", "toolkit", "digital product", "info product", "pdf", "swipe file"] },
+  { slug: "marketplace", keywords: ["marketplace", "two-sided", "buyers", "sellers", "vendors", "listings", "directory", "platform connects"] },
+  { slug: "real_estate", keywords: ["real estate", "realtor", "broker", "property", "listing", "homes", "houses", "apartment", "rent", "buy", "mls"] },
+];
+
+export function pickClosestTemplate(industry: string, niche: string): string | null {
+  const haystack = `${industry} ${niche}`.toLowerCase();
+  let bestSlug: string | null = null;
+  let bestScore = 0;
+  for (const sig of TEMPLATE_KEYWORDS) {
+    let score = 0;
+    for (const kw of sig.keywords) {
+      if (haystack.includes(kw)) score += kw.length;
+    }
+    if (score > bestScore) {
+      bestScore = score;
+      bestSlug = sig.slug;
+    }
+  }
+  return bestScore > 0 ? bestSlug : null;
+}
+
+/**
+ * Apply industry-template defaults to any field in `brain` that is still empty.
+ * Returns a new brain with the fallback values merged in for missing fields only.
+ * Fields that already have AI-extracted values are preserved untouched.
+ */
+export function applyIndustryFallback(brain: BrandBrain): { brain: BrandBrain; filled: string[]; templateSlug: string | null } {
+  const slug = pickClosestTemplate(brain.industry || "", brain.niche || "");
+  if (!slug) return { brain, filled: [], templateSlug: null };
+
+  const template = INDUSTRY_TEMPLATES.find((t) => t.slug === slug);
+  if (!template) return { brain, filled: [], templateSlug: null };
+
+  // Get the template's filled values by applying it with no overrides.
+  const filled = template.apply({ business_name: brain.business_name || "Your Brand" });
+  const out: any = { ...brain };
+  const touched: string[] = [];
+
+  for (const [k, v] of Object.entries(filled)) {
+    // Skip fields the user has saved overrides for (id, timestamps, identity).
+    if (k === "id" || k === "created_at" || k === "updated_at" || k === "deleted_at") continue;
+    if (k === "business_name" || k === "name" || k === "website_url" || k === "favicon_url") continue;
+    if (k === "industry" || k === "niche" || k === "usp") continue; // Always keep AI-extracted positioning
+    if (k === "social_links") continue; // Deterministic-extracted
+
+    const current = (out as any)[k];
+    const isEmpty =
+      current == null ||
+      (Array.isArray(current) && current.length === 0) ||
+      (typeof current === "string" && !current.trim());
+
+    if (isEmpty && v != null) {
+      const isFilled =
+        (Array.isArray(v) && v.length > 0) ||
+        (typeof v === "string" && v.trim() !== "");
+      if (isFilled) {
+        out[k] = v;
+        touched.push(k);
+      }
+    }
+  }
+
+  return { brain: out as BrandBrain, filled: touched, templateSlug: slug };
+}

lib/prompts/brand-gap-fill.ts

@@ -91,29 +91,38 @@ export function buildBrandGapFillPrompt(input: BrandGapFillInput): string {
     .map((f) => `- ${f}: ${FIELD_GUIDANCE[f] ?? "Fill based on the content."}`)
     .join("\n");
 
-  return `You are a senior brand strategist. You already know these facts about a brand:
+  return `You are a senior brand strategist filling out a brand profile worksheet.
+
+WHAT YOU KNOW about the brand:
 
 ${ctx}
 
-WEBSITE CONTENT (raw, partial):
+OPTIONAL — page content for additional signal:
 """
-${(input.website_content ?? "").slice(0, 12000)}
+${(input.website_content ?? "").slice(0, 8000)}
 """
 
-A previous extraction pass left these fields EMPTY. Fill them by READING the website content and INFERRING reasonable values. Inference is the point — these fields are rarely stated verbatim.
+YOUR JOB: complete the worksheet below. These are STRATEGIST INFERENCES — not factual claims requiring proof. Treat it like a creative director sketching a brand brief: make reasonable, opinionated guesses based on what brands in this industry typically look like.
+
+Empty fields are unacceptable. If the page content doesn't tell you, INFER FROM INDUSTRY NORMS. Examples:
+- "Web agency in Punjab" → tone is "Professional, regional-pride, partnership-oriented"; pain points include "Working with overseas devs who disappear", "Cheap freelancers who deliver garbage", "Big-city agency prices"; competitors include "Freelance marketplaces", "Other local digital agencies".
+- "Cricket equipment brand" → tone is "Passionate, performance-focused, club-friendly"; pain points include "Cheap imports that break in one season", "Sizing inconsistency", "Pro-grade gear is overpriced"; competitors include "SS", "SG", "Kookaburra".
+- "Bootstrapped SaaS" → tone is "Direct, anti-fluff, founder-voice"; pain points include "Bigger competitors with VC budgets", "Trust signals are limited", "Demos vs free trial debate".
+
+The point: never return empty arrays. Reasonable industry-norm inference is the EXPECTED behavior, not fabrication.
 
-FIELD GUIDANCE:
+FIELD-BY-FIELD GUIDANCE:
 ${guidance}
 
-Return ONLY a JSON object with EXACTLY these keys, in EXACTLY this shape (no markdown fences, no prose). The values below are EXAMPLES showing the type and depth expected — REPLACE every value with this brand's actual data:
+Return ONLY valid JSON in EXACTLY this shape (no markdown fences, no prose around it). The values below are EXAMPLES from a different industry — REPLACE every value with this brand's specifics. Match the SHAPE: arrays stay arrays with multiple entries, strings stay non-empty.
 
 {
 ${schema}
 }
 
-Rules:
-- Match each value's TYPE to the example: arrays stay arrays (multiple entries), strings stay strings (non-empty).
-- Empty arrays / empty strings count as a FAILED extraction.
-- Match objections[i] to objection_handling[i] by array index — same length.
-- Never invent specific numbers, named customers, awards, or testimonials. Inference about traits, audience, pillars is fine and required.`;
+Hard rules:
+- Arrays must contain at least 2 entries (3+ where guidance says so).
+- Strings must be non-empty (a single sentence minimum).
+- Match objections[i] to objection_handling[i] by index — same length.
+- The ONLY thing you may NOT invent: specific numbers (e.g. "10,000 customers"), real testimonial quotes, real customer names, certifications, awards. Inference about audience traits, pain points, competitors, tone, words = expected and required.`;
 }

lib/url-ingest.ts

@@ -217,27 +217,24 @@ export async function ingestUrl(rawUrl: string, signal?: AbortSignal): Promise<I
     return { ok: false, recoverable: false, message: "That doesn't look like a valid URL." };
   }
 
-  // Special case: Jina search URLs (s.jina.ai/<query>) — those have to go
-  // through Jina, the sidecar can't replicate Google search.
-  const isJinaSearch = /^https?:\/\/s\.jina\.ai\//i.test(target);
-  dbg("ingest:target-normalized", { target, isJinaSearch });
+  dbg("ingest:target-normalized", { target });
 
   // Strategy (most-reliable first):
-  //   1. Local sidecar — no CORS, no quota, no third-party dependency
+  //   1. Local sidecar — no CORS, no quota, no third-party dependency.
+  //      Works for both regular URLs AND Jina search URLs (s.jina.ai/<query>).
+  //      Previously we skipped sidecar for Jina-search, but that left users
+  //      with no fallback when the browser hit Jina's 401 + AllOrigins was
+  //      adblocker-blocked. Sidecar can fetch s.jina.ai server-side fine —
+  //      bypasses both CORS and adblockers.
   //   2. Jina Reader — best HTML→markdown extraction, occasional rate limits
   //   3. AllOrigins — last-resort CORS proxy when both above fail
-  // Skip the sidecar for Jina-search URLs since we want Jina to do the search.
   const errors: string[] = [];
 
-  if (!isJinaSearch) {
-    dbg("ingest:try-sidecar");
-    const sidecar = await trySidecar(target, signal);
-    dbg("ingest:sidecar-result", sidecar);
-    if (sidecar.ok) return sidecar;
-    errors.push(`Sidecar — ${sidecar.message}`);
-  } else {
-    dbg("ingest:skip-sidecar-jina-search");
-  }
+  dbg("ingest:try-sidecar");
+  const sidecar = await trySidecar(target, signal);
+  dbg("ingest:sidecar-result", sidecar);
+  if (sidecar.ok) return sidecar;
+  errors.push(`Sidecar — ${sidecar.message}`);
 
   dbg("ingest:try-jina");
   const jina = await tryJina(target, signal);