fix(link-preview): save MissAV OG metadata fallback

Author: ThaddeusJiang

CHANGELOG.md

@@ -16,6 +16,7 @@ before the CalVer migration retain their original version labels.
 
 ### Fixed
 
+- Save searchable MissAV `missav.ai` Open Graph metadata by falling back to readable mirror metadata when the main page blocks preview fetches.
 - Render successful resource creation info logs in green in runtime logger output.
 - Save a webpage `og:image` preview when a URL resolves to a non-media resource instead of storing the resolved HTML or document file.
 - Send and index a generated preview image when a downloaded URL video is too large for Telegram video upload, while still saving the downloaded video locally.

docs/postmortem/2026-06-16-missav-ai-og-metadata-fallback.md

@@ -0,0 +1,21 @@
+# MissAV AI Open Graph Metadata Fallback
+
+## What happened
+
+A `missav.ai` URL could show a rich Telegram client preview, but `save_it` could not reliably save the page's Open Graph title, description, keywords, or image URL into Typesense. URL-only saves therefore had no user caption and no searchable URL metadata.
+
+## Root cause
+
+Telegram client previews do not mean the Bot API exposes the resolved Open Graph fields to the bot. `save_it` still has to fetch the preview page itself. Direct server-side requests to the affected `missav.ai` page returned a Cloudflare 403 page, so `SmallSdk.LinkPreview.get_metadata/1` returned an error and the bot indexed the saved media without URL metadata.
+
+The MissAV provider strategy recognized `missav.ai`, but it did not have a metadata fallback URL when the primary host blocked preview fetches.
+
+## Fix applied
+
+`SmallSdk.LinkPreview` now tries the normal preview URL first. When a `missav.ai` metadata fetch fails, it retries the same path on a configurable MissAV metadata fallback host, defaulting to `https://missav.ws`. Metadata parsed from the fallback page still flows through the existing `title`, `description`, `keywords`, and `thumbnail_url` fields, while the saved document keeps the original `missav.ai` URL.
+
+Regression coverage now verifies both the `LinkPreview` fallback behavior and the full bot save path that writes MissAV Open Graph metadata into the Typesense document.
+
+## What we learned
+
+Telegram's visible link preview is not a reliable data source for bot indexing. Provider-specific metadata recovery should live at the link-preview boundary so the rest of the save pipeline can keep the same field semantics and continue preserving the user's original URL.

docs/specs/001-typesense-saved-content.md

@@ -157,7 +157,7 @@ Supported provider strategies:
 | Instagram | `instagram.com` and subdomains | Use the default URL metadata strategy. Provider-specific handling may be added when public metadata is blocked or incomplete. |
 | YouTube Shorts | `youtube.com/shorts/*`, `youtu.be/*` when resolved as Shorts | Use the default URL metadata strategy. The Shorts provider may normalize canonical YouTube URLs before metadata fetches. |
 | `bad.news` | `bad.news` and subdomains | Use `bad.news` download resolution for media and the default URL metadata strategy for page metadata. |
-| `missav.ai` | `missav.ai` and subdomains | Use the default URL metadata strategy. Provider-specific media and metadata handling may be added without changing document field semantics. |
+| `missav.ai` | `missav.ai` and subdomains | Use the default URL metadata strategy first. If the `missav.ai` preview HTML fetch fails, fetch the same path from the configured MissAV metadata fallback host, defaulting to `https://missav.ws`, and store any readable Open Graph metadata while preserving the original `missav.ai` URL as `url`. |
 | Other URL | Any supported URL not matched above | Use the default URL metadata strategy. |
 
 ### Default URL Metadata Strategy

lib/small_sdk/link_preview.ex

@@ -6,6 +6,8 @@ defmodule SmallSdk.LinkPreview do
   alias SmallSdk.WebDownloader
   alias SmallSdk.XMetadata
 
+  @missav_metadata_fallback_base_url "https://missav.ws"
+
   @preview_patterns [
     ~r/<meta[^>]+property=["']og:image["'][^>]+(?:content|value)=["']([^"']+)["']/i,
     ~r/<meta[^>]+(?:content|value)=["']([^"']+)["'][^>]+property=["']og:image["']/i,
@@ -72,18 +74,20 @@ defmodule SmallSdk.LinkPreview do
 
   def get_title(_page_url), do: {:error, :missing_preview_url}
 
-  def get_metadata(page_url) when is_binary(page_url) do
+  def get_metadata(page_url, opts \\ [])
+
+  def get_metadata(page_url, opts) when is_binary(page_url) do
     case get_authenticated_metadata(page_url) do
       {:ok, metadata} ->
         log_metadata(page_url, metadata)
         {:ok, metadata}
 
       {:error, _reason} ->
-        get_html_metadata(page_url)
+        get_html_metadata(page_url, opts)
     end
   end
 
-  def get_metadata(_page_url), do: {:error, :missing_preview_url}
+  def get_metadata(_page_url, _opts), do: {:error, :missing_preview_url}
 
   defp get_authenticated_metadata(page_url) do
     if XMetadata.x_url?(page_url) do
@@ -93,8 +97,15 @@ defmodule SmallSdk.LinkPreview do
     end
   end
 
-  defp get_html_metadata(page_url) do
-    case Req.get(page_url, headers: [{"User-Agent", "Mozilla/5.0"}]) do
+  defp get_html_metadata(page_url, opts) do
+    case fetch_html_metadata(page_url, opts) do
+      {:ok, metadata} -> {:ok, metadata}
+      {:error, reason} -> maybe_get_provider_fallback_metadata(page_url, reason, opts)
+    end
+  end
+
+  defp fetch_html_metadata(page_url, opts) do
+    case Req.get(page_url, request_options(opts)) do
       {:ok, %{status: status, body: body}} when status in 200..209 and is_binary(body) ->
         metadata = get_metadata_from_html(page_url, body)
         log_metadata(page_url, metadata)
@@ -111,6 +122,71 @@ defmodule SmallSdk.LinkPreview do
     end
   end
 
+  defp maybe_get_provider_fallback_metadata(page_url, reason, opts) do
+    if Keyword.get(opts, :provider_fallback?, true) do
+      case missav_metadata_fallback_url(page_url, opts) do
+        fallback_url when is_binary(fallback_url) ->
+          log_metadata_fallback(page_url, fallback_url, reason)
+          fetch_html_metadata(fallback_url, Keyword.put(opts, :provider_fallback?, false))
+
+        nil ->
+          {:error, reason}
+      end
+    else
+      {:error, reason}
+    end
+  end
+
+  defp request_options(opts) do
+    opts
+    |> Keyword.get(:req_options, Application.get_env(:save_it, :link_preview_req_options, []))
+    |> Keyword.put_new(:retry, false)
+    |> put_request_headers([{"User-Agent", "Mozilla/5.0"}])
+  end
+
+  defp put_request_headers(req_options, headers) do
+    Keyword.update(req_options, :headers, headers, fn existing_headers ->
+      headers ++ existing_headers
+    end)
+  end
+
+  defp missav_metadata_fallback_url(page_url, opts) do
+    with %URI{host: host} = page_uri when is_binary(host) <- URI.parse(page_url),
+         true <- missav_ai_host?(String.downcase(host)),
+         fallback_base_url when is_binary(fallback_base_url) <-
+           missav_metadata_fallback_base_url(opts),
+         %URI{host: fallback_host} = fallback_uri when is_binary(fallback_host) <-
+           URI.parse(fallback_base_url) do
+      %URI{
+        page_uri
+        | scheme: fallback_uri.scheme || "https",
+          userinfo: fallback_uri.userinfo,
+          host: fallback_host,
+          port: fallback_uri.port
+      }
+      |> URI.to_string()
+    else
+      _ -> nil
+    end
+  rescue
+    _ -> nil
+  end
+
+  defp missav_metadata_fallback_base_url(opts) do
+    opts
+    |> Keyword.get(
+      :missav_metadata_fallback_base_url,
+      Application.get_env(
+        :save_it,
+        :missav_metadata_fallback_base_url,
+        @missav_metadata_fallback_base_url
+      )
+    )
+    |> blank_to_nil()
+  end
+
+  defp missav_ai_host?(host), do: host == "missav.ai" or String.ends_with?(host, ".missav.ai")
+
   def get_metadata_from_html(page_url, html) when is_binary(page_url) and is_binary(html) do
     %{
       title: html |> extract_title() |> blank_to_nil(),
@@ -226,6 +302,16 @@ defmodule SmallSdk.LinkPreview do
     )
   end
 
+  defp log_metadata_fallback(page_url, fallback_url, reason) do
+    Logger.warning(
+      "Link preview metadata fallback selected: " <>
+        "page_url=#{format_log_url(page_url)} " <>
+        "fallback_url=#{format_log_url(fallback_url)} " <>
+        "reason=#{format_log_reason(reason)}",
+      kind: :link_preview
+    )
+  end
+
   defp format_log_url(nil), do: "nil"
 
   defp format_log_url(url) when is_binary(url) do

test/bot_test.exs

@@ -335,6 +335,48 @@ defmodule SaveIt.BotTest do
            end)
   end
 
+  test "stores missav.ai mirror OG metadata when direct preview fetch is blocked",
+       %{base_url: base_url} do
+    original_url = "https://missav.ai/ja/sdam-101-uncensored-leak"
+    download_url = base_url <> "/downloaded/photo.jpg"
+
+    Application.put_env(:save_it, :test_pid, self())
+
+    Application.put_env(:save_it, :link_preview_req_options,
+      adapter: &__MODULE__.MissavLinkPreviewAdapter.request/1
+    )
+
+    Application.put_env(:save_it, :missav_metadata_fallback_base_url, "https://missav.ws")
+
+    message = %{
+      chat: %{id: 12_345, username: "save_it_test_chat"},
+      date: 1_717_170_000,
+      message_id: 103,
+      text: original_url
+    }
+
+    assert {:ok, true} = Bot.handle({:text, original_url, message}, nil)
+
+    assert_receive {:test_http_request, :post, "/", cobalt_body}
+    assert Jason.decode!(cobalt_body) == %{"url" => original_url}
+    assert_receive {:test_http_request, :get, "/downloaded/photo.jpg", ""}
+    assert_receive {:link_preview_request, "https://missav.ai/ja/sdam-101-uncensored-leak"}
+    assert_receive {:link_preview_request, "https://missav.ws/ja/sdam-101-uncensored-leak"}
+    assert_receive {:test_http_request, :post, "/collections/photos/documents", typesense_body}
+
+    document = Jason.decode!(typesense_body)
+
+    assert document["url"] == original_url
+    assert document["download_url"] == download_url
+    assert document["thumbnail_url"] == "https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg"
+    assert document["caption"] == ""
+    assert document["title"] == "MissAV Mirror OG Title"
+    assert document["description"] == "MissAV Mirror OG Description"
+    assert document["keywords"] == ["missav", "metadata", "fallback"]
+    assert document["file_id"] == "telegram-photo-file-id"
+    assert document["belongs_to_id"] == "12345"
+  end
+
   test "sends downloaded URL media to the source topic and stores a topic message URL",
        %{base_url: base_url} do
     Application.put_env(:ex_gram, :adapter, __MODULE__.TopicSendPhotoAdapter)
@@ -1882,6 +1924,41 @@ defmodule SaveIt.BotTest do
     end
   end
 
+  defmodule MissavLinkPreviewAdapter do
+    def request(%Req.Request{url: url} = request) do
+      send(
+        Application.fetch_env!(:save_it, :test_pid),
+        {:link_preview_request, URI.to_string(url)}
+      )
+
+      response =
+        case {url.host, url.path} do
+          {"missav.ai", "/ja/sdam-101-uncensored-leak"} ->
+            %Req.Response{status: 403, body: "Just a moment..."}
+
+          {"missav.ws", "/ja/sdam-101-uncensored-leak"} ->
+            %Req.Response{
+              status: 200,
+              body: """
+              <html>
+                <head>
+                  <meta property="og:title" content="MissAV Mirror OG Title" />
+                  <meta property="og:description" content="MissAV Mirror OG Description" />
+                  <meta name="keywords" content="missav, metadata, fallback" />
+                  <meta property="og:image" content="https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg" />
+                </head>
+              </html>
+              """
+            }
+
+          unexpected ->
+            raise "Unexpected MissAV preview request: #{inspect(unexpected)}"
+        end
+
+      {request, response}
+    end
+  end
+
   defmodule TopicSendPhotoAdapter do
     @behaviour ExGram.Adapter
 
@@ -2587,6 +2664,10 @@ defmodule SaveIt.BotTest do
       json_response(%{"url" => "http://127.0.0.1:#{port}/downloaded/video.mp4"})
     end
 
+    defp cobalt_response(%{"url" => "https://missav.ai/ja/sdam-101-uncensored-leak"}, port) do
+      json_response(%{"url" => "http://127.0.0.1:#{port}/downloaded/photo.jpg"})
+    end
+
     defp cobalt_response(%{"url" => "https://example.com/unavailable"}, _port) do
       error_response(%{"error" => "unsupported url"})
     end

test/small_sdk/link_preview_test.exs

@@ -132,6 +132,24 @@ defmodule SmallSdk.LinkPreviewTest do
     refute log =~ "token=secret"
   end
 
+  test "uses missav mirror metadata when missav.ai blocks preview HTML" do
+    Application.put_env(:save_it, :test_pid, self())
+
+    assert {:ok, metadata} =
+             LinkPreview.get_metadata("https://missav.ai/ja/sdam-101-uncensored-leak",
+               req_options: [adapter: &__MODULE__.MissavMetadataAdapter.request/1],
+               missav_metadata_fallback_base_url: "https://missav.ws"
+             )
+
+    assert metadata.title == "MissAV Mirror OG Title"
+    assert metadata.description == "MissAV Mirror OG Description"
+    assert metadata.keywords == ["missav", "metadata", "fallback"]
+    assert metadata.image_url == "https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg"
+
+    assert_receive {:link_preview_request, "https://missav.ai/ja/sdam-101-uncensored-leak"}
+    assert_receive {:link_preview_request, "https://missav.ws/ja/sdam-101-uncensored-leak"}
+  end
+
   test "uses authenticated X metadata when cobalt cookies are available", %{tmp_dir: tmp_dir} do
     cookie_path = Path.join(tmp_dir, "cobalt-cookies.json")
 
@@ -172,6 +190,38 @@ defmodule SmallSdk.LinkPreviewTest do
     end)
   end
 
+  defmodule MissavMetadataAdapter do
+    def request(%Req.Request{url: url} = request) do
+      send(
+        Application.fetch_env!(:save_it, :test_pid),
+        {:link_preview_request, URI.to_string(url)}
+      )
+
+      response =
+        case {url.host, url.path} do
+          {"missav.ai", "/ja/sdam-101-uncensored-leak"} ->
+            %Req.Response{status: 403, body: "Just a moment..."}
+
+          {"missav.ws", "/ja/sdam-101-uncensored-leak"} ->
+            %Req.Response{
+              status: 200,
+              body: """
+              <html>
+                <head>
+                  <meta property="og:title" content="MissAV Mirror OG Title" />
+                  <meta property="og:description" content="MissAV Mirror OG Description" />
+                  <meta name="keywords" content="missav, metadata, fallback" />
+                  <meta property="og:image" content="https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg" />
+                </head>
+              </html>
+              """
+            }
+        end
+
+      {request, response}
+    end
+  end
+
   defmodule TestHttpServer do
     use GenServer