refactor: isolate MissAV metadata fallback

CHANGELOG.md

@@ -16,6 +16,7 @@ before the CalVer migration retain their original version labels.
 
 ### Fixed
 
+- Save searchable MissAV `missav.ai` Open Graph metadata by falling back to readable mirror metadata when the main page blocks preview fetches.
 - Render successful resource creation info logs in green in runtime logger output.
 - Save a webpage `og:image` preview when a URL resolves to a non-media resource instead of storing the resolved HTML or document file.
 - Send and index a generated preview image when a downloaded URL video is too large for Telegram video upload, while still saving the downloaded video locally.

docs/postmortem/2026-06-16-missav-ai-og-metadata-fallback.md

@@ -0,0 +1,21 @@
+# MissAV AI Open Graph Metadata Fallback
+
+## What happened
+
+A `missav.ai` URL could show a rich Telegram client preview, but `save_it` could not reliably save the page's Open Graph title, description, keywords, or image URL into Typesense. URL-only saves therefore had no user caption and no searchable URL metadata.
+
+## Root cause
+
+Telegram client previews do not mean the Bot API exposes the resolved Open Graph fields to the bot. `save_it` still has to fetch the preview page itself. Direct server-side requests to the affected `missav.ai` page returned a Cloudflare 403 page, so `SmallSdk.LinkPreview.get_metadata/1` returned an error and the bot indexed the saved media without URL metadata.
+
+The MissAV provider strategy recognized `missav.ai`, but it did not have a metadata fallback URL when the primary host blocked preview fetches.
+
+## Fix applied
+
+`SmallSdk.LinkPreview` now tries the normal preview URL first, then delegates provider-specific fallback handling to isolated provider modules. `SmallSdk.MissavMetadata` owns the `missav.ai` fallback behavior: when a MissAV metadata fetch fails, it retries the same path on `https://missav.ws`. Metadata parsed from the fallback page still flows through the existing `title`, `description`, `keywords`, and `thumbnail_url` fields, while the saved document keeps the original `missav.ai` URL.
+
+Regression coverage now verifies the isolated MissAV metadata provider and the full bot save path that writes MissAV Open Graph metadata into the Typesense document.
+
+## What we learned
+
+Telegram's visible link preview is not a reliable data source for bot indexing. Provider-specific metadata recovery should live at the link-preview boundary so the rest of the save pipeline can keep the same field semantics and continue preserving the user's original URL.

docs/specs/001-typesense-saved-content.md

@@ -157,7 +157,7 @@ Supported provider strategies:
 | Instagram | `instagram.com` and subdomains | Use the default URL metadata strategy. Provider-specific handling may be added when public metadata is blocked or incomplete. |
 | YouTube Shorts | `youtube.com/shorts/*`, `youtu.be/*` when resolved as Shorts | Use the default URL metadata strategy. The Shorts provider may normalize canonical YouTube URLs before metadata fetches. |
 | `bad.news` | `bad.news` and subdomains | Use `bad.news` download resolution for media and the default URL metadata strategy for page metadata. |
-| `missav.ai` | `missav.ai` and subdomains | Use the default URL metadata strategy. Provider-specific media and metadata handling may be added without changing document field semantics. |
+| `missav.ai` | `missav.ai` and subdomains | Use the default URL metadata strategy first. If the `missav.ai` preview HTML fetch fails, fetch the same path from `https://missav.ws` and store any readable Open Graph metadata while preserving the original `missav.ai` URL as `url`. |
 | Other URL | Any supported URL not matched above | Use the default URL metadata strategy. |
 
 ### Default URL Metadata Strategy

lib/small_sdk/link_preview.ex

@@ -4,6 +4,7 @@ defmodule SmallSdk.LinkPreview do
   require Logger
 
   alias SmallSdk.WebDownloader
+  alias SmallSdk.LinkPreview.ProviderFallback
   alias SmallSdk.XMetadata
 
   @preview_patterns [
@@ -72,18 +73,20 @@ defmodule SmallSdk.LinkPreview do
 
   def get_title(_page_url), do: {:error, :missing_preview_url}
 
-  def get_metadata(page_url) when is_binary(page_url) do
+  def get_metadata(page_url, opts \\ [])
+
+  def get_metadata(page_url, opts) when is_binary(page_url) do
     case get_authenticated_metadata(page_url) do
       {:ok, metadata} ->
         log_metadata(page_url, metadata)
         {:ok, metadata}
 
       {:error, _reason} ->
-        get_html_metadata(page_url)
+        get_html_metadata(page_url, opts)
     end
   end
 
-  def get_metadata(_page_url), do: {:error, :missing_preview_url}
+  def get_metadata(_page_url, _opts), do: {:error, :missing_preview_url}
 
   defp get_authenticated_metadata(page_url) do
     if XMetadata.x_url?(page_url) do
@@ -93,8 +96,18 @@ defmodule SmallSdk.LinkPreview do
     end
   end
 
-  defp get_html_metadata(page_url) do
-    case Req.get(page_url, headers: [{"User-Agent", "Mozilla/5.0"}]) do
+  defp get_html_metadata(page_url, opts) do
+    case fetch_html_metadata(page_url, opts) do
+      {:ok, metadata} ->
+        {:ok, metadata}
+
+      {:error, reason} ->
+        ProviderFallback.fetch_metadata(page_url, reason, opts, &fetch_html_metadata/2)
+    end
+  end
+
+  defp fetch_html_metadata(page_url, opts) do
+    case Req.get(page_url, request_options(opts)) do
       {:ok, %{status: status, body: body}} when status in 200..209 and is_binary(body) ->
         metadata = get_metadata_from_html(page_url, body)
         log_metadata(page_url, metadata)
@@ -111,6 +124,19 @@ defmodule SmallSdk.LinkPreview do
     end
   end
 
+  defp request_options(opts) do
+    opts
+    |> Keyword.get(:req_options, Application.get_env(:save_it, :link_preview_req_options, []))
+    |> Keyword.put_new(:retry, false)
+    |> put_request_headers([{"User-Agent", "Mozilla/5.0"}])
+  end
+
+  defp put_request_headers(req_options, headers) do
+    Keyword.update(req_options, :headers, headers, fn existing_headers ->
+      headers ++ existing_headers
+    end)
+  end
+
   def get_metadata_from_html(page_url, html) when is_binary(page_url) and is_binary(html) do
     %{
       title: html |> extract_title() |> blank_to_nil(),

lib/small_sdk/link_preview/provider_fallback.ex

@@ -0,0 +1,15 @@
+defmodule SmallSdk.LinkPreview.ProviderFallback do
+  @moduledoc false
+
+  @providers [SmallSdk.MissavMetadata]
+
+  def fetch_metadata(page_url, reason, opts, fetch_metadata) do
+    case Enum.find(@providers, & &1.supports?(page_url)) do
+      nil ->
+        {:error, reason}
+
+      provider ->
+        provider.fetch_fallback_metadata(page_url, reason, opts, fetch_metadata)
+    end
+  end
+end

lib/small_sdk/missav_metadata.ex

@@ -0,0 +1,106 @@
+defmodule SmallSdk.MissavMetadata do
+  @moduledoc false
+
+  @fallback_base_url "https://missav.ws"
+
+  require Logger
+
+  def supports?(url) when is_binary(url) do
+    case URI.parse(url) do
+      %URI{host: host} when is_binary(host) ->
+        missav_ai_host?(String.downcase(host))
+
+      _ ->
+        false
+    end
+  rescue
+    _ -> false
+  end
+
+  def supports?(_url), do: false
+
+  def fetch_fallback_metadata(page_url, reason, opts, fetch_metadata)
+      when is_binary(page_url) and is_function(fetch_metadata, 2) do
+    case fallback_url(page_url) do
+      fallback_url when is_binary(fallback_url) ->
+        log_metadata_fallback(page_url, fallback_url, reason)
+        fetch_metadata.(fallback_url, opts)
+
+      nil ->
+        {:error, reason}
+    end
+  end
+
+  def fetch_fallback_metadata(_page_url, reason, _opts, _fetch_metadata), do: {:error, reason}
+
+  defp fallback_url(page_url) do
+    with true <- supports?(page_url),
+         %URI{host: fallback_host} = fallback_uri when is_binary(fallback_host) <-
+           URI.parse(@fallback_base_url),
+         %URI{} = page_uri <- URI.parse(page_url) do
+      %URI{
+        page_uri
+        | scheme: fallback_uri.scheme || "https",
+          userinfo: fallback_uri.userinfo,
+          host: fallback_host,
+          port: fallback_uri.port
+      }
+      |> URI.to_string()
+    else
+      _ -> nil
+    end
+  rescue
+    _ -> nil
+  end
+
+  defp missav_ai_host?(host), do: host == "missav.ai" or String.ends_with?(host, ".missav.ai")
+
+  defp log_metadata_fallback(page_url, fallback_url, reason) do
+    Logger.warning(
+      "Link preview metadata fallback selected: " <>
+        "page_url=#{format_log_url(page_url)} " <>
+        "fallback_url=#{format_log_url(fallback_url)} " <>
+        "reason=#{format_log_reason(reason)}",
+      kind: :link_preview
+    )
+  end
+
+  defp format_log_url(nil), do: "nil"
+
+  defp format_log_url(url) when is_binary(url) do
+    url
+    |> remove_query_and_fragment()
+    |> format_log_value()
+  end
+
+  defp format_log_value(nil), do: "nil"
+
+  defp format_log_value(value) when is_binary(value) do
+    value
+    |> truncate_log_value()
+    |> inspect()
+  end
+
+  defp format_log_reason(reason) do
+    reason
+    |> inspect()
+    |> truncate_log_value()
+  end
+
+  defp truncate_log_value(value) do
+    if String.length(value) > 160 do
+      String.slice(value, 0, 160) <> "..."
+    else
+      value
+    end
+  end
+
+  defp remove_query_and_fragment(url) do
+    uri = URI.parse(url)
+
+    %URI{uri | query: nil, fragment: nil}
+    |> URI.to_string()
+  rescue
+    _ -> url
+  end
+end

test/bot_test.exs

@@ -335,6 +335,46 @@ defmodule SaveIt.BotTest do
            end)
   end
 
+  test "stores missav.ai mirror OG metadata when direct preview fetch is blocked",
+       %{base_url: base_url} do
+    original_url = "https://missav.ai/ja/sdam-101-uncensored-leak"
+    download_url = base_url <> "/downloaded/photo.jpg"
+
+    Application.put_env(:save_it, :test_pid, self())
+
+    Application.put_env(:save_it, :link_preview_req_options,
+      adapter: &__MODULE__.MissavLinkPreviewAdapter.request/1
+    )
+
+    message = %{
+      chat: %{id: 12_345, username: "save_it_test_chat"},
+      date: 1_717_170_000,
+      message_id: 103,
+      text: original_url
+    }
+
+    assert {:ok, true} = Bot.handle({:text, original_url, message}, nil)
+
+    assert_receive {:test_http_request, :post, "/", cobalt_body}
+    assert Jason.decode!(cobalt_body) == %{"url" => original_url}
+    assert_receive {:test_http_request, :get, "/downloaded/photo.jpg", ""}
+    assert_receive {:link_preview_request, "https://missav.ai/ja/sdam-101-uncensored-leak"}
+    assert_receive {:link_preview_request, "https://missav.ws/ja/sdam-101-uncensored-leak"}
+    assert_receive {:test_http_request, :post, "/collections/photos/documents", typesense_body}
+
+    document = Jason.decode!(typesense_body)
+
+    assert document["url"] == original_url
+    assert document["download_url"] == download_url
+    assert document["thumbnail_url"] == "https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg"
+    assert document["caption"] == ""
+    assert document["title"] == "MissAV Mirror OG Title"
+    assert document["description"] == "MissAV Mirror OG Description"
+    assert document["keywords"] == ["missav", "metadata", "fallback"]
+    assert document["file_id"] == "telegram-photo-file-id"
+    assert document["belongs_to_id"] == "12345"
+  end
+
   test "sends downloaded URL media to the source topic and stores a topic message URL",
        %{base_url: base_url} do
     Application.put_env(:ex_gram, :adapter, __MODULE__.TopicSendPhotoAdapter)
@@ -1882,6 +1922,41 @@ defmodule SaveIt.BotTest do
     end
   end
 
+  defmodule MissavLinkPreviewAdapter do
+    def request(%Req.Request{url: url} = request) do
+      send(
+        Application.fetch_env!(:save_it, :test_pid),
+        {:link_preview_request, URI.to_string(url)}
+      )
+
+      response =
+        case {url.host, url.path} do
+          {"missav.ai", "/ja/sdam-101-uncensored-leak"} ->
+            %Req.Response{status: 403, body: "Just a moment..."}
+
+          {"missav.ws", "/ja/sdam-101-uncensored-leak"} ->
+            %Req.Response{
+              status: 200,
+              body: """
+              <html>
+                <head>
+                  <meta property="og:title" content="MissAV Mirror OG Title" />
+                  <meta property="og:description" content="MissAV Mirror OG Description" />
+                  <meta name="keywords" content="missav, metadata, fallback" />
+                  <meta property="og:image" content="https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg" />
+                </head>
+              </html>
+              """
+            }
+
+          unexpected ->
+            raise "Unexpected MissAV preview request: #{inspect(unexpected)}"
+        end
+
+      {request, response}
+    end
+  end
+
   defmodule TopicSendPhotoAdapter do
     @behaviour ExGram.Adapter
 
@@ -2587,6 +2662,10 @@ defmodule SaveIt.BotTest do
       json_response(%{"url" => "http://127.0.0.1:#{port}/downloaded/video.mp4"})
     end
 
+    defp cobalt_response(%{"url" => "https://missav.ai/ja/sdam-101-uncensored-leak"}, port) do
+      json_response(%{"url" => "http://127.0.0.1:#{port}/downloaded/photo.jpg"})
+    end
+
     defp cobalt_response(%{"url" => "https://example.com/unavailable"}, _port) do
       error_response(%{"error" => "unsupported url"})
     end

test/small_sdk/missav_metadata_test.exs

@@ -0,0 +1,59 @@
+defmodule SmallSdk.MissavMetadataTest do
+  use ExUnit.Case, async: false
+
+  import ExUnit.CaptureLog
+
+  alias SmallSdk.MissavMetadata
+
+  test "fetches mirror metadata for missav.ai URLs" do
+    reason = {:preview_page_status, 403}
+
+    fetch_metadata = fn fallback_url, opts ->
+      send(self(), {:fallback_fetch, fallback_url, opts})
+
+      {:ok,
+       %{
+         title: "MissAV Mirror OG Title",
+         description: "MissAV Mirror OG Description",
+         keywords: ["missav", "metadata", "fallback"],
+         image_url: "https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg"
+       }}
+    end
+
+    log =
+      capture_log(fn ->
+        assert {:ok, metadata} =
+                 MissavMetadata.fetch_fallback_metadata(
+                   "https://missav.ai/ja/sdam-101-uncensored-leak?token=secret",
+                   reason,
+                   [],
+                   fetch_metadata
+                 )
+
+        assert metadata.title == "MissAV Mirror OG Title"
+      end)
+
+    assert_receive {:fallback_fetch, "https://missav.ws/ja/sdam-101-uncensored-leak?token=secret",
+                    []}
+
+    assert log =~ "Link preview metadata fallback selected"
+    assert log =~ ~s(page_url="https://missav.ai/ja/sdam-101-uncensored-leak")
+    assert log =~ ~s(fallback_url="https://missav.ws/ja/sdam-101-uncensored-leak")
+    assert log =~ "reason={:preview_page_status, 403}"
+    refute log =~ "token=secret"
+  end
+
+  test "does not handle non-missav URLs" do
+    fetch_metadata = fn _url, _opts ->
+      flunk("non-missav URL should not be fetched")
+    end
+
+    assert {:error, :blocked} =
+             MissavMetadata.fetch_fallback_metadata(
+               "https://example.com/ja/sdam-101-uncensored-leak",
+               :blocked,
+               [],
+               fetch_metadata
+             )
+  end
+end