refactor(link-preview): simplify missav metadata fallback host

Author: ThaddeusJiang

docs/postmortem/2026-06-16-missav-ai-og-metadata-fallback.md

@@ -12,9 +12,9 @@ The MissAV provider strategy recognized `missav.ai`, but it did not have a metad
 
 ## Fix applied
 
-`SmallSdk.LinkPreview` now tries the normal preview URL first. When a `missav.ai` metadata fetch fails, it retries the same path on a configurable MissAV metadata fallback host, defaulting to `https://missav.ws`. Metadata parsed from the fallback page still flows through the existing `title`, `description`, `keywords`, and `thumbnail_url` fields, while the saved document keeps the original `missav.ai` URL.
+`SmallSdk.LinkPreview` now tries the normal preview URL first, then delegates provider-specific fallback handling to isolated provider modules. `SmallSdk.MissavMetadata` owns the `missav.ai` fallback behavior: when a MissAV metadata fetch fails, it retries the same path on `https://missav.ws`. Metadata parsed from the fallback page still flows through the existing `title`, `description`, `keywords`, and `thumbnail_url` fields, while the saved document keeps the original `missav.ai` URL.
 
-Regression coverage now verifies both the `LinkPreview` fallback behavior and the full bot save path that writes MissAV Open Graph metadata into the Typesense document.
+Regression coverage now verifies the isolated MissAV metadata provider and the full bot save path that writes MissAV Open Graph metadata into the Typesense document.
 
 ## What we learned
 

docs/specs/001-typesense-saved-content.md

@@ -157,7 +157,7 @@ Supported provider strategies:
 | Instagram | `instagram.com` and subdomains | Use the default URL metadata strategy. Provider-specific handling may be added when public metadata is blocked or incomplete. |
 | YouTube Shorts | `youtube.com/shorts/*`, `youtu.be/*` when resolved as Shorts | Use the default URL metadata strategy. The Shorts provider may normalize canonical YouTube URLs before metadata fetches. |
 | `bad.news` | `bad.news` and subdomains | Use `bad.news` download resolution for media and the default URL metadata strategy for page metadata. |
-| `missav.ai` | `missav.ai` and subdomains | Use the default URL metadata strategy first. If the `missav.ai` preview HTML fetch fails, fetch the same path from the configured MissAV metadata fallback host, defaulting to `https://missav.ws`, and store any readable Open Graph metadata while preserving the original `missav.ai` URL as `url`. |
+| `missav.ai` | `missav.ai` and subdomains | Use the default URL metadata strategy first. If the `missav.ai` preview HTML fetch fails, fetch the same path from `https://missav.ws` and store any readable Open Graph metadata while preserving the original `missav.ai` URL as `url`. |
 | Other URL | Any supported URL not matched above | Use the default URL metadata strategy. |
 
 ### Default URL Metadata Strategy

lib/small_sdk/link_preview.ex

@@ -4,10 +4,9 @@ defmodule SmallSdk.LinkPreview do
   require Logger
 
   alias SmallSdk.WebDownloader
+  alias SmallSdk.LinkPreview.ProviderFallback
   alias SmallSdk.XMetadata
 
-  @missav_metadata_fallback_base_url "https://missav.ws"
-
   @preview_patterns [
     ~r/<meta[^>]+property=["']og:image["'][^>]+(?:content|value)=["']([^"']+)["']/i,
     ~r/<meta[^>]+(?:content|value)=["']([^"']+)["'][^>]+property=["']og:image["']/i,
@@ -99,8 +98,11 @@ defmodule SmallSdk.LinkPreview do
 
   defp get_html_metadata(page_url, opts) do
     case fetch_html_metadata(page_url, opts) do
-      {:ok, metadata} -> {:ok, metadata}
-      {:error, reason} -> maybe_get_provider_fallback_metadata(page_url, reason, opts)
+      {:ok, metadata} ->
+        {:ok, metadata}
+
+      {:error, reason} ->
+        ProviderFallback.fetch_metadata(page_url, reason, opts, &fetch_html_metadata/2)
     end
   end
 
@@ -122,21 +124,6 @@ defmodule SmallSdk.LinkPreview do
     end
   end
 
-  defp maybe_get_provider_fallback_metadata(page_url, reason, opts) do
-    if Keyword.get(opts, :provider_fallback?, true) do
-      case missav_metadata_fallback_url(page_url, opts) do
-        fallback_url when is_binary(fallback_url) ->
-          log_metadata_fallback(page_url, fallback_url, reason)
-          fetch_html_metadata(fallback_url, Keyword.put(opts, :provider_fallback?, false))
-
-        nil ->
-          {:error, reason}
-      end
-    else
-      {:error, reason}
-    end
-  end
-
   defp request_options(opts) do
     opts
     |> Keyword.get(:req_options, Application.get_env(:save_it, :link_preview_req_options, []))
@@ -150,43 +137,6 @@ defmodule SmallSdk.LinkPreview do
     end)
   end
 
-  defp missav_metadata_fallback_url(page_url, opts) do
-    with %URI{host: host} = page_uri when is_binary(host) <- URI.parse(page_url),
-         true <- missav_ai_host?(String.downcase(host)),
-         fallback_base_url when is_binary(fallback_base_url) <-
-           missav_metadata_fallback_base_url(opts),
-         %URI{host: fallback_host} = fallback_uri when is_binary(fallback_host) <-
-           URI.parse(fallback_base_url) do
-      %URI{
-        page_uri
-        | scheme: fallback_uri.scheme || "https",
-          userinfo: fallback_uri.userinfo,
-          host: fallback_host,
-          port: fallback_uri.port
-      }
-      |> URI.to_string()
-    else
-      _ -> nil
-    end
-  rescue
-    _ -> nil
-  end
-
-  defp missav_metadata_fallback_base_url(opts) do
-    opts
-    |> Keyword.get(
-      :missav_metadata_fallback_base_url,
-      Application.get_env(
-        :save_it,
-        :missav_metadata_fallback_base_url,
-        @missav_metadata_fallback_base_url
-      )
-    )
-    |> blank_to_nil()
-  end
-
-  defp missav_ai_host?(host), do: host == "missav.ai" or String.ends_with?(host, ".missav.ai")
-
   def get_metadata_from_html(page_url, html) when is_binary(page_url) and is_binary(html) do
     %{
       title: html |> extract_title() |> blank_to_nil(),
@@ -302,16 +252,6 @@ defmodule SmallSdk.LinkPreview do
     )
   end
 
-  defp log_metadata_fallback(page_url, fallback_url, reason) do
-    Logger.warning(
-      "Link preview metadata fallback selected: " <>
-        "page_url=#{format_log_url(page_url)} " <>
-        "fallback_url=#{format_log_url(fallback_url)} " <>
-        "reason=#{format_log_reason(reason)}",
-      kind: :link_preview
-    )
-  end
-
   defp format_log_url(nil), do: "nil"
 
   defp format_log_url(url) when is_binary(url) do

lib/small_sdk/link_preview/provider_fallback.ex

@@ -0,0 +1,15 @@
+defmodule SmallSdk.LinkPreview.ProviderFallback do
+  @moduledoc false
+
+  @providers [SmallSdk.MissavMetadata]
+
+  def fetch_metadata(page_url, reason, opts, fetch_metadata) do
+    case Enum.find(@providers, & &1.supports?(page_url)) do
+      nil ->
+        {:error, reason}
+
+      provider ->
+        provider.fetch_fallback_metadata(page_url, reason, opts, fetch_metadata)
+    end
+  end
+end

lib/small_sdk/missav_metadata.ex

@@ -0,0 +1,106 @@
+defmodule SmallSdk.MissavMetadata do
+  @moduledoc false
+
+  @fallback_base_url "https://missav.ws"
+
+  require Logger
+
+  def supports?(url) when is_binary(url) do
+    case URI.parse(url) do
+      %URI{host: host} when is_binary(host) ->
+        missav_ai_host?(String.downcase(host))
+
+      _ ->
+        false
+    end
+  rescue
+    _ -> false
+  end
+
+  def supports?(_url), do: false
+
+  def fetch_fallback_metadata(page_url, reason, opts, fetch_metadata)
+      when is_binary(page_url) and is_function(fetch_metadata, 2) do
+    case fallback_url(page_url) do
+      fallback_url when is_binary(fallback_url) ->
+        log_metadata_fallback(page_url, fallback_url, reason)
+        fetch_metadata.(fallback_url, opts)
+
+      nil ->
+        {:error, reason}
+    end
+  end
+
+  def fetch_fallback_metadata(_page_url, reason, _opts, _fetch_metadata), do: {:error, reason}
+
+  defp fallback_url(page_url) do
+    with true <- supports?(page_url),
+         %URI{host: fallback_host} = fallback_uri when is_binary(fallback_host) <-
+           URI.parse(@fallback_base_url),
+         %URI{} = page_uri <- URI.parse(page_url) do
+      %URI{
+        page_uri
+        | scheme: fallback_uri.scheme || "https",
+          userinfo: fallback_uri.userinfo,
+          host: fallback_host,
+          port: fallback_uri.port
+      }
+      |> URI.to_string()
+    else
+      _ -> nil
+    end
+  rescue
+    _ -> nil
+  end
+
+  defp missav_ai_host?(host), do: host == "missav.ai" or String.ends_with?(host, ".missav.ai")
+
+  defp log_metadata_fallback(page_url, fallback_url, reason) do
+    Logger.warning(
+      "Link preview metadata fallback selected: " <>
+        "page_url=#{format_log_url(page_url)} " <>
+        "fallback_url=#{format_log_url(fallback_url)} " <>
+        "reason=#{format_log_reason(reason)}",
+      kind: :link_preview
+    )
+  end
+
+  defp format_log_url(nil), do: "nil"
+
+  defp format_log_url(url) when is_binary(url) do
+    url
+    |> remove_query_and_fragment()
+    |> format_log_value()
+  end
+
+  defp format_log_value(nil), do: "nil"
+
+  defp format_log_value(value) when is_binary(value) do
+    value
+    |> truncate_log_value()
+    |> inspect()
+  end
+
+  defp format_log_reason(reason) do
+    reason
+    |> inspect()
+    |> truncate_log_value()
+  end
+
+  defp truncate_log_value(value) do
+    if String.length(value) > 160 do
+      String.slice(value, 0, 160) <> "..."
+    else
+      value
+    end
+  end
+
+  defp remove_query_and_fragment(url) do
+    uri = URI.parse(url)
+
+    %URI{uri | query: nil, fragment: nil}
+    |> URI.to_string()
+  rescue
+    _ -> url
+  end
+end

test/bot_test.exs

@@ -346,8 +346,6 @@ defmodule SaveIt.BotTest do
       adapter: &__MODULE__.MissavLinkPreviewAdapter.request/1
     )
 
-    Application.put_env(:save_it, :missav_metadata_fallback_base_url, "https://missav.ws")
-
     message = %{
       chat: %{id: 12_345, username: "save_it_test_chat"},
       date: 1_717_170_000,

test/small_sdk/link_preview_test.exs

@@ -132,24 +132,6 @@ defmodule SmallSdk.LinkPreviewTest do
     refute log =~ "token=secret"
   end
 
-  test "uses missav mirror metadata when missav.ai blocks preview HTML" do
-    Application.put_env(:save_it, :test_pid, self())
-
-    assert {:ok, metadata} =
-             LinkPreview.get_metadata("https://missav.ai/ja/sdam-101-uncensored-leak",
-               req_options: [adapter: &__MODULE__.MissavMetadataAdapter.request/1],
-               missav_metadata_fallback_base_url: "https://missav.ws"
-             )
-
-    assert metadata.title == "MissAV Mirror OG Title"
-    assert metadata.description == "MissAV Mirror OG Description"
-    assert metadata.keywords == ["missav", "metadata", "fallback"]
-    assert metadata.image_url == "https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg"
-
-    assert_receive {:link_preview_request, "https://missav.ai/ja/sdam-101-uncensored-leak"}
-    assert_receive {:link_preview_request, "https://missav.ws/ja/sdam-101-uncensored-leak"}
-  end
-
   test "uses authenticated X metadata when cobalt cookies are available", %{tmp_dir: tmp_dir} do
     cookie_path = Path.join(tmp_dir, "cobalt-cookies.json")
 
@@ -190,38 +172,6 @@ defmodule SmallSdk.LinkPreviewTest do
     end)
   end
 
-  defmodule MissavMetadataAdapter do
-    def request(%Req.Request{url: url} = request) do
-      send(
-        Application.fetch_env!(:save_it, :test_pid),
-        {:link_preview_request, URI.to_string(url)}
-      )
-
-      response =
-        case {url.host, url.path} do
-          {"missav.ai", "/ja/sdam-101-uncensored-leak"} ->
-            %Req.Response{status: 403, body: "Just a moment..."}
-
-          {"missav.ws", "/ja/sdam-101-uncensored-leak"} ->
-            %Req.Response{
-              status: 200,
-              body: """
-              <html>
-                <head>
-                  <meta property="og:title" content="MissAV Mirror OG Title" />
-                  <meta property="og:description" content="MissAV Mirror OG Description" />
-                  <meta name="keywords" content="missav, metadata, fallback" />
-                  <meta property="og:image" content="https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg" />
-                </head>
-              </html>
-              """
-            }
-        end
-
-      {request, response}
-    end
-  end
-
   defmodule TestHttpServer do
     use GenServer
 

test/small_sdk/missav_metadata_test.exs

@@ -0,0 +1,59 @@
+defmodule SmallSdk.MissavMetadataTest do
+  use ExUnit.Case, async: false
+
+  import ExUnit.CaptureLog
+
+  alias SmallSdk.MissavMetadata
+
+  test "fetches mirror metadata for missav.ai URLs" do
+    reason = {:preview_page_status, 403}
+
+    fetch_metadata = fn fallback_url, opts ->
+      send(self(), {:fallback_fetch, fallback_url, opts})
+
+      {:ok,
+       %{
+         title: "MissAV Mirror OG Title",
+         description: "MissAV Mirror OG Description",
+         keywords: ["missav", "metadata", "fallback"],
+         image_url: "https://fourhoi.com/sdam-101-uncensored-leak/cover-n.jpg"
+       }}
+    end
+
+    log =
+      capture_log(fn ->
+        assert {:ok, metadata} =
+                 MissavMetadata.fetch_fallback_metadata(
+                   "https://missav.ai/ja/sdam-101-uncensored-leak?token=secret",
+                   reason,
+                   [],
+                   fetch_metadata
+                 )
+
+        assert metadata.title == "MissAV Mirror OG Title"
+      end)
+
+    assert_receive {:fallback_fetch, "https://missav.ws/ja/sdam-101-uncensored-leak?token=secret",
+                    []}
+
+    assert log =~ "Link preview metadata fallback selected"
+    assert log =~ ~s(page_url="https://missav.ai/ja/sdam-101-uncensored-leak")
+    assert log =~ ~s(fallback_url="https://missav.ws/ja/sdam-101-uncensored-leak")
+    assert log =~ "reason={:preview_page_status, 403}"
+    refute log =~ "token=secret"
+  end
+
+  test "does not handle non-missav URLs" do
+    fetch_metadata = fn _url, _opts ->
+      flunk("non-missav URL should not be fetched")
+    end
+
+    assert {:error, :blocked} =
+             MissavMetadata.fetch_fallback_metadata(
+               "https://example.com/ja/sdam-101-uncensored-leak",
+               :blocked,
+               [],
+               fetch_metadata
+             )
+  end
+end