feat: add ArchiveSearch for full-text search across archived snapshots

- Searches text content of archived snapshots for query strings
- Case-insensitive by default with case_sensitive option
- Returns SearchResult with surrounding context for each match
- Configurable max_results limit and date range filtering
- SearchResult value object with serialization

Author: ronaldtse

lib/archaeo/archive_search.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Archaeo
+  # Value object for a single search match within an archived snapshot.
+  SearchResult = Struct.new(
+    :url, :snapshot, :context, :match_offset,
+    keyword_init: true
+  ) do
+    def to_h
+      {
+        url: url,
+        snapshot: snapshot.as_json,
+        context: context,
+        match_offset: match_offset,
+      }
+    end
+
+    def as_json(*)
+      to_h
+    end
+  end
+
+  # Full-text search across archived snapshots.
+  #
+  # Fetches snapshots from CDX, downloads their content, and
+  # searches for the given query string. Returns matches with
+  # surrounding context for each hit.
+  class ArchiveSearch
+    CONTEXT_RADIUS = 80
+
+    def initialize(cdx_api: CdxApi.new, fetcher: Fetcher.new)
+      @cdx = cdx_api
+      @fetcher = fetcher
+    end
+
+    def search(url, query:, from: nil, to: nil,
+               max_results: nil, case_sensitive: false)
+      if query.nil? || query.empty?
+        raise ArgumentError,
+              "query must not be empty"
+      end
+
+      url = UrlNormalizer.normalize(url)
+      opts = build_options(from, to)
+
+      snapshots = @cdx.snapshots(url, **opts)
+        .select { |s| s.success? && s.mimetype.to_s.include?("text") }
+        .to_a
+
+      find_matches(snapshots, query, case_sensitive, max_results)
+    end
+
+    private
+
+    def build_options(from, to)
+      opts = { collapse: ["digest"] }
+      opts[:from] = Timestamp.coerce(from).to_s if from
+      opts[:to] = Timestamp.coerce(to).to_s if to
+      opts
+    end
+
+    def find_matches(snapshots, query, case_sensitive, max_results)
+      results = []
+      pattern = build_pattern(query, case_sensitive)
+
+      snapshots.each do |snap|
+        break if max_results && results.size >= max_results
+
+        content = fetch_content(snap)
+        next unless content
+
+        scan_content(content, pattern).each do |match_offset|
+          results << SearchResult.new(
+            url: snap.original_url,
+            snapshot: snap,
+            context: extract_context(content, match_offset, query.length),
+            match_offset: match_offset,
+          )
+          break if max_results && results.size >= max_results
+        end
+      end
+
+      results
+    end
+
+    def build_pattern(query, case_sensitive)
+      escaped = Regexp.escape(query)
+      return /#{escaped}/im unless case_sensitive
+
+      /#{escaped}/m
+    end
+
+    def fetch_content(snapshot)
+      page = @fetcher.fetch(
+        snapshot.original_url, timestamp: snapshot.timestamp
+      )
+      page.content if page.text?
+    rescue Error
+      nil
+    end
+
+    def scan_content(content, pattern)
+      offsets = []
+      content.scan(pattern) do
+        offsets << Regexp.last_match.offset(0).first
+      end
+      offsets
+    end
+
+    def extract_context(content, offset, length)
+      start_pos = [0, offset - CONTEXT_RADIUS].max
+      end_pos = [content.length, offset + length + CONTEXT_RADIUS].min
+
+      ctx = content[start_pos...end_pos]
+      ctx = "...#{ctx}" if start_pos.positive?
+      ctx = "#{ctx}..." if end_pos < content.length
+      ctx.tr("\n\r", " ").strip
+    end
+  end
+end

spec/archaeo/archive_search_spec.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe Archaeo::ArchiveSearch do
+  let(:snapshot) do
+    Archaeo::Snapshot.new(
+      urlkey: "com,example)/",
+      timestamp: "20220615000000",
+      original_url: "https://example.com/",
+      mimetype: "text/html", status_code: 200,
+      digest: "SHA1-abc", length: 200
+    )
+  end
+
+  let(:page) do
+    Archaeo::Page.new(
+      content: "<html><body>Contact us at info@example.com</body></html>",
+      content_type: "text/html",
+      status_code: 200,
+      archive_url: "https://web.archive.org/web/20220615000000/https://example.com/",
+      original_url: "https://example.com/",
+      timestamp: Archaeo::Timestamp.new(year: 2022, month: 6, day: 15),
+    )
+  end
+
+  let(:fake_cdx) do
+    cdx = instance_double(Archaeo::CdxApi)
+    allow(cdx).to receive(:snapshots).and_return([snapshot])
+    cdx
+  end
+
+  let(:fake_fetcher) do
+    fetcher = instance_double(Archaeo::Fetcher)
+    allow(fetcher).to receive(:fetch).and_return(page)
+    fetcher
+  end
+
+  let(:searcher) do
+    described_class.new(cdx_api: fake_cdx, fetcher: fake_fetcher)
+  end
+
+  it "finds matching text in snapshots" do
+    results = searcher.search("example.com", query: "Contact us")
+    expect(results.size).to be >= 1
+    expect(results.first.context).to include("Contact us")
+  end
+
+  it "returns empty results for no matches" do
+    results = searcher.search("example.com", query: "xyzzy-no-match")
+    expect(results).to be_empty
+  end
+
+  it "raises on empty query" do
+    expect { searcher.search("example.com", query: "") }
+      .to raise_error(ArgumentError)
+  end
+
+  it "respects max_results limit" do
+    results = searcher.search("example.com", query: "example", max_results: 1)
+    expect(results.size).to be <= 1
+  end
+
+  it "performs case-insensitive search by default" do
+    results = searcher.search("example.com", query: "contact us")
+    expect(results).not_to be_empty
+  end
+
+  it "respects case_sensitive option" do
+    results = searcher.search("example.com", query: "contact us",
+                                             case_sensitive: true)
+    expect(results).to be_empty
+  end
+end
+
+RSpec.describe Archaeo::SearchResult do
+  it "serializes to hash" do
+    snap = Archaeo::Snapshot.new(
+      urlkey: "com,example)/",
+      timestamp: "20220615000000",
+      original_url: "https://example.com/",
+      mimetype: "text/html", status_code: 200
+    )
+    result = described_class.new(
+      url: "https://example.com/",
+      snapshot: snap,
+      context: "...Contact us...",
+      match_offset: 10,
+    )
+    h = result.to_h
+    expect(h[:url]).to eq("https://example.com/")
+    expect(h[:context]).to eq("...Contact us...")
+  end
+
+  it "serializes to JSON" do
+    snap = Archaeo::Snapshot.new(
+      urlkey: "com,example)/",
+      timestamp: "20220615000000",
+      original_url: "https://example.com/",
+      mimetype: "text/html", status_code: 200
+    )
+    result = described_class.new(
+      url: "https://example.com/",
+      snapshot: snap,
+      context: "test",
+      match_offset: 0,
+    )
+    expect { JSON.generate(result.as_json) }.not_to raise_error
+  end
+end