Cover the pure helpers with tests; drop two copied functions

compare-feed-urls.py predates feed_probe.py and carried its own copies of
validate_feed and dir_sha1, identical line for line apart from a hardcoded
timeout. It now imports the module, which also gets it the caching variant of
dir_sha1, worth having since each call costs a download.

Tests for the three functions most able to be quietly wrong:

normalise_url decides every match in every discovery source, and had none. The
collapses it makes are pinned, and so are the ones it refuses: a trailing dot
must keep two URLs apart, because a real submission differed from the
registered URL by exactly that and guessing would have hidden it.

calendar_status is where expiry decisions come from. Boundaries are covered,
including that a feed expiring today is still active. Writing these corrected a
wrong assumption of mine: GTFS YYYYMMDD dates parse fine, because
date.fromisoformat accepts ISO 8601 basic format, so the test now pins that
rather than asserting it fails.

The feed-list extractors are covered because one already failed silently: a
catalogue yielding nothing looks exactly like a catalogue that has not changed,
which is how MnDOT's hub sat unwatched. Both the URL and json-values paths are
pinned, including that URL extraction finds nothing in that shape.

policy_for gained a branch when a policy was allowed to name no feeds; a
position-only policy must never match by accident.

59 tests, all offline.

Author: Automated Bot

scripts/compare-feed-urls.py

@@ -10,54 +10,19 @@
 import os
 import sys
 import json
-import subprocess
 import concurrent.futures
 import urllib.request
 import urllib.error
 from datetime import date
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import feed_probe
 from rich.console import Console
 from rich.table import Table
 from rich.text import Text
 from rich.panel import Panel
 
 
-def dir_sha1_of(url: str) -> str | None:
-    """Content checksum, independent of how the archive was built."""
-    try:
-        proc = subprocess.run(["transitland", "checksum", "--raw-dir-sha1", url],
-                              capture_output=True, text=True, timeout=300)
-        return proc.stdout.strip() if proc.returncode == 0 else None
-    except (subprocess.TimeoutExpired, FileNotFoundError):
-        return None
-
-
-def validate_feed(url: str) -> dict:
-    try:
-        result = subprocess.run(
-            [
-                "transitland", "validate",
-                "-o", "-",
-                "--include-entities",
-                "--include-service-levels",
-                url,
-            ],
-            capture_output=True,
-            text=True,
-            timeout=180,
-        )
-        if result.returncode != 0 and not result.stdout.strip():
-            return {"_error": result.stderr.strip() or "Command failed", "_url": url}
-        data = json.loads(result.stdout)
-        data["_url"] = url
-        return data
-    except subprocess.TimeoutExpired:
-        return {"_error": "Timeout after 180s", "_url": url}
-    except json.JSONDecodeError as e:
-        return {"_error": f"Invalid JSON: {e}", "_url": url}
-    except FileNotFoundError:
-        return {"_error": "'transitland' command not found in PATH", "_url": url}
-
-
 ROUTE_TYPE_NAMES = {
     0: "Tram/Streetcar",
     1: "Subway/Metro",
@@ -247,14 +212,14 @@ def main():
     console.print("[dim]Validating feeds in parallel (this may take a minute)...[/dim]\n")
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=n * 2) as executor:
-        val_futs = [executor.submit(validate_feed, url) for url in urls]
+        val_futs = [executor.submit(feed_probe.validate_feed, url) for url in urls]
         results = [f.result() for f in val_futs]
         sha1s = [(r.get("details") or {}).get("sha1", "N/A") for r in results]
         arc_futs = [executor.submit(lookup_feed_version, sha1, api_key) for sha1 in sha1s]
         # The zip SHA1 above is the archive's primary key, and fragile: a server
         # that rebuilds the archive per request changes it while the data stays
         # put. The directory SHA1 is what actually answers "same feed?".
-        dir_futs = [executor.submit(dir_sha1_of, url) for url in urls]
+        dir_futs = [executor.submit(feed_probe.dir_sha1, url) for url in urls]
         arc_results = [f.result() for f in arc_futs]
         dir_sha1s = [f.result() for f in dir_futs]
 

scripts/test_atlas_registry.py

@@ -103,3 +103,59 @@ def test_split_ids_accepts_either_separator():
     assert atlas_registry.split_ids("") == []
     assert atlas_registry.split_ids(None) == []
     assert atlas_registry.split_ids("   ") == []
+
+
+# --- normalise_url ---------------------------------------------------------
+# Every match decision in every discovery source runs through this, so the
+# collapses it makes and the ones it refuses are both worth pinning.
+
+@pytest.mark.parametrize("a,b", [
+    ("https://example.com/a.zip", "http://example.com/a.zip"),          # scheme
+    ("https://example.com/a.zip", "https://www.example.com/a.zip"),      # www.
+    ("https://example.com/a/", "https://example.com/a"),                 # trailing slash
+    ("https://EXAMPLE.com/A.zip", "https://example.com/a.zip"),          # host and path case
+    ("  https://example.com/a.zip  ", "https://example.com/a.zip"),      # surrounding space
+    ("example.com/a.zip", "https://example.com/a.zip"),                  # missing scheme
+])
+def test_normalise_url_collapses_differences_that_do_not_change_the_file(a, b):
+    assert atlas_registry.normalise_url(a) == atlas_registry.normalise_url(b)
+
+
+@pytest.mark.parametrize("a,b", [
+    # A trailing dot is a different path, and a real submission once differed from
+    # the registered URL by exactly this. Guessing here would merge two URLs that
+    # a human needs to see separately.
+    ("https://example.com/a.zip.", "https://example.com/a.zip"),
+    ("https://example.com/a.zip?v=1", "https://example.com/a.zip"),      # query is significant
+    ("https://example.com/a.zip", "https://example.com/b.zip"),
+    ("https://example.com/a.zip", "https://other.com/a.zip"),
+    # www. is stripped only as a prefix, never mid-host
+    ("https://www.example.com/a", "https://example.www.com/a"),
+])
+def test_normalise_url_keeps_differences_that_might_matter(a, b):
+    assert atlas_registry.normalise_url(a) != atlas_registry.normalise_url(b)
+
+
+def test_normalise_url_handles_empty_input():
+    assert atlas_registry.normalise_url("") == ""
+    assert atlas_registry.normalise_url(None) == ""
+
+
+# --- normalise_ntd_id ------------------------------------------------------
+
+@pytest.mark.parametrize("raw,expected", [
+    ("1", "00001"),
+    ("90252", "90252"),
+    (" 123 ", "00123"),
+    ("", ""),
+    ("R-12", "R-12"),      # not all digits, so left alone rather than padded
+])
+def test_normalise_ntd_id(raw, expected):
+    assert atlas_registry.normalise_ntd_id(raw) == expected
+
+
+def test_operator_feeds_can_filter_by_spec(feeds_dir):
+    db = atlas_registry.load(feeds_dir)
+    assert atlas_registry.operator_feeds(db, "o-one") == {"f-one", "f-one~rt"}
+    assert atlas_registry.operator_feeds(db, "o-one", spec="gtfs-rt") == {"f-one~rt"}
+    assert atlas_registry.operator_feeds(db, "o-one", spec="gtfs") == {"f-one"}

scripts/test_feed_probe.py

@@ -0,0 +1,95 @@
+"""Tests for the pure parts of feed_probe.
+
+Everything here runs offline. The functions that shell out to `transitland` or
+hit the network are deliberately not covered: their failure modes are timeouts
+and a missing binary, both already handled and neither cheap to simulate
+usefully.
+
+Run: cd scripts && uv run --with pytest pytest -q
+"""
+
+import os
+import sys
+from datetime import date
+
+import pytest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import feed_probe  # noqa: E402
+
+TODAY = date(2026, 8, 8)
+
+
+# --- calendar_status -------------------------------------------------------
+# Expiry is the field most registration decisions turn on, and the boundaries
+# are where date arithmetic goes wrong.
+
+def test_calendar_active_midway():
+    status, expired = feed_probe.calendar_status("2026-01-01", "2026-12-31", TODAY)
+    assert status.startswith("active")
+    assert expired is None
+
+
+def test_calendar_expiring_today_is_still_active():
+    # The last day of service is inclusive; a feed is not expired until the day
+    # after its latest date.
+    status, expired = feed_probe.calendar_status("2026-01-01", "2026-08-08", TODAY)
+    assert status.startswith("active")
+    assert expired is None
+
+
+def test_calendar_expired_yesterday_reports_one_day():
+    status, expired = feed_probe.calendar_status("2026-01-01", "2026-08-07", TODAY)
+    assert expired == 1
+    assert status == "expired 1d"
+
+
+def test_calendar_long_expired_reports_the_distance():
+    # The distinction the docstring cares about: mid-refresh versus abandoned.
+    _, expired = feed_probe.calendar_status("2021-09-01", "2024-10-01", TODAY)
+    assert expired == 676
+
+
+def test_calendar_starting_in_future():
+    status, expired = feed_probe.calendar_status("2026-09-01", "2026-12-31", TODAY)
+    assert status.startswith("future")
+    assert expired is None
+
+
+def test_calendar_single_day_span_does_not_divide_by_zero():
+    status, expired = feed_probe.calendar_status("2026-08-08", "2026-08-08", TODAY)
+    assert status.startswith("active")
+    assert expired is None
+
+
+@pytest.mark.parametrize("earliest,latest", [
+    ("", "2026-12-31"),
+    ("2026-01-01", ""),
+    ("", ""),
+    (None, None),
+])
+def test_calendar_missing_dates_are_unknown_not_expired(earliest, latest):
+    assert feed_probe.calendar_status(earliest, latest, TODAY) == ("unknown", None)
+
+
+def test_calendar_accepts_gtfs_wire_format_dates():
+    # GTFS writes dates as YYYYMMDD, which date.fromisoformat parses as ISO 8601
+    # basic format. Worth pinning: it means dates lifted straight out of a feed
+    # need no conversion, and a future tightening of the parser would break that
+    # silently.
+    status, expired = feed_probe.calendar_status("20260101", "20261231", TODAY)
+    assert status.startswith("active")
+    assert expired is None
+    assert feed_probe.calendar_status("20260101", "20260807", TODAY)[1] == 1
+
+
+@pytest.mark.parametrize("earliest,latest", [
+    ("2026-13-01", "2026-12-31"),  # not a real month
+    ("garbage", "2026-12-31"),
+    ("2026-01-01\r", "2026-12-31"),  # stray carriage return, seen in a real feed
+])
+def test_calendar_unparseable_dates_are_reported_as_such(earliest, latest):
+    # A feed whose dates cannot be read is a different finding from one with no
+    # dates at all: a real archive was published with stray carriage returns in
+    # its date fields, and reporting that as "unknown" would have hidden it.
+    assert feed_probe.calendar_status(earliest, latest, TODAY) == ("unparseable", None)

scripts/test_scan_feed_sources.py

@@ -0,0 +1,133 @@
+"""Tests for the pure helpers in scan-feed-sources.
+
+Only the offline parts: extraction from a fetched body, and policy matching.
+The sources themselves talk to the network and to the registry, and are
+exercised by running them.
+
+Imported by path because the script is named with hyphens and so is not a
+importable module name.
+
+Run: cd scripts && uv run --with pytest pytest -q
+"""
+
+import importlib.util
+import os
+import pathlib
+import sys
+
+import pytest
+
+HERE = pathlib.Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+
+_spec = importlib.util.spec_from_file_location("scan_feed_sources", HERE / "scan-feed-sources.py")
+scan = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(scan)
+
+
+# --- feed-list extraction --------------------------------------------------
+# A catalogue that yields nothing looks identical to a catalogue that has not
+# changed, which is how a real source sat silently unwatched: MnDOT's hub
+# contains no absolute URLs at all, so URL extraction returned an empty set for
+# a perfectly healthy page.
+
+def test_urls_extracted_from_html_anchors():
+    body = '''<a href="https://example.com/gtfs.zip">feed</a>
+               <a href="https://example.com/about.html">about</a>'''
+    assert scan._feed_list_urls(body) == ["https://example.com/gtfs.zip"]
+
+
+def test_urls_extracted_from_plain_text_and_csv():
+    body = "agency,url\nExample,https://example.com/google_transit.zip\n"
+    assert scan._feed_list_urls(body) == ["https://example.com/google_transit.zip"]
+
+
+@pytest.mark.parametrize("url", [
+    "https://example.com/logo.png",
+    "https://example.com/app.js",
+    "https://example.com/style.css",
+    "https://example.com/timetable.pdf",
+])
+def test_assets_that_match_the_hint_are_still_excluded(url):
+    # "download" and "gtfs" appear in plenty of asset paths.
+    assert scan._feed_list_urls(f'<a href="{url}">x</a>') == []
+
+
+def test_urls_are_deduplicated_and_sorted():
+    body = ('https://b.example.com/gtfs.zip https://a.example.com/gtfs.zip '
+            'https://b.example.com/gtfs.zip')
+    assert scan._feed_list_urls(body) == [
+        "https://a.example.com/gtfs.zip", "https://b.example.com/gtfs.zip"]
+
+
+def test_trailing_punctuation_is_stripped_from_prose_urls():
+    body = "The feed is at https://example.com/gtfs.zip, updated weekly."
+    assert scan._feed_list_urls(body) == ["https://example.com/gtfs.zip"]
+
+
+def test_json_values_extraction_finds_feeds_identified_by_id():
+    # The MnDOT shape: identifiers, no URLs anywhere.
+    body = ('[{"feed_id":"browncounty-mn-us","fileNames":["brown_county.zip"]},'
+            '{"feed_id":"metro-mn-us","fileNames":["metro.zip"]}]')
+    assert scan._feed_list_json_values(body) == [
+        "brown_county.zip", "browncounty-mn-us", "metro-mn-us", "metro.zip"]
+
+
+def test_json_values_walks_nested_structures():
+    body = '{"a":{"b":["x",{"c":"y"}]},"d":"z"}'
+    assert scan._feed_list_json_values(body) == ["x", "y", "z"]
+
+
+def test_json_values_rejects_non_json_loudly():
+    # Better to report a source as unreadable than to record an empty set and
+    # report every feed as removed on the next run.
+    with pytest.raises(ValueError):
+        scan._feed_list_json_values("<html>not json</html>")
+
+
+def test_url_extraction_returns_nothing_for_the_json_shape():
+    # Pins the bug that motivated json-values mode: this page is healthy, and
+    # URL extraction still finds nothing.
+    body = '[{"feed_id":"browncounty-mn-us","fileNames":["brown_county.zip"]}]'
+    assert scan._feed_list_urls(body) == []
+
+
+# --- policy matching -------------------------------------------------------
+
+def _policies():
+    return [
+        {"name": "by-operator", "operators": {"o-one"}, "url_prefixes": ()},
+        {"name": "by-url", "operators": set(),
+         "url_prefixes": ("https://api.example.org/transit/",)},
+        # A policy that states a position rather than a place: it names no feeds
+        # and no prefixes, so it must never match anything by accident.
+        {"name": "position-only", "operators": set()},
+    ]
+
+
+def test_policy_matches_on_operator():
+    assert scan.policy_for(_policies(), {"o-one"})["name"] == "by-operator"
+
+
+def test_policy_matches_on_url_prefix():
+    p = scan.policy_for(_policies(), set(), "https://api.example.org/transit/x/gtfs")
+    assert p["name"] == "by-url"
+
+
+def test_policy_url_match_ignores_scheme_and_host_case():
+    p = scan.policy_for(_policies(), set(), "HTTP://API.Example.ORG/transit/x/gtfs")
+    assert p["name"] == "by-url"
+
+
+def test_policy_accepts_a_bare_string_of_urls():
+    assert scan.policy_for(_policies(), set(), "https://elsewhere.example/x") is None
+
+
+def test_policy_without_feeds_or_prefixes_never_matches():
+    # position-only is last, so anything reaching it would return it.
+    assert scan.policy_for(_policies(), {"o-unknown"}, ()) is None
+    assert scan.policy_for(_policies(), set(), "https://nothing.example/") is None
+
+
+def test_policy_returns_none_when_nothing_applies():
+    assert scan.policy_for([], {"o-one"}, "https://example.com/x") is None