collect_5min_bars: isolate per-symbol failures so one bad CSV can't stall the rest

A corrupt last line in XRPUSDT.csv (truncated "2026" string) and an empty
UNIUSDT.csv had been raising in append_bars on every cycle, with the
existing cycle-level try/except catching the exception AFTER the for-loop
had already moved on, so all symbols *after* the bad CSV in the rotation
were silently skipped. 11 of 17 binance symbols had been stale by 27-34
days as a result.

Move the per-symbol fetch + append into its own try/except inside the
loop, pull the cycle into a testable run_collection_cycle helper, and
also wrap the daily backfill scheduler so a backfill exception cannot
suppress the next cycle either.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: lee101

scripts/collect_5min_bars.py

@@ -79,6 +79,27 @@ def daily_backfill(symbols: list, out_root: Path):
         logger.warning(f"backfill failed: {e}")
 
 
+def run_collection_cycle(symbols: list[str], out_root: Path, last_backfill):
+    for sym in symbols:
+        try:
+            bars = fetch_recent_5m(sym, limit=3)
+            path = out_root / f"{sym}.csv"
+            n = append_bars(path, bars)
+            if n > 0:
+                logger.info(f"{sym}: +{n} bars")
+        except Exception as e:
+            logger.error(f"{sym} fetch/append failed: {e}")
+
+    try:
+        now = datetime.now(timezone.utc)
+        if last_backfill is None or (now - last_backfill).total_seconds() > 86400:
+            daily_backfill(symbols, out_root)
+            last_backfill = now
+    except Exception as e:
+        logger.error(f"backfill scheduler error: {e}")
+    return last_backfill
+
+
 def main():
     p = argparse.ArgumentParser()
     p.add_argument("--symbols", nargs="+", default=["DOGEUSDT"])
@@ -92,22 +113,7 @@ def main():
     last_backfill = None
 
     while True:
-        try:
-            for sym in args.symbols:
-                bars = fetch_recent_5m(sym, limit=3)
-                path = args.out_root / f"{sym}.csv"
-                n = append_bars(path, bars)
-                if n > 0:
-                    logger.info(f"{sym}: +{n} bars")
-
-            now = datetime.now(timezone.utc)
-            if last_backfill is None or (now - last_backfill).total_seconds() > 86400:
-                daily_backfill(args.symbols, args.out_root)
-                last_backfill = now
-
-        except Exception as e:
-            logger.error(f"cycle error: {e}")
-
+        last_backfill = run_collection_cycle(args.symbols, args.out_root, last_backfill)
         time.sleep(args.interval_seconds)
 
 

tests/test_collect_5min_bars.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+import pandas as pd
+
+from scripts import collect_5min_bars
+
+
+def test_collection_cycle_continues_after_symbol_failure(
+    monkeypatch,
+    tmp_path: Path,
+) -> None:
+    fetched: list[str] = []
+    appended: list[str] = []
+
+    def fake_fetch(symbol: str, *, limit: int) -> pd.DataFrame:
+        fetched.append(symbol)
+        if symbol == "BADUSDT":
+            raise RuntimeError("temporary fetch failure")
+        return pd.DataFrame([{"timestamp": pd.Timestamp("2026-01-01T00:00:00Z")}])
+
+    def fake_append(path: Path, new: pd.DataFrame) -> int:
+        appended.append(path.stem)
+        assert not new.empty
+        return len(new)
+
+    monkeypatch.setattr(collect_5min_bars, "fetch_recent_5m", fake_fetch)
+    monkeypatch.setattr(collect_5min_bars, "append_bars", fake_append)
+    monkeypatch.setattr(collect_5min_bars, "daily_backfill", lambda symbols, out_root: None)
+
+    last_backfill = collect_5min_bars.run_collection_cycle(
+        ["BADUSDT", "GOODUSDT"],
+        tmp_path,
+        None,
+    )
+
+    assert fetched == ["BADUSDT", "GOODUSDT"]
+    assert appended == ["GOODUSDT"]
+    assert isinstance(last_backfill, datetime)
+
+
+def test_collection_cycle_preserves_backfill_time_after_scheduler_failure(
+    monkeypatch,
+    tmp_path: Path,
+) -> None:
+    previous_backfill = datetime(2026, 1, 1, tzinfo=timezone.utc)
+
+    monkeypatch.setattr(
+        collect_5min_bars,
+        "fetch_recent_5m",
+        lambda symbol, *, limit: pd.DataFrame(),
+    )
+    monkeypatch.setattr(collect_5min_bars, "append_bars", lambda path, new: 0)
+
+    def fail_backfill(symbols: list[str], out_root: Path) -> None:
+        raise RuntimeError("backfill scheduler failure")
+
+    monkeypatch.setattr(collect_5min_bars, "daily_backfill", fail_backfill)
+
+    last_backfill = collect_5min_bars.run_collection_cycle(
+        ["DOGEUSDT"],
+        tmp_path,
+        previous_backfill,
+    )
+
+    assert last_backfill == previous_backfill