Merge pull request #6074 from colophon-group/fix-crawler/error-review-private-identifiers

Redact private identifiers from error review evidence

Author: viktor-shcherb

apps/crawler/VERSION

@@ -1 +1 @@
-0.13.193
+0.13.194

apps/crawler/tests/test_error_review_bundle.py

@@ -13,6 +13,42 @@
 SPEC.loader.exec_module(bundle)
 
 
+def test_redact_removes_private_identifiers_and_quoted_credentials():
+    raw_resource_id = "a" * 64
+    text = bundle._redact(
+        """
+        endpoint=192.0.2.4 peer=[2001:db8::1]:8108
+        posting=123e4567-e89b-12d3-a456-426614174000
+        image=sha256:{raw_resource_id}
+        TOKEN=plain-secret
+        "api_key": "json-secret"
+        "Authorization": "Bearer bearer-secret"
+        """.replace("{raw_resource_id}", raw_resource_id).strip()
+    )
+
+    for private_value in (
+        "192.0.2.4",
+        "2001:db8::1",
+        "123e4567-e89b-12d3-a456-426614174000",
+        raw_resource_id,
+        "plain-secret",
+        "json-secret",
+        "bearer-secret",
+    ):
+        assert private_value not in text
+    assert text.count("<redacted-host-address>") == 2
+    assert text.count("<redacted-resource-id>") == 2
+    assert text.count("<redacted>") == 3
+
+    redacted_json = bundle._redact(
+        '{"api_key": "json-secret", "Authorization": "Bearer bearer-secret"}'
+    )
+    assert json.loads(redacted_json) == {
+        "api_key": "<redacted>",
+        "Authorization": "Bearer <redacted>",
+    }
+
+
 def test_parse_cgroup_key_values_ignores_malformed_rows():
     assert bundle._parse_cgroup_key_values("low 0\nhigh 2\nmalformed\noom nope\noom_kill 3\n") == {
         "low": 0,

apps/crawler/tests/test_maintenance_provenance.py

@@ -111,7 +111,7 @@ def test_graceful_sigterm_exit_is_not_reported_as_forced_termination():
     assert pause["forced_termination"] is False
 
 
-def test_expected_marker_sigterm_brackets_window_without_becoming_a_failed_oneoff():
+def test_expected_marker_termination_brackets_window_without_becoming_a_failed_oneoff():
     provenance_contract = {
         "operation": "crawler-deploy",
         "issue": 3409,
@@ -151,7 +151,7 @@ def test_expected_marker_sigterm_brackets_window_without_becoming_a_failed_oneof
             "container_generation": "marker-generation",
             "compose_service": "maintenance-window",
             "compose_oneoff": "True",
-            "event_exit_code": "143",
+            "event_exit_code": "137",
             "maintenance_provenance": provenance_contract,
         },
     ]
@@ -165,6 +165,105 @@ def test_expected_marker_sigterm_brackets_window_without_becoming_a_failed_oneof
     assert window["service_pauses"][0]["downtime_seconds"] == 50.0
 
 
+def test_transient_unmonitored_service_does_not_stretch_a_maintenance_window():
+    provenance_contract = {
+        "operation": "crawler-deploy",
+        "issue": 3409,
+        "revision": "d" * 40,
+        "budget_seconds": 1800,
+    }
+    events = [
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T04:00:00+00:00",
+            "action": "die",
+            "container_generation": "old-init-generation",
+            "compose_service": "murmur-shim-runtime-init",
+            "compose_oneoff": "False",
+            "event_exit_code": "0",
+        },
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T05:00:00+00:00",
+            "action": "start",
+            "container_generation": "marker-generation",
+            "compose_service": "maintenance-window",
+            "compose_oneoff": "True",
+            "maintenance_provenance": provenance_contract,
+        },
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T05:01:00+00:00",
+            "action": "die",
+            "container_generation": "marker-generation",
+            "compose_service": "maintenance-window",
+            "compose_oneoff": "True",
+            "event_exit_code": "137",
+            "maintenance_provenance": provenance_contract,
+        },
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T05:01:01+00:00",
+            "action": "start",
+            "container_generation": "new-init-generation",
+            "compose_service": "murmur-shim-runtime-init",
+            "compose_oneoff": "False",
+        },
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T05:01:05+00:00",
+            "action": "die",
+            "container_generation": "new-init-generation",
+            "compose_service": "murmur-shim-runtime-init",
+            "compose_oneoff": "False",
+            "event_exit_code": "0",
+        },
+    ]
+
+    summary = provenance.correlate_events(events)
+
+    assert summary["unattributed_service_pauses"] == []
+    window = summary["maintenance_windows"][0]
+    assert window["duration_seconds"] == 60.0
+    assert window["service_pauses"] == []
+    assert window["status"] == "completed"
+
+
+def test_marker_oom_remains_actionable():
+    provenance_contract = {
+        "operation": "crawler-deploy",
+        "issue": 3409,
+        "revision": "d" * 40,
+        "budget_seconds": 1800,
+    }
+    events = [
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T05:00:00+00:00",
+            "action": "start",
+            "container_generation": "marker-generation",
+            "compose_service": "maintenance-window",
+            "compose_oneoff": "True",
+            "maintenance_provenance": provenance_contract,
+        },
+        {
+            "source": "docker_event",
+            "event_at": "2026-07-23T05:00:10+00:00",
+            "action": "oom",
+            "container_generation": "marker-generation",
+            "compose_service": "maintenance-window",
+            "compose_oneoff": "True",
+            "state": {"oom_killed": True},
+            "maintenance_provenance": provenance_contract,
+        },
+    ]
+
+    window = provenance.correlate_events(events)["maintenance_windows"][0]
+
+    assert window["status"] == "actionable"
+    assert window["actionable_reasons"] == ["oom"]
+
+
 def test_missing_service_restoration_remains_actionable():
     events = [
         event

docs/14-error-review-routine.md

@@ -60,6 +60,12 @@ survive Docker's volatile event buffer and container recreation. The root
 collector allowlists the journal again and one-way transforms raw IDs from
 legacy schema rows before the bundle becomes readable by `codex-runner`.
 
+Every file written to the runner bundle passes the same final redaction
+boundary. In addition to credential shapes, it removes IPv4/IPv6 host
+addresses, UUID-shaped domain/resource identifiers, and raw 64-hex
+Docker/image identifiers. The 16-hex `container_generation` remains available
+for same-generation comparisons without exposing the underlying resource ID.
+
 Production maintenance attribution is deterministic rather than inferred from
 container names. Repository-owned one-offs and the
 `/usr/local/sbin/jobseek-maintenance` wrapper attach an all-or-nothing
@@ -73,15 +79,21 @@ classifying instability:
 
 - A single validated maintenance window owns only service pauses that overlap
   it or its bounded two-minute correlation edge.
+- Service-pause correlation is limited to the eight monitored long-running
+  crawler services: Redis, three HTTP workers, the browser worker, exporter,
+  drain, and Alloy. Transient Compose init services cannot stretch a crawler
+  maintenance window.
 - Missing, partial, invalid, or conflicting provenance remains unattributed;
   never infer authorization from a name or an adjacent unlabelled one-off.
 - Authorized maintenance is reported as a maintenance outcome, with its
   tracking issue, revision, downtime, termination mode, and restoration
   health, rather than as spontaneous worker instability.
 - OOM/native exits, forced termination, nonzero maintenance one-offs, budget
   overruns, and failed restoration remain actionable even in an authorized
-  window. Update the maintenance issue or create a deduplicated operational
-  follow-up instead of reopening an unrelated instability issue.
+  window. The wrapper-owned marker's expected termination is not a failed
+  maintenance one-off; marker OOM still is. Update the maintenance issue or
+  create a deduplicated operational follow-up instead of reopening an
+  unrelated instability issue.
 
 Compatibility fallback:
 [`.claude/commands/jobseek-error-review.md`](../.claude/commands/jobseek-error-review.md).

scripts/codex-error-review-bundle.py

@@ -9,6 +9,7 @@
 from __future__ import annotations
 
 import argparse
+import ipaddress
 import json
 import os
 import re
@@ -47,17 +48,17 @@
 REDACTIONS: tuple[tuple[re.Pattern[str], str], ...] = (
     (
         re.compile(
-            r"(?i)\b(authorization|proxy-authorization)\s*[:=]\s*"
-            r"(bearer|basic)\s+[A-Za-z0-9._~+/\-]+=*"
+            r"""(?i)(\b(?:authorization|proxy-authorization)["']?\s*[:=]\s*"""
+            r"""["']?(?:bearer|basic)\s+)[A-Za-z0-9._~+/\-]+=*"""
         ),
-        r"\1: <redacted>",
+        r"\1<redacted>",
     ),
     (
         re.compile(
-            r"(?i)\b([A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|API[_-]?KEY|PRIVATE[_-]?KEY)"
-            r"[A-Z0-9_]*)\s*[:=]\s*([^\s,;\"']+)"
+            r"""(?i)(\b[A-Z0-9_]*(?:TOKEN|SECRET|PASSWORD|API[_-]?KEY|PRIVATE[_-]?KEY)"""
+            r"""[A-Z0-9_]*["']?\s*[:=]\s*["']?)(?!<redacted>)[^\s,;"']+"""
         ),
-        r"\1=<redacted>",
+        r"\1<redacted>",
     ),
     (
         re.compile(r"(?i)\b(bearer)\s+[A-Za-z0-9._~+/\-]+=*"),
@@ -75,11 +76,30 @@
         "-----BEGIN PRIVATE KEY-----<redacted>-----END PRIVATE KEY-----",
     ),
 )
+IPV4_RE = re.compile(r"(?<![\d.])(?:\d{1,3}\.){3}\d{1,3}(?![\d.])")
+IPV6_CANDIDATE_RE = re.compile(
+    r"(?<![0-9A-Fa-f:])(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}"
+    r"(?![0-9A-Fa-f:])"
+)
+UUID_RE = re.compile(r"(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b")
+RAW_RESOURCE_ID_RE = re.compile(r"(?i)(?<![0-9a-f])[0-9a-f]{64}(?![0-9a-f])")
+
+
+def _redact_ipv6(match: re.Match[str]) -> str:
+    try:
+        address = ipaddress.ip_address(match.group(0))
+    except ValueError:
+        return match.group(0)
+    return "<redacted-host-address>" if address.version == 6 else match.group(0)
 
 
 def _redact(text: str) -> str:
     for pattern, replacement in REDACTIONS:
         text = pattern.sub(replacement, text)
+    text = IPV4_RE.sub("<redacted-host-address>", text)
+    text = IPV6_CANDIDATE_RE.sub(_redact_ipv6, text)
+    text = UUID_RE.sub("<redacted-resource-id>", text)
+    text = RAW_RESOURCE_ID_RE.sub("<redacted-resource-id>", text)
     return text
 
 

scripts/jobseek_maintenance_provenance.py

@@ -30,6 +30,18 @@
 
 CORRELATION_PADDING_SECONDS = 120
 MERGE_GAP_SECONDS = 180
+MONITORED_SERVICES = frozenset(
+    {
+        "redis",
+        "worker-1",
+        "worker-2",
+        "worker-3",
+        "browser-1",
+        "exporter",
+        "drain",
+        "alloy",
+    }
+)
 
 LIFECYCLE_FIELDS = frozenset(
     {
@@ -336,7 +348,7 @@ def _service_pauses(
     by_service: dict[str, list[tuple[datetime, dict[str, Any]]]] = {}
     for event_at, event in events:
         service = str(event.get("compose_service", ""))
-        if not service or _is_oneoff(event) or service == "maintenance-window":
+        if service not in MONITORED_SERVICES or _is_oneoff(event):
             continue
         action = str(event.get("action", ""))
         if action not in {"kill", "stop", "die", "oom", "start", "restart"}:
@@ -438,7 +450,7 @@ def _public_window(window: dict[str, Any]) -> dict[str, object]:
     for interval in sorted(window["oneoffs"], key=lambda item: item["start"]):
         code = interval.get("exit_code")
         is_marker = interval["compose_service"] == "maintenance-window"
-        if code not in (None, 0) and not (is_marker and code == 143):
+        if code not in (None, 0) and not is_marker:
             reasons.add("oneoff_nonzero_exit")
         if interval.get("oom"):
             reasons.add("oom")