Skip to content

Commit df47275

Browse files
druvusclaude
andcommitted
docs: document accession_inputs and seed in GenerateConfig docstring
Round 9 (wheel hygiene + test stability + docstring coverage) found the GenerateConfig dataclass docstring was missing entries for the two recent additions: - accession_inputs (round 7, --accession feature) - seed (round 8, byte-reproducible runs) Added both Attribute entries; round 9 docstring-coverage check now passes (ReplayConfig: 18/18 fields documented; GenerateConfig: 28/28). Round 9 audit driver bin/audit_wheel_stability.py covers: - Y: wheel ships only the importable package (no bin/, examples/, tests/, docs/, or __pycache__); the dist-info METADATA / RECORD / WHEEL files are present. - Y: sdist contains no build artefacts (__pycache__, .pyc, /build/, /.git/). The standard setuptools nanorunner.egg-info/ is allowed. - Z: three consecutive pytest runs produce identical normalized summaries (count + warnings, timing stripped). - AA: every public symbol re-exported from nanopore_simulator has a docstring; ReplayConfig and GenerateConfig dataclass docstrings cover every field. All 6 scenarios pass; the only code change was the docstring fix. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 733df93 commit df47275

2 files changed

Lines changed: 363 additions & 0 deletions

File tree

bin/audit_wheel_stability.py

Lines changed: 357 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,357 @@
1+
#!/usr/bin/env python
2+
"""Round 9: wheel hygiene + test-suite stability + API docstring coverage.
3+
4+
Round 7 confirmed that the wheel builds and installs into a fresh
5+
venv. This round looks one layer deeper:
6+
7+
- Y (wheel hygiene): what does the wheel/sdist actually contain? Are
8+
there developer artifacts (audit drivers, examples, tests, stray
9+
__pycache__) shipped to end users that shouldn't be?
10+
- Z (test stability): is the suite deterministic? Three consecutive
11+
runs should produce the same pass count and the same set of test
12+
IDs in the same order.
13+
- AA (docstring coverage): every public symbol re-exported from
14+
``nanopore_simulator`` should have a docstring.
15+
16+
Usage:
17+
python bin/audit_wheel_stability.py --root /tmp/audit-2026-05-28-round9
18+
"""
19+
20+
from __future__ import annotations
21+
22+
import argparse
23+
import inspect
24+
import json
25+
import logging
26+
import shutil
27+
import subprocess
28+
import sys
29+
import time
30+
import traceback
31+
import zipfile
32+
from dataclasses import dataclass, field
33+
from pathlib import Path
34+
from typing import Dict, List, Optional, Set
35+
36+
logger = logging.getLogger("audit-r9")
37+
REPO_ROOT = Path(__file__).resolve().parent.parent
38+
39+
40+
@dataclass
41+
class Finding:
42+
scenario: str
43+
phase: str
44+
passed: bool = True
45+
details: List[str] = field(default_factory=list)
46+
duration_s: float = 0.0
47+
48+
def add(self, m: str) -> None:
49+
self.details.append(m)
50+
51+
def fail(self, m: str) -> None:
52+
self.passed = False
53+
self.details.append(f"FAIL: {m}")
54+
55+
56+
def _reset(p: Path) -> Path:
57+
if p.exists():
58+
shutil.rmtree(p, ignore_errors=True)
59+
p.mkdir(parents=True)
60+
return p
61+
62+
63+
# ---------------------------------------------------------------------
64+
# Phase Y: wheel hygiene
65+
# ---------------------------------------------------------------------
66+
67+
68+
def _build_artifacts(root: Path) -> Optional[Path]:
69+
dist = _reset(root / "dist")
70+
proc = subprocess.run(
71+
[sys.executable, "-m", "build", "--outdir", str(dist)],
72+
cwd=REPO_ROOT,
73+
capture_output=True,
74+
text=True,
75+
timeout=300,
76+
)
77+
if proc.returncode != 0:
78+
return None
79+
return dist
80+
81+
82+
def _wheel_members(wheel: Path) -> List[str]:
83+
with zipfile.ZipFile(wheel) as zf:
84+
return zf.namelist()
85+
86+
87+
def _sdist_members(sdist: Path) -> List[str]:
88+
proc = subprocess.run(
89+
["tar", "-tzf", str(sdist)], capture_output=True, text=True, timeout=30
90+
)
91+
return proc.stdout.splitlines()
92+
93+
94+
def phase_y_wheel(root: Path) -> List[Finding]:
95+
findings: List[Finding] = []
96+
97+
dist = _build_artifacts(root)
98+
if dist is None:
99+
f = Finding(scenario="wheel/build_succeeded", phase="Y", passed=False)
100+
f.fail("python -m build failed")
101+
return [f]
102+
103+
wheel = next(iter(dist.glob("*.whl")), None)
104+
sdist = next(iter(dist.glob("*.tar.gz")), None)
105+
106+
# Y1: wheel must only ship the importable package -- no audit
107+
# drivers, examples, tests, docs, or stray __pycache__.
108+
f = Finding(scenario="wheel/contents_minimal", phase="Y")
109+
t0 = time.perf_counter()
110+
try:
111+
if wheel is None:
112+
f.fail("no wheel produced")
113+
else:
114+
members = _wheel_members(wheel)
115+
disallowed_prefixes = ("bin/", "examples/", "tests/", "docs/")
116+
disallowed = [
117+
m for m in members if any(m.startswith(p) for p in disallowed_prefixes)
118+
]
119+
pycache = [m for m in members if "__pycache__" in m]
120+
f.add(f"total_entries={len(members)}")
121+
f.add(f"disallowed={disallowed[:5]}")
122+
f.add(f"pycache={len(pycache)}")
123+
if disallowed:
124+
f.fail(f"wheel ships developer-only paths: {disallowed[:5]}")
125+
if pycache:
126+
f.fail(f"wheel contains {len(pycache)} __pycache__ entries")
127+
except Exception as exc:
128+
f.fail(f"exception: {exc}\n{traceback.format_exc()}")
129+
f.duration_s = time.perf_counter() - t0
130+
findings.append(f)
131+
132+
# Y2: wheel must include the top-level metadata files declared in
133+
# pyproject (README + LICENSE) so PyPI page renders correctly.
134+
f = Finding(scenario="wheel/metadata_files_present", phase="Y")
135+
t0 = time.perf_counter()
136+
try:
137+
if wheel is None:
138+
f.fail("no wheel produced")
139+
else:
140+
members = _wheel_members(wheel)
141+
dist_info = [m for m in members if "dist-info" in m]
142+
f.add(f"dist_info_entries={len(dist_info)}")
143+
need = {"METADATA": False, "RECORD": False, "WHEEL": False}
144+
for m in dist_info:
145+
for key in need:
146+
if m.endswith(f"/{key}"):
147+
need[key] = True
148+
missing = [k for k, v in need.items() if not v]
149+
if missing:
150+
f.fail(f"wheel missing dist-info files: {missing}")
151+
# The METADATA file should embed the README (long
152+
# description). Quick check that "nanorunner" appears.
153+
metadata_path = next(
154+
(m for m in dist_info if m.endswith("/METADATA")), None
155+
)
156+
if metadata_path:
157+
with zipfile.ZipFile(wheel) as zf:
158+
body = zf.read(metadata_path).decode("utf-8", "replace")
159+
if "nanopore" not in body.lower():
160+
f.fail("wheel METADATA does not mention 'nanopore'")
161+
except Exception as exc:
162+
f.fail(f"exception: {exc}")
163+
f.duration_s = time.perf_counter() - t0
164+
findings.append(f)
165+
166+
# Y3: sdist may legitimately ship tests/examples/bin/docs (it's a
167+
# source distribution), but should NOT ship __pycache__ or build
168+
# artefacts.
169+
f = Finding(scenario="sdist/no_build_artifacts", phase="Y")
170+
t0 = time.perf_counter()
171+
try:
172+
if sdist is None:
173+
f.fail("no sdist produced")
174+
else:
175+
members = _sdist_members(sdist)
176+
# nanorunner.egg-info/ is a standard sdist artefact
177+
# produced by setuptools (PKG-INFO, SOURCES.txt, etc.) --
178+
# not pollution. Only flag things that genuinely should
179+
# not be in a clean sdist.
180+
polluted = [
181+
m
182+
for m in members
183+
if "__pycache__" in m
184+
or m.endswith(".pyc")
185+
or "/build/" in m
186+
or "/.git/" in m
187+
]
188+
f.add(f"total_entries={len(members)} polluted={len(polluted)}")
189+
if polluted:
190+
f.fail(f"sdist contains build artefacts: {polluted[:5]}")
191+
except Exception as exc:
192+
f.fail(f"exception: {exc}")
193+
f.duration_s = time.perf_counter() - t0
194+
findings.append(f)
195+
196+
return findings
197+
198+
199+
# ---------------------------------------------------------------------
200+
# Phase Z: test-suite stability
201+
# ---------------------------------------------------------------------
202+
203+
204+
def phase_z_stability(root: Path) -> List[Finding]:
205+
findings: List[Finding] = []
206+
207+
f = Finding(scenario="suite/three_runs_identical", phase="Z")
208+
t0 = time.perf_counter()
209+
try:
210+
import re
211+
212+
# Compare a normalized form of the summary -- pass/fail counts
213+
# and warning count, with the timing tail stripped so that
214+
# natural runtime jitter does not look like flakiness.
215+
def _normalize(tail: str) -> str:
216+
return re.sub(r"\s*in\s*[\d.]+s.*$", "", tail).strip()
217+
218+
run_summaries: List[str] = []
219+
for i in range(3):
220+
proc = subprocess.run(
221+
[sys.executable, "-m", "pytest", "-q", "--tb=no"],
222+
cwd=REPO_ROOT,
223+
capture_output=True,
224+
text=True,
225+
timeout=300,
226+
)
227+
tail = proc.stdout.strip().split("\n")[-1]
228+
run_summaries.append(_normalize(tail))
229+
if proc.returncode != 0:
230+
f.fail(f"run {i} exited non-zero: {tail}")
231+
break
232+
f.add(f"normalized_summaries={run_summaries}")
233+
if len(set(run_summaries)) > 1:
234+
f.fail(f"summary varied across runs: {sorted(set(run_summaries))}")
235+
except Exception as exc:
236+
f.fail(f"exception: {exc}\n{traceback.format_exc()}")
237+
f.duration_s = time.perf_counter() - t0
238+
findings.append(f)
239+
240+
return findings
241+
242+
243+
# ---------------------------------------------------------------------
244+
# Phase AA: public API docstring coverage
245+
# ---------------------------------------------------------------------
246+
247+
248+
def phase_aa_docstrings(root: Path) -> List[Finding]:
249+
findings: List[Finding] = []
250+
251+
f = Finding(scenario="api/public_symbols_documented", phase="AA")
252+
t0 = time.perf_counter()
253+
try:
254+
# Import the top-level package (the public re-export surface).
255+
import nanopore_simulator as pkg
256+
257+
public = [
258+
(name, getattr(pkg, name)) for name in dir(pkg) if not name.startswith("_")
259+
]
260+
f.add(f"public_symbols={len(public)}")
261+
missing: List[str] = []
262+
for name, obj in public:
263+
# Modules are documented in their own docstring; we check
264+
# them too. Built-in types (str, int) shouldn't appear here.
265+
doc = inspect.getdoc(obj)
266+
if not doc or len(doc.strip()) < 5:
267+
missing.append(name)
268+
if missing:
269+
f.fail(f"no/empty docstring on: {missing}")
270+
except Exception as exc:
271+
f.fail(f"exception: {exc}\n{traceback.format_exc()}")
272+
f.duration_s = time.perf_counter() - t0
273+
findings.append(f)
274+
275+
# AA2: public API methods of ReplayConfig / GenerateConfig (the
276+
# dataclass docstrings should describe every field).
277+
f = Finding(scenario="api/config_dataclass_docstrings", phase="AA")
278+
t0 = time.perf_counter()
279+
try:
280+
from nanopore_simulator.config import GenerateConfig, ReplayConfig
281+
282+
for cls in (ReplayConfig, GenerateConfig):
283+
doc = inspect.getdoc(cls) or ""
284+
field_names = list(cls.__dataclass_fields__.keys())
285+
undocumented = [fn for fn in field_names if fn not in doc and fn != "self"]
286+
f.add(
287+
f"{cls.__name__}: fields={len(field_names)} undocumented={undocumented}"
288+
)
289+
if undocumented:
290+
f.fail(f"{cls.__name__} docstring missing fields: {undocumented}")
291+
except Exception as exc:
292+
f.fail(f"exception: {exc}\n{traceback.format_exc()}")
293+
f.duration_s = time.perf_counter() - t0
294+
findings.append(f)
295+
296+
return findings
297+
298+
299+
def write_report(root: Path, findings: List[Finding]) -> Path:
300+
path = root / "reports" / "wheel-stability-report.md"
301+
path.parent.mkdir(parents=True, exist_ok=True)
302+
failed = [f for f in findings if not f.passed]
303+
by_phase: Dict[str, List[Finding]] = {}
304+
for f in findings:
305+
by_phase.setdefault(f.phase, []).append(f)
306+
lines = [
307+
"# nanorunner wheel + stability + docstring audit (round 9)",
308+
"",
309+
f"- total scenarios: {len(findings)}",
310+
f"- failed: {len(failed)}",
311+
f"- passed: {len(findings) - len(failed)}",
312+
"",
313+
]
314+
for phase in sorted(by_phase):
315+
lines.append(f"## Phase {phase}")
316+
lines.append("")
317+
lines.append("| scenario | result | seconds | details |")
318+
lines.append("|---|---|---|---|")
319+
for f in by_phase[phase]:
320+
status = "PASS" if f.passed else "FAIL"
321+
joined = "<br>".join(d.replace("|", "\\|") for d in f.details)
322+
lines.append(f"| {f.scenario} | {status} | {f.duration_s:.2f} | {joined} |")
323+
lines.append("")
324+
if failed:
325+
lines.append("## Failure detail")
326+
for f in failed:
327+
lines.append(f"### {f.scenario}")
328+
for d in f.details:
329+
lines.append(f"- {d}")
330+
lines.append("")
331+
path.write_text("\n".join(lines))
332+
return path
333+
334+
335+
def main(argv: Optional[List[str]] = None) -> int:
336+
p = argparse.ArgumentParser()
337+
p.add_argument("--root", required=True, type=Path)
338+
args = p.parse_args(argv)
339+
logging.basicConfig(
340+
level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s"
341+
)
342+
root: Path = args.root
343+
root.mkdir(parents=True, exist_ok=True)
344+
(root / "reports").mkdir(exist_ok=True)
345+
346+
findings: List[Finding] = []
347+
findings.extend(phase_y_wheel(root))
348+
findings.extend(phase_z_stability(root))
349+
findings.extend(phase_aa_docstrings(root))
350+
351+
report = write_report(root, findings)
352+
logger.info("wrote %s", report)
353+
return 1 if any(not f.passed for f in findings) else 0
354+
355+
356+
if __name__ == "__main__":
357+
sys.exit(main())

nanopore_simulator/config.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ class GenerateConfig:
179179
species_inputs: Species names to resolve via GTDB/NCBI.
180180
mock_name: Preset mock community name.
181181
taxid_inputs: NCBI taxonomy IDs.
182+
accession_inputs: Explicit NCBI assembly accessions (e.g.
183+
``GCA_000005845.2`` or ``GCF_...``). Mutually exclusive
184+
with the other genome sources.
182185
abundances: Per-genome abundance fractions (must sum to ~1.0).
183186
read_count: Total reads to generate across all organisms.
184187
interval: Base seconds between batch operations.
@@ -200,6 +203,9 @@ class GenerateConfig:
200203
adapter: Pipeline adapter name, or None.
201204
structure: Output directory layout -- "singleplex" or "multiplex".
202205
offline_mode: Use only cached genomes, no network requests.
206+
seed: Explicit RNG seed for byte-reproducible runs. When None,
207+
the runner derives a deterministic seed from ``target_dir``
208+
so the same target still yields the same output.
203209
"""
204210

205211
target_dir: Path

0 commit comments

Comments
 (0)