Merge dev: gz inspect.txt support + unmapped-validation-taxid warning

Author: druvus

nanometa_live/app/tabs/kraken2_helpers.py

@@ -203,25 +203,41 @@ def load_kraken2_taxonomy(kraken_db_path: str) -> dict:
         Dict mapping taxid -> parent_taxid. Returns empty dict if the
         inspect.txt file is missing or cannot be read.
     """
+    import gzip
     import os
 
     if not kraken_db_path:
         return {}
 
-    inspect_path = os.path.join(kraken_db_path, "inspect.txt")
-    if inspect_path in _TAXONOMY_CACHE:
-        return _TAXONOMY_CACHE[inspect_path]
-
-    if not os.path.exists(inspect_path):
-        logging.debug(f"Kraken2 inspect.txt not found at {inspect_path}")
-        _TAXONOMY_CACHE[inspect_path] = {}
+    # Cache on the database directory so a gz-only DB is cached too (the cache
+    # key used to be the plain inspect.txt path).
+    if kraken_db_path in _TAXONOMY_CACHE:
+        return _TAXONOMY_CACHE[kraken_db_path]
+
+    # Prefer a plain inspect.txt; fall back to a gzipped inspect.txt.gz. Some
+    # Kraken2 builds (e.g. GTDB-derived / size-conscious DBs) ship only the
+    # compressed form. The DB indexer already reads the .gz
+    # (database_indexer.build_from_inspect_gz); mirror that here so the
+    # authoritative-taxonomy correction for Sankey/Sunburst is not silently
+    # disabled on those databases.
+    plain_path = os.path.join(kraken_db_path, "inspect.txt")
+    gz_path = os.path.join(kraken_db_path, "inspect.txt.gz")
+    if os.path.exists(plain_path):
+        inspect_path = plain_path
+        _opener = lambda p: open(p)
+    elif os.path.exists(gz_path):
+        inspect_path = gz_path
+        _opener = lambda p: gzip.open(p, "rt", encoding="utf-8")
+    else:
+        logging.debug(f"Kraken2 inspect.txt[.gz] not found in {kraken_db_path}")
+        _TAXONOMY_CACHE[kraken_db_path] = {}
         return {}
 
     taxid_to_parent: dict = {}
     indent_stack = []  # list of (indent, taxid)
 
     try:
-        with open(inspect_path) as f:
+        with _opener(inspect_path) as f:
             for line in f:
                 if line.startswith("#"):
                     continue
@@ -241,16 +257,18 @@ def load_kraken2_taxonomy(kraken_db_path: str) -> dict:
                 parent = indent_stack[-1][1] if indent_stack else 0
                 taxid_to_parent[taxid] = parent
                 indent_stack.append((indent, taxid))
-    except OSError as exc:
-        logging.warning(f"Failed to read Kraken2 inspect.txt: {exc}")
-        _TAXONOMY_CACHE[inspect_path] = {}
+    except (OSError, EOFError) as exc:
+        # EOFError / gzip.BadGzipFile (an OSError subclass) cover a truncated or
+        # malformed .gz; a plain-text OSError covers an unreadable inspect.txt.
+        logging.warning(f"Failed to read Kraken2 {os.path.basename(inspect_path)}: {exc}")
+        _TAXONOMY_CACHE[kraken_db_path] = {}
         return {}
 
     logging.info(
         f"Loaded Kraken2 taxonomy from {inspect_path}: "
         f"{len(taxid_to_parent)} taxa"
     )
-    _TAXONOMY_CACHE[inspect_path] = taxid_to_parent
+    _TAXONOMY_CACHE[kraken_db_path] = taxid_to_parent
     return taxid_to_parent
 
 

nanometa_live/core/config/parameter_mapping.py

@@ -148,6 +148,12 @@ def get_validation_species_from_watchlist(
 
         species_list = []
         genome_paths = []
+        # Track entries whose Kraken2 taxid fell back to the raw NCBI taxid
+        # because nothing mapped them. On a GTDB/custom-taxid database (where the
+        # NCBI taxid is absent), read extraction by that taxid yields ZERO reads
+        # and validation silently produces nothing -- the failure mode found
+        # auditing a custom-taxid DB. We surface it as a warning below.
+        unmapped_names = []
 
         for entry in entries:
             ncbi_taxid = getattr(entry, 'taxid', 0)
@@ -159,9 +165,11 @@ def get_validation_species_from_watchlist(
             # so the operator does not have to run "Scan Database"; otherwise
             # fall back to the auto-mapping collection, then to the NCBI taxid.
             kraken_taxid = ncbi_taxid  # Default to NCBI taxid
+            mapped = False
             explicit_db_taxid = getattr(entry, 'db_taxid', None)
             if explicit_db_taxid:
                 kraken_taxid = explicit_db_taxid
+                mapped = True
                 logging.debug(
                     f"Using explicit db_taxid {explicit_db_taxid} for "
                     f"{getattr(entry, 'name', '')}"
@@ -170,10 +178,14 @@ def get_validation_species_from_watchlist(
                 db_taxid = mapping_collection.get_db_taxid(ncbi_taxid)
                 if db_taxid:
                     kraken_taxid = db_taxid
+                    mapped = True
                     logging.debug(
                         f"Mapped NCBI {ncbi_taxid} -> Kraken2 {db_taxid} for {getattr(entry, 'name', '')}"
                     )
 
+            if not mapped:
+                unmapped_names.append(getattr(entry, 'name', '') or str(ncbi_taxid))
+
             species_info = {
                 'taxid': ncbi_taxid,
                 'kraken_taxid': kraken_taxid,
@@ -190,6 +202,31 @@ def get_validation_species_from_watchlist(
             f"Found {len(species_list)} enabled watchlist species, "
             f"{len(genome_paths)} with downloaded genomes"
         )
+
+        # Warn loudly when validation taxids fell back to raw NCBI taxids: on a
+        # custom/GTDB database this silently extracts 0 reads. Distinguish "never
+        # scanned" (no mapping at all) from "scanned but these did not map".
+        if unmapped_names:
+            preview = ", ".join(unmapped_names[:5]) + (
+                f" (+{len(unmapped_names) - 5} more)" if len(unmapped_names) > 5 else "")
+            if mapping_collection is None:
+                logging.warning(
+                    "No Kraken2 taxid mapping has been generated for this database, "
+                    "so validation will use raw NCBI taxids (%d species: %s). If this "
+                    "is a GTDB or custom database whose taxids differ from NCBI, read "
+                    "extraction will find ZERO reads and validation will produce "
+                    "nothing. Run 'Scan Database' / 'Verify Taxonomy IDs' in the "
+                    "Watchlist & Preparation tab before starting.",
+                    len(unmapped_names), preview,
+                )
+            else:
+                logging.warning(
+                    "%d watchlist species could not be mapped to this Kraken2 "
+                    "database (%s); validation will fall back to their NCBI taxids "
+                    "and may find no reads if the database does not contain them.",
+                    len(unmapped_names), preview,
+                )
+
         return species_list, genome_paths
 
     except (ImportError, AttributeError) as e:

tests/test_kraken2_helpers.py

@@ -63,6 +63,30 @@ def test_missing_inspect_returns_empty(self, tmp_path):
     def test_empty_path_returns_empty(self):
         assert load_kraken2_taxonomy("") == {}
 
+    def test_parses_gzipped_inspect(self, tmp_path):
+        # GTDB-derived / size-conscious DBs ship only inspect.txt.gz. The
+        # authoritative-taxonomy correction must still load (was silently empty).
+        import gzip
+        content = (
+            "100.00\t1000\t0\tR\t1\troot\n"
+            "90.00\t900\t0\tD\t2\t  Bacteria\n"
+            "50.00\t500\t0\tG\t561\t    Escherichia\n"
+            "25.00\t250\t250\tS\t562\t      Escherichia coli\n"
+        )
+        with gzip.open(tmp_path / "inspect.txt.gz", "wt", encoding="utf-8") as fh:
+            fh.write(content)
+        mapping = load_kraken2_taxonomy(str(tmp_path))
+        assert mapping == {1: 0, 2: 1, 561: 2, 562: 561}
+
+    def test_plain_inspect_preferred_over_gz(self, tmp_path):
+        import gzip
+        (tmp_path / "inspect.txt").write_text(
+            "100.00\t1000\t0\tR\t1\troot\n90.00\t900\t0\tD\t2\t  Bacteria\n"
+        )
+        with gzip.open(tmp_path / "inspect.txt.gz", "wt", encoding="utf-8") as fh:
+            fh.write("bogus\tnot\tparsed\n")  # would corrupt if gz were read
+        assert load_kraken2_taxonomy(str(tmp_path)) == {1: 0, 2: 1}
+
 
 class TestApplyAuthoritativeTaxonomy:
     def test_replaces_parent_taxid_from_mapping(self):

tests/test_parameter_mapping_extra.py

@@ -149,6 +149,65 @@ def test_empty_watchlist(self):
             assert get_validation_species({}) == ([], [])
 
 
+class TestUnmappedValidationTaxidWarning:
+    """A custom/GTDB DB whose taxids differ from NCBI silently extracts 0 reads
+    when the watchlist was never mapped. get_validation_species_from_watchlist
+    must WARN at that fallback instead of failing silently."""
+
+    def _entry(self, taxid, name, db_taxid=None):
+        e = MagicMock()
+        e.taxid = taxid
+        e.name = name
+        e.db_taxid = db_taxid
+        e.names_alt = []
+        return e
+
+    def _run(self, entries, mapping_collection):
+        wm = MagicMock()
+        wm._loaded = True
+        wm.get_active_entries.return_value = {e.taxid: e for e in entries}
+        gm = MagicMock()
+        gm.get_genome_path.return_value = None
+        with patch.object(pm, "get_watchlist_manager", return_value=wm), \
+             patch.object(pm, "get_genome_manager", return_value=gm), \
+             patch("nanometa_live.core.taxonomy.taxid_mapping.get_mapping_collection",
+                   return_value=mapping_collection):
+            return pm.get_validation_species_from_watchlist({"kraken_db": "/db"})
+
+    def test_no_mapping_warns_run_scan(self, caplog):
+        import logging
+        with caplog.at_level(logging.WARNING):
+            species, _ = self._run([self._entry(263, "Francisella tularensis")], None)
+        assert species and species[0]["kraken_taxid"] == 263  # raw NCBI fallback
+        assert any("Scan Database" in r.message for r in caplog.records)
+
+    def test_scanned_all_mapped_no_warning(self, caplog):
+        import logging
+        coll = MagicMock()
+        coll.get_db_taxid.return_value = 4007169  # 263 -> custom DB taxid
+        with caplog.at_level(logging.WARNING):
+            species, _ = self._run([self._entry(263, "Francisella tularensis")], coll)
+        assert species[0]["kraken_taxid"] == 4007169
+        assert not any("could not be mapped" in r.message or "Scan Database" in r.message
+                       for r in caplog.records)
+
+    def test_scanned_partial_warns_specific(self, caplog):
+        import logging
+        coll = MagicMock()
+        coll.get_db_taxid.side_effect = lambda t: 4007169 if t == 263 else None
+        with caplog.at_level(logging.WARNING):
+            self._run([self._entry(263, "F. tularensis"),
+                       self._entry(1392, "B. anthracis")], coll)
+        assert any("could not be mapped" in r.message for r in caplog.records)
+
+    def test_explicit_db_taxid_not_warned(self, caplog):
+        import logging
+        with caplog.at_level(logging.WARNING):
+            species, _ = self._run([self._entry(263, "F. tularensis", db_taxid=4007169)], None)
+        assert species[0]["kraken_taxid"] == 4007169
+        assert not any("Scan Database" in r.message for r in caplog.records)
+
+
 class TestPathogenGenomesLocation:
     """Operator feedback #2: archive/rerun crashed with 'No such file:
     .../validation/pathogen_genomes.json'. The launch input must live OUTSIDE