fix: impactful 'Low-tier' data-loss/crash bugs from the audit backlog

Triaged the audit's Low tier; these ~12 were really data-loss/crash bugs, not
cosmetic. Fixed on this PR (deferred 4 larger items to issues #405-#408):

- DOC-15: width/height attrs like '100%'/'50px' crashed int() (html/epub) or
  silently dropped the image (word). New shared scraper_utils.parse_leading_int.
- DOC-13: table extraction dropped any body row equal to the header text; now
  skips <thead> rows STRUCTURALLY (shared helper + html_scraper copy). +tests.
- DOC-11: EPUB section images with empty bytes wrote 0-byte PNGs + broken links;
  require non-empty bytes.
- DOC-17: PDF image branch raised KeyError/TypeError (missing/empty data) and
  aborted the reference-file write; guard with .get + non-empty bytes.
- DOC-14: checkpoint write is now atomic (temp + os.replace) so a second Ctrl-C
  can't truncate it and silently lose all crawl progress.
- MED-14: video AI ref-cleaning overwrote the reference with a truncated reply;
  require stop_reason == 'end_turn'.
- MED-17: Slack conversations_list is now paginated (was single-page; >200
  channels truncated).
- ENH-17: assembled PDF content was silently discarded when the base SKILL.md had
  no Code Examples / API Reference / Reference heading AND no footer; append at end.
- MCP-15: install_skill aborts when packaging fails instead of fabricating a
  zip_path and uploading a missing file.
- MCP-16: generate_config refuses to clobber an existing config without force=true
  (exposed on the FastMCP tool too).
- ENH-15: prompt_file.write_text now uses encoding='utf-8' (was UnicodeEncodeError
  on non-UTF-8 locales for emoji/CJK prompts).
- ADP-03: cap weaviate-client <4 (the adaptor uses the v3 API; v4 breaks it).

MED-16 (Discord batch[-1]['id'] KeyError) was already fixed in the audit's batch 2.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: yusufkaraaslan

pyproject.toml

@@ -178,7 +178,7 @@ chroma = [
 ]
 
 weaviate = [
-    "weaviate-client>=3.25.0",
+    "weaviate-client>=3.25.0,<4",
 ]
 
 sentence-transformers = [
@@ -191,7 +191,7 @@ pinecone = [
 
 rag-upload = [
     "chromadb>=0.4.0",
-    "weaviate-client>=3.25.0",
+    "weaviate-client>=3.25.0,<4",
     "sentence-transformers>=2.2.0",
     "pinecone>=5.0.0",
 ]
@@ -267,7 +267,7 @@ all = [
     "google-cloud-storage>=2.10.0",
     "azure-storage-blob>=12.19.0",
     "chromadb>=0.4.0",
-    "weaviate-client>=3.25.0",
+    "weaviate-client>=3.25.0,<4",
     "pinecone>=5.0.0",
     "fastapi>=0.109.0",
     "sentence-transformers>=2.3.0",

src/skill_seekers/cli/agent_client.py

@@ -448,7 +448,7 @@ def _call_local(
                 if output_file:
                     full_prompt += f"\n\nWrite your response to: {resp_file}\n"
 
-                prompt_file.write_text(full_prompt)
+                prompt_file.write_text(full_prompt, encoding="utf-8")
 
                 # Build command from preset
                 cmd = []

src/skill_seekers/cli/chat_scraper.py

@@ -651,12 +651,21 @@ def _extract_slack_api(self) -> list[dict]:
                 channel_ids = [self.channel]
                 channel_names = {self.channel: self.channel}
             else:
-                # List all accessible channels
-                result = client.conversations_list(
-                    types="public_channel,private_channel",
-                    limit=200,
-                )
-                channels = result.get("channels", [])
+                # List ALL accessible channels — paginate via next_cursor. A
+                # single conversations_list call caps at the page limit and
+                # silently truncates workspaces with >limit channels.
+                channels = []
+                cursor = None
+                while True:
+                    result = client.conversations_list(
+                        types="public_channel,private_channel",
+                        limit=200,
+                        cursor=cursor,
+                    )
+                    channels.extend(result.get("channels", []))
+                    cursor = (result.get("response_metadata") or {}).get("next_cursor")
+                    if not cursor:
+                        break
                 channel_ids = [ch["id"] for ch in channels]
                 channel_names = {ch["id"]: ch.get("name", ch["id"]) for ch in channels}
                 print(f"   Found {len(channel_ids)} channel(s)")

src/skill_seekers/cli/doc_scraper.py

@@ -372,8 +372,14 @@ def save_checkpoint(self) -> None:
         }
 
         try:
-            with open(self.checkpoint_file, "w", encoding="utf-8") as f:
+            # Atomic write: a second Ctrl-C during a direct open(...,'w') can
+            # truncate the checkpoint, and load_checkpoint then silently starts
+            # fresh — losing all crawl progress. Write a temp file, then
+            # os.replace() it into place (atomic on the same filesystem).
+            tmp_file = f"{self.checkpoint_file}.tmp"
+            with open(tmp_file, "w", encoding="utf-8") as f:
                 json.dump(checkpoint_data, f, indent=2)
+            os.replace(tmp_file, self.checkpoint_file)
             logger.info("  💾 Checkpoint saved (%d pages)", self.pages_scraped)
         except Exception as e:
             logger.warning("  ⚠️  Failed to save checkpoint: %s", e)

src/skill_seekers/cli/epub_scraper.py

@@ -31,6 +31,7 @@
 from .skill_converter import SkillConverter
 from skill_seekers.cli.scraper_utils import score_code_quality as _score_code_quality
 from skill_seekers.cli.scraper_utils import extract_table_from_html as _extract_table_from_html
+from skill_seekers.cli.scraper_utils import parse_leading_int as _parse_leading_int
 
 logger = logging.getLogger(__name__)
 
@@ -603,7 +604,9 @@ def _generate_reference_file(self, _cat_key, cat_data, section_num, total_sectio
                         img_filename = f"section_{sec_num}_img_{img_index}.png"
                         img_path = os.path.join(assets_dir, img_filename)
 
-                        if isinstance(img_data, (bytes, bytearray)):
+                        # Require NON-EMPTY bytes — b"" passed the isinstance
+                        # check and produced a 0-byte PNG plus a broken ![] link.
+                        if isinstance(img_data, (bytes, bytearray)) and len(img_data) > 0:
                             with open(img_path, "wb") as img_file:
                                 img_file.write(img_data)
                             f.write(f"![Image {img_index}](../assets/{img_filename})\n\n")
@@ -974,8 +977,8 @@ def _build_section(
                     {
                         "index": len(images),
                         "data": b"",  # EPUB images handled separately via manifest
-                        "width": int(elem.get("width", 0) or 0),
-                        "height": int(elem.get("height", 0) or 0),
+                        "width": _parse_leading_int(elem.get("width")),
+                        "height": _parse_leading_int(elem.get("height")),
                     }
                 )
             continue

src/skill_seekers/cli/html_scraper.py

@@ -26,6 +26,7 @@
 from bs4 import BeautifulSoup, Comment, Tag
 
 from .skill_converter import SkillConverter
+from skill_seekers.cli.scraper_utils import parse_leading_int as _parse_leading_int
 from skill_seekers.cli.scraper_utils import score_code_quality as _score_code_quality
 
 logger = logging.getLogger(__name__)
@@ -863,12 +864,17 @@ def _extract_tables(self, table_elem: Tag) -> dict | None:
             if header_row:
                 headers = [th.get_text(strip=True) for th in header_row.find_all(["th", "td"])]
 
-        # Body rows
-        tbody = table_elem.find("tbody") or table_elem
-        for row in tbody.find_all("tr"):
+        # Body rows. Prefer an explicit <tbody>; otherwise take rows directly
+        # under the table but skip <thead> rows STRUCTURALLY (not by value) so a
+        # legitimate body row that duplicates the header text isn't dropped.
+        tbody = table_elem.find("tbody")
+        if tbody is not None:
+            body_rows = tbody.find_all("tr")
+        else:
+            body_rows = [r for r in table_elem.find_all("tr") if r.find_parent("thead") is None]
+        for row in body_rows:
             cells = [td.get_text(strip=True) for td in row.find_all(["td", "th"])]
-            # Skip the header row we already captured
-            if cells and cells != headers:
+            if cells:
                 rows.append(cells)
 
         # If no explicit thead, use first row as header
@@ -908,8 +914,8 @@ def _extract_image_info(self, img_elem: Tag, source_file: Path) -> dict | None:
             "src": resolved_src,
             "alt": img_elem.get("alt", ""),
             "title": img_elem.get("title", ""),
-            "width": int(img_elem.get("width", 0) or 0),
-            "height": int(img_elem.get("height", 0) or 0),
+            "width": _parse_leading_int(img_elem.get("width")),
+            "height": _parse_leading_int(img_elem.get("height")),
             "data": b"",  # Placeholder; actual image data loaded separately
         }
 

src/skill_seekers/cli/pdf_scraper.py

@@ -343,12 +343,20 @@ def _generate_reference_file(self, _cat_key, cat_data, section_num, total_sectio
 
                     f.write("### Images\n\n")
                     for img in page["images"]:
+                        # Guard the raw image data: a missing "data" key raised
+                        # KeyError and a non-bytes/empty value either crashed
+                        # (TypeError) or wrote a 0-byte PNG with a broken ![] link
+                        # — aborting the whole reference-file write.
+                        img_data = img.get("data")
+                        if not (isinstance(img_data, (bytes, bytearray)) and len(img_data) > 0):
+                            continue
+
                         # Save image to assets
                         img_filename = f"page_{page['page_number']}_img_{img['index']}.png"
                         img_path = os.path.join(assets_dir, img_filename)
 
                         with open(img_path, "wb") as img_file:
-                            img_file.write(img["data"])
+                            img_file.write(img_data)
 
                         # Add markdown image reference
                         f.write(f"![Image {img['index']}](../assets/{img_filename})\n\n")

src/skill_seekers/cli/scraper_utils.py

@@ -15,6 +15,21 @@
 import re
 
 
+def parse_leading_int(value, default: int = 0) -> int:
+    """Parse the leading integer from a dimension-ish value, defensively.
+
+    HTML/EPUB/Word width/height attributes can be ``"100%"``, ``"50px"``,
+    ``""`` or ``None``; a bare ``int("100%")`` raises ``ValueError`` (crashing
+    image extraction) or silently drops the image. Returns the leading integer
+    (``"100%"`` -> 100, ``"50px"`` -> 50) or ``default`` when there's none
+    (``"auto"``/``""``/``None`` -> ``default``).
+    """
+    if value is None:
+        return default
+    match = re.match(r"\s*(-?\d+)", str(value))
+    return int(match.group(1)) if match else default
+
+
 def score_code_quality(code: str, *, notebook_mode: bool = False) -> float:
     """Heuristic quality score for a code block (0.0-10.0).
 
@@ -92,12 +107,18 @@ def extract_table_from_html(table_elem) -> dict | None:
         if header_row:
             headers = [th.get_text(strip=True) for th in header_row.find_all(["th", "td"])]
 
-    # Body rows
-    tbody = table_elem.find("tbody") or table_elem
-    for row in tbody.find_all("tr"):
+    # Body rows. Prefer an explicit <tbody>; otherwise take rows directly under
+    # the table but skip any that belong to <thead> — skipping STRUCTURALLY, not
+    # by value, so a legitimate body row that merely duplicates the header text
+    # isn't dropped.
+    tbody = table_elem.find("tbody")
+    if tbody is not None:
+        body_rows = tbody.find_all("tr")
+    else:
+        body_rows = [r for r in table_elem.find_all("tr") if r.find_parent("thead") is None]
+    for row in body_rows:
         cells = [td.get_text(strip=True) for td in row.find_all(["td", "th"])]
-        # Skip the header row we already captured
-        if cells and cells != headers:
+        if cells:
             rows.append(cells)
 
     # If no explicit thead, use first row as header

src/skill_seekers/cli/unified_skill_builder.py

@@ -451,14 +451,19 @@ def _synthesize_docs_github_pdf(self, skill_mds: dict[str, str]) -> str:
         if pdf_content_lines and insertion_index != -1:
             lines[insertion_index:insertion_index] = pdf_content_lines
         elif pdf_content_lines:
-            # Append at end before footer
+            # Append before the trailing footer if present, otherwise at the very
+            # end. ENH-17: when the base SKILL.md had no Code Examples / API
+            # Reference / Reference Documentation heading AND no trailing footer,
+            # the assembled PDF content was silently discarded.
             footer_index = -1
             for i, line in enumerate(lines):
                 if line.startswith("---") and i > len(lines) - 5:
                     footer_index = i
                     break
             if footer_index != -1:
                 lines[footer_index:footer_index] = pdf_content_lines
+            else:
+                lines.extend(pdf_content_lines)
 
         # Update reference documentation to include PDF
         final_content = "\n".join(lines)

src/skill_seekers/cli/video_scraper.py

@@ -304,7 +304,14 @@ def _ai_clean_reference(ref_path: str, content: str, api_key: str | None = None)
             messages=[{"role": "user", "content": prompt}],
         )
         result = response.content[0].text
-        if result and len(result) > len(content) * 0.5:
+        # Require a COMPLETE response before overwriting the reference file: a
+        # response truncated at max_tokens can still exceed 50% of the input yet
+        # silently corrupt the reference. (stop_reason == "end_turn" => complete.)
+        if (
+            result
+            and getattr(response, "stop_reason", None) == "end_turn"
+            and len(result) > len(content) * 0.5
+        ):
             with open(ref_path, "w", encoding="utf-8") as f:
                 f.write(result)
             logger.info(f"AI-cleaned reference: {os.path.basename(ref_path)}")