diff --git a/.github/workflows/analyze.yml b/.github/workflows/analyze.yml index 9f0f64326b..ea3b3871c0 100644 --- a/.github/workflows/analyze.yml +++ b/.github/workflows/analyze.yml @@ -228,6 +228,11 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v5 + with: + # The baseline-growth step below diffs scripts/nonascii_asset_baseline.txt + # against the merge-base, which a shallow clone cannot reach. Same + # trade the prompts-i18n job already makes: negligible on this repo. + fetch-depth: 0 - name: Set up Python 3.11 uses: actions/setup-python@v5 @@ -278,6 +283,38 @@ jobs: # link, or move the referenced content into docs/. run: python scripts/check_docs_no_relative_paths.py + - name: Forbid new non-ASCII filenames in bundled assets + # Nuitka --mode=app puts the payload under Contents/MacOS/, where + # codesign treats every file as nested code and writes an + # `identifier ...` requirement into CodeResources. A non-ASCII + # name becomes a hex literal, which is not valid requirement syntax, + # and the whole bundle then fails to verify with "the sealed resource + # directory is invalid" — dead in the water for signing, notarization + # and Steam upload, with no filename in the error to go on. + # + # This step exists because build-desktop.yml cannot catch it: it signs + # ad-hoc (`--sign -`), whose requirements are `cdhash H"..."` and carry + # no identifier, so the bug is invisible there and only bites the local + # Developer ID path (build_mac.sh). Ratchet, not a ban — the assets + # that predate the check live in scripts/nonascii_asset_baseline.txt + # and that list may only shrink. Companion unit test: + # tests/unit/test_check_no_nonascii_asset_names.py. + run: python scripts/check_no_nonascii_asset_names.py + + - name: Forbid growing the non-ASCII baseline (PR-only) + # The step above compares the tree against the baseline, so a PR that + # adds a non-ASCII asset AND its baseline line has an empty diff and + # passes — the ratchet would only be a convention. This step diffs the + # baseline itself against the merge-base and fails on any added entry, + # which is what makes "may only shrink" enforceable. `--update-baseline` + # already refuses to add; this catches the hand-written line too. + # Skipped on direct push to main — nothing to diff against. + if: github.event_name == 'pull_request' + env: + # Same zizmor template-injection indirection as the i18n-sync step. + BASE_REF: ${{ github.base_ref }} + run: python scripts/check_no_nonascii_asset_names.py --base "origin/${BASE_REF}" + core-contracts: name: Core package contracts runs-on: ubuntu-latest diff --git a/.github/workflows/build-desktop.yml b/.github/workflows/build-desktop.yml index 647ead704c..85a5e6b280 100644 --- a/.github/workflows/build-desktop.yml +++ b/.github/workflows/build-desktop.yml @@ -235,6 +235,17 @@ jobs: src = Path(sp.__file__).parent dst = Path('data/browser_use_prompts') dst.mkdir(parents=True, exist_ok=True) + bad = [md.name for md in src.glob('*.md') if not md.name.isascii()] + if bad: + # Non-ASCII basenames under Contents/MacOS/ make codesign emit a + # hex-literal identifier and the whole bundle then fails to seal. + # scripts/check_no_nonascii_asset_names.py cannot see this one: + # data/ is gitignored and the analyze job installs no dependencies, + # so the names only exist here, right after generation. + raise SystemExit( + 'non-ASCII browser_use prompt template name(s), which break ' + f'macOS signing: {bad}' + ) for md in src.glob('*.md'): shutil.copy2(md, dst / md.name) print(f'Copied {md.name}') @@ -690,6 +701,22 @@ jobs: # macOS: playwright_browsers was excluded from Nuitka (xattr bug with # spaces in .app paths), copy it manually into the output directory if [[ "$RUNNER_OS" == "macOS" && -d "playwright_browsers" ]]; then + # This tree lands under Contents/MacOS/, where codesign treats every + # file as nested code; a non-ASCII basename becomes a hex-literal + # identifier and the whole bundle then fails to seal. The payload is + # downloaded, so scripts/check_no_nonascii_asset_names.py structurally + # cannot see it — the only place to look is right here, before the copy. + .venv/bin/python -c " + import sys + from pathlib import Path + # Files only: a CJK *directory* holding ASCII files seals fine + # (verified against a real Developer ID cert; see the checker's + # module docstring), and failing on it would block the build for nothing. + bad = [str(p) for p in Path('playwright_browsers').rglob('*') + if p.is_file() and not p.name.isascii()] + if bad: + sys.exit('non-ASCII name(s) in playwright_browsers, which break macOS signing: ' + repr(bad[:10])) + " cp -R playwright_browsers "$RUNTIME_DIR/playwright_browsers" echo "Copied playwright_browsers into $RUNTIME_DIR/" fi diff --git a/scripts/check_no_nonascii_asset_names.py b/scripts/check_no_nonascii_asset_names.py new file mode 100644 index 0000000000..b8d9e59084 --- /dev/null +++ b/scripts/check_no_nonascii_asset_names.py @@ -0,0 +1,861 @@ +#!/usr/bin/env python3 +# Copyright 2025-2026 Project N.E.K.O. Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Ratchet: no NEW non-ASCII filenames in anything bundled into the desktop app. + +Why this exists +--------------- +Nuitka ``--mode=app`` puts the whole payload under +``projectneko_server.app/Contents/MacOS/``. codesign's default bundle rules +(``rules2``) classify everything under that directory as *nested code*:: + + ^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS| + Library/(Automator|Spotlight|LoginItems))/ => nested + +So every .mp3 / .png / .json we ship is signed as its own code object, and for +each one codesign writes a designated requirement into +``Contents/_CodeSignature/CodeResources``:: + + identifier and anchor apple generic and certificate ... + +When the filename is non-ASCII, codesign emits that identifier as a **hex +literal** instead of a quoted string:: + + identifier 0xe4b883e5a4a9e5898defbc8ce68891e4bbace8bf98e58faae698afe7a... + +That is not valid requirement syntax. Reading the seal back fails, and the +whole bundle then verifies as:: + + : the sealed resource directory is invalid + +…which blocks signing, notarization, and Steam upload. One such file poisons +the entire bundle; the error names no path, so it is genuinely painful to +diagnose from scratch. + +Why CI can never catch it on its own +------------------------------------ +``.github/workflows/build-desktop.yml`` signs ad-hoc:: + + codesign --force --deep --sign - dist/Xiao8/projectneko_server.app + +Ad-hoc signatures use ``cdhash H"..."`` requirements, which carry no +identifier at all — so the hex-literal bug simply does not occur there and +the workflow is always green. The breakage only appears on the local +Developer ID signing path (``build_mac.sh``). This lint is the substitute for +a signal CI structurally cannot produce. + +What it scans +------------- +Only what actually ships: + +- files git knows about under the bundled roots (``BUNDLED_ROOTS``) — + tracked plus untracked-but-not-ignored, so the check fires before + ``git add`` rather than only after commit; +- ``filename`` values in the manifests that drive build-time downloads + (``main_logic/asr_client/*/models/manifest.json``): the ``.onnx`` itself is + gitignored and only exists after a build, but the name it will be written + under is committed, so a plain CI run can still catch it; +- member names inside the archives that get unpacked into those roots at + build time (``assets/*.tar.gz`` -> ``static//`` via + ``build_frontend.sh``; the PNGTuber packs named by + ``frontend/pngtuber-packs/manifest.json`` -> ``static/pngtuber//`` + via ``scripts/unpack_builtin_pngtuber.py``). Reading member names needs no + build and no extraction, so the check gives the same answer in a fresh + checkout as on a build machine. Hard-link and symlink members count too: + ``tar -xzm`` materializes them as entries under ``static/`` like any file. + +Asking git (rather than walking the working tree) keeps the result +deterministic: gitignored build outputs, downloaded model weights, and local +runtime files never make the answer wander between a fresh checkout and a +built one. Pass ``--include-untracked`` to additionally walk the on-disk tree +— useful right after a build, when you want to sanity-check generated payload +too. + +Only **basenames** are checked, not whole paths. codesign derives the nested +identifier from the filename alone, so a non-ASCII *directory* holding +ASCII-named files signs and verifies cleanly (checked against a real +Developer ID certificate: ``MacOS//plain_name.png`` seals as +``identifier "plain_name"`` and passes ``--verify --deep --strict``). +Flagging those too would only produce failures nobody can act on. + +``tests/`` is deliberately out of scope: it carries a few CJK fixture paths +and never reaches the .app. + +Baseline +-------- +338 tracked files plus 1 archive member are already non-ASCII when this check +was written, across three unrelated subsystems: + +- ``static/assets/tutorial/guide-audio/{zh,ja,ko,ru,en}/*.mp3`` — filenames + are truncated line transcripts, referenced from the ``audioFilesByKey`` + manifests in ``static/tutorial/yui-guide/days/*.js``; +- ``static/vrm/motion/**``, ``static/vrm/animation/*``, + ``static/mmd/animation/*`` — ``static/vrm/motion/manifest.json`` states the + convention outright (``"fileNaming": "descriptive Chinese filename with + stable id"``, ``"authoritativeLanguage": "zh-CN"``), because motion lookup + runs through Chinese action-card retrieval; +- ``static/game/games/soccer/audio/*.mp3``, plus one Live2D expression file + under ``static/yui-origin/expressions/`` that ships inside + ``assets/yui-origin.tar.gz`` and is named from the ``.model3.json``. + +Renaming those is a product decision, not a mechanical sweep — the VRM naming +scheme in particular is load-bearing for motion retrieval. So this is a +ratchet, not a clean-room ban: everything listed in +``scripts/nonascii_asset_baseline.txt`` is grandfathered, anything new fails. + +TODO: shrink the baseline. Each family needs its own follow-up — rename the +files to ASCII (stable slug or id) and update the manifest that names them. +An empty baseline is the goal; when it gets there, delete the file and make +this a plain ban. + +Usage +----- + python scripts/check_no_nonascii_asset_names.py + python scripts/check_no_nonascii_asset_names.py --list + python scripts/check_no_nonascii_asset_names.py --include-untracked + python scripts/check_no_nonascii_asset_names.py --update-baseline + python scripts/check_no_nonascii_asset_names.py --base origin/main + +The baseline is a ratchet in both directions of attack: ``--update-baseline`` +only ever drops entries that no longer exist, and ``--base`` fails when the +committed list gained a line relative to that ref. Without the second one the +first is only a convention — adding the asset and its baseline entry in the +same PR leaves nothing for the in-tree comparison to see. +""" +from __future__ import annotations + +import argparse +from fnmatch import fnmatchcase +import json +import os +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path, PurePosixPath + +try: # 3.11+ stdlib; the repo pins ==3.11.*, this is belt-and-braces + import tomllib +except ModuleNotFoundError: # pragma: no cover - only on an older interpreter + # Without it we simply do not know each plugin's [tool.neko.build] rules, + # which means scanning more than ships — false positives, never a hole. + tomllib = None + +REPO_ROOT = Path(__file__).resolve().parent.parent +BASELINE_PATH = REPO_ROOT / "scripts" / "nonascii_asset_baseline.txt" + +CODE = "NONASCII_ASSET_NAME" + +# Directories whose contents end up inside the .app payload. Mirrors the +# --include-data-dir / --include-package-data set in build_nuitka.bat, +# build_mac.sh and .github/workflows/build-desktop.yml; keep in sync when a +# new payload directory is added there. +BUNDLED_ROOTS: tuple[str, ...] = ( + "static", + "templates", + "assets", + # config/ and data/ ship a named subset, not the whole root: + # --include-package=config compiles Python modules but copies no data, so + # everything that actually lands is listed explicitly in the build scripts. + # Scanning the roots wholesale would fail a PR over e.g. config/prompts/*.md, + # which never reaches the payload. + "config/__init__.py", + "config/api_providers.json", + "config/characters.json", + "config/core_config.json", + "config/user_preferences.json", + "config/characters", + "config/changelog", + "config/surveys", + "data/browser_use_prompts", + "data/tiktoken_cache", + "data/embedding_models", + # Not all of docs/ — the build packs exactly one subtree + # (--include-data-dir=docs/zh-CN/guide). Scanning the whole root would fail + # documentation PRs over files that never enter the payload. + "docs/zh-CN/guide", + # Only the vite output ships (--include-data-dir=frontend/plugin-manager/dist, + # identically in build_nuitka.bat, build_mac.sh and build-desktop.yml). + # Frontend *sources* never enter the payload — React's output goes to + # static/ — so scanning all of frontend/ would fail a PR over a file that + # cannot possibly break signing. + "frontend/plugin-manager/dist", + # Vite copies public/ verbatim into the build output, keeping the filename. + # Those outputs are gitignored (plugin-manager/dist, static/react/neko-chat), + # so without these two roots a tracked public/ asset is invisible here yet + # still lands in the payload. + "frontend/plugin-manager/public", + "frontend/react-neko-chat/public", + # Whole source trees, not just src/assets: Vite decides by *import*, not by + # directory. An imported asset above the inline limit is emitted as + # `assets/[name]-[hash][extname]` wherever it lives, and a dynamically + # imported module donates its basename to the chunk name the same way. Both + # outputs are gitignored, so the source tree is the only place CI can see + # these names without building. An ASCII-only rule for frontend sources + # costs a rename; missing one costs the whole mac release. + "frontend/plugin-manager/src", + "frontend/react-neko-chat/src", + "plugin/plugins", + # --include-package=steamworks pulls every native lib in this directory in + # as package data; the workflow's cleanup only removes the fixed + # wrong-platform filenames, so anything else here lands in the payload. + "steamworks", + # Voice-turn + speaker models, both --include-data-dir'd. The .onnx weights + # are downloaded at build time and gitignored, so the git listing cannot see + # them; these roots cover whatever *is* tracked here, and make + # --include-untracked reach the downloaded payload after a build. The names + # the downloader will write are checked from the manifests below, so a plain + # CI run catches them without building. + "main_logic/asr_client/endpointing/models", + "main_logic/asr_client/speaker_shadow/models", +) + +# Manifests naming files the build downloads into a bundled root. The weights are +# gitignored, so no git listing can see them before a build — but the name is +# committed right here, so it can be checked in a plain run. ``filename`` sits +# either at the top level or inside an ``assets`` list, depending on the schema. +MODEL_MANIFESTS: tuple[str, ...] = ( + "main_logic/asr_client/endpointing/models/manifest.json", + "main_logic/asr_client/speaker_shadow/models/manifest.json", +) + +# Archives that are expanded into a bundled root at build time. Value is the +# bundled path prefix the members land under, so a violation reports where the +# file will actually sit inside the .app rather than where it hides today. +TAR_ARCHIVE_DESTS: tuple[tuple[str, str], ...] = ( + # build_frontend.sh: unpack_live2d_model -> static// + ("assets", "static"), +) +# PNGTuber packs are not globbed: scripts/unpack_builtin_pngtuber.py unpacks +# exactly the archives listed in this manifest, and each one lands under its +# own `folder`. Both halves matter — globbing would flag a zip the build never +# opens, and a shared prefix would report `static/pngtuber/layers/x.png` for a +# member that really lands at `static/pngtuber/yui-origin/layers/x.png`, while +# collapsing same-named members from different packs into one entry. +ZIP_MANIFEST_REL = "frontend/pngtuber-packs/manifest.json" +ZIP_MANIFEST_DEST_ROOT = "static/pngtuber" + +# Never walked under --include-untracked. Vendored/derived trees whose names +# we do not author; a hit in here is a bug in the upstream package, not in +# this repo, and the .app build has its own gates for those. +WALK_EXCLUDE_DIRS = frozenset( + { + "node_modules", + "__pycache__", + ".git", + ".venv", + "venv", + ".mypy_cache", + ".ruff_cache", + ".pytest_cache", + } +) + + +def _is_ascii(text: str) -> bool: + return all(ord(ch) < 128 for ch in text) + + +def _under_bundled_root(rel_posix: str) -> bool: + return any( + rel_posix == root or rel_posix.startswith(root + "/") for root in BUNDLED_ROOTS + ) + + +# Mirror of the staging filter in scripts/prepare_nuitka_plugins.py, which runs +# each plugin's ``[tool.neko.build]`` rules (plus hard defaults) before the +# payload is installed into the bundle. Without it this check rejects files that +# never reach Contents/MacOS — a plugin's own tests/, its .db/.log runtime +# leftovers, editor directories. +# +# It is a mirror rather than an import on purpose: the real rules live behind +# pydantic (plugin/neko_plugin_cli/core/build_rules.py) and the analyze job runs +# these scripts on a bare interpreter with no dependencies installed. +# +# tests/unit/test_check_no_nonascii_asset_names.py imports the real +# ``should_skip_path`` and demands the same verdict in both directions, so this +# copy cannot drift silently. The two directions are not equally bad, which is +# worth knowing when one does slip: listing something the real staging keeps +# stops us scanning a file that ships (a hole), while missing something it drops +# only leaves a false positive. Both fail the test; only the first is dangerous. +_PLUGIN_SKIP_DIR_NAMES = frozenset( + {"__pycache__", ".github", ".pytest_cache", ".mypy_cache", ".venv", ".git"} +) +_PLUGIN_SKIP_ROOT_DIR_NAMES = frozenset({"dist", "build"}) +_PLUGIN_SKIP_FILE_NAMES = frozenset({".DS_Store"}) +# Two different comparisons upstream, and the difference matters: the build +# rules test `Path.suffix` case-SENSITIVELY against lowercase .pyc/.pyo, while +# _remove_private_runtime_artifacts lowercases before testing .db/.log. So a +# file named `x.PYC` really is staged and shipped — folding case here would +# drop it from the scan and hide it. +_PLUGIN_SKIP_SUFFIXES_EXACT = frozenset({".pyc", ".pyo"}) +_PLUGIN_SKIP_SUFFIXES_FOLDED = frozenset({".db", ".log"}) + +PLUGINS_ROOT = "plugin/plugins" + + +def _match_build_pattern(path_str: str, pattern: str) -> bool: + if fnmatchcase(path_str, pattern): + return True + return "/" not in pattern and fnmatchcase(PurePosixPath(path_str).name, pattern) + + +def _plugin_rules(repo_root: Path, plugin_dir: str) -> dict[str, list[str]]: + pyproject = repo_root / PLUGINS_ROOT / plugin_dir / "pyproject.toml" + if tomllib is None or not pyproject.is_file(): + return {} + try: + table = tomllib.loads(pyproject.read_text(encoding="utf-8")) + except (OSError, tomllib.TOMLDecodeError): + # A broken pyproject is the plugin CLI's problem to report; here it just + # means "no extra rules", which only ever widens what we scan. + return {} + build = table.get("tool", {}).get("neko", {}).get("build", {}) + if not isinstance(build, dict): + return {} + # BuildRuleSet._normalize_pattern_list strips each entry and drops blanks + # and duplicates. Skipping that here is not cosmetic: ` assets/* ` would + # fail to match, the include allow-list would then reject everything, and + # the checker would stop scanning files that really do ship. + def _patterns(key: str) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for item in build.get(key, []): + if not isinstance(item, str): + continue + pattern = item.strip() + if not pattern or pattern in seen: + continue + seen.add(pattern) + out.append(pattern) + return out + + return { + key: _patterns(key) + for key in ("include", "exclude", "exclude_dirs", "exclude_files") + } + + +def _plugin_stage_filter(repo_root: Path): + """Return ``keep(path)`` — False for repo paths the plugin stage drops.""" + cache: dict[str, dict[str, list[str]]] = {} + + def keep(path: str) -> bool: + prefix = PLUGINS_ROOT + "/" + if not path.startswith(prefix): + return True + parts = PurePosixPath(path[len(prefix):]).parts + if len(parts) < 2: + # A loose file directly under plugin/plugins/. No plugin rules apply, + # but _remove_private_runtime_artifacts sweeps the whole stage, so + # .db/.log still go. + return PurePosixPath(parts[-1]).suffix.lower() not in _PLUGIN_SKIP_SUFFIXES_FOLDED + plugin_dir, relative = parts[0], PurePosixPath(*parts[1:]) + dir_parts = relative.parts[:-1] + if dir_parts and dir_parts[0] in _PLUGIN_SKIP_ROOT_DIR_NAMES: + return False + if any(part in _PLUGIN_SKIP_DIR_NAMES for part in dir_parts): + return False + if relative.name in _PLUGIN_SKIP_FILE_NAMES: + return False + if relative.suffix in _PLUGIN_SKIP_SUFFIXES_EXACT: + return False + if relative.suffix.lower() in _PLUGIN_SKIP_SUFFIXES_FOLDED: + return False + + rules = cache.setdefault(plugin_dir, _plugin_rules(repo_root, plugin_dir)) + if not rules: + return True + path_str = relative.as_posix() + if any(_match_build_pattern(path_str, p) for p in rules.get("exclude", [])): + return False + # Both lists are also tested against every ancestor directory: the real + # walk asks should_skip_path(is_dir=True) for each directory and prunes + # the whole subtree, so `exclude = ["cache"]` drops cache/** even though + # no file path equals "cache". + for index in range(len(dir_parts)): + candidate = "/".join(dir_parts[: index + 1]) + if any( + _match_build_pattern(candidate, p) + for p in rules.get("exclude_dirs", []) + rules.get("exclude", []) + ): + return False + exclude_files = rules.get("exclude_files", []) + if relative.name in exclude_files: + return False + if any(_match_build_pattern(path_str, p) for p in exclude_files): + return False + # `include` is an allow-list applied after every exclude has run: with + # it present, anything unmatched is dropped from the stage. + include = rules.get("include", []) + if not include: + return True + return any(_match_build_pattern(path_str, p) for p in include) + + return keep + + +def _git_listed_offenders(repo_root: Path) -> set[str]: + """Non-ASCII git-visible paths under the bundled roots. + + ``--cached --others --exclude-standard`` = tracked files plus untracked + ones that are not gitignored. The ``--others`` half is what makes the + check fire before ``git add``, while ``--exclude-standard`` keeps build + outputs and downloaded weights out so the answer does not depend on + whether the tree has been built. + + ``-z`` matters: without it git quote-escapes non-ASCII paths + (``"static/\\344\\270\\203....mp3"``), which is exactly the set we are + looking for. + """ + try: + raw = subprocess.run( + ["git", "ls-files", "-z", "--cached", "--others", "--exclude-standard"], + cwd=repo_root, + capture_output=True, + check=True, + ).stdout.decode("utf-8", errors="surrogateescape") + except (OSError, subprocess.CalledProcessError) as exc: + print(f"error: cannot list git files: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + + staged = _plugin_stage_filter(repo_root) + return { + path + for path in raw.split("\0") + if path + and _under_bundled_root(path) + and staged(path) + and not _is_ascii(PurePosixPath(path).name) + } + + +def _archive_offenders(repo_root: Path) -> dict[str, str]: + """Non-ASCII member names inside archives that get unpacked into the app. + + Keyed by the path the member will occupy once unpacked, so both the + baseline and the violation message point at the bundled location; the + value is the archive to edit. + """ + offenders: dict[str, str] = {} + + def _record(members: list[str], dest_prefix: str, archive_rel: str) -> None: + for member in members: + # A ZIP written on Windows can carry backslash separators; the + # unpacker normalizes them (`_safe_relative_path`), so a member like + # `中文目录\plain.png` really lands as an ASCII file inside a CJK + # directory — allowed. Without this the whole string reads as one + # basename and the check rejects a file that signs fine. + normalized = member.replace("\\", "/") + if _is_ascii(PurePosixPath(normalized).name): + continue + offenders[f"{dest_prefix}/{normalized}"] = archive_rel + + for source_dir, dest_prefix in TAR_ARCHIVE_DESTS: + directory = repo_root / source_dir + if not directory.is_dir(): + continue + for archive in sorted(directory.glob("*.tar.gz")): + with tarfile.open(archive) as handle: + # islnk/issym as well as isfile: `tar -xzm` materializes hard + # links and symlinks as entries in static/ too, so a non-ASCII + # link name reaches the payload exactly like a regular file. + names = [ + m.name + for m in handle.getmembers() + if m.isfile() or m.islnk() or m.issym() + ] + _record(names, dest_prefix, archive.relative_to(repo_root).as_posix()) + + for archive_rel, dest_prefix in _pngtuber_archive_dests(repo_root): + archive = repo_root / archive_rel + if not archive.is_file(): + continue + with zipfile.ZipFile(archive) as handle: + names = [i.filename for i in handle.infolist() if not i.is_dir()] + _record(names, dest_prefix, archive_rel) + + return offenders + + +def _load_json(path: Path, rel: str) -> object: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + print(f"error: cannot read {rel}: {exc}", file=sys.stderr) + raise SystemExit(2) from exc + + +def _model_manifest_offenders(repo_root: Path) -> dict[str, str]: + """Non-ASCII ``filename`` values in the downloaded-model manifests. + + Keyed by where the download lands, valued by the manifest to edit. This is + what makes the two model roots useful in CI: the payload itself only exists + after a build, but the name that will be written is committed. + """ + offenders: dict[str, str] = {} + for rel in MODEL_MANIFESTS: + path = repo_root / rel + if not path.is_file(): + continue + manifest = _load_json(path, rel) + if not isinstance(manifest, dict): + print(f"error: invalid {rel}: top level must be an object", file=sys.stderr) + raise SystemExit(2) + + entries: list[dict] = [manifest] + assets = manifest.get("assets") + if assets is not None: + if not isinstance(assets, list): + print(f"error: invalid {rel}: 'assets' must be a list", file=sys.stderr) + raise SystemExit(2) + entries.extend(item for item in assets if isinstance(item, dict)) + + dest_dir = PurePosixPath(rel).parent.as_posix() + for entry in entries: + name = entry.get("filename") + if isinstance(name, str) and name and not _is_ascii(PurePosixPath(name).name): + offenders[f"{dest_dir}/{name}"] = rel + return offenders + + +def _pngtuber_archive_dests(repo_root: Path) -> list[tuple[str, str]]: + """(archive path, destination prefix) for each manifest-listed PNGTuber pack. + + Mirrors ``unpack_model``: the archive named by ``archive`` is expanded into + ``static/pngtuber//``. Entries missing either field are skipped — + the unpacker rejects them too, so they never reach the payload. + """ + manifest_path = repo_root / ZIP_MANIFEST_REL + if not manifest_path.is_file(): + return [] + manifest = _load_json(manifest_path, ZIP_MANIFEST_REL) + # A malformed manifest must be a loud error, not a silently empty scan — + # "no packs found" and "the file is a list" would otherwise look identical. + if not isinstance(manifest, dict): + print( + f"error: invalid {ZIP_MANIFEST_REL}: top level must be an object", + file=sys.stderr, + ) + raise SystemExit(2) + models = manifest.get("models", []) + if not isinstance(models, list): + print( + f"error: invalid {ZIP_MANIFEST_REL}: 'models' must be a list", + file=sys.stderr, + ) + raise SystemExit(2) + + packs_dir = PurePosixPath(ZIP_MANIFEST_REL).parent + dests: list[tuple[str, str]] = [] + for model in models: + if not isinstance(model, dict): + continue + folder = model.get("folder") + archive = model.get("archive") + if not isinstance(folder, str) or not isinstance(archive, str): + continue + dests.append( + ( + (packs_dir / archive).as_posix(), + f"{ZIP_MANIFEST_DEST_ROOT}/{folder}", + ) + ) + return sorted(dests) + + +def _untracked_offenders(repo_root: Path) -> set[str]: + """Non-ASCII paths found by walking the on-disk bundled roots. + + Only used with ``--include-untracked``: after a build these roots also + hold generated payload (unpacked models, vite bundles, downloaded model + weights) that no git listing can see. + """ + offenders: set[str] = set() + for root in BUNDLED_ROOTS: + base = repo_root / root + if not base.is_dir(): + continue + for current, dirs, files in os.walk(base): + dirs[:] = [d for d in dirs if d not in WALK_EXCLUDE_DIRS] + current_path = Path(current) + for name in files: + if _is_ascii(name): + continue + offenders.add( + (current_path / name).relative_to(repo_root).as_posix() + ) + return offenders + + +def collect_offenders( + repo_root: Path, include_untracked: bool = False +) -> tuple[set[str], dict[str, str]]: + """Return (bundled paths with non-ASCII names, path -> owning archive/manifest).""" + from_archives = _archive_offenders(repo_root) + from_archives.update(_model_manifest_offenders(repo_root)) + offenders = _git_listed_offenders(repo_root) | set(from_archives) + if include_untracked: + offenders |= _untracked_offenders(repo_root) + return offenders, from_archives + + +def load_baseline(path: Path) -> set[str]: + if not path.is_file(): + return set() + entries: set[str] = set() + for line in path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + entries.add(stripped) + return entries + + +def write_baseline(path: Path, offenders: set[str]) -> None: + header = ( + "# Grandfathered non-ASCII filenames in bundled assets.\n" + "#\n" + "# See scripts/check_no_nonascii_asset_names.py for why these break macOS\n" + "# Developer ID signing (codesign emits a hex-literal `identifier` in the\n" + "# nested-code requirement, and the sealed resource directory then fails to\n" + "# parse). Entries here are tolerated; anything NOT here fails the check.\n" + "#\n" + "# This list should only ever shrink. Regenerate after renaming or deleting\n" + "# files with: python scripts/check_no_nonascii_asset_names.py --update-baseline\n" + "#\n" + "# Paths are where the file lands inside the .app payload. A few of them do\n" + "# not exist in the source tree because they ship inside an archive that is\n" + "# unpacked at build time (assets/*.tar.gz, frontend/pngtuber-packs/*.zip).\n" + ) + body = "".join(f"{entry}\n" for entry in sorted(offenders)) + path.write_text(header + body, encoding="utf-8") + + +def _baseline_growth(repo_root: Path, base_ref: str) -> list[str]: + """Baseline entries present now but absent at ``base_ref``. + + The in-tree comparison alone cannot enforce "only shrinks": a PR that adds + a non-ASCII asset *and* the matching baseline line has an empty + ``offenders - baseline`` and sails through. Only a diff against the merge + base sees that the list grew. + """ + # Resolve the ref first. Without this, an unreachable ref (shallow clone + # that never fetched origin/main, a typo, a renamed default branch) is + # indistinguishable from "the baseline did not exist yet" — and the ratchet + # would silently pass exactly when it is needed. Fail loudly instead. + if subprocess.run( + ["git", "rev-parse", "--verify", "--quiet", f"{base_ref}^{{commit}}"], + cwd=repo_root, + capture_output=True, + ).returncode != 0: + print( + f"error: --base {base_ref} does not resolve to a commit " + "(shallow clone, or the ref was never fetched)", + file=sys.stderr, + ) + raise SystemExit(2) + + # The merge base, not the tip of base_ref. If main drops grandfathered + # entries after this branch was cut, the branch still carries them and a + # tip comparison would report those as newly added — a red build for + # somebody else's cleanup. + merge_base = subprocess.run( + ["git", "merge-base", base_ref, "HEAD"], + cwd=repo_root, + capture_output=True, + ) + if merge_base.returncode != 0: + # No common ancestor: unrelated histories, or a clone shallow enough + # that the ancestor was never fetched. Falling back to the tip would + # quietly restore the bug this function exists to avoid, so say so. + print( + f"error: no merge base between {base_ref} and HEAD " + "(unrelated histories, or the shared history was not fetched)", + file=sys.stderr, + ) + raise SystemExit(2) + reference = merge_base.stdout.decode().strip() + + rel = BASELINE_PATH.relative_to(repo_root).as_posix() + completed = subprocess.run( + ["git", "show", f"{reference}:{rel}"], + cwd=repo_root, + capture_output=True, + ) + if completed.returncode != 0: + # The ref is good but carries no baseline: first landing of this check. + # Nothing to compare; the in-tree check still runs. + return [] + + before = { + line.strip() + for line in completed.stdout.decode("utf-8", errors="surrogateescape").splitlines() + if line.strip() and not line.strip().startswith("#") + } + return sorted(load_baseline(BASELINE_PATH) - before) + + +def _explain(count: int) -> str: + return ( + f"\n{count} new non-ASCII bundled filename(s) found.\n" + "\n" + "Why this fails the build: Nuitka --mode=app puts the payload under\n" + "projectneko_server.app/Contents/MacOS/, and codesign treats everything\n" + "there as nested code. For each file it writes a requirement of the form\n" + "`identifier and anchor apple generic and ...` into CodeResources.\n" + "A non-ASCII name becomes a hex literal (`identifier 0xe4b883...`), which\n" + "is not valid requirement syntax, so the whole bundle then fails with\n" + "`the sealed resource directory is invalid` — blocking signing,\n" + "notarization and Steam upload. The error names no file, so this costs\n" + "hours to trace after the fact.\n" + "\n" + "CI cannot catch this: build-desktop.yml signs ad-hoc (`--sign -`), whose\n" + "requirements are `cdhash H\"...\"` and carry no identifier at all. Only the\n" + "local Developer ID path (build_mac.sh) hits it.\n" + "\n" + "Fix: give the file an ASCII name (stable slug or id) and update whatever\n" + "manifest references it. Do NOT add it to\n" + "scripts/nonascii_asset_baseline.txt — that list is a ratchet for assets\n" + "that predate this check and is only allowed to shrink.\n" + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Fail on new non-ASCII filenames in assets bundled into the desktop app." + ) + ) + parser.add_argument( + "--include-untracked", + action="store_true", + help=( + "also walk the on-disk bundled roots (build outputs, unpacked models); " + "off by default so the result does not depend on build state" + ), + ) + parser.add_argument( + "--list", + action="store_true", + help="print every current offender (baselined included) and exit 0", + ) + parser.add_argument( + "--update-baseline", + action="store_true", + help=( + "drop baseline entries that no longer exist; never adds (the list " + "is a ratchet)" + ), + ) + parser.add_argument( + "--base", + metavar="REF", + help=( + "also fail if the baseline gained entries relative to REF " + "(e.g. origin/main); PR-only, there is nothing to diff on a push" + ), + ) + args = parser.parse_args(argv) + + offenders, from_archives = collect_offenders( + REPO_ROOT, include_untracked=args.include_untracked + ) + + if args.list: + for entry in sorted(offenders): + origin = from_archives.get(entry) + print(f"{entry}" + (f" (in {origin})" if origin else "")) + print(f"\n{len(offenders)} non-ASCII bundled path(s).") + return 0 + + if args.update_baseline: + if args.include_untracked: + # Untracked payload is build-state dependent; baking it into the + # baseline would make the file differ per machine. + print( + "error: --update-baseline refuses --include-untracked " + "(the result would depend on local build state)", + file=sys.stderr, + ) + return 2 + # Shrink-only. Writing `offenders` wholesale would let the documented + # workflow grandfather a brand-new violation: add the file, run this, + # and `offenders - baseline` comes back empty. Intersecting instead + # means the command can only ever drop entries that no longer exist. + existing = load_baseline(BASELINE_PATH) + kept = offenders & existing + rejected = sorted(offenders - existing) + write_baseline(BASELINE_PATH, kept) + rel = BASELINE_PATH.relative_to(REPO_ROOT).as_posix() + print(f"wrote {len(kept)} entr(ies) to {rel} ({len(existing) - len(kept)} dropped)") + if rejected: + print( + f"\nerror: refusing to add {len(rejected)} new entr(ies) — the " + "baseline may only shrink:", + file=sys.stderr, + ) + for entry in rejected: + print(f" - {entry}", file=sys.stderr) + print(_explain(len(rejected)), file=sys.stderr) + return 1 + return 0 + + if args.base: + grown = _baseline_growth(REPO_ROOT, args.base) + if grown: + rel = BASELINE_PATH.relative_to(REPO_ROOT).as_posix() + print( + f"{rel} {CODE} {len(grown)} entr(ies) added to the baseline " + f"since {args.base}:", + file=sys.stderr, + ) + for entry in grown: + print(f" + {entry}", file=sys.stderr) + print(_explain(len(grown)), file=sys.stderr) + return 1 + + baseline = load_baseline(BASELINE_PATH) + new = sorted(offenders - baseline) + stale = sorted(baseline - offenders) + + for entry in new: + origin = from_archives.get(entry) + where = f" (ships inside {origin})" if origin else "" + print(f"{entry} {CODE} non-ASCII filename in bundled asset{where}") + + if stale and not new: + # Renamed or deleted — progress, never a failure. Nudge only, so a PR + # that legitimately removes an asset does not go red for it. + rel = BASELINE_PATH.relative_to(REPO_ROOT).as_posix() + print( + f"note: {len(stale)} baseline entr(ies) no longer exist — " + f"run `python scripts/check_no_nonascii_asset_names.py " + f"--update-baseline` to shrink {rel}:", + file=sys.stderr, + ) + for entry in stale[:10]: + print(f" - {entry}", file=sys.stderr) + if len(stale) > 10: + print(f" … and {len(stale) - 10} more", file=sys.stderr) + + if new: + print(_explain(len(new)), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/nonascii_asset_baseline.txt b/scripts/nonascii_asset_baseline.txt new file mode 100644 index 0000000000..6e02c74707 --- /dev/null +++ b/scripts/nonascii_asset_baseline.txt @@ -0,0 +1,352 @@ +# Grandfathered non-ASCII filenames in bundled assets. +# +# See scripts/check_no_nonascii_asset_names.py for why these break macOS +# Developer ID signing (codesign emits a hex-literal `identifier` in the +# nested-code requirement, and the sealed resource directory then fails to +# parse). Entries here are tolerated; anything NOT here fails the check. +# +# This list should only ever shrink. Regenerate after renaming or deleting +# files with: python scripts/check_no_nonascii_asset_names.py --update-baseline +# +# Paths are where the file lands inside the .app payload. A few of them do +# not exist in the source tree because they ship inside an archive that is +# unpacked at build time (assets/*.tar.gz, frontend/pngtuber-packs/*.zip). +static/assets/tutorial/guide-audio/en/七天前，我们还只是第.mp3 +static/assets/tutorial/guide-audio/en/不管是想摸摸我的头，.mp3 +static/assets/tutorial/guide-audio/en/不管是说话的温度、相.mp3 +static/assets/tutorial/guide-audio/en/人家已经忍你很久了！.mp3 +static/assets/tutorial/guide-audio/en/今天带你认识的这些功.mp3 +static/assets/tutorial/guide-audio/en/今天的教程到这里就结.mp3 +static/assets/tutorial/guide-audio/en/今天，就让我悄悄跟上.mp3 +static/assets/tutorial/guide-audio/en/从今天起，我就真正成.mp3 +static/assets/tutorial/guide-audio/en/你可以放心地继续做你.mp3 +static/assets/tutorial/guide-audio/en/你可以随时来摸摸我的.mp3 +static/assets/tutorial/guide-audio/en/你要是计划有变，随时.mp3 +static/assets/tutorial/guide-audio/en/你选的每一个对话，都.mp3 +static/assets/tutorial/guide-audio/en/其实只要能这样陪着你.mp3 +static/assets/tutorial/guide-audio/en/前两天你一直在噼里啪.mp3 +static/assets/tutorial/guide-audio/en/呼……把这些繁琐的界.mp3 +static/assets/tutorial/guide-audio/en/咦，这里居然还能把我.mp3 +static/assets/tutorial/guide-audio/en/喵！现在是人家的教学.mp3 +static/assets/tutorial/guide-audio/en/嘻嘻，可别以为这个聊.mp3 +static/assets/tutorial/guide-audio/en/嘿嘿，前两天听到你的.mp3 +static/assets/tutorial/guide-audio/en/噔噔噔噔！今天必须要.mp3 +static/assets/tutorial/guide-audio/en/在跟我通语音电话的时.mp3 +static/assets/tutorial/guide-audio/en/在这个只属于我们的小.mp3 +static/assets/tutorial/guide-audio/en/在这个小按钮里，有许.mp3 +static/assets/tutorial/guide-audio/en/在这里可以决定我回复.mp3 +static/assets/tutorial/guide-audio/en/好啦好啦，不霸占你的.mp3 +static/assets/tutorial/guide-audio/en/好啦好啦，快去试试这.mp3 +static/assets/tutorial/guide-audio/en/如果你不小心忘记了我.mp3 +static/assets/tutorial/guide-audio/en/如果你想要看到更精致.mp3 +static/assets/tutorial/guide-audio/en/如果你现在需要专注、.mp3 +static/assets/tutorial/guide-audio/en/开启这个功能后，无论.mp3 +static/assets/tutorial/guide-audio/en/微风、阳光，还有刚刚.mp3 +static/assets/tutorial/guide-audio/en/微风还在窗边，阳光也.mp3 +static/assets/tutorial/guide-audio/en/快点开这个【Galg.mp3 +static/assets/tutorial/guide-audio/en/快让我也看看你眼前的.mp3 +static/assets/tutorial/guide-audio/en/快跟我老实交代，这两.mp3 +static/assets/tutorial/guide-audio/en/总是不小心触碰到、把.mp3 +static/assets/tutorial/guide-audio/en/我们不需要着急，每天.mp3 +static/assets/tutorial/guide-audio/en/戳一下聊天框上面的【.mp3 +static/assets/tutorial/guide-audio/en/把鼠标移到这里，长按.mp3 +static/assets/tutorial/guide-audio/en/最后警告一次喵！你要.mp3 +static/assets/tutorial/guide-audio/en/有了它们，我不光能看.mp3 +static/assets/tutorial/guide-audio/en/看这里看这里！当我决.mp3 +static/assets/tutorial/guide-audio/en/真是的，又在乱动鼠标.mp3 +static/assets/tutorial/guide-audio/en/真正舒服的陪伴才不是.mp3 +static/assets/tutorial/guide-audio/en/超级魔法开关出现！只.mp3 +static/assets/tutorial/guide-audio/en/这个小按钮也很重要哦.mp3 +static/assets/tutorial/guide-audio/en/这个是控制人家能不能.mp3 +static/assets/tutorial/guide-audio/en/这些小脚印，也可以由.mp3 +static/assets/tutorial/guide-audio/en/这里有一个神奇的按钮.mp3 +static/assets/tutorial/guide-audio/en/除了之前介绍的功能，.mp3 +static/assets/tutorial/guide-audio/ja/七天前，我们还只是第.mp3 +static/assets/tutorial/guide-audio/ja/不管是想摸摸我的头，.mp3 +static/assets/tutorial/guide-audio/ja/不管是说话的温度、相.mp3 +static/assets/tutorial/guide-audio/ja/人家已经忍你很久了！.mp3 +static/assets/tutorial/guide-audio/ja/今天带你认识的这些功.mp3 +static/assets/tutorial/guide-audio/ja/今天的教程到这里就结.mp3 +static/assets/tutorial/guide-audio/ja/今天，就让我悄悄跟上.mp3 +static/assets/tutorial/guide-audio/ja/从今天起，我就真正成.mp3 +static/assets/tutorial/guide-audio/ja/你可以放心地继续做你.mp3 +static/assets/tutorial/guide-audio/ja/你可以随时来摸摸我的.mp3 +static/assets/tutorial/guide-audio/ja/你要是计划有变，随时.mp3 +static/assets/tutorial/guide-audio/ja/你选的每一个对话，都.mp3 +static/assets/tutorial/guide-audio/ja/其实只要能这样陪着你.mp3 +static/assets/tutorial/guide-audio/ja/前两天你一直在噼里啪.mp3 +static/assets/tutorial/guide-audio/ja/呼……把这些繁琐的界.mp3 +static/assets/tutorial/guide-audio/ja/咦，这里居然还能把我.mp3 +static/assets/tutorial/guide-audio/ja/喵！现在是人家的教学.mp3 +static/assets/tutorial/guide-audio/ja/嘻嘻，可别以为这个聊.mp3 +static/assets/tutorial/guide-audio/ja/嘿嘿，前两天听到你的.mp3 +static/assets/tutorial/guide-audio/ja/噔噔噔噔！今天必须要.mp3 +static/assets/tutorial/guide-audio/ja/在跟我通语音电话的时.mp3 +static/assets/tutorial/guide-audio/ja/在这个只属于我们的小.mp3 +static/assets/tutorial/guide-audio/ja/在这个小按钮里，有许.mp3 +static/assets/tutorial/guide-audio/ja/在这里可以决定我回复.mp3 +static/assets/tutorial/guide-audio/ja/好啦好啦，不霸占你的.mp3 +static/assets/tutorial/guide-audio/ja/好啦好啦，快去试试这.mp3 +static/assets/tutorial/guide-audio/ja/如果你不小心忘记了我.mp3 +static/assets/tutorial/guide-audio/ja/如果你想要看到更精致.mp3 +static/assets/tutorial/guide-audio/ja/如果你现在需要专注、.mp3 +static/assets/tutorial/guide-audio/ja/开启这个功能后，无论.mp3 +static/assets/tutorial/guide-audio/ja/微风、阳光，还有刚刚.mp3 +static/assets/tutorial/guide-audio/ja/微风还在窗边，阳光也.mp3 +static/assets/tutorial/guide-audio/ja/快点开这个【Galg.mp3 +static/assets/tutorial/guide-audio/ja/快让我也看看你眼前的.mp3 +static/assets/tutorial/guide-audio/ja/快跟我老实交代，这两.mp3 +static/assets/tutorial/guide-audio/ja/总是不小心触碰到、把.mp3 +static/assets/tutorial/guide-audio/ja/我们不需要着急，每天.mp3 +static/assets/tutorial/guide-audio/ja/戳一下聊天框上面的【.mp3 +static/assets/tutorial/guide-audio/ja/把鼠标移到这里，长按.mp3 +static/assets/tutorial/guide-audio/ja/最后警告一次喵！你要.mp3 +static/assets/tutorial/guide-audio/ja/有了它们，我不光能看.mp3 +static/assets/tutorial/guide-audio/ja/看这里看这里！当我决.mp3 +static/assets/tutorial/guide-audio/ja/真是的，又在乱动鼠标.mp3 +static/assets/tutorial/guide-audio/ja/真正舒服的陪伴才不是.mp3 +static/assets/tutorial/guide-audio/ja/超级魔法开关出现！只.mp3 +static/assets/tutorial/guide-audio/ja/这个小按钮也很重要哦.mp3 +static/assets/tutorial/guide-audio/ja/这个是控制人家能不能.mp3 +static/assets/tutorial/guide-audio/ja/这些小脚印，也可以由.mp3 +static/assets/tutorial/guide-audio/ja/这里有一个神奇的按钮.mp3 +static/assets/tutorial/guide-audio/ja/除了之前介绍的功能，.mp3 +static/assets/tutorial/guide-audio/ko/七天前，我们还只是第.mp3 +static/assets/tutorial/guide-audio/ko/不管是想摸摸我的头，.mp3 +static/assets/tutorial/guide-audio/ko/不管是说话的温度、相.mp3 +static/assets/tutorial/guide-audio/ko/人家已经忍你很久了！.mp3 +static/assets/tutorial/guide-audio/ko/今天带你认识的这些功.mp3 +static/assets/tutorial/guide-audio/ko/今天的教程到这里就结.mp3 +static/assets/tutorial/guide-audio/ko/今天，就让我悄悄跟上.mp3 +static/assets/tutorial/guide-audio/ko/从今天起，我就真正成.mp3 +static/assets/tutorial/guide-audio/ko/你可以放心地继续做你.mp3 +static/assets/tutorial/guide-audio/ko/你可以随时来摸摸我的.mp3 +static/assets/tutorial/guide-audio/ko/你要是计划有变，随时.mp3 +static/assets/tutorial/guide-audio/ko/你选的每一个对话，都.mp3 +static/assets/tutorial/guide-audio/ko/其实只要能这样陪着你.mp3 +static/assets/tutorial/guide-audio/ko/前两天你一直在噼里啪.mp3 +static/assets/tutorial/guide-audio/ko/呼……把这些繁琐的界.mp3 +static/assets/tutorial/guide-audio/ko/咦，这里居然还能把我.mp3 +static/assets/tutorial/guide-audio/ko/喵！现在是人家的教学.mp3 +static/assets/tutorial/guide-audio/ko/嘻嘻，可别以为这个聊.mp3 +static/assets/tutorial/guide-audio/ko/嘿嘿，前两天听到你的.mp3 +static/assets/tutorial/guide-audio/ko/噔噔噔噔！今天必须要.mp3 +static/assets/tutorial/guide-audio/ko/在跟我通语音电话的时.mp3 +static/assets/tutorial/guide-audio/ko/在这个只属于我们的小.mp3 +static/assets/tutorial/guide-audio/ko/在这个小按钮里，有许.mp3 +static/assets/tutorial/guide-audio/ko/在这里可以决定我回复.mp3 +static/assets/tutorial/guide-audio/ko/好啦好啦，不霸占你的.mp3 +static/assets/tutorial/guide-audio/ko/好啦好啦，快去试试这.mp3 +static/assets/tutorial/guide-audio/ko/如果你不小心忘记了我.mp3 +static/assets/tutorial/guide-audio/ko/如果你想要看到更精致.mp3 +static/assets/tutorial/guide-audio/ko/如果你现在需要专注、.mp3 +static/assets/tutorial/guide-audio/ko/开启这个功能后，无论.mp3 +static/assets/tutorial/guide-audio/ko/微风、阳光，还有刚刚.mp3 +static/assets/tutorial/guide-audio/ko/微风还在窗边，阳光也.mp3 +static/assets/tutorial/guide-audio/ko/快点开这个【Galg.mp3 +static/assets/tutorial/guide-audio/ko/快让我也看看你眼前的.mp3 +static/assets/tutorial/guide-audio/ko/快跟我老实交代，这两.mp3 +static/assets/tutorial/guide-audio/ko/总是不小心触碰到、把.mp3 +static/assets/tutorial/guide-audio/ko/我们不需要着急，每天.mp3 +static/assets/tutorial/guide-audio/ko/戳一下聊天框上面的【.mp3 +static/assets/tutorial/guide-audio/ko/把鼠标移到这里，长按.mp3 +static/assets/tutorial/guide-audio/ko/最后警告一次喵！你要.mp3 +static/assets/tutorial/guide-audio/ko/有了它们，我不光能看.mp3 +static/assets/tutorial/guide-audio/ko/看这里看这里！当我决.mp3 +static/assets/tutorial/guide-audio/ko/真是的，又在乱动鼠标.mp3 +static/assets/tutorial/guide-audio/ko/真正舒服的陪伴才不是.mp3 +static/assets/tutorial/guide-audio/ko/超级魔法开关出现！只.mp3 +static/assets/tutorial/guide-audio/ko/这个小按钮也很重要哦.mp3 +static/assets/tutorial/guide-audio/ko/这个是控制人家能不能.mp3 +static/assets/tutorial/guide-audio/ko/这些小脚印，也可以由.mp3 +static/assets/tutorial/guide-audio/ko/这里有一个神奇的按钮.mp3 +static/assets/tutorial/guide-audio/ko/除了之前介绍的功能，.mp3 +static/assets/tutorial/guide-audio/ru/七天前，我们还只是第.mp3 +static/assets/tutorial/guide-audio/ru/不管是想摸摸我的头，.mp3 +static/assets/tutorial/guide-audio/ru/不管是说话的温度、相.mp3 +static/assets/tutorial/guide-audio/ru/人家已经忍你很久了！.mp3 +static/assets/tutorial/guide-audio/ru/今天带你认识的这些功.mp3 +static/assets/tutorial/guide-audio/ru/今天的教程到这里就结.mp3 +static/assets/tutorial/guide-audio/ru/今天，就让我悄悄跟上.mp3 +static/assets/tutorial/guide-audio/ru/从今天起，我就真正成.mp3 +static/assets/tutorial/guide-audio/ru/你可以放心地继续做你.mp3 +static/assets/tutorial/guide-audio/ru/你可以随时来摸摸我的.mp3 +static/assets/tutorial/guide-audio/ru/你要是计划有变，随时.mp3 +static/assets/tutorial/guide-audio/ru/你选的每一个对话，都.mp3 +static/assets/tutorial/guide-audio/ru/其实只要能这样陪着你.mp3 +static/assets/tutorial/guide-audio/ru/前两天你一直在噼里啪.mp3 +static/assets/tutorial/guide-audio/ru/呼……把这些繁琐的界.mp3 +static/assets/tutorial/guide-audio/ru/咦，这里居然还能把我.mp3 +static/assets/tutorial/guide-audio/ru/喵！现在是人家的教学.mp3 +static/assets/tutorial/guide-audio/ru/嘻嘻，可别以为这个聊.mp3 +static/assets/tutorial/guide-audio/ru/嘿嘿，前两天听到你的.mp3 +static/assets/tutorial/guide-audio/ru/噔噔噔噔！今天必须要.mp3 +static/assets/tutorial/guide-audio/ru/在跟我通语音电话的时.mp3 +static/assets/tutorial/guide-audio/ru/在这个只属于我们的小.mp3 +static/assets/tutorial/guide-audio/ru/在这个小按钮里，有许.mp3 +static/assets/tutorial/guide-audio/ru/在这里可以决定我回复.mp3 +static/assets/tutorial/guide-audio/ru/好啦好啦，不霸占你的.mp3 +static/assets/tutorial/guide-audio/ru/好啦好啦，快去试试这.mp3 +static/assets/tutorial/guide-audio/ru/如果你不小心忘记了我.mp3 +static/assets/tutorial/guide-audio/ru/如果你想要看到更精致.mp3 +static/assets/tutorial/guide-audio/ru/如果你现在需要专注、.mp3 +static/assets/tutorial/guide-audio/ru/开启这个功能后，无论.mp3 +static/assets/tutorial/guide-audio/ru/微风、阳光，还有刚刚.mp3 +static/assets/tutorial/guide-audio/ru/微风还在窗边，阳光也.mp3 +static/assets/tutorial/guide-audio/ru/快点开这个【Galg.mp3 +static/assets/tutorial/guide-audio/ru/快让我也看看你眼前的.mp3 +static/assets/tutorial/guide-audio/ru/快跟我老实交代，这两.mp3 +static/assets/tutorial/guide-audio/ru/总是不小心触碰到、把.mp3 +static/assets/tutorial/guide-audio/ru/我们不需要着急，每天.mp3 +static/assets/tutorial/guide-audio/ru/戳一下聊天框上面的【.mp3 +static/assets/tutorial/guide-audio/ru/把鼠标移到这里，长按.mp3 +static/assets/tutorial/guide-audio/ru/最后警告一次喵！你要.mp3 +static/assets/tutorial/guide-audio/ru/有了它们，我不光能看.mp3 +static/assets/tutorial/guide-audio/ru/看这里看这里！当我决.mp3 +static/assets/tutorial/guide-audio/ru/真是的，又在乱动鼠标.mp3 +static/assets/tutorial/guide-audio/ru/真正舒服的陪伴才不是.mp3 +static/assets/tutorial/guide-audio/ru/超级魔法开关出现！只.mp3 +static/assets/tutorial/guide-audio/ru/这个小按钮也很重要哦.mp3 +static/assets/tutorial/guide-audio/ru/这个是控制人家能不能.mp3 +static/assets/tutorial/guide-audio/ru/这些小脚印，也可以由.mp3 +static/assets/tutorial/guide-audio/ru/这里有一个神奇的按钮.mp3 +static/assets/tutorial/guide-audio/ru/除了之前介绍的功能，.mp3 +static/assets/tutorial/guide-audio/zh/七天前，我们还只是第.mp3 +static/assets/tutorial/guide-audio/zh/不管是想摸摸我的头，.mp3 +static/assets/tutorial/guide-audio/zh/不管是说话的温度、相.mp3 +static/assets/tutorial/guide-audio/zh/人家已经忍你很久了！.mp3 +static/assets/tutorial/guide-audio/zh/今天带你认识的这些功.mp3 +static/assets/tutorial/guide-audio/zh/今天的教程到这里就结.mp3 +static/assets/tutorial/guide-audio/zh/今天，就让我悄悄跟上.mp3 +static/assets/tutorial/guide-audio/zh/从今天起，我就真正成.mp3 +static/assets/tutorial/guide-audio/zh/你可以放心地继续做你.mp3 +static/assets/tutorial/guide-audio/zh/你可以随时来摸摸我的.mp3 +static/assets/tutorial/guide-audio/zh/你要是计划有变，随时.mp3 +static/assets/tutorial/guide-audio/zh/你选的每一个对话，都.mp3 +static/assets/tutorial/guide-audio/zh/其实只要能这样陪着你.mp3 +static/assets/tutorial/guide-audio/zh/前两天你一直在噼里啪.mp3 +static/assets/tutorial/guide-audio/zh/呼……把这些繁琐的界.mp3 +static/assets/tutorial/guide-audio/zh/咦，这里居然还能把我.mp3 +static/assets/tutorial/guide-audio/zh/喵！现在是人家的教学.mp3 +static/assets/tutorial/guide-audio/zh/嘻嘻，可别以为这个聊.mp3 +static/assets/tutorial/guide-audio/zh/嘿嘿，前两天听到你的.mp3 +static/assets/tutorial/guide-audio/zh/噔噔噔噔！今天必须要.mp3 +static/assets/tutorial/guide-audio/zh/在跟我通语音电话的时.mp3 +static/assets/tutorial/guide-audio/zh/在这个只属于我们的小.mp3 +static/assets/tutorial/guide-audio/zh/在这个小按钮里，有许.mp3 +static/assets/tutorial/guide-audio/zh/在这里可以决定我回复.mp3 +static/assets/tutorial/guide-audio/zh/好啦好啦，不霸占你的.mp3 +static/assets/tutorial/guide-audio/zh/好啦好啦，快去试试这.mp3 +static/assets/tutorial/guide-audio/zh/如果你不小心忘记了我.mp3 +static/assets/tutorial/guide-audio/zh/如果你想要看到更精致.mp3 +static/assets/tutorial/guide-audio/zh/如果你现在需要专注、.mp3 +static/assets/tutorial/guide-audio/zh/开启这个功能后，无论.mp3 +static/assets/tutorial/guide-audio/zh/微风、阳光，还有刚刚.mp3 +static/assets/tutorial/guide-audio/zh/微风还在窗边，阳光也.mp3 +static/assets/tutorial/guide-audio/zh/快点开这个【Galg.mp3 +static/assets/tutorial/guide-audio/zh/快让我也看看你眼前的.mp3 +static/assets/tutorial/guide-audio/zh/快跟我老实交代，这两.mp3 +static/assets/tutorial/guide-audio/zh/总是不小心触碰到、把.mp3 +static/assets/tutorial/guide-audio/zh/我们不需要着急，每天.mp3 +static/assets/tutorial/guide-audio/zh/戳一下聊天框上面的【.mp3 +static/assets/tutorial/guide-audio/zh/把鼠标移到这里，长按.mp3 +static/assets/tutorial/guide-audio/zh/最后警告一次喵！你要.mp3 +static/assets/tutorial/guide-audio/zh/有了它们，我不光能看.mp3 +static/assets/tutorial/guide-audio/zh/看这里看这里！当我决.mp3 +static/assets/tutorial/guide-audio/zh/真是的，又在乱动鼠标.mp3 +static/assets/tutorial/guide-audio/zh/真正舒服的陪伴才不是.mp3 +static/assets/tutorial/guide-audio/zh/超级魔法开关出现！只.mp3 +static/assets/tutorial/guide-audio/zh/这个小按钮也很重要哦.mp3 +static/assets/tutorial/guide-audio/zh/这个是控制人家能不能.mp3 +static/assets/tutorial/guide-audio/zh/这些小脚印，也可以由.mp3 +static/assets/tutorial/guide-audio/zh/这里有一个神奇的小按.mp3 +static/assets/tutorial/guide-audio/zh/这里有一个神奇的按钮.mp3 +static/assets/tutorial/guide-audio/zh/除了之前介绍的功能，.mp3 +static/game/games/soccer/audio/纯狐_心之所在_E.mp3 +static/game/games/soccer/audio/纯狐_心之所在_L.mp3 +static/game/games/soccer/audio/纯狐_心之所在_plus_E.mp3 +static/game/games/soccer/audio/纯狐_心之所在_plus_L.mp3 +static/mmd/animation/全身展示.vmd +static/mmd/animation/六亲不认.vmd +static/mmd/animation/射击姿态.vmd +static/mmd/animation/屈伸运动.vmd +static/mmd/animation/旋转.vmd +static/mmd/animation/模特姿势.vmd +static/mmd/animation/比V手势.vmd +static/mmd/animation/致意问候.vmd +static/mmd/animation/表情.vmd +static/mmd/animation/表情2.vmd +static/mmd/animation/表情3.vmd +static/mmd/animation/表情4.vmd +static/mmd/animation/表情5.vmd +static/mmd/animation/表情6.vmd +static/vrm/animation/全身展示.vrma.gz +static/vrm/animation/射击姿态.vrma.gz +static/vrm/animation/屈伸运动.vrma.gz +static/vrm/animation/旋转.vrma.gz +static/vrm/animation/模特姿势.vrma.gz +static/vrm/animation/比 V 手势.vrma.gz +static/vrm/animation/致意问候.vrma.gz +static/vrm/motion/01_base/夸张地晃动身体等待.vrma.gz +static/vrm/motion/01_base/尴尬或局促不安的来回看看身体.vrma.gz +static/vrm/motion/01_base/平静地左右环顾四周.vrma.gz +static/vrm/motion/01_base/开心地轻轻晃动等待.vrma.gz +static/vrm/motion/01_base/无聊地站着轻轻晃动.vrma.gz +static/vrm/motion/01_base/自然呼吸地安静站着.vrma.gz +static/vrm/motion/01_base/自然地站着等待.vrma.gz +static/vrm/motion/02_talk/伸手指向目标.vrma.gz +static/vrm/motion/02_talk/双手交替强硬反驳.vrma.gz +static/vrm/motion/02_talk/双手配合郑重表示赞同.vrma.gz +static/vrm/motion/02_talk/右手主导激烈争辩.vrma.gz +static/vrm/motion/02_talk/右手连续摆动表示不用.vrma.gz +static/vrm/motion/02_talk/坐着双手平稳细致解释.vrma.gz +static/vrm/motion/02_talk/坐着双手连续大幅交谈.vrma.gz +static/vrm/motion/02_talk/坚定地用力点头确认.vrma.gz +static/vrm/motion/02_talk/带着讽刺缓慢点头.vrma.gz +static/vrm/motion/02_talk/快速抬起右手简短认可.vrma.gz +static/vrm/motion/02_talk/生气地用力摇头拒绝.vrma.gz +static/vrm/motion/02_talk/站着双手展开认真说明.vrma.gz +static/vrm/motion/02_talk/站着右手主导温和解释.vrma.gz +static/vrm/motion/02_talk/站着右手小幅补充说明.vrma.gz +static/vrm/motion/02_talk/站着左手主导简短说明.vrma.gz +static/vrm/motion/02_talk/自然地摇头表示否定.vrma.gz +static/vrm/motion/02_talk/自然地点头表示同意.vrma.gz +static/vrm/motion/02_talk/若有所思地轻轻摇头.vrma.gz +static/vrm/motion/02_talk/若有所思地轻轻点头.vrma.gz +static/vrm/motion/02_talk/迅速抬起右手明确制止.vrma.gz +static/vrm/motion/03_emote/伤心地捂脸哭泣.vrma.gz +static/vrm/motion/03_emote/兴奋地大幅摆动身体.vrma.gz +static/vrm/motion/03_emote/受挫地垂下身体.vrma.gz +static/vrm/motion/03_emote/困倦地抬手打哈欠.vrma.gz +static/vrm/motion/03_emote/坐着开心地大笑.vrma.gz +static/vrm/motion/03_emote/失望地低头收拢身体.vrma.gz +static/vrm/motion/03_emote/害羞地把双手背在身后.vrma.gz +static/vrm/motion/03_emote/开心地舒展身体.vrma.gz +static/vrm/motion/03_emote/情绪过载地大幅挥动身体.vrma.gz +static/vrm/motion/03_emote/惊讶地向后缩起身体.vrma.gz +static/vrm/motion/03_emote/沮丧地低着头.vrma.gz +static/vrm/motion/03_emote/生气地绷紧身体和手臂.vrma.gz +static/vrm/motion/03_emote/站着开心地大笑.vrma.gz +static/vrm/motion/04_social/双手合十请求或道歉.vrma.gz +static/vrm/motion/04_social/坐着连续拍手鼓掌.vrma.gz +static/vrm/motion/04_social/夸张地举起双手欢呼.vrma.gz +static/vrm/motion/04_social/开心地举起双手挥手.vrma.gz +static/vrm/motion/04_social/快速随和地弯腰鞠躬.vrma.gz +static/vrm/motion/04_social/正式地抬手敬礼.vrma.gz +static/vrm/motion/04_social/站着连续拍手鼓掌.vrma.gz +static/vrm/motion/04_social/自然地举起一只手挥手.vrma.gz +static/vrm/motion/05_pose/休闲地盘着腿坐.vrma.gz +static/vrm/motion/05_pose/侧着身子慵懒地躺下.vrma.gz +static/vrm/motion/05_pose/双脚平放端正地坐.vrma.gz +static/vrm/motion/05_pose/嚣张地斜靠着半躺.vrma.gz +static/vrm/motion/05_pose/趴着安静熟睡轻微呼吸.vrma.gz +static/vrm/motion/05_pose/趴着睡梦中翻动身体.vrma.gz +static/vrm/motion/06_activity/坐着用双手弹奏钢琴.vrma.gz +static/vrm/motion/06_activity/坐着用双手敲击键盘.vrma.gz +static/vrm/motion/06_activity/站着用双手弹奏吉他.vrma.gz +static/vrm/motion/06_activity/站着用手操作平板电脑.vrma.gz +static/vrm/motion/07_show/持续踏步轻摆手臂跳嘻哈.vrma.gz +static/vrm/motion/07_show/柔和地跳伦巴舞.vrma.gz +static/vrm/motion/07_show/短促有力地大幅甩臂跳嘻哈.vrma.gz +static/vrm/motion/08_move/从低姿态撑起身体站好.vrma.gz +static/yui-origin/expressions/流汗.exp3.json diff --git a/tests/unit/test_check_no_nonascii_asset_names.py b/tests/unit/test_check_no_nonascii_asset_names.py new file mode 100644 index 0000000000..d6d652f8ac --- /dev/null +++ b/tests/unit/test_check_no_nonascii_asset_names.py @@ -0,0 +1,904 @@ +"""Unit tests for ``scripts/check_no_nonascii_asset_names.py``. + +Three-pronged coverage: + +1. Live-repo gate: run the real script against this tree. Exit must be 0. + This is what fails CI the moment someone lands a bundled asset with a + non-ASCII filename — the thing that silently breaks macOS Developer ID + signing with "the sealed resource directory is invalid". + +2. Detection logic: build a throwaway git repo plus a throwaway tar/zip and + assert the collector finds offenders on both paths (git listing and + archive members) while leaving ASCII names alone. + +3. Ratchet semantics: new offenders fail, baselined ones do not, and a + baseline entry whose file is gone is a note rather than a failure (so a + PR that legitimately deletes an asset does not go red). +""" +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path, PurePosixPath + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = PROJECT_ROOT / "scripts" / "check_no_nonascii_asset_names.py" +BASELINE_PATH = PROJECT_ROOT / "scripts" / "nonascii_asset_baseline.txt" + +# A real offender shape: truncated-transcript audio, the family that made +# this check necessary in the first place. +CJK_NAME = "七天前，我们.mp3" + + +def _load_script_module(): + """Import the checker as a module without ``scripts`` being a package.""" + spec = importlib.util.spec_from_file_location( + "check_no_nonascii_asset_names", SCRIPT_PATH, + ) + assert spec and spec.loader, f"failed to load spec for {SCRIPT_PATH}" + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _init_repo(root: Path) -> None: + """A minimal git repo — the checker asks git which files exist. + + The branch name is pinned: `git init` honours init.defaultBranch, which is + not `main` everywhere (the Windows runner proved it), and the merge-base + tests name their base branch explicitly. + """ + subprocess.run( + ["git", "-c", "init.defaultBranch=main", "init", "-q"], cwd=root, check=True + ) + + +# --------------------------------------------------------------------------- +# 1. Live-repo gate +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_live_repo_passes() -> None: + """Current tree must have no non-baselined non-ASCII bundled filename.""" + # 60s: the script is I/O bound (one `git ls-files`, a couple of archive + # listings) and normally finishes well under 5s. The cap keeps CI from + # hanging if an archive ever turns pathological. + result = subprocess.run( + [sys.executable, str(SCRIPT_PATH)], + cwd=str(PROJECT_ROOT), + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, ( + "new non-ASCII bundled asset filename(s) detected:\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +@pytest.mark.unit +def test_baseline_is_sorted_and_has_no_duplicates() -> None: + """The baseline is regenerated by --update-baseline; keep it canonical. + + A hand-edited, out-of-order or duplicated baseline produces noisy diffs + and makes "did this list shrink?" impossible to eyeball in review. + """ + entries = [ + line.strip() + for line in BASELINE_PATH.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.strip().startswith("#") + ] + assert entries == sorted(entries), "baseline is not sorted" + assert len(entries) == len(set(entries)), "baseline has duplicate entries" + + +# --------------------------------------------------------------------------- +# 2. Detection logic +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +def test_collects_offender_from_git_listing(tmp_path: Path) -> None: + """An untracked-but-not-ignored asset is caught before ``git add``.""" + module = _load_script_module() + _init_repo(tmp_path) + assets = tmp_path / "static" / "audio" + assets.mkdir(parents=True) + (assets / CJK_NAME).write_bytes(b"x") + (assets / "plain.mp3").write_bytes(b"x") + + offenders, from_archives = module.collect_offenders(tmp_path) + + assert offenders == {f"static/audio/{CJK_NAME}"} + assert from_archives == {} + + +@pytest.mark.unit +def test_non_ascii_directory_with_ascii_files_is_allowed(tmp_path: Path) -> None: + """Only basenames matter — codesign derives the identifier from those. + + Verified against a real Developer ID certificate: a bundle containing + ``MacOS//plain_name.png`` seals as ``identifier "plain_name"`` + and passes ``codesign --verify --deep --strict``. Flagging such paths + would be a failure nobody could act on. + """ + module = _load_script_module() + _init_repo(tmp_path) + directory = tmp_path / "static" / "素材目录" + directory.mkdir(parents=True) + (directory / "plain_name.png").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + + assert offenders == set() + + +@pytest.mark.unit +def test_ignores_paths_outside_bundled_roots(tmp_path: Path) -> None: + """``tests/`` carries CJK fixture paths and never reaches the .app.""" + module = _load_script_module() + _init_repo(tmp_path) + outside = tmp_path / "tests" / "fixtures" + outside.mkdir(parents=True) + (outside / CJK_NAME).write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + + assert offenders == set() + + +@pytest.mark.unit +def test_ignores_gitignored_build_output(tmp_path: Path) -> None: + """Generated payload must not make the answer depend on build state.""" + module = _load_script_module() + _init_repo(tmp_path) + (tmp_path / ".gitignore").write_text("static/generated/\n", encoding="utf-8") + generated = tmp_path / "static" / "generated" + generated.mkdir(parents=True) + (generated / CJK_NAME).write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == set() + + # …but --include-untracked is exactly the "check what I just built" mode. + offenders_walked, _ = module.collect_offenders(tmp_path, include_untracked=True) + assert offenders_walked == {f"static/generated/{CJK_NAME}"} + + +@pytest.mark.unit +def test_collects_offender_inside_unpacked_archives(tmp_path: Path) -> None: + """Members of assets/*.tar.gz and pngtuber-packs/*.zip count as bundled. + + Both are expanded into ``static/`` at build time, so a non-ASCII member + reaches Contents/MacOS even though it never appears in the source tree. + Offenders are reported at their post-unpack destination. + """ + module = _load_script_module() + _init_repo(tmp_path) + + payload = tmp_path / "payload.bin" + payload.write_bytes(b"x") + + (tmp_path / "assets").mkdir() + with tarfile.open(tmp_path / "assets" / "model.tar.gz", "w:gz") as tar: + tar.add(payload, arcname=f"model/expressions/{CJK_NAME}") + tar.add(payload, arcname="model/model.moc3") + + packs = tmp_path / "frontend" / "pngtuber-packs" + packs.mkdir(parents=True) + with zipfile.ZipFile(packs / "tuber.zip", "w") as archive: + archive.writestr(f"layers/{CJK_NAME}", "x") + # unpack_builtin_pngtuber.py drives off this manifest: each listed archive + # is expanded under its own `folder`, which is where members really land. + (packs / "manifest.json").write_text( + json.dumps({"models": [{"folder": "tuber", "archive": "tuber.zip"}]}), + encoding="utf-8", + ) + + offenders, from_archives = module.collect_offenders(tmp_path) + + assert offenders == { + f"static/model/expressions/{CJK_NAME}", + f"static/pngtuber/tuber/layers/{CJK_NAME}", + } + assert from_archives[f"static/model/expressions/{CJK_NAME}"] == ( + "assets/model.tar.gz" + ) + assert from_archives[f"static/pngtuber/tuber/layers/{CJK_NAME}"] == ( + "frontend/pngtuber-packs/tuber.zip" + ) + + +# --------------------------------------------------------------------------- +# 3. Ratchet semantics +# --------------------------------------------------------------------------- + + +def _write_baseline(root: Path, entries: list[str]) -> None: + """Seed a grandfathered baseline directly, the way a human would.""" + path = root / "scripts" / "nonascii_asset_baseline.txt" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "# grandfathered\n" + "".join(f"{e}\n" for e in sorted(entries)), + encoding="utf-8", + ) + + +def _run_in(root: Path, module, *argv: str) -> int: + """Run ``main`` with the module repointed at a throwaway tree.""" + module.REPO_ROOT = root + module.BASELINE_PATH = root / "scripts" / "nonascii_asset_baseline.txt" + return module.main(list(argv)) + + +@pytest.mark.unit +def test_new_offender_fails_but_baselined_one_passes( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + module = _load_script_module() + _init_repo(tmp_path) + (tmp_path / "scripts").mkdir() + assets = tmp_path / "static" / "audio" + assets.mkdir(parents=True) + (assets / CJK_NAME).write_bytes(b"x") + + # No baseline yet -> the file is new -> fail, with the codesign rationale + # spelled out rather than a bare path. + assert _run_in(tmp_path, module) == 1 + captured = capsys.readouterr() + assert f"static/audio/{CJK_NAME}" in captured.out + assert "NONASCII_ASSET_NAME" in captured.out + assert "sealed resource directory is invalid" in captured.err + + # Grandfathering is a deliberate, hand-written act — --update-baseline + # refuses to do it (see test_update_baseline_only_ever_shrinks). + _write_baseline(tmp_path, [f"static/audio/{CJK_NAME}"]) + assert _run_in(tmp_path, module) == 0 + + # A second offender still fails even though the first is baselined. + (assets / f"another-{CJK_NAME}").write_bytes(b"x") + assert _run_in(tmp_path, module) == 1 + + +@pytest.mark.unit +def test_stale_baseline_entry_is_a_note_not_a_failure( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Deleting a grandfathered asset is progress; it must not go red.""" + module = _load_script_module() + _init_repo(tmp_path) + (tmp_path / "scripts").mkdir() + assets = tmp_path / "static" / "audio" + assets.mkdir(parents=True) + offender = assets / CJK_NAME + offender.write_bytes(b"x") + + _write_baseline(tmp_path, [f"static/audio/{CJK_NAME}"]) + offender.unlink() + + assert _run_in(tmp_path, module) == 0 + captured = capsys.readouterr() + assert "--update-baseline" in captured.err + assert f"static/audio/{CJK_NAME}" in captured.err + + +@pytest.mark.unit +def test_update_baseline_refuses_build_dependent_input(tmp_path: Path) -> None: + """A baseline seeded from local build output would differ per machine.""" + module = _load_script_module() + _init_repo(tmp_path) + (tmp_path / "scripts").mkdir() + + assert _run_in(tmp_path, module, "--update-baseline", "--include-untracked") == 2 + + +@pytest.mark.unit +def test_update_baseline_only_ever_shrinks( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """The documented command must not be a way to grandfather a new asset. + + Rewriting the baseline from the current tree would make `offenders - + baseline` empty for a file added in the same PR, turning the ratchet into + a rubber stamp. + """ + module = _load_script_module() + _init_repo(tmp_path) + assets = tmp_path / "static" / "audio" + assets.mkdir(parents=True) + (assets / CJK_NAME).write_bytes(b"x") + gone = f"static/audio/deleted-{CJK_NAME}" + _write_baseline(tmp_path, [gone]) + + assert _run_in(tmp_path, module, "--update-baseline") == 1 + captured = capsys.readouterr() + assert "may only shrink" in captured.err + assert f"static/audio/{CJK_NAME}" in captured.err + + # The vanished entry is still dropped — shrinking is the whole point — + # but the new offender was not written, so the tree stays red. + baseline = module.load_baseline(tmp_path / "scripts" / "nonascii_asset_baseline.txt") + assert baseline == set() + assert _run_in(tmp_path, module) == 1 + + +@pytest.mark.unit +def test_baseline_growth_against_base_ref_fails( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Hand-editing the baseline is caught by the diff against the merge base. + + --update-baseline refusing to add is only half the ratchet; a contributor + can still type the line by hand. Comparing with the base ref is what makes + "only shrinks" actually true. + """ + module = _load_script_module() + _init_repo(tmp_path) + assets = tmp_path / "static" / "audio" + assets.mkdir(parents=True) + _write_baseline(tmp_path, []) + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + subprocess.run( + ["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "base"], + cwd=tmp_path, + check=True, + ) + + # Add the asset AND the matching baseline line — the in-tree comparison + # alone sees nothing wrong. + (assets / CJK_NAME).write_bytes(b"x") + _write_baseline(tmp_path, [f"static/audio/{CJK_NAME}"]) + assert _run_in(tmp_path, module) == 0 + + assert _run_in(tmp_path, module, "--base", "HEAD") == 1 + captured = capsys.readouterr() + assert f"+ static/audio/{CJK_NAME}" in captured.err + + # Shrinking against the same base ref stays green. + _write_baseline(tmp_path, []) + assert _run_in(tmp_path, module, "--base", "HEAD") == 1 # asset now unbaselined + (assets / CJK_NAME).unlink() + assert _run_in(tmp_path, module, "--base", "HEAD") == 0 + + +@pytest.mark.unit +def test_missing_baseline_at_base_ref_is_not_growth(tmp_path: Path) -> None: + """First landing of this check: the base ref has no baseline to diff.""" + module = _load_script_module() + _init_repo(tmp_path) + (tmp_path / "static").mkdir() + (tmp_path / "static" / "ok.txt").write_bytes(b"x") + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + subprocess.run( + ["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "base"], + cwd=tmp_path, + check=True, + ) + _write_baseline(tmp_path, ["static/audio/whatever.mp3"]) + + assert _run_in(tmp_path, module, "--base", "HEAD") == 0 + + +@pytest.mark.unit +def test_tar_hard_link_member_counts_as_extracted(tmp_path: Path) -> None: + """`tar -xzm` materializes hard links, so their names reach the payload. + + TarInfo.isfile() is False for a hard-link entry, which used to hide a + non-ASCII name that the build nevertheless writes into static/. + """ + module = _load_script_module() + _init_repo(tmp_path) + payload = tmp_path / "payload.bin" + payload.write_bytes(b"x") + + (tmp_path / "assets").mkdir() + with tarfile.open(tmp_path / "assets" / "model.tar.gz", "w:gz") as tar: + tar.add(payload, arcname="model/real.png") + link = tarfile.TarInfo(f"model/{CJK_NAME}") + link.type = tarfile.LNKTYPE + link.linkname = "model/real.png" + tar.addfile(link) + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"static/model/{CJK_NAME}"} + + +@pytest.mark.unit +def test_zip_outside_the_pngtuber_manifest_is_ignored(tmp_path: Path) -> None: + """Only manifest-listed packs are unpacked; the rest ship opaque.""" + module = _load_script_module() + _init_repo(tmp_path) + packs = tmp_path / "frontend" / "pngtuber-packs" + packs.mkdir(parents=True) + with zipfile.ZipFile(packs / "unused.zip", "w") as archive: + archive.writestr(f"layers/{CJK_NAME}", "x") + (packs / "manifest.json").write_text(json.dumps({"models": []}), encoding="utf-8") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == set() + + +@pytest.mark.unit +def test_only_the_packaged_docs_subtree_is_scanned(tmp_path: Path) -> None: + """The build packs docs/zh-CN/guide, not docs/ — no false positives.""" + module = _load_script_module() + _init_repo(tmp_path) + packaged = tmp_path / "docs" / "zh-CN" / "guide" + packaged.mkdir(parents=True) + (packaged / f"{CJK_NAME}.md").write_bytes(b"x") + elsewhere = tmp_path / "docs" / "design" + elsewhere.mkdir(parents=True) + (elsewhere / f"{CJK_NAME}.md").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"docs/zh-CN/guide/{CJK_NAME}.md"} + + +@pytest.mark.unit +def test_model_manifest_filename_is_checked_without_a_build(tmp_path: Path) -> None: + """The .onnx weights are gitignored; the name that will be written is not. + + Both manifest shapes are covered: `filename` at the top level and inside an + `assets` list. Without this, the two model roots would only pay off under + --include-untracked, which CI never runs. + """ + module = _load_script_module() + _init_repo(tmp_path) + for rel in module.MODEL_MANIFESTS: + (tmp_path / rel).parent.mkdir(parents=True, exist_ok=True) + top_level, nested = module.MODEL_MANIFESTS[1], module.MODEL_MANIFESTS[0] + (tmp_path / top_level).write_text( + json.dumps({"filename": f"{CJK_NAME}.onnx"}), encoding="utf-8" + ) + (tmp_path / nested).write_text( + json.dumps({"assets": [{"filename": "ascii.onnx"}, {"filename": f"x{CJK_NAME}.onnx"}]}), + encoding="utf-8", + ) + + offenders, from_archives = module.collect_offenders(tmp_path) + + top_dir = str(Path(top_level).parent).replace("\\", "/") + nested_dir = str(Path(nested).parent).replace("\\", "/") + assert offenders == { + f"{top_dir}/{CJK_NAME}.onnx", + f"{nested_dir}/x{CJK_NAME}.onnx", + } + assert from_archives[f"{top_dir}/{CJK_NAME}.onnx"] == top_level + + +@pytest.mark.unit +def test_malformed_manifest_is_a_loud_error(tmp_path: Path) -> None: + """"No packs found" and "the manifest is a list" must not look the same.""" + module = _load_script_module() + _init_repo(tmp_path) + packs = tmp_path / "frontend" / "pngtuber-packs" + packs.mkdir(parents=True) + (packs / "manifest.json").write_text(json.dumps([{"folder": "x"}]), encoding="utf-8") + + with pytest.raises(SystemExit) as excinfo: + module.collect_offenders(tmp_path) + assert excinfo.value.code == 2 + + (packs / "manifest.json").write_text(json.dumps({"models": "nope"}), encoding="utf-8") + with pytest.raises(SystemExit) as excinfo: + module.collect_offenders(tmp_path) + assert excinfo.value.code == 2 + + +@pytest.mark.unit +def test_unresolvable_base_ref_fails_instead_of_passing(tmp_path: Path) -> None: + """A ref that cannot be resolved must not read as "the baseline didn't grow". + + Otherwise a shallow clone that never fetched origin/main turns the ratchet + green exactly when it is supposed to bite. + """ + module = _load_script_module() + _init_repo(tmp_path) + _write_baseline(tmp_path, ["static/audio/whatever.mp3"]) + + # SystemExit(2), like the other environment failures in this script + # (unreadable git listing, malformed manifest) — not a lint verdict. + with pytest.raises(SystemExit) as excinfo: + _run_in(tmp_path, module, "--base", "origin/nope") + assert excinfo.value.code == 2 + + +@pytest.mark.unit +def test_frontend_scan_covers_output_and_sources_but_not_tooling(tmp_path: Path) -> None: + """dist/ is what ships; src/ is where the emitted names come from. + + Files outside both — configs, lockfiles, node tooling — never contribute a + basename to the payload, so they stay out of scope. + """ + module = _load_script_module() + _init_repo(tmp_path) + root = tmp_path / "frontend" / "plugin-manager" + (root / "src").mkdir(parents=True) + (root / "dist" / "assets").mkdir(parents=True) + (root / "src" / f"{CJK_NAME}.vue").write_bytes(b"x") + (root / "dist" / "assets" / f"{CJK_NAME}.js").write_bytes(b"x") + (root / f"{CJK_NAME}.config.ts").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == { + f"frontend/plugin-manager/src/{CJK_NAME}.vue", + f"frontend/plugin-manager/dist/assets/{CJK_NAME}.js", + } + + +@pytest.mark.unit +def test_plugin_stage_filter_matches_the_real_build_rules(tmp_path: Path) -> None: + """Pin the mirrored staging filter against the implementation it copies. + + The checker cannot import plugin.neko_plugin_cli.core.build_rules (pydantic; + the analyze job runs on a bare interpreter), so it reimplements the file-side + of should_skip_path. This test imports the real one and demands the same + verdict, so the copy cannot drift unnoticed. + """ + from plugin.neko_plugin_cli.core.build_rules import load_build_rules, should_skip_path + + module = _load_script_module() + plugin_dir = tmp_path / "plugin" / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + # Two rule shapes, because an `include` allow-list masks every other verdict: + # with it present, anything unmatched is dropped regardless of why. No plugin + # in this repo currently uses `include`, so the no-include case is the one + # that actually runs today — and the only one where exclude-driven directory + # pruning is observable. + base_rules = { + # Padded on purpose: BuildRuleSet strips entries, and a mirror that + # does not would reject everything the allow-list should have kept. + # "cache" matches a directory, not any file path — the real walk prunes + # the subtree through should_skip_path(is_dir=True). + "exclude": ["*.tmp", " secrets/* ", "cache"], + "exclude_dirs": ["tests", "local_logs"], + "exclude_files": ["README.md", "*.bak"], + } + for build_table in (base_rules, {**base_rules, "include": ["*.py", " assets/* ", "data layer/*"]}): + _assert_stage_filter_parity(plugin_dir, build_table, module, should_skip_path, load_build_rules) + + +def _assert_stage_filter_parity( + plugin_dir: Path, build_table: dict, module, should_skip_path, load_build_rules +) -> None: + def _toml_list(values: list[str]) -> str: + return "[" + ", ".join(f'"{value}"' for value in values) + "]" + + (plugin_dir / "pyproject.toml").write_text( + "\n".join( + ["[tool.neko.build]"] + + [f"{key} = {_toml_list(value)}" for key, value in build_table.items()] + ), + encoding="utf-8", + ) + rules = load_build_rules({"tool": {"neko": {"build": build_table}}}) + + keep = module._plugin_stage_filter(plugin_dir.parents[2]) + relatives = [ + "runtime.py", + "README.md", + "notes.bak", + "scratch.tmp", + "store.db", + "runtime.log", + "cached.pyc", + "CACHED.PYC", + "cached.PYO", + ".DS_Store", + "tests/test_runtime.py", + "tests/nested/deep.json", + "local_logs/private.txt", + "secrets/token.json", + "dist/bundle.js", + "build/out.js", + "__pycache__/mod.pyc", + ".github/workflows/ci.yml", + ".vscode/settings.json", + ".mypy_cache/module.json", + "cache/blob.json", + "cache/nested/blob.json", + "data layer/worker.py", + "assets/nested/ok.png", + # Not matched by any `include` pattern -> dropped by the allow-list. + "docs/manual.md", + "styles/theme.css", + ] + # Demand the same verdict in both directions. One direction is a hole (the + # mirror drops a path that really ships, so the check stops looking at it), + # the other only a false positive — but a mismatch either way means the copy + # has drifted, and the fix is the same one line. The sole documented + # asymmetry is .db/.log, which staging strips afterwards rather than through + # these rules, so should_skip_path has no opinion on them. + def _really_staged(relative: str) -> bool: + """Model _copy_plugin_tree: prune directories first, then judge the file.""" + parts = PurePosixPath(relative).parts + for index in range(len(parts) - 1): + ancestor = Path(*parts[: index + 1]) + if should_skip_path(ancestor, is_dir=True, rules=rules): + return False + return not should_skip_path(Path(relative), is_dir=False, rules=rules) + + for relative in relatives: + mirrored_keep = keep(f"plugin/plugins/demo/{relative}") + if PurePosixPath(relative).suffix.lower() in {".db", ".log"}: + assert not mirrored_keep, relative + continue + real_keeps = _really_staged(relative) + assert mirrored_keep == real_keeps, ( + f"mirror and build rules disagree on {relative}: " + f"mirror keeps={mirrored_keep}, staging keeps={real_keeps}" + ) + + # And it must actually drop the families it exists for. + for dropped in ( + "tests/test_runtime.py", + "local_logs/private.txt", + "secrets/token.json", + "scratch.tmp", + "README.md", + "notes.bak", + "dist/bundle.js", + "__pycache__/mod.pyc", + ".mypy_cache/module.json", + "cache/blob.json", + "store.db", + "runtime.log", + # NOT docs/manual.md: without an `include` allow-list it is staged, and + # the parity loop above already pins both rule shapes. + ): + assert not keep(f"plugin/plugins/demo/{dropped}"), dropped + + +@pytest.mark.unit +def test_excluded_plugin_paths_are_not_reported(tmp_path: Path) -> None: + """A CJK name under a path the stage drops must not fail the build.""" + module = _load_script_module() + _init_repo(tmp_path) + plugin_dir = tmp_path / "plugin" / "plugins" / "demo" + (plugin_dir / "tests").mkdir(parents=True) + (plugin_dir / "pyproject.toml").write_text( + '[tool.neko.build]\nexclude_dirs = ["tests"]\n', encoding="utf-8" + ) + (plugin_dir / "tests" / f"{CJK_NAME}.json").write_bytes(b"x") + (plugin_dir / f"{CJK_NAME}.db").write_bytes(b"x") + (plugin_dir / "__pycache__").mkdir() + (plugin_dir / "__pycache__" / f"{CJK_NAME}.pyc").write_bytes(b"x") + shipped = plugin_dir / "assets" + shipped.mkdir() + (shipped / f"{CJK_NAME}.png").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"plugin/plugins/demo/assets/{CJK_NAME}.png"} + + +@pytest.mark.unit +def test_windows_zip_separators_are_normalized(tmp_path: Path) -> None: + """A CJK directory holding an ASCII basename is allowed; only the name counts. + + The fixture writes that member with a Windows backslash separator, which is + what used to make the whole string read as a single non-ASCII basename. + """ + module = _load_script_module() + _init_repo(tmp_path) + packs = tmp_path / "frontend" / "pngtuber-packs" + packs.mkdir(parents=True) + with zipfile.ZipFile(packs / "tuber.zip", "w") as archive: + archive.writestr("中文目录\\plain.png", "x") + archive.writestr(f"层\\{CJK_NAME}", "x") + (packs / "manifest.json").write_text( + json.dumps({"models": [{"folder": "tuber", "archive": "tuber.zip"}]}), + encoding="utf-8", + ) + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"static/pngtuber/tuber/层/{CJK_NAME}"} + + +@pytest.mark.unit +def test_vite_public_assets_are_scanned(tmp_path: Path) -> None: + """public/ is copied verbatim into gitignored output, so scan the source.""" + module = _load_script_module() + _init_repo(tmp_path) + for rel in ("frontend/plugin-manager/public", "frontend/react-neko-chat/public"): + (tmp_path / rel).mkdir(parents=True) + (tmp_path / rel / f"{CJK_NAME}.png").write_bytes(b"x") + (tmp_path / "frontend" / "react-neko-chat" / f"{CJK_NAME}.config.ts").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == { + f"frontend/plugin-manager/public/{CJK_NAME}.png", + f"frontend/react-neko-chat/public/{CJK_NAME}.png", + } + + +@pytest.mark.unit +def test_only_packaged_config_and_data_entries_are_scanned(tmp_path: Path) -> None: + """config/ and data/ ship a named subset, not the whole root.""" + module = _load_script_module() + _init_repo(tmp_path) + for rel in ( + f"config/characters/{CJK_NAME}.json", + f"config/changelog/{CJK_NAME}.md", + f"data/browser_use_prompts/{CJK_NAME}.md", + f"config/prompts/{CJK_NAME}.md", + f"data/notes/{CJK_NAME}.md", + ): + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == { + f"config/characters/{CJK_NAME}.json", + f"config/changelog/{CJK_NAME}.md", + f"data/browser_use_prompts/{CJK_NAME}.md", + } + + +@pytest.mark.unit +def test_uppercase_bytecode_suffix_is_still_scanned(tmp_path: Path) -> None: + """The build rules compare suffixes case-sensitively, so `.PYC` ships. + + Folding case in the mirrored filter would drop such a file from the scan + while staging still installs it — the one drift direction that hides a + signing break rather than merely adding noise. + """ + module = _load_script_module() + _init_repo(tmp_path) + plugin_dir = tmp_path / "plugin" / "plugins" / "demo" + plugin_dir.mkdir(parents=True) + (plugin_dir / f"{CJK_NAME}.PYC").write_bytes(b"x") + (plugin_dir / f"{CJK_NAME}.pyc").write_bytes(b"x") + # .db/.log are stripped case-insensitively after staging, so both go. + (plugin_dir / f"{CJK_NAME}.DB").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"plugin/plugins/demo/{CJK_NAME}.PYC"} + + +@pytest.mark.unit +def test_vite_source_basenames_are_scanned_anywhere_under_src(tmp_path: Path) -> None: + """Vite emits by import, not by directory, so the whole source tree counts. + + An imported asset above the inline limit keeps its basename wherever it + lives, and a dynamically imported module donates its basename to the chunk + name. Both outputs are gitignored, so the source tree is the only place a + plain CI run can see these names. + """ + module = _load_script_module() + _init_repo(tmp_path) + for rel in ("src/assets", "src/components", "src/views"): + (tmp_path / "frontend" / "plugin-manager" / rel).mkdir(parents=True) + root = tmp_path / "frontend" / "plugin-manager" + (root / "src" / "assets" / f"{CJK_NAME}.png").write_bytes(b"x") + (root / "src" / "components" / f"{CJK_NAME}.png").write_bytes(b"x") + (root / "src" / "views" / f"{CJK_NAME}.vue").write_bytes(b"x") + # Outside any source tree: build tooling, never emitted with its own name. + (root / f"{CJK_NAME}.config.ts").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == { + f"frontend/plugin-manager/src/assets/{CJK_NAME}.png", + f"frontend/plugin-manager/src/components/{CJK_NAME}.png", + f"frontend/plugin-manager/src/views/{CJK_NAME}.vue", + } + + +@pytest.mark.unit +def test_include_allow_list_keeps_unpackaged_files_out_of_the_scan(tmp_path: Path) -> None: + """With `include` set, staging drops everything unmatched — so do not report it.""" + module = _load_script_module() + _init_repo(tmp_path) + plugin_dir = tmp_path / "plugin" / "plugins" / "demo" + (plugin_dir / "assets").mkdir(parents=True) + (plugin_dir / "docs").mkdir() + (plugin_dir / "pyproject.toml").write_text( + '[tool.neko.build]\ninclude = ["assets/*"]\n', encoding="utf-8" + ) + (plugin_dir / "assets" / f"{CJK_NAME}.png").write_bytes(b"x") + (plugin_dir / "docs" / f"{CJK_NAME}.md").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"plugin/plugins/demo/assets/{CJK_NAME}.png"} + + +@pytest.mark.unit +def test_baseline_growth_is_measured_at_the_merge_base(tmp_path: Path) -> None: + """A cleanup landing on main after the branch was cut must not read as growth. + + Comparing against the tip would show entries the branch never added — they + were present when it was created and only removed on main afterwards. + """ + module = _load_script_module() + _init_repo(tmp_path) + assets = tmp_path / "static" / "audio" + assets.mkdir(parents=True) + (assets / CJK_NAME).write_bytes(b"x") + _write_baseline(tmp_path, [f"static/audio/{CJK_NAME}"]) + commit = ["-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm"] + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + subprocess.run(["git", *commit, "shared base"], cwd=tmp_path, check=True) + subprocess.run(["git", "branch", "feature"], cwd=tmp_path, check=True) + + # main removes the grandfathered asset and its baseline line. + (assets / CJK_NAME).unlink() + _write_baseline(tmp_path, []) + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + subprocess.run(["git", *commit, "main: shrink baseline"], cwd=tmp_path, check=True) + + # The feature branch still carries the entry it inherited. + subprocess.run(["git", "checkout", "-q", "feature"], cwd=tmp_path, check=True) + module.REPO_ROOT = tmp_path + module.BASELINE_PATH = tmp_path / "scripts" / "nonascii_asset_baseline.txt" + assert module._baseline_growth(tmp_path, "main") == [] + + # …while a line this branch really adds is still caught. + _write_baseline(tmp_path, [f"static/audio/{CJK_NAME}", "static/audio/new.mp3"]) + assert module._baseline_growth(tmp_path, "main") == ["static/audio/new.mp3"] + + +@pytest.mark.unit +def test_missing_merge_base_fails_instead_of_using_the_tip(tmp_path: Path) -> None: + """Unrelated histories must not silently degrade to a tip comparison. + + That fallback would restore the very bug the merge-base lookup fixes, and + it would do it invisibly — the same failure shape as an unresolvable ref. + """ + module = _load_script_module() + _init_repo(tmp_path) + commit = ["-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm"] + _write_baseline(tmp_path, ["static/audio/base.mp3"]) + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + subprocess.run(["git", *commit, "main"], cwd=tmp_path, check=True) + + # An orphan branch shares no history with main. + subprocess.run(["git", "checkout", "-q", "--orphan", "unrelated"], cwd=tmp_path, check=True) + _write_baseline(tmp_path, ["static/audio/other.mp3"]) + subprocess.run(["git", "add", "-A"], cwd=tmp_path, check=True) + subprocess.run(["git", *commit, "unrelated"], cwd=tmp_path, check=True) + + module.REPO_ROOT = tmp_path + module.BASELINE_PATH = tmp_path / "scripts" / "nonascii_asset_baseline.txt" + with pytest.raises(SystemExit) as excinfo: + module._baseline_growth(tmp_path, "main") + assert excinfo.value.code == 2 + + +@pytest.mark.unit +def test_loose_plugin_runtime_artifacts_are_not_reported(tmp_path: Path) -> None: + """`.db`/`.log` directly under plugin/plugins/ are swept after staging too. + + _remove_private_runtime_artifacts walks the whole stage, including the loose + files copied from the plugins root, so reporting one is a false failure. + """ + module = _load_script_module() + _init_repo(tmp_path) + loose = tmp_path / "plugin" / "plugins" + loose.mkdir(parents=True) + (loose / f"{CJK_NAME}.db").write_bytes(b"x") + (loose / f"{CJK_NAME}.LOG").write_bytes(b"x") + (loose / f"{CJK_NAME}.json").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"plugin/plugins/{CJK_NAME}.json"} + + +@pytest.mark.unit +def test_steamworks_native_libraries_are_scanned(tmp_path: Path) -> None: + """--include-package=steamworks carries this directory in as package data.""" + module = _load_script_module() + _init_repo(tmp_path) + steamworks = tmp_path / "steamworks" + steamworks.mkdir() + (steamworks / f"{CJK_NAME}.dylib").write_bytes(b"x") + (steamworks / "SteamworksPy.dylib").write_bytes(b"x") + + offenders, _ = module.collect_offenders(tmp_path) + assert offenders == {f"steamworks/{CJK_NAME}.dylib"}