From f64e72cf70a0bc4fe69582f4a7597884c30a7a23 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Tue, 16 Jun 2026 23:11:06 +0545 Subject: [PATCH 1/8] skip tokenizer tests for gated model temporarily --- tests/test_tokenizer.py | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index fcf1c1d1d8..84c8d825a8 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -6,6 +6,7 @@ from unittest import mock import pytest +from huggingface_hub.errors import GatedRepoError from tokenizers import Tokenizer as HFTokenizer from tokenizers.models import BPE from transformers import AutoTokenizer @@ -15,6 +16,13 @@ from litgpt import PromptStyle, Tokenizer +def _is_gated_hf_error(ex: Exception) -> bool: + if isinstance(ex, GatedRepoError): + return True + status = getattr(getattr(ex, "response", None), "status_code", None) + return status in (401, 403) + + # @pytest.mark.flaky(reruns=3, rerun_except=["AssertionError", "assert", "TypeError"]) @pytest.mark.flaky(reruns=3, reruns_delay=120) @pytest.mark.parametrize("config", config_module.configs, ids=[c["hf_config"]["name"] for c in config_module.configs]) @@ -22,18 +30,25 @@ def test_tokenizer_against_hf(config, tmp_path): config = config_module.Config(**config) repo_id = f"{config.hf_config['org']}/{config.hf_config['name']}" - theirs = AutoTokenizer.from_pretrained(repo_id, token=os.getenv("HF_TOKEN")) - - # create a checkpoint directory that points to the HF files - hf_files = {} - for filename in ("tokenizer.json", "generation_config.json", "tokenizer.model", "tokenizer_config.json"): - try: # download the HF tokenizer config - hf_file = cached_file(path_or_repo_id=repo_id, filename=filename) - hf_files[filename] = str(hf_file) - except Exception as ex: - warnings.warn(str(ex), RuntimeWarning) - if "tokenizer.json" not in hf_files and "tokenizer.model" not in hf_files: - raise ConnectionError("Unable to download any tokenizer files from HF") + + try: + theirs = AutoTokenizer.from_pretrained(repo_id, token=os.getenv("HF_TOKEN")) + + # create a checkpoint directory that points to the HF files + hf_files = {} + for filename in ("tokenizer.json", "generation_config.json", "tokenizer.model", "tokenizer_config.json"): + try: # download the HF tokenizer config + hf_file = cached_file(path_or_repo_id=repo_id, filename=filename) + hf_files[filename] = str(hf_file) + except Exception as ex: + warnings.warn(str(ex), RuntimeWarning) + if "tokenizer.json" not in hf_files and "tokenizer.model" not in hf_files: + raise ConnectionError("Unable to download any tokenizer files from HF") + except Exception as ex: + # TODO: Resolve with Lightning Registry model for gated HF tokenizer tests. + if not _is_gated_hf_error(ex): + raise + pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded in this environment.") # Create a clean, model-specific subdirectory for this test run. # This avoids errors if previous runs or retries left files behind, ensuring the directory is always ready for fresh downloads and comparisons. From 233917e9fbe0f20db2e78df4f25c56749f0b8abd Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Wed, 17 Jun 2026 23:30:03 +0545 Subject: [PATCH 2/8] fix(tests): use snapshot_download to properly trigger gated repo skip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AutoTokenizer.from_pretrained raises a bare OSError for gated repos, which _is_gated_hf_error doesn't recognise — causing the test to fail and retry 3×120s per model (6 min each) until the 35-min job times out. Switch to huggingface_hub.snapshot_download with allow_patterns limited to tokenizer/config files (no weights, no safetensors). It raises the typed GatedRepoError that the existing skip check already handles, so fork PRs without HF_TOKEN skip cleanly in seconds. Using the default HF cache (no local_dir) also means retries and CI caching reuse downloads. --- tests/test_tokenizer.py | 43 ++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 84c8d825a8..f4e9f4fa11 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,20 +1,27 @@ # Copyright Lightning AI. Licensed under the Apache License 2.0, see LICENSE file. import os -import shutil -import warnings from types import SimpleNamespace from unittest import mock import pytest +from huggingface_hub import snapshot_download from huggingface_hub.errors import GatedRepoError from tokenizers import Tokenizer as HFTokenizer from tokenizers.models import BPE from transformers import AutoTokenizer -from transformers.utils import cached_file import litgpt.config as config_module from litgpt import PromptStyle, Tokenizer +# Tokenizer/config assets only — never weights, so `*.safetensors`/`*.bin` are skipped. +_TOKENIZER_FILES = ( + "tokenizer.json", + "tokenizer.model", + "tokenizer_config.json", + "generation_config.json", + "special_tokens_map.json", +) + def _is_gated_hf_error(ex: Exception) -> bool: if isinstance(ex, GatedRepoError): @@ -26,39 +33,23 @@ def _is_gated_hf_error(ex: Exception) -> bool: # @pytest.mark.flaky(reruns=3, rerun_except=["AssertionError", "assert", "TypeError"]) @pytest.mark.flaky(reruns=3, reruns_delay=120) @pytest.mark.parametrize("config", config_module.configs, ids=[c["hf_config"]["name"] for c in config_module.configs]) -def test_tokenizer_against_hf(config, tmp_path): +def test_tokenizer_against_hf(config): config = config_module.Config(**config) repo_id = f"{config.hf_config['org']}/{config.hf_config['name']}" try: + # Fetch tokenizer/config files only (no weights) into the default HF cache, so retries + # and CI cache reuse the download. `snapshot_download` raises a typed `GatedRepoError` + # for gated repos, so we skip cleanly instead of failing — and retrying for minutes — + # on fork PRs that have no HF_TOKEN. + model_dir = snapshot_download(repo_id, allow_patterns=list(_TOKENIZER_FILES), token=os.getenv("HF_TOKEN")) theirs = AutoTokenizer.from_pretrained(repo_id, token=os.getenv("HF_TOKEN")) - - # create a checkpoint directory that points to the HF files - hf_files = {} - for filename in ("tokenizer.json", "generation_config.json", "tokenizer.model", "tokenizer_config.json"): - try: # download the HF tokenizer config - hf_file = cached_file(path_or_repo_id=repo_id, filename=filename) - hf_files[filename] = str(hf_file) - except Exception as ex: - warnings.warn(str(ex), RuntimeWarning) - if "tokenizer.json" not in hf_files and "tokenizer.model" not in hf_files: - raise ConnectionError("Unable to download any tokenizer files from HF") except Exception as ex: # TODO: Resolve with Lightning Registry model for gated HF tokenizer tests. if not _is_gated_hf_error(ex): raise - pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded in this environment.") - - # Create a clean, model-specific subdirectory for this test run. - # This avoids errors if previous runs or retries left files behind, ensuring the directory is always ready for fresh downloads and comparisons. - model_dir = tmp_path / config.hf_config["name"] - if model_dir.exists(): - shutil.rmtree(model_dir) - os.makedirs(model_dir, exist_ok=True) - - for filename, hf_file in hf_files.items(): - shutil.copy(hf_file, model_dir / filename) + pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded without HF_TOKEN.") ours = Tokenizer(model_dir) From 7be407f036a3dc0271f063334b81e76abf70d7f9 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Wed, 17 Jun 2026 23:30:12 +0545 Subject: [PATCH 3/8] ci: pin pytest<9 in check-links workflow pytest 9 removed the `path` argument from the pytest_collect_file hookspec. pytest-check-links 0.9.1 still declares the old signature, causing a PluginValidationError on startup. The uv migration (#2257) un-pinned pytest, which let 9.x get resolved. Pin to <9 until pytest-check-links ships a compatible release. --- .github/workflows/check-links.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index da2b969d6b..ea5a74c702 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -24,7 +24,8 @@ jobs: - name: Install dependencies # a newer version of mistune is incompatible with nbconvert - run: uv pip install "mistune<3.1" pytest pytest-check-links + # pytest>=9 removed the `path` arg from pytest_collect_file; pytest-check-links still uses it + run: uv pip install "mistune<3.1" "pytest<9" pytest-check-links - name: Check links run: pytest --check-links README.md tutorials --check-links-ignore "http*" From 36b3120f4959cd6642627deee7b522e88a1e8d67 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Thu, 18 Jun 2026 14:10:39 +0545 Subject: [PATCH 4/8] fix(tests): skip on 429 rate-limits from anonymous HF downloads Fork PRs have no HF_TOKEN, so 6 parallel CI jobs hit HF anonymously and get rate-limited (HTTP 429). The HfHubHTTPError wasn't caught, causing each rate-limited model to fail and retry 3x120s until the job timed out. Extend the skip check to include 429 and rename to _is_hf_skip_error to reflect that it now covers both gated repos and anonymous rate-limits. --- tests/test_tokenizer.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index f4e9f4fa11..70219c8bad 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -23,11 +23,16 @@ ) -def _is_gated_hf_error(ex: Exception) -> bool: +def _is_hf_skip_error(ex: Exception) -> bool: + """Returns True for errors that should skip the test rather than fail it. + + Covers gated repos (401/403) and anonymous rate-limits (429) that occur when + fork PRs have no HF_TOKEN and multiple CI jobs download concurrently. + """ if isinstance(ex, GatedRepoError): return True status = getattr(getattr(ex, "response", None), "status_code", None) - return status in (401, 403) + return status in (401, 403, 429) # @pytest.mark.flaky(reruns=3, rerun_except=["AssertionError", "assert", "TypeError"]) @@ -47,7 +52,7 @@ def test_tokenizer_against_hf(config): theirs = AutoTokenizer.from_pretrained(repo_id, token=os.getenv("HF_TOKEN")) except Exception as ex: # TODO: Resolve with Lightning Registry model for gated HF tokenizer tests. - if not _is_gated_hf_error(ex): + if not _is_hf_skip_error(ex): raise pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded without HF_TOKEN.") From 9a645dc893b336a0813343feba0dbb09235c0031 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Thu, 18 Jun 2026 14:12:44 +0545 Subject: [PATCH 5/8] update --- .github/workflows/cpu-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml index b6debc69e6..6fedb770bf 100644 --- a/.github/workflows/cpu-tests.yml +++ b/.github/workflows/cpu-tests.yml @@ -120,7 +120,7 @@ jobs: - name: Run tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: pytest -v litgpt/ tests/ --timeout=180 --durations=100 + run: pytest -sv -rs litgpt/ tests/ --timeout=180 --durations=100 - name: Show cache run: uvx py-tree -d 1 .cache-HF From 0eaaa7c37a2298b4e7adc432f983b043a8be6741 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Tue, 23 Jun 2026 11:23:38 +0545 Subject: [PATCH 6/8] fix(tests): restore model-named dir so Tokenizer BOS heuristic works snapshot_download returns a commit-hash path; litgpt's Tokenizer infers BOS from the directory name (SmolLM2-*-Instruct, Llama-3*, etc.) so symlink the cached files into tmp_path/ to preserve that. Also temporarily scope pytester to test_tokenizer.py for faster CI iteration while this is being validated. --- .github/workflows/cpu-tests.yml | 3 ++- tests/test_tokenizer.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml index 6fedb770bf..42392cc554 100644 --- a/.github/workflows/cpu-tests.yml +++ b/.github/workflows/cpu-tests.yml @@ -120,7 +120,8 @@ jobs: - name: Run tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: pytest -sv -rs litgpt/ tests/ --timeout=180 --durations=100 + # TODO: temporary — scoped to tokenizer tests for faster iteration; restore full suite before merge. + run: pytest -sv -rs tests/test_tokenizer.py --timeout=180 --durations=100 - name: Show cache run: uvx py-tree -d 1 .cache-HF diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 70219c8bad..8ea28c0c9d 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,5 +1,6 @@ # Copyright Lightning AI. Licensed under the Apache License 2.0, see LICENSE file. import os +from pathlib import Path from types import SimpleNamespace from unittest import mock @@ -38,7 +39,7 @@ def _is_hf_skip_error(ex: Exception) -> bool: # @pytest.mark.flaky(reruns=3, rerun_except=["AssertionError", "assert", "TypeError"]) @pytest.mark.flaky(reruns=3, reruns_delay=120) @pytest.mark.parametrize("config", config_module.configs, ids=[c["hf_config"]["name"] for c in config_module.configs]) -def test_tokenizer_against_hf(config): +def test_tokenizer_against_hf(config, tmp_path): config = config_module.Config(**config) repo_id = f"{config.hf_config['org']}/{config.hf_config['name']}" @@ -48,7 +49,7 @@ def test_tokenizer_against_hf(config): # and CI cache reuse the download. `snapshot_download` raises a typed `GatedRepoError` # for gated repos, so we skip cleanly instead of failing — and retrying for minutes — # on fork PRs that have no HF_TOKEN. - model_dir = snapshot_download(repo_id, allow_patterns=list(_TOKENIZER_FILES), token=os.getenv("HF_TOKEN")) + cache_dir = snapshot_download(repo_id, allow_patterns=list(_TOKENIZER_FILES), token=os.getenv("HF_TOKEN")) theirs = AutoTokenizer.from_pretrained(repo_id, token=os.getenv("HF_TOKEN")) except Exception as ex: # TODO: Resolve with Lightning Registry model for gated HF tokenizer tests. @@ -56,6 +57,14 @@ def test_tokenizer_against_hf(config): raise pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded without HF_TOKEN.") + # litgpt's Tokenizer infers BOS behavior from the directory name (e.g. `SmolLM2-*-Instruct`, + # `Llama-3*`), so expose the files under a model-named dir rather than the HF cache's + # commit-hash snapshot path — otherwise the heuristic misfires and BOS handling diverges. + model_dir = tmp_path / config.hf_config["name"] + model_dir.mkdir(parents=True, exist_ok=True) + for file in Path(cache_dir).iterdir(): + (model_dir / file.name).symlink_to(file) + ours = Tokenizer(model_dir) assert ours.vocab_size == theirs.vocab_size From 62b772bd93505e2eda322442bf9722d8dd755a21 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Tue, 23 Jun 2026 12:05:35 +0545 Subject: [PATCH 7/8] fix(tests): copy tokenizer files instead of symlinking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI sets a relative HF_HOME (.cache-HF), so snapshot_download returns a relative path; symlinking from the test's tmp_path produced dangling links and every tokenizer test failed. Copy the small tokenizer/config files into the model-named dir instead — robust regardless of HF_HOME. --- tests/test_tokenizer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 8ea28c0c9d..44266638af 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -1,5 +1,6 @@ # Copyright Lightning AI. Licensed under the Apache License 2.0, see LICENSE file. import os +import shutil from pathlib import Path from types import SimpleNamespace from unittest import mock @@ -58,12 +59,15 @@ def test_tokenizer_against_hf(config, tmp_path): pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded without HF_TOKEN.") # litgpt's Tokenizer infers BOS behavior from the directory name (e.g. `SmolLM2-*-Instruct`, - # `Llama-3*`), so expose the files under a model-named dir rather than the HF cache's + # `Llama-3*`), so copy the files into a model-named dir rather than pointing at the HF cache's # commit-hash snapshot path — otherwise the heuristic misfires and BOS handling diverges. + # We copy (not symlink) because CI sets a relative HF_HOME, so the cache path is relative and + # symlinks from the test's tmp_path would dangle. model_dir = tmp_path / config.hf_config["name"] model_dir.mkdir(parents=True, exist_ok=True) for file in Path(cache_dir).iterdir(): - (model_dir / file.name).symlink_to(file) + if file.is_file(): + shutil.copy(file, model_dir / file.name) ours = Tokenizer(model_dir) From c1f06fbe6772d0f950d280fa947aa065e94ff443 Mon Sep 17 00:00:00 2001 From: bhimrazy Date: Tue, 23 Jun 2026 14:16:41 +0545 Subject: [PATCH 8/8] chore: revert CI workflow changes and tighten test comments Drop the check-links and cpu-tests.yml edits (check-links fix lives in a separate PR; restore the full CPU test suite) and trim the tokenizer test comments to the essentials. --- .github/workflows/check-links.yml | 3 +-- .github/workflows/cpu-tests.yml | 3 +-- tests/test_tokenizer.py | 15 ++++++--------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index ea5a74c702..da2b969d6b 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -24,8 +24,7 @@ jobs: - name: Install dependencies # a newer version of mistune is incompatible with nbconvert - # pytest>=9 removed the `path` arg from pytest_collect_file; pytest-check-links still uses it - run: uv pip install "mistune<3.1" "pytest<9" pytest-check-links + run: uv pip install "mistune<3.1" pytest pytest-check-links - name: Check links run: pytest --check-links README.md tutorials --check-links-ignore "http*" diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml index 42392cc554..b6debc69e6 100644 --- a/.github/workflows/cpu-tests.yml +++ b/.github/workflows/cpu-tests.yml @@ -120,8 +120,7 @@ jobs: - name: Run tests env: HF_TOKEN: ${{ secrets.HF_TOKEN }} - # TODO: temporary — scoped to tokenizer tests for faster iteration; restore full suite before merge. - run: pytest -sv -rs tests/test_tokenizer.py --timeout=180 --durations=100 + run: pytest -v litgpt/ tests/ --timeout=180 --durations=100 - name: Show cache run: uvx py-tree -d 1 .cache-HF diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py index 44266638af..c950de6043 100644 --- a/tests/test_tokenizer.py +++ b/tests/test_tokenizer.py @@ -46,10 +46,9 @@ def test_tokenizer_against_hf(config, tmp_path): repo_id = f"{config.hf_config['org']}/{config.hf_config['name']}" try: - # Fetch tokenizer/config files only (no weights) into the default HF cache, so retries - # and CI cache reuse the download. `snapshot_download` raises a typed `GatedRepoError` - # for gated repos, so we skip cleanly instead of failing — and retrying for minutes — - # on fork PRs that have no HF_TOKEN. + # Download only tokenizer/config files (no weights). `snapshot_download` raises a typed + # `GatedRepoError`, so we skip gated repos cleanly instead of retrying for minutes on + # fork PRs that have no HF_TOKEN. cache_dir = snapshot_download(repo_id, allow_patterns=list(_TOKENIZER_FILES), token=os.getenv("HF_TOKEN")) theirs = AutoTokenizer.from_pretrained(repo_id, token=os.getenv("HF_TOKEN")) except Exception as ex: @@ -58,11 +57,9 @@ def test_tokenizer_against_hf(config, tmp_path): raise pytest.skip(f"{repo_id} is gated on Hugging Face and cannot be loaded without HF_TOKEN.") - # litgpt's Tokenizer infers BOS behavior from the directory name (e.g. `SmolLM2-*-Instruct`, - # `Llama-3*`), so copy the files into a model-named dir rather than pointing at the HF cache's - # commit-hash snapshot path — otherwise the heuristic misfires and BOS handling diverges. - # We copy (not symlink) because CI sets a relative HF_HOME, so the cache path is relative and - # symlinks from the test's tmp_path would dangle. + # litgpt's Tokenizer infers BOS from the directory name (e.g. `SmolLM2-*-Instruct`, `Llama-3*`), + # so copy the files into a model-named dir, not the cache's commit-hash snapshot path. + # Copy, not symlink: CI's relative HF_HOME makes the cache path relative, so symlinks would dangle. model_dir = tmp_path / config.hf_config["name"] model_dir.mkdir(parents=True, exist_ok=True) for file in Path(cache_dir).iterdir():