From f4641303aa73b59c547b598973b642f7e6078459 Mon Sep 17 00:00:00 2001 From: emme1t <149944796+emme1t@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:07:06 -0700 Subject: [PATCH 1/2] fix: load cached pretrained models in offline mode --- CHANGELOG.md | 2 + llama_cpp/llama.py | 40 ++++++-- tests/test_llama_from_pretrained.py | 142 ++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 tests/test_llama_from_pretrained.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c34e2729..e13290fc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- fix: load cached pretrained models in offline mode by @emme1t + ## [0.3.36] - feat: update llama.cpp to ggml-org/llama.cpp@0c1e57098 diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 14e2f8500..e29b63300 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -2341,6 +2341,7 @@ def from_pretrained( """Create a Llama model from a pretrained model name or path. This method requires the huggingface-hub package. You can install it with `pip install huggingface-hub`. + With HF_HUB_OFFLINE=1, filenames are matched against cached files or a populated local_dir. Args: repo_id: The model repo id. @@ -2353,7 +2354,12 @@ def from_pretrained( Returns: A Llama model.""" try: - from huggingface_hub import hf_hub_download, HfFileSystem + from huggingface_hub import ( + hf_hub_download, + HfFileSystem, + snapshot_download, + constants, + ) from huggingface_hub.utils import validate_repo_id except ImportError: raise ImportError( @@ -2363,12 +2369,32 @@ def from_pretrained( validate_repo_id(repo_id) - hffs = HfFileSystem() - - files = [ - file["name"] if isinstance(file, dict) else file - for file in hffs.ls(repo_id, recursive=True) - ] + if constants.HF_HUB_OFFLINE: + if ( + local_dir is not None + and Path(local_dir).is_dir() + and any(Path(local_dir).iterdir()) + ): + snapshot_path = Path(local_dir) + else: + snapshot_path = Path( + snapshot_download( + repo_id=repo_id, + cache_dir=cache_dir, + local_files_only=True, + ) + ) + files = [ + f"{repo_id}/{file.relative_to(snapshot_path).as_posix()}" + for file in snapshot_path.rglob("*") + if file.is_file() + ] + else: + hffs = HfFileSystem() + files = [ + file["name"] if isinstance(file, dict) else file + for file in hffs.ls(repo_id, recursive=True) + ] # split each file into repo_id, subfolder, filename file_list: List[str] = [] diff --git a/tests/test_llama_from_pretrained.py b/tests/test_llama_from_pretrained.py new file mode 100644 index 000000000..8aa1e7f44 --- /dev/null +++ b/tests/test_llama_from_pretrained.py @@ -0,0 +1,142 @@ +from pathlib import Path + +import huggingface_hub +import pytest +from huggingface_hub import constants +from huggingface_hub.utils import LocalEntryNotFoundError + +from llama_cpp import Llama + + +class LoadedPath(Llama): + def __init__(self, model_path, **kwargs): + self.model_path = model_path + self.options = kwargs + + def close(self): + pass + + +@pytest.fixture +def offline_cache(tmp_path, monkeypatch): + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + + def unexpected_listing(*args, **kwargs): + pytest.fail("Offline loading must use the local snapshot") + + monkeypatch.setattr(huggingface_hub.HfFileSystem, "ls", unexpected_listing) + repo = tmp_path / "models--test--model" + revision = "a" * 40 + (repo / "refs").mkdir(parents=True) + (repo / "refs" / "main").write_text(revision) + snapshot = repo / "snapshots" / revision + snapshot.mkdir(parents=True) + return tmp_path, snapshot + + +@pytest.mark.parametrize("filename", ["weights/model.gguf", "weights/*.gguf"]) +def test_from_pretrained_uses_offline_snapshot(offline_cache, filename): + cache_dir, snapshot = offline_cache + model = snapshot / "weights" / "model.gguf" + model.parent.mkdir() + model.write_bytes(b"cached model") + + loaded = LoadedPath.from_pretrained( + "test/model", filename, cache_dir=cache_dir, n_ctx=128 + ) + + assert Path(loaded.model_path) == model + assert loaded.options == {"n_ctx": 128} + + +@pytest.mark.parametrize("additional_file", ["model-00002.gguf", "*-00002.gguf"]) +def test_from_pretrained_offline_additional_files(offline_cache, additional_file): + cache_dir, snapshot = offline_cache + for name in ["model-00001.gguf", "model-00002.gguf"]: + (snapshot / name).write_bytes(b"cached shard") + + loaded = LoadedPath.from_pretrained( + "test/model", + "model-00001.gguf", + additional_files=[additional_file], + cache_dir=cache_dir, + ) + + assert Path(loaded.model_path) == snapshot / "model-00001.gguf" + + +@pytest.mark.parametrize( + "filenames, pattern, message", + [ + (["model.gguf"], "missing*.gguf", "No file found"), + (["one.gguf", "two.gguf"], "*.gguf", "Multiple files found"), + ], +) +def test_from_pretrained_offline_matching_errors( + offline_cache, filenames, pattern, message +): + cache_dir, snapshot = offline_cache + for name in filenames: + (snapshot / name).write_bytes(b"cached model") + + with pytest.raises(ValueError, match=message): + LoadedPath.from_pretrained("test/model", pattern, cache_dir=cache_dir) + + +def test_from_pretrained_offline_missing_cache(tmp_path, monkeypatch): + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + + with pytest.raises(LocalEntryNotFoundError): + LoadedPath.from_pretrained("test/missing", "*.gguf", cache_dir=tmp_path) + + +def test_from_pretrained_offline_local_dir(tmp_path, monkeypatch): + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True) + local_dir = tmp_path / "downloaded" + local_dir.mkdir() + model = local_dir / "model.gguf" + model.write_bytes(b"downloaded model") + + loaded = LoadedPath.from_pretrained( + "test/model", "*.gguf", cache_dir=tmp_path / "cache", local_dir=local_dir + ) + + assert Path(loaded.model_path) == model + + +@pytest.mark.parametrize("existing_dir", [False, True]) +def test_from_pretrained_offline_requires_files_in_local_dir( + offline_cache, existing_dir +): + cache_dir, snapshot = offline_cache + (snapshot / "model.gguf").write_bytes(b"cached model") + local_dir = cache_dir / "downloaded" + if existing_dir: + local_dir.mkdir() + + with pytest.raises(LocalEntryNotFoundError): + LoadedPath.from_pretrained( + "test/model", "*.gguf", cache_dir=cache_dir, local_dir=local_dir + ) + + +def test_from_pretrained_online_lists_repository(tmp_path, monkeypatch): + monkeypatch.setattr(constants, "HF_HUB_OFFLINE", False) + calls = [] + + def list_repository(self, repo_id, recursive): + calls.append((repo_id, recursive)) + return [{"name": "test/model/model.gguf"}, "test/model/README.md"] + + def download(**kwargs): + assert kwargs["repo_id"] == "test/model" + assert kwargs["filename"] == "model.gguf" + return str(tmp_path / "model.gguf") + + monkeypatch.setattr(huggingface_hub.HfFileSystem, "ls", list_repository) + monkeypatch.setattr(huggingface_hub, "hf_hub_download", download) + + loaded = LoadedPath.from_pretrained("test/model", "*.gguf") + + assert Path(loaded.model_path) == tmp_path / "model.gguf" + assert calls == [("test/model", True)] From 8dd5e44b6097bfbf3e4b8a8a8ec4cf5ed65d3bf0 Mon Sep 17 00:00:00 2001 From: emme1t <149944796+emme1t@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:09:00 -0700 Subject: [PATCH 2/2] docs: link offline loading changelog to PR --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e13290fc1..4a231dbb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- fix: load cached pretrained models in offline mode by @emme1t +- fix: load cached pretrained models in offline mode by @emme1t in #2396 ## [0.3.36]