This commit is contained in:
Dobromir Popov
2026-07-07 17:37:38 +03:00
parent 640ef78711
commit e81d989f39
12 changed files with 1392 additions and 358 deletions

View File

@@ -566,6 +566,78 @@ def test_download_shard_prefers_tracker_model_source_over_huggingface(
assert hf_calls == []
def test_download_shard_prefers_tracker_full_model_source_over_huggingface(
tmp_path,
monkeypatch,
):
"""A tracker-advertised full snapshot is sufficient on its own — HF is never contacted."""
contents = {
"config.json": b"{}",
"weights-a.safetensors": b"tracker-a",
"weights-b.safetensors": b"tracker-b",
}
class FakeFileResponse:
def __init__(self, payload: bytes):
self._payload = io.BytesIO(payload)
self._length = len(payload)
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def getheader(self, name: str):
if name == "Content-Length":
return str(self._length)
if name == "Content-Type":
return "application/octet-stream"
return None
def read(self, size: int = -1) -> bytes:
return self._payload.read(size)
def fake_urlopen(url, *args, **kwargs):
query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
rel = query.get("file", [None])[0]
assert rel in contents, f"unexpected per-file request: {url}"
return FakeFileResponse(contents[rel])
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
hf_calls = []
def fake_snapshot_download(**kwargs):
hf_calls.append(kwargs)
raise AssertionError("HuggingFace should not be contacted when tracker full_files are available")
monkeypatch.setitem(
sys.modules,
"huggingface_hub",
types.SimpleNamespace(snapshot_download=fake_snapshot_download),
)
shard_dir = download_shard(
"tiny-llama",
0,
3,
cache_dir=tmp_path / "cache",
hf_repo="org/tiny-llama-shards",
model_sources=[{
"type": "tracker-full",
"url": "http://tracker/v1/model-files/download?model=tiny-llama&full=1",
"files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
"full_files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
}],
progress=False,
)
assert (shard_dir / "config.json").read_text() == "{}"
assert (shard_dir / "weights-a.safetensors").read_text() == "tracker-a"
assert (shard_dir / "weights-b.safetensors").read_text() == "tracker-b"
assert hf_calls == []
def test_download_shard_falls_back_to_huggingface_when_tracker_source_fails(
tmp_path,
monkeypatch,