dash QOL
This commit is contained in:
@@ -566,6 +566,78 @@ def test_download_shard_prefers_tracker_model_source_over_huggingface(
|
||||
assert hf_calls == []
|
||||
|
||||
|
||||
def test_download_shard_prefers_tracker_full_model_source_over_huggingface(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""A tracker-advertised full snapshot is sufficient on its own — HF is never contacted."""
|
||||
contents = {
|
||||
"config.json": b"{}",
|
||||
"weights-a.safetensors": b"tracker-a",
|
||||
"weights-b.safetensors": b"tracker-b",
|
||||
}
|
||||
|
||||
class FakeFileResponse:
|
||||
def __init__(self, payload: bytes):
|
||||
self._payload = io.BytesIO(payload)
|
||||
self._length = len(payload)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def getheader(self, name: str):
|
||||
if name == "Content-Length":
|
||||
return str(self._length)
|
||||
if name == "Content-Type":
|
||||
return "application/octet-stream"
|
||||
return None
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
return self._payload.read(size)
|
||||
|
||||
def fake_urlopen(url, *args, **kwargs):
|
||||
query = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
|
||||
rel = query.get("file", [None])[0]
|
||||
assert rel in contents, f"unexpected per-file request: {url}"
|
||||
return FakeFileResponse(contents[rel])
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
|
||||
hf_calls = []
|
||||
|
||||
def fake_snapshot_download(**kwargs):
|
||||
hf_calls.append(kwargs)
|
||||
raise AssertionError("HuggingFace should not be contacted when tracker full_files are available")
|
||||
|
||||
monkeypatch.setitem(
|
||||
sys.modules,
|
||||
"huggingface_hub",
|
||||
types.SimpleNamespace(snapshot_download=fake_snapshot_download),
|
||||
)
|
||||
|
||||
shard_dir = download_shard(
|
||||
"tiny-llama",
|
||||
0,
|
||||
3,
|
||||
cache_dir=tmp_path / "cache",
|
||||
hf_repo="org/tiny-llama-shards",
|
||||
model_sources=[{
|
||||
"type": "tracker-full",
|
||||
"url": "http://tracker/v1/model-files/download?model=tiny-llama&full=1",
|
||||
"files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
|
||||
"full_files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
|
||||
}],
|
||||
progress=False,
|
||||
)
|
||||
|
||||
assert (shard_dir / "config.json").read_text() == "{}"
|
||||
assert (shard_dir / "weights-a.safetensors").read_text() == "tracker-a"
|
||||
assert (shard_dir / "weights-b.safetensors").read_text() == "tracker-b"
|
||||
assert hf_calls == []
|
||||
|
||||
|
||||
def test_download_shard_falls_back_to_huggingface_when_tracker_source_fails(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
|
||||
Reference in New Issue
Block a user