tracker download fix

This commit is contained in:
Dobromir Popov
2026-07-06 18:36:42 +03:00
parent 2e696be80f
commit d151dd5484
6 changed files with 159 additions and 43 deletions

View File

@@ -213,6 +213,47 @@ def _allow_patterns_from_sources(model_sources: list[dict]) -> list[str] | None:
return sorted(patterns) if patterns else None
def _allow_patterns_from_remote_index(
hf_repo: str,
cache_dir: Path,
shard_start: int,
shard_end: int,
) -> list[str] | None:
"""Fetch just the SafeTensors index + config (a few KB) from HF and compute
which weight files the assigned layer range needs, so a HuggingFace fallback
download stays layer-scoped even when the tracker has no model_sources
(e.g. it has no local snapshot for this repo cached yet)."""
try:
from huggingface_hub import hf_hub_download # type: ignore[import]
from .safetensors_selection import (
INDEX_FILENAME,
METADATA_FILENAMES,
layers_from_config_dict,
select_files_for_layers_from_index,
)
index_path = hf_hub_download(repo_id=hf_repo, filename=INDEX_FILENAME, cache_dir=str(cache_dir))
weight_map = json.loads(Path(index_path).read_text(encoding="utf-8")).get("weight_map")
except Exception:
return None
if not isinstance(weight_map, dict):
return None
total_layers: int | None = None
try:
config_path = hf_hub_download(repo_id=hf_repo, filename="config.json", cache_dir=str(cache_dir))
config = json.loads(Path(config_path).read_text(encoding="utf-8"))
total_layers = layers_from_config_dict(config)
except Exception:
pass
selected = select_files_for_layers_from_index(
weight_map, shard_start, shard_end, total_layers=total_layers
)
return sorted(selected | METADATA_FILENAMES)
def download_shard(
model: str,
shard_start: int,
@@ -285,7 +326,14 @@ def download_shard(
if raced is not None:
return raced[1]
allow_patterns = _allow_patterns_from_remote_index(hf_repo, cache_dir, shard_start, shard_end)
if progress:
print(" download source: HuggingFace", flush=True)
if allow_patterns:
print(" download source: HuggingFace (layer-filtered)", flush=True)
else:
print(
" download source: HuggingFace (full snapshot — no SafeTensors index found)",
flush=True,
)
return _download_huggingface_subset(hf_repo, cache_dir, shard_dir, None)
return _download_huggingface_subset(hf_repo, cache_dir, shard_dir, allow_patterns)