tracker download fix
This commit is contained in:
@@ -213,6 +213,47 @@ def _allow_patterns_from_sources(model_sources: list[dict]) -> list[str] | None:
|
||||
return sorted(patterns) if patterns else None
|
||||
|
||||
|
||||
def _allow_patterns_from_remote_index(
|
||||
hf_repo: str,
|
||||
cache_dir: Path,
|
||||
shard_start: int,
|
||||
shard_end: int,
|
||||
) -> list[str] | None:
|
||||
"""Fetch just the SafeTensors index + config (a few KB) from HF and compute
|
||||
which weight files the assigned layer range needs, so a HuggingFace fallback
|
||||
download stays layer-scoped even when the tracker has no model_sources
|
||||
(e.g. it has no local snapshot for this repo cached yet)."""
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download # type: ignore[import]
|
||||
|
||||
from .safetensors_selection import (
|
||||
INDEX_FILENAME,
|
||||
METADATA_FILENAMES,
|
||||
layers_from_config_dict,
|
||||
select_files_for_layers_from_index,
|
||||
)
|
||||
|
||||
index_path = hf_hub_download(repo_id=hf_repo, filename=INDEX_FILENAME, cache_dir=str(cache_dir))
|
||||
weight_map = json.loads(Path(index_path).read_text(encoding="utf-8")).get("weight_map")
|
||||
except Exception:
|
||||
return None
|
||||
if not isinstance(weight_map, dict):
|
||||
return None
|
||||
|
||||
total_layers: int | None = None
|
||||
try:
|
||||
config_path = hf_hub_download(repo_id=hf_repo, filename="config.json", cache_dir=str(cache_dir))
|
||||
config = json.loads(Path(config_path).read_text(encoding="utf-8"))
|
||||
total_layers = layers_from_config_dict(config)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
selected = select_files_for_layers_from_index(
|
||||
weight_map, shard_start, shard_end, total_layers=total_layers
|
||||
)
|
||||
return sorted(selected | METADATA_FILENAMES)
|
||||
|
||||
|
||||
def download_shard(
|
||||
model: str,
|
||||
shard_start: int,
|
||||
@@ -285,7 +326,14 @@ def download_shard(
|
||||
if raced is not None:
|
||||
return raced[1]
|
||||
|
||||
allow_patterns = _allow_patterns_from_remote_index(hf_repo, cache_dir, shard_start, shard_end)
|
||||
if progress:
|
||||
print(" download source: HuggingFace", flush=True)
|
||||
if allow_patterns:
|
||||
print(" download source: HuggingFace (layer-filtered)", flush=True)
|
||||
else:
|
||||
print(
|
||||
" download source: HuggingFace (full snapshot — no SafeTensors index found)",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
return _download_huggingface_subset(hf_repo, cache_dir, shard_dir, None)
|
||||
return _download_huggingface_subset(hf_repo, cache_dir, shard_dir, allow_patterns)
|
||||
|
||||
Reference in New Issue
Block a user