fix model registration and anouncement. added console panel
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
"""Shard downloader — fetches model shards from peers or HuggingFace Hub.
|
||||
|
||||
Cache layout: ~/.cache/meshnet/shards/<model>/layers_<start>-<end>/
|
||||
Cache layout: ~/.cache/meshnet/shards/<model>/
|
||||
|
||||
For "stub-model" (no HF repo), a placeholder JSON file is written so the
|
||||
test suite never touches the network.
|
||||
@@ -106,9 +106,7 @@ def _download_shard_from_peer(
|
||||
_safe_extract_shard(archive_path, extract_dir)
|
||||
if compute_shard_checksum(extract_dir) != checksum:
|
||||
return False
|
||||
if shard_dir.exists():
|
||||
shutil.rmtree(shard_dir)
|
||||
shutil.move(str(extract_dir), str(shard_dir))
|
||||
_merge_tree(extract_dir, shard_dir)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -217,6 +215,38 @@ def _reuse_local_file(expected: int, dest: Path, final: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _valid_source_rel_files(source: dict) -> list[str]:
|
||||
return [
|
||||
rel for rel in (source.get("files") or [])
|
||||
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
|
||||
]
|
||||
|
||||
|
||||
def _source_files_cached(source: dict, shard_dir: Path) -> bool:
|
||||
rel_files = _valid_source_rel_files(source)
|
||||
if not rel_files:
|
||||
return False
|
||||
sizes = source.get("file_sizes")
|
||||
if not isinstance(sizes, dict):
|
||||
return False
|
||||
for rel in rel_files:
|
||||
expected = sizes.get(rel)
|
||||
path = shard_dir / rel
|
||||
if not isinstance(expected, int) or not path.exists() or path.stat().st_size != expected:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _merge_tree(src: Path, dest: Path) -> None:
|
||||
dest.mkdir(parents=True, exist_ok=True)
|
||||
for path in sorted(p for p in src.rglob("*") if p.is_file()):
|
||||
target = dest / path.relative_to(src)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if target.exists():
|
||||
target.unlink()
|
||||
shutil.move(str(path), str(target))
|
||||
|
||||
|
||||
def _download_source_files(
|
||||
source: dict,
|
||||
shard_dir: Path,
|
||||
@@ -231,10 +261,7 @@ def _download_source_files(
|
||||
the download is retried or the node is restarted.
|
||||
"""
|
||||
url = source.get("url")
|
||||
rel_files = [
|
||||
rel for rel in (source.get("files") or [])
|
||||
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
|
||||
]
|
||||
rel_files = _valid_source_rel_files(source)
|
||||
if not isinstance(url, str) or not url or not rel_files:
|
||||
return None
|
||||
sizes = source.get("file_sizes")
|
||||
@@ -283,9 +310,11 @@ def _download_source_files(
|
||||
time.sleep(1.0 * attempt)
|
||||
finally:
|
||||
tracker_bar.close()
|
||||
if shard_dir.exists():
|
||||
shutil.rmtree(shard_dir)
|
||||
shutil.move(str(partial_dir), str(shard_dir))
|
||||
_merge_tree(partial_dir, shard_dir)
|
||||
try:
|
||||
partial_dir.rmdir()
|
||||
except OSError:
|
||||
pass
|
||||
return shard_dir
|
||||
|
||||
|
||||
@@ -331,9 +360,7 @@ def _download_model_source(
|
||||
if progress:
|
||||
print(f" {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
|
||||
_safe_extract_shard(archive_path, extract_dir)
|
||||
if shard_dir.exists():
|
||||
shutil.rmtree(shard_dir)
|
||||
shutil.move(str(extract_dir), str(shard_dir))
|
||||
_merge_tree(extract_dir, shard_dir)
|
||||
return shard_dir
|
||||
except Exception as exc:
|
||||
if progress:
|
||||
@@ -431,10 +458,17 @@ def download_shard(
|
||||
the test suite hermetic while the real download path is exercised by
|
||||
passing a non-stub *hf_repo*.
|
||||
"""
|
||||
shard_dir = cache_dir / model / f"layers_{shard_start}-{shard_end}"
|
||||
shard_dir = cache_dir / model
|
||||
if progress:
|
||||
print(f" Target location: {shard_dir}", flush=True)
|
||||
|
||||
for source in model_sources or []:
|
||||
label = str(source.get("type") or "model-source")
|
||||
if _source_files_cached(source, shard_dir):
|
||||
if progress:
|
||||
print(f" [{label}] requested files already cached at {shard_dir}", flush=True)
|
||||
return shard_dir
|
||||
|
||||
for peer in peers or []:
|
||||
if progress:
|
||||
print(f" Trying peer shard download from {peer.get('endpoint')} ...", flush=True)
|
||||
|
||||
Reference in New Issue
Block a user