fix model registration and anouncement. added console panel

This commit is contained in:
Dobromir Popov
2026-07-07 14:24:37 +02:00
parent 0e8acf5d59
commit 339577a26c
9 changed files with 732 additions and 190 deletions

View File

@@ -1,6 +1,6 @@
"""Shard downloader — fetches model shards from peers or HuggingFace Hub.
Cache layout: ~/.cache/meshnet/shards/<model>/layers_<start>-<end>/
Cache layout: ~/.cache/meshnet/shards/<model>/
For "stub-model" (no HF repo), a placeholder JSON file is written so the
test suite never touches the network.
@@ -106,9 +106,7 @@ def _download_shard_from_peer(
_safe_extract_shard(archive_path, extract_dir)
if compute_shard_checksum(extract_dir) != checksum:
return False
if shard_dir.exists():
shutil.rmtree(shard_dir)
shutil.move(str(extract_dir), str(shard_dir))
_merge_tree(extract_dir, shard_dir)
return True
except Exception:
return False
@@ -217,6 +215,38 @@ def _reuse_local_file(expected: int, dest: Path, final: Path) -> bool:
return False
def _valid_source_rel_files(source: dict) -> list[str]:
return [
rel for rel in (source.get("files") or [])
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
]
def _source_files_cached(source: dict, shard_dir: Path) -> bool:
rel_files = _valid_source_rel_files(source)
if not rel_files:
return False
sizes = source.get("file_sizes")
if not isinstance(sizes, dict):
return False
for rel in rel_files:
expected = sizes.get(rel)
path = shard_dir / rel
if not isinstance(expected, int) or not path.exists() or path.stat().st_size != expected:
return False
return True
def _merge_tree(src: Path, dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for path in sorted(p for p in src.rglob("*") if p.is_file()):
target = dest / path.relative_to(src)
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
target.unlink()
shutil.move(str(path), str(target))
def _download_source_files(
source: dict,
shard_dir: Path,
@@ -231,10 +261,7 @@ def _download_source_files(
the download is retried or the node is restarted.
"""
url = source.get("url")
rel_files = [
rel for rel in (source.get("files") or [])
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
]
rel_files = _valid_source_rel_files(source)
if not isinstance(url, str) or not url or not rel_files:
return None
sizes = source.get("file_sizes")
@@ -283,9 +310,11 @@ def _download_source_files(
time.sleep(1.0 * attempt)
finally:
tracker_bar.close()
if shard_dir.exists():
shutil.rmtree(shard_dir)
shutil.move(str(partial_dir), str(shard_dir))
_merge_tree(partial_dir, shard_dir)
try:
partial_dir.rmdir()
except OSError:
pass
return shard_dir
@@ -331,9 +360,7 @@ def _download_model_source(
if progress:
print(f" {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
_safe_extract_shard(archive_path, extract_dir)
if shard_dir.exists():
shutil.rmtree(shard_dir)
shutil.move(str(extract_dir), str(shard_dir))
_merge_tree(extract_dir, shard_dir)
return shard_dir
except Exception as exc:
if progress:
@@ -431,10 +458,17 @@ def download_shard(
the test suite hermetic while the real download path is exercised by
passing a non-stub *hf_repo*.
"""
shard_dir = cache_dir / model / f"layers_{shard_start}-{shard_end}"
shard_dir = cache_dir / model
if progress:
print(f" Target location: {shard_dir}", flush=True)
for source in model_sources or []:
label = str(source.get("type") or "model-source")
if _source_files_cached(source, shard_dir):
if progress:
print(f" [{label}] requested files already cached at {shard_dir}", flush=True)
return shard_dir
for peer in peers or []:
if progress:
print(f" Trying peer shard download from {peer.get('endpoint')} ...", flush=True)