5-th DL fix

packages/node/meshnet_node/downloader.py

@@ -13,7 +13,6 @@ import tarfile
 import tempfile
 import urllib.parse
 import urllib.request
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from typing import Any
 
@@ -147,11 +146,13 @@ def _download_model_source(
             if progress:
                 print(f"  {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
             _safe_extract_shard(archive_path, extract_dir)
+            if shard_dir.exists():
+                shutil.rmtree(shard_dir)
             shutil.move(str(extract_dir), str(shard_dir))
             return shard_dir
         except Exception as exc:
             if progress:
-                print(f"  {label}: download failed: {exc!r}", flush=True)
+                print(f"  {label}: download failed ({url}): {exc!r}", flush=True)
             return None
 
 
@@ -177,55 +178,6 @@ def _download_huggingface_subset(
         return Path(snapshot_download(**kwargs))
 
 
-def _download_from_fastest_source(
-    *,
-    model_sources: list[dict],
-    hf_repo: str,
-    cache_dir: Path,
-    shard_dir: Path,
-    progress: bool,
-    timeout: float,
-) -> tuple[str, Path] | None:
-    shard_dir.parent.mkdir(parents=True, exist_ok=True)
-    with tempfile.TemporaryDirectory(prefix="meshnet-race-", dir=shard_dir.parent) as tmp:
-        tmp_root = Path(tmp)
-        jobs: dict[Any, tuple[str, Path]] = {}
-        pool = ThreadPoolExecutor(max_workers=min(4, len(model_sources) + 1))
-        try:
-            for index, source in enumerate(model_sources):
-                label = str(source.get("type") or "model-source")
-                candidate = tmp_root / f"source-{index}"
-                jobs[pool.submit(
-                    _download_model_source, source, candidate, timeout, progress, label,
-                )] = (label, candidate)
-            allow_patterns = _allow_patterns_from_sources(model_sources)
-            hf_candidate = tmp_root / "huggingface"
-            jobs[pool.submit(_download_huggingface_subset, hf_repo, cache_dir, hf_candidate, allow_patterns)] = (
-                "HuggingFace",
-                hf_candidate,
-            )
-            for future in as_completed(jobs):
-                label, candidate = jobs[future]
-                try:
-                    result = future.result()
-                except Exception as exc:
-                    if progress:
-                        print(f"  {label}: download failed: {exc!r}", flush=True)
-                    continue
-                if result is None:
-                    continue
-                if shard_dir.exists():
-                    shutil.rmtree(shard_dir)
-                shutil.move(str(candidate), str(shard_dir))
-                if progress:
-                    print(f"  download source: {label}", flush=True)
-                pool.shutdown(wait=False, cancel_futures=True)
-                return label, shard_dir
-        finally:
-            pool.shutdown(wait=False, cancel_futures=True)
-    return None
-
-
 def _allow_patterns_from_sources(model_sources: list[dict]) -> list[str] | None:
     patterns: set[str] = set()
     for source in model_sources:
@@ -336,21 +288,31 @@ def download_shard(
             f"  Downloading layers {shard_start}-{shard_end} from {hf_repo} ...",
             flush=True,
         )
-    if model_sources:
+    # Tracker (or peer) model sources are preferred outright — usually LAN-fast.
+    # HuggingFace is only the fallback when every advertised source fails.
+    for source in model_sources or []:
+        label = str(source.get("type") or "model-source")
         if progress:
-            print("  Racing tracker model source against HuggingFace ...", flush=True)
-        raced = _download_from_fastest_source(
-            model_sources=model_sources,
-            hf_repo=hf_repo,
-            cache_dir=cache_dir,
-            shard_dir=shard_dir,
-            progress=progress,
+            print(f"  Downloading from {label} model source (HuggingFace is the fallback) ...", flush=True)
+        fetched = _download_model_source(
+            source,
+            shard_dir,
             timeout=max(peer_timeout, _MODEL_SOURCE_TIMEOUT_SECONDS),
+            progress=progress,
+            label=label,
         )
-        if raced is not None:
-            return raced[1]
+        if fetched is not None:
+            if progress:
+                print(f"  download source: {label}", flush=True)
+            return fetched
+    if model_sources and progress:
+        print("  All model sources failed — falling back to HuggingFace ...", flush=True)
 
-    allow_patterns = _allow_patterns_from_remote_index(hf_repo, cache_dir, shard_start, shard_end)
+    allow_patterns = None
+    if model_sources:
+        allow_patterns = _allow_patterns_from_sources(model_sources)
+    if allow_patterns is None:
+        allow_patterns = _allow_patterns_from_remote_index(hf_repo, cache_dir, shard_start, shard_end)
     if progress:
         if allow_patterns:
             print("  download source: HuggingFace (layer-filtered)", flush=True)