Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai

.claude/memory/project-status.md

@@ -35,3 +35,10 @@ Historical handoff note: `/mnt/c/Users/popov/Downloads/neuron-tai-alpha-handoff-
 - Run: `meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct`
 - Known: tracker registration fails with `http://` — must use `https://`
 - pynvml deprecation warning is harmless (use nvidia-ml-py to silence it)
+
+## Model cache layout (2026-07-07)
+- Node downloads now cache files directly under `<download_dir>/<model>/`, not `<model>/layers_<start>-<end>/`, so a wider cached layer assignment can satisfy a later narrower assignment without duplicate shard folders.
+- Downloader checks tracker-advertised `files` + `file_sizes` before peer/HF download; complete local files return immediately and preserve any extra files already in the model folder.
+- Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures.
+- Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`.
+- Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events.

packages/node/meshnet_node/downloader.py

@@ -1,6 +1,6 @@
 """Shard downloader — fetches model shards from peers or HuggingFace Hub.
 
-Cache layout: ~/.cache/meshnet/shards/<model>/layers_<start>-<end>/
+Cache layout: ~/.cache/meshnet/shards/<model>/
 
 For "stub-model" (no HF repo), a placeholder JSON file is written so the
 test suite never touches the network.
@@ -106,9 +106,7 @@ def _download_shard_from_peer(
             _safe_extract_shard(archive_path, extract_dir)
             if compute_shard_checksum(extract_dir) != checksum:
                 return False
-            if shard_dir.exists():
+            _merge_tree(extract_dir, shard_dir)
-                shutil.rmtree(shard_dir)
-            shutil.move(str(extract_dir), str(shard_dir))
             return True
         except Exception:
             return False
@@ -217,6 +215,38 @@ def _reuse_local_file(expected: int, dest: Path, final: Path) -> bool:
     return False
 
 
+def _valid_source_rel_files(source: dict) -> list[str]:
+    return [
+        rel for rel in (source.get("files") or [])
+        if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
+    ]
+
+
+def _source_files_cached(source: dict, shard_dir: Path) -> bool:
+    rel_files = _valid_source_rel_files(source)
+    if not rel_files:
+        return False
+    sizes = source.get("file_sizes")
+    if not isinstance(sizes, dict):
+        return False
+    for rel in rel_files:
+        expected = sizes.get(rel)
+        path = shard_dir / rel
+        if not isinstance(expected, int) or not path.exists() or path.stat().st_size != expected:
+            return False
+    return True
+
+
+def _merge_tree(src: Path, dest: Path) -> None:
+    dest.mkdir(parents=True, exist_ok=True)
+    for path in sorted(p for p in src.rglob("*") if p.is_file()):
+        target = dest / path.relative_to(src)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if target.exists():
+            target.unlink()
+        shutil.move(str(path), str(target))
+
+
 def _download_source_files(
     source: dict,
     shard_dir: Path,
@@ -231,10 +261,7 @@ def _download_source_files(
     the download is retried or the node is restarted.
     """
     url = source.get("url")
-    rel_files = [
+    rel_files = _valid_source_rel_files(source)
-        rel for rel in (source.get("files") or [])
-        if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
-    ]
     if not isinstance(url, str) or not url or not rel_files:
         return None
     sizes = source.get("file_sizes")
@@ -283,9 +310,11 @@ def _download_source_files(
                     time.sleep(1.0 * attempt)
     finally:
         tracker_bar.close()
-    if shard_dir.exists():
+    _merge_tree(partial_dir, shard_dir)
-        shutil.rmtree(shard_dir)
+    try:
-    shutil.move(str(partial_dir), str(shard_dir))
+        partial_dir.rmdir()
+    except OSError:
+        pass
     return shard_dir
 
 
@@ -331,9 +360,7 @@ def _download_model_source(
             if progress:
                 print(f"  {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
             _safe_extract_shard(archive_path, extract_dir)
-            if shard_dir.exists():
+            _merge_tree(extract_dir, shard_dir)
-                shutil.rmtree(shard_dir)
-            shutil.move(str(extract_dir), str(shard_dir))
             return shard_dir
         except Exception as exc:
             if progress:
@@ -431,10 +458,17 @@ def download_shard(
     the test suite hermetic while the real download path is exercised by
     passing a non-stub *hf_repo*.
     """
-    shard_dir = cache_dir / model / f"layers_{shard_start}-{shard_end}"
+    shard_dir = cache_dir / model
     if progress:
         print(f"  Target location: {shard_dir}", flush=True)
 
+    for source in model_sources or []:
+        label = str(source.get("type") or "model-source")
+        if _source_files_cached(source, shard_dir):
+            if progress:
+                print(f"  [{label}] requested files already cached at {shard_dir}", flush=True)
+            return shard_dir
+
     for peer in peers or []:
         if progress:
             print(f"  Trying peer shard download from {peer.get('endpoint')} ...", flush=True)

packages/node/meshnet_node/model_catalog.py

@@ -64,6 +64,17 @@ def _load_model_metadata() -> dict[str, dict]:
 _MODEL_METADATA = _load_model_metadata()
 
 
+def _local_model_path(hf_repo: str, cache_dir: Path | None) -> Path | None:
+    if cache_dir is None:
+        return None
+    if (cache_dir / "config.json").exists():
+        return cache_dir
+    candidate = cache_dir / hf_repo.split("/")[-1]
+    if (candidate / "config.json").exists():
+        return candidate
+    return None
+
+
 CURATED_MODELS: list[ModelPreset] = [
     ModelPreset(
         name="Qwen2.5-0.5B-Instruct",
@@ -215,9 +226,11 @@ def model_metadata_for(
     try:
         from transformers import AutoConfig  # type: ignore[import]
 
+        local_model = _local_model_path(hf_repo, cache_dir)
+        load_source = str(local_model) if local_model is not None else hf_repo
         cfg = AutoConfig.from_pretrained(
-            hf_repo,
+            load_source,
-            cache_dir=str(cache_dir) if cache_dir is not None else None,
+            cache_dir=str(cache_dir) if cache_dir is not None and local_model is None else None,
         )
         # Composite configs (VLM/MoE) nest decoder fields in text_config.
         text_cfg = getattr(cfg, "text_config", None) or cfg

packages/node/meshnet_node/startup.py

@@ -26,6 +26,86 @@ from .wallet import load_or_create_wallet
 _DEFAULT_BYTES_PER_LAYER = 30 * 1024 * 1024
 
 
+def _downloaded_model_inventory(
+    model: str,
+    shard_start: int,
+    shard_end: int,
+    shard_path: Path,
+    hf_repo: str | None = None,
+    model_sources: list[dict] | None = None,
+) -> list[dict]:
+    """Return a cheap local inventory record without reading model file contents."""
+    file_count = 0
+    total_bytes = 0
+    existing_rel_files: set[str] = set()
+    if shard_path.exists():
+        for path in shard_path.rglob("*"):
+            if not path.is_file():
+                continue
+            file_count += 1
+            try:
+                existing_rel_files.add(path.relative_to(shard_path).as_posix())
+            except ValueError:
+                pass
+            try:
+                total_bytes += path.stat().st_size
+            except OSError:
+                pass
+    record = {
+        "model": model,
+        "shard_start": shard_start,
+        "shard_end": shard_end,
+        "path": str(shard_path),
+        "file_count": file_count,
+        "total_bytes": total_bytes,
+    }
+    if hf_repo is not None:
+        record["hf_repo"] = hf_repo
+    expected_files: set[str] = set()
+    file_sizes: dict[str, int] = {}
+    for source in model_sources or []:
+        for rel in source.get("full_files") or source.get("files") or []:
+            if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts:
+                expected_files.add(rel)
+        sizes = source.get("file_sizes")
+        if isinstance(sizes, dict):
+            for rel, size in sizes.items():
+                if isinstance(rel, str) and isinstance(size, int):
+                    file_sizes[rel] = size
+    if expected_files:
+        expected_bytes = sum(file_sizes.get(rel, 0) for rel in expected_files)
+        local_expected_files = existing_rel_files & expected_files
+        local_expected_bytes = sum(file_sizes.get(rel, 0) for rel in local_expected_files)
+        record["expected_file_count"] = len(expected_files)
+        record["local_expected_file_count"] = len(local_expected_files)
+        record["expected_bytes"] = expected_bytes
+        record["local_expected_bytes"] = local_expected_bytes
+        record["local_model_percentage"] = (
+            round((local_expected_bytes / expected_bytes) * 100, 4)
+            if expected_bytes > 0
+            else 0.0
+        )
+    return [record]
+
+
+def _registration_shard_checksum(model: str, shard_path: Path) -> str | None:
+    """Only checksum tiny stub shards; real model folders are too large to hash at startup."""
+    if model != "stub-model":
+        return None
+    return compute_shard_checksum(shard_path)
+
+
+def _model_cache_path(model_id: str, cache_dir: Path | None) -> Path | None:
+    if cache_dir is None:
+        return None
+    if (cache_dir / "config.json").exists():
+        return cache_dir
+    candidate = cache_dir / model_id.split("/")[-1]
+    if candidate.exists():
+        return candidate
+    return None
+
+
 def _memory_budget(device: str, vram_mb: int, ram_mb: int, shared_vram_mb: int = 0) -> tuple[int, str]:
     """Return the capacity budget in MB and whether it came from VRAM or RAM."""
     if device == "cuda" and vram_mb > 0:
@@ -502,6 +582,7 @@ def run_startup(
 
     if model_id:  # treat "" the same as None — no explicit model given
         user_pinned_shard = shard_start is not None or shard_end is not None
+        full_sources: list[dict] = []
         # Auto-detect shard range from model config if not explicitly provided
         if shard_start is None or shard_end is None:
             try:
@@ -581,6 +662,7 @@ def run_startup(
         _attach_relay_bridge(node, relay_bridge)
         # Register with tracker so other nodes can auto-join this model.
         total_layers = getattr(getattr(node, "backend", None), "total_layers", None)
+        model_cache_path = _model_cache_path(model_id, cache_dir)
         reg_payload = {
             "endpoint": endpoint,
             "model": model_id.split("/")[-1],
@@ -595,6 +677,18 @@ def run_startup(
             "tracker_mode": (shard_start == 0),
             "managed_assignment": not user_pinned_shard,
             "model_metadata": model_metadata_for(model_id, total_layers, cache_dir=cache_dir),
+            "downloaded_models": (
+                _downloaded_model_inventory(
+                    model_id.split("/")[-1],
+                    shard_start,
+                    shard_end,
+                    model_cache_path,
+                    hf_repo=model_id,
+                    model_sources=full_sources,
+                )
+                if model_cache_path is not None
+                else []
+            ),
             **registration_capabilities,
             **relay_fields,
         }
@@ -693,6 +787,7 @@ def run_startup(
             relay_url=relay_url,
         )
         _attach_relay_bridge(node, relay_bridge)
+        model_cache_path = _model_cache_path(assigned_hf_repo, cache_dir)
         auto_reg_payload = {
             "endpoint": endpoint,
             "model": assigned_hf_repo.split("/")[-1],
@@ -707,6 +802,18 @@ def run_startup(
             "tracker_mode": (assigned_shard_start == 0),
             "managed_assignment": True,
             "model_metadata": model_metadata_for(assigned_hf_repo, assigned_num_layers, cache_dir=cache_dir),
+            "downloaded_models": (
+                _downloaded_model_inventory(
+                    assigned_hf_repo.split("/")[-1],
+                    assigned_shard_start,
+                    assigned_shard_end,
+                    model_cache_path,
+                    hf_repo=assigned_hf_repo,
+                    model_sources=full_sources,
+                )
+                if model_cache_path is not None
+                else []
+            ),
             **registration_capabilities,
             **relay_fields,
         }
@@ -776,7 +883,15 @@ def run_startup(
     if model_sources:
         dl_kwargs["model_sources"] = model_sources
     shard_path = download_shard(assigned_model, shard_start, shard_end, **dl_kwargs)
-    shard_checksum = compute_shard_checksum(shard_path)
+    shard_checksum = _registration_shard_checksum(assigned_model, shard_path)
+    downloaded_models = _downloaded_model_inventory(
+        assigned_model,
+        shard_start,
+        shard_end,
+        shard_path,
+        hf_repo=hf_repo,
+        model_sources=model_sources,
+    )
     print(f"  Cached at: {shard_path}", flush=True)
 
     # 5. Start HTTP server
@@ -790,6 +905,7 @@ def run_startup(
         model=assigned_model,
         shard_path=shard_path,
     )
+    _node_start_time = time.monotonic()
     actual_port = node.start()
     public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
     endpoint = f"http://{public_host}:{actual_port}"
@@ -805,24 +921,27 @@ def run_startup(
 
     # 6. Register with tracker
     print("Registering with tracker...", flush=True)
+    reg_payload = {
+        "endpoint": endpoint,
+        "model": assigned_model,
+        "shard_start": shard_start,
+        "shard_end": shard_end,
+        "shard_checksum": shard_checksum,
+        "downloaded_models": downloaded_models,
+        "hardware_profile": hw,
+        "wallet_address": address,
+        "score": 1.0,
+        **registration_capabilities,
+        **relay_fields,
+    }
     try:
         reg_resp = _post_json(
             f"{tracker_url}/v1/nodes/register",
-            {
+            reg_payload,
-                "endpoint": endpoint,
-                "model": assigned_model,
-                "shard_start": shard_start,
-                "shard_end": shard_end,
-                "shard_checksum": shard_checksum,
-                "hardware_profile": hw,
-                "wallet_address": address,
-                "score": 1.0,
-                **registration_capabilities,
-                **relay_fields,
-            },
         )
         node_id = str(reg_resp["node_id"])
         setattr(node, "tracker_node_id", node_id)
+        _start_heartbeat(tracker_url, node_id, reg_payload, node_ref=node, start_time=_node_start_time)
     except Exception:
         node.stop()
         raise
@@ -855,9 +974,11 @@ def _detect_num_layers(model_id: str, cache_dir: Path | None = None) -> int | No
 
         from .model_catalog import layers_from_config
 
+        local_model = _model_cache_path(model_id, cache_dir)
+        load_source = str(local_model) if local_model is not None else model_id
         cfg = AutoConfig.from_pretrained(
-            model_id,
+            load_source,
-            cache_dir=str(cache_dir) if cache_dir is not None else None,
+            cache_dir=str(cache_dir) if cache_dir is not None and local_model is None else None,
         )
         layers = layers_from_config(cfg)
         if layers is None:

packages/tracker/meshnet_tracker/dashboard.html

@@ -44,6 +44,17 @@
   .tabs { display:flex; gap:10px; margin-bottom:8px; }
   .tabs a { color:var(--dim); cursor:pointer; }
   .tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
+  .wide { grid-column:1 / -1; }
+  .console {
+    background:var(--bg); border:1px solid var(--border); border-radius:6px;
+    min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
+    white-space:pre-wrap; word-break:break-word; font-size:11px;
+  }
+  .console-line { padding:1px 0; border-bottom:1px solid #161b22; }
+  .console-time { color:var(--dim); }
+  .console-level-info { color:var(--accent); }
+  .console-level-warn { color:var(--warn); }
+  .console-level-error { color:var(--bad); }
 </style>
 </head>
 <body>
@@ -63,6 +74,7 @@
   <section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
   <section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
   <section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
+  <section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
 </main>
 <script>
 "use strict";
@@ -212,6 +224,21 @@ function renderThroughput(stats) {
   $("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
 }
 
+function renderConsole(data) {
+  const events = (data && data.events) || [];
+  if (!events.length) {
+    $("console").innerHTML = '<div class="empty">no console events</div>';
+    return;
+  }
+  $("console").innerHTML = events.slice(-120).map(e => {
+    const level = String(e.level || "info");
+    const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
+    const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
+    return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` +
+      `<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`;
+  }).join("");
+}
+
 // ---- account panel (registration / login / balance / usage / API keys) ----
 
 let sessionToken = localStorage.getItem("meshnet_session") || null;
@@ -366,13 +393,14 @@ async function renderAdminPanel() {
 
 async function refresh() {
   $("self-url").textContent = location.host;
-  const [raft, map, summary, settlements, wallets, stats] = await Promise.all([
+  const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
     fetchJson("/v1/raft/status"),
     fetchJson("/v1/network/map"),
     fetchJson("/v1/billing/summary"),
     fetchJson("/v1/billing/settlements"),
     fetchJson("/v1/registry/wallets"),
     fetchJson("/v1/stats"),
+    fetchJson("/v1/console"),
   ]);
   renderHive(raft);
   renderNodes(map);
@@ -381,6 +409,7 @@ async function refresh() {
   renderFraud(wallets, summary);
   renderStats(stats);
   renderThroughput(stats);
+  renderConsole(consoleData);
   $("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
 }
 refresh();

packages/tracker/meshnet_tracker/server.py

@@ -34,6 +34,7 @@ import time
 import urllib.parse
 import urllib.request
 import uuid
+from collections import deque
 from importlib.resources import files
 from pathlib import Path
 from typing import Any
@@ -49,6 +50,9 @@ from .model_files import files_for_layer_range, snapshot_dir_for_repo
 from .raft import RaftNode
 
 
+_CONSOLE_LIMIT = 300
+
+
 def _preset_price_keys(name: str, preset: dict) -> set[str]:
     """All model strings a client may bill under for one preset.
 
@@ -529,7 +533,7 @@ class _StatsCollector:
 class _NodeEntry:
     __slots__ = (
         "node_id", "endpoint", "shard_start", "shard_end",
-        "model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "hardware_profile", "wallet_address",
+        "model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "downloaded_models", "hardware_profile", "wallet_address",
         "score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
         "benchmark_tokens_per_sec", "quantization", "managed_assignment",
         "model_tokens_per_sec",
@@ -565,6 +569,7 @@ class _NodeEntry:
         hf_repo: str | None = None,
         num_layers: int | None = None,
         model_metadata: dict | None = None,
+        downloaded_models: list[dict] | None = None,
         relay_addr: str | None = None,
         cert_fingerprint: str | None = None,
         peer_id: str | None = None,
@@ -590,6 +595,7 @@ class _NodeEntry:
         self.hf_repo = hf_repo
         self.num_layers = num_layers
         self.model_metadata = dict(model_metadata or {})
+        self.downloaded_models = [dict(item) for item in (downloaded_models or []) if isinstance(item, dict)]
         self.relay_addr = relay_addr
         self.cert_fingerprint = cert_fingerprint
         self.peer_id = peer_id
@@ -1247,10 +1253,15 @@ def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
     ]
     for node_id in expired_ids:
         entry = server.registry.pop(node_id)
-        print(
+        _tracker_log(
-            f"[tracker] node expired: {node_id}  {entry.endpoint}  "
+            server,
-            f"(no heartbeat for >{server.heartbeat_timeout:.0f}s)",
+            "warn",
-            flush=True,
+            "node expired",
+            node_id=node_id,
+            endpoint=entry.endpoint,
+            model=entry.model,
+            shard=f"{entry.shard_start}-{entry.shard_end}",
+            heartbeat_timeout_seconds=server.heartbeat_timeout,
         )
     if expired_ids:
         _rebalance_all_locked(server)
@@ -1617,6 +1628,24 @@ def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -
     return None
 
 
+def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
+    event = {
+        "ts": time.time(),
+        "level": level,
+        "message": message,
+        "fields": {
+            key: value
+            for key, value in fields.items()
+            if value is not None
+        },
+    }
+    with server.console_lock:
+        server.console_events.append(event)
+    extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
+    suffix = f"  {extras}" if extras else ""
+    print(f"[tracker] {level}: {message}{suffix}", flush=True)
+
+
 def _node_id_for_registration(
     endpoint: str,
     model: str,
@@ -1700,6 +1729,8 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
         self.toploc_backend = toploc_backend
         self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
         self.models_dir = models_dir
+        self.console_events = deque(maxlen=_CONSOLE_LIMIT)
+        self.console_lock = threading.Lock()
 
 
 class _TrackerHandler(http.server.BaseHTTPRequestHandler):
@@ -1887,6 +1918,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             self._handle_raft_status()
         elif parsed.path == "/v1/stats":
             self._handle_stats()
+        elif parsed.path == "/v1/console":
+            self._handle_console()
         elif parsed.path == "/v1/billing/summary":
             self._handle_billing_summary()
         elif parsed.path == "/v1/billing/settlements":
@@ -2118,6 +2151,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                     "hf_repo": node.hf_repo,
                     "num_layers": node.num_layers,
                     "model_metadata": dict(node.model_metadata),
+                    "downloaded_models": [dict(item) for item in node.downloaded_models],
                     "shard_start": node.shard_start,
                     "shard_end": node.shard_end,
                     "tracker_mode": node.tracker_mode,
@@ -2264,6 +2298,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                     ]
 
             if not candidates:
+                with server.lock:
+                    registered = [
+                        {
+                            "node_id": n.node_id,
+                            "model": n.model,
+                            "hf_repo": n.hf_repo,
+                            "shard": f"{n.shard_start}-{n.shard_end}",
+                            "tracker_mode": n.tracker_mode,
+                        }
+                        for n in server.registry.values()
+                    ]
+                _tracker_log(
+                    server,
+                    "warn",
+                    "no nodes available for model",
+                    model=model,
+                    registered_nodes=registered,
+                )
                 self._send_json(503, {"error": {
                     "message": f"no nodes available for model {model!r}",
                     "type": "service_unavailable",
@@ -2296,6 +2348,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         else:
             route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
             if route_error:
+                _tracker_log(
+                    server,
+                    "warn",
+                    "route unavailable",
+                    model=model,
+                    route_model=route_model,
+                    error=route_error,
+                    candidate_count=len(all_nodes),
+                )
                 self._send_json(503, {"error": {
                     "message": route_error,
                     "type": "service_unavailable",
@@ -2322,11 +2383,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
             for n in route_nodes
         )
-        print(
+        _tracker_log(
-            f"[tracker] proxy route {request_id}: model={model!r} "
+            server,
-            f"head={node.node_id}@{node.endpoint} downstream={downstream_urls} "
+            "info",
-            f"route={route_debug or '<empty>'}",
+            "proxy route selected",
-            flush=True,
+            request_id=request_id,
+            model=model,
+            head_node_id=node.node_id,
+            head_endpoint=node.endpoint,
+            downstream=downstream_urls,
+            route=route_debug or "<empty>",
         )
 
         req = urllib.request.Request(
@@ -2350,10 +2416,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         }
 
         if node.relay_addr:
-            print(
+            _tracker_log(
-                f"[tracker] proxy via relay {request_id}: {node.relay_addr} "
+                server,
-                f"(direct endpoint {target_url})",
+                "info",
-                flush=True,
+                "proxy via relay",
+                request_id=request_id,
+                relay_addr=node.relay_addr,
+                direct_endpoint=target_url,
             )
             started = time.monotonic()
             frames = _relay_http_request_frames(
@@ -2388,16 +2457,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                         input_tokens=in_tokens, output_tokens=out_tokens,
                     )
                 return
-            print(
+            _tracker_log(
-                f"[tracker] relay proxy failed {request_id}: {node.relay_addr}; "
+                server,
-                f"trying direct {target_url}",
+                "warn",
-                flush=True,
+                "relay proxy failed, trying direct",
+                request_id=request_id,
+                relay_addr=node.relay_addr,
+                direct_endpoint=target_url,
             )
 
         try:
             started = time.monotonic()
             upstream = urllib.request.urlopen(req, timeout=300.0)
-            print(f"[tracker] proxy connected {request_id}: {target_url}", flush=True)
+            _tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
         except urllib.error.HTTPError as exc:
             # Relay error status + body from node
             err_body = exc.read()
@@ -2412,13 +2484,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             return
         except Exception as exc:
             if node.relay_addr:
-                print(
+                _tracker_log(
-                    f"[tracker] direct proxy failed {request_id}: {target_url}: {exc}; "
+                    server,
-                    f"relay already attempted for {node.relay_addr}",
+                    "error",
-                    flush=True,
+                    "direct proxy failed after relay",
+                    request_id=request_id,
+                    target_url=target_url,
+                    relay_addr=node.relay_addr,
+                    error=repr(exc),
                 )
             else:
-                print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True)
+                _tracker_log(
+                    server,
+                    "error",
+                    "proxy failed",
+                    request_id=request_id,
+                    target_url=target_url,
+                    error=repr(exc),
+                )
             self._send_json(503, {"error": {
                 "message": f"upstream node unreachable: {exc}",
                 "type": "service_unavailable",
@@ -2467,9 +2550,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                 # Non-streaming: buffer and relay
                 resp_body = upstream.read()
                 elapsed = time.monotonic() - started
-                print(
+                _tracker_log(
-                    f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
+                    server,
-                    flush=True,
+                    "info",
+                    "proxy complete",
+                    request_id=request_id,
+                    target_url=target_url,
+                    bytes=len(resp_body),
                 )
                 observed_output = ""
                 try:
@@ -2812,6 +2899,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if not isinstance(model_metadata, dict):
             self._send_json(400, {"error": "model_metadata must be an object"})
             return
+        downloaded_models = body.get("downloaded_models", [])
+        if downloaded_models is None:
+            downloaded_models = []
+        if not (
+            isinstance(downloaded_models, list)
+            and all(isinstance(item, dict) for item in downloaded_models)
+        ):
+            self._send_json(400, {"error": "downloaded_models must be an array of objects"})
+            return
         relay_addr = body.get("relay_addr") or None
         cert_fingerprint = body.get("cert_fingerprint") or None
         peer_id = body.get("peer_id") or None
@@ -2845,6 +2941,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             hf_repo=hf_repo,
             num_layers=num_layers,
             model_metadata=model_metadata,
+            downloaded_models=downloaded_models,
             relay_addr=relay_addr,
             cert_fingerprint=cert_fingerprint,
             peer_id=peer_id,
@@ -2858,10 +2955,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             ]
             for eid in stale_ids:
                 old = server.registry.pop(eid)
-                print(
+                _tracker_log(
-                    f"[tracker] node re-registered: replaced {eid} with {node_id}"
+                    server,
-                    f"  {old.endpoint}",
+                    "info",
-                    flush=True,
+                    "node re-registered",
+                    old_node_id=eid,
+                    node_id=node_id,
+                    endpoint=old.endpoint,
                 )
             server.registry[node_id] = entry
             if entry.managed_assignment:
@@ -2873,6 +2973,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             if assignment_directive is not None:
                 entry.pending_directives.clear()
 
+            _tracker_log(
+                server,
+                "info",
+                "node registered",
+                node_id=node_id,
+                endpoint=entry.endpoint,
+                model=entry.model,
+                hf_repo=entry.hf_repo,
+                shard=f"{entry.shard_start}-{entry.shard_end}",
+                tracker_mode=entry.tracker_mode,
+            )
+
         shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
         repo_info = f" [{hf_repo}]" if hf_repo else ""
         budget_bytes, budget_source = _node_memory_budget_bytes(entry)
@@ -2913,6 +3025,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             self._purge_expired_nodes()
             entry = server.registry.get(node_id)
             if entry is None:
+                _tracker_log(server, "warn", "heartbeat for unknown node", node_id=node_id)
                 self._send_json(404, {"error": "node not found"})
                 return
             entry.last_heartbeat = time.monotonic()
@@ -3041,6 +3154,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         except BrokenPipeError:
             pass
 
+    def _handle_console(self):
+        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
+        with server.console_lock:
+            events = [dict(event) for event in server.console_events]
+        self._send_json(200, {"events": events})
+
     def _handle_registry_wallets(self):
         server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
         if not self._require_role("admin"):
@@ -4195,6 +4314,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                     "hf_repo": e.hf_repo,
                     "num_layers": e.num_layers,
                     "model_metadata": dict(e.model_metadata),
+                    "downloaded_models": [dict(item) for item in e.downloaded_models],
                     "shard_checksum": e.shard_checksum,
                     "score": e.score,
                 }
@@ -4260,6 +4380,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                         "shard_start": e.shard_start,
                         "shard_end": e.shard_end,
                         "model": e.model,
+                        "downloaded_models": [dict(item) for item in e.downloaded_models],
                         "shard_checksum": e.shard_checksum,
                         "score": e.score,
                     }
@@ -4681,6 +4802,11 @@ class TrackerServer:
             hf_repo=payload.get("hf_repo"),
             num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
             model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
+            downloaded_models=(
+                payload.get("downloaded_models")
+                if isinstance(payload.get("downloaded_models"), list)
+                else None
+            ),
         )
         with self._lock:
             self._registry[node_id] = entry

tests/test_dashboard.py

@@ -12,6 +12,7 @@ PANELS = [
     "Tracker hive", "Nodes & coverage", "Client balances",
     "Node pending payouts", "Settlement history",
     "Strikes / bans / forfeitures", "Model usage", "Node throughput",
+    "Console output",
 ]
 
 
@@ -64,3 +65,29 @@ def test_registry_wallets_endpoint():
         assert data["wallets"]["wallet-a"]["banned"] is False
     finally:
         tracker.stop()
+
+
+def test_console_endpoint_exposes_tracker_events():
+    tracker = TrackerServer()
+    port = tracker.start()
+    try:
+        body = json.dumps({
+            "endpoint": "http://127.0.0.1:9001",
+            "model": "stub-model",
+            "shard_start": 0,
+            "shard_end": 3,
+            "hardware_profile": {},
+        }).encode()
+        req = urllib.request.Request(
+            f"http://127.0.0.1:{port}/v1/nodes/register",
+            data=body,
+            headers={"Content-Type": "application/json"},
+            method="POST",
+        )
+        urllib.request.urlopen(req).read()
+
+        data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
+    finally:
+        tracker.stop()
+
+    assert any(event["message"] == "node registered" for event in data["events"])

tests/test_node_startup.py

@@ -438,7 +438,7 @@ def test_download_shard_uses_huggingface_when_repo_is_assigned(tmp_path, monkeyp
         progress=False,
     )
 
-    assert shard_dir == tmp_path / "tiny-llama" / "layers_0-3"
+    assert shard_dir == tmp_path / "tiny-llama"
     assert calls == [{
         "repo_id": "org/tiny-llama-shards",
         "cache_dir": str(tmp_path),
@@ -446,6 +446,54 @@ def test_download_shard_uses_huggingface_when_repo_is_assigned(tmp_path, monkeyp
     }]
 
 
+def test_download_shard_reuses_model_cache_for_narrower_layer_range(
+    tmp_path,
+    monkeypatch,
+):
+    """A wider cached shard satisfies a later narrower assignment for the same model."""
+    cache_dir = tmp_path / "cache"
+    model_dir = cache_dir / "tiny-llama"
+    model_dir.mkdir(parents=True)
+    (model_dir / "config.json").write_bytes(b"{}")
+    (model_dir / "model-00001-of-00002.safetensors").write_bytes(b"a" * 3)
+    (model_dir / "model-00002-of-00002.safetensors").write_bytes(b"b" * 5)
+
+    def unexpected_urlopen(*args, **kwargs):
+        raise AssertionError("cached files should avoid tracker download")
+
+    def unexpected_snapshot_download(*args, **kwargs):
+        raise AssertionError("cached files should avoid HuggingFace download")
+
+    monkeypatch.setattr(urllib.request, "urlopen", unexpected_urlopen)
+    monkeypatch.setitem(
+        sys.modules,
+        "huggingface_hub",
+        types.SimpleNamespace(snapshot_download=unexpected_snapshot_download),
+    )
+
+    shard_dir = download_shard(
+        "tiny-llama",
+        0,
+        1,
+        cache_dir=cache_dir,
+        hf_repo="org/tiny-llama-shards",
+        model_sources=[{
+            "type": "tracker",
+            "url": "http://tracker/v1/model-files/download?model=tiny-llama",
+            "files": ["config.json", "model-00001-of-00002.safetensors"],
+            "file_sizes": {
+                "config.json": 2,
+                "model-00001-of-00002.safetensors": 3,
+            },
+        }],
+        peers=[{"endpoint": "http://peer", "checksum": "unused"}],
+        progress=False,
+    )
+
+    assert shard_dir == model_dir
+    assert (model_dir / "model-00002-of-00002.safetensors").read_bytes() == b"b" * 5
+
+
 def test_download_shard_prefers_tracker_model_source_over_huggingface(
     tmp_path,
     monkeypatch,
@@ -1368,12 +1416,7 @@ def test_full_startup_sequence(tmp_path):
         tracker.stop()
 
 
-def test_second_node_downloads_same_shard_from_peer_without_huggingface(
+def test_preset_model_startup_starts_heartbeat(tmp_path, monkeypatch):
-    tmp_path,
-    monkeypatch,
-    capsys,
-):
-    """Node A downloads from HF stub; node B downloads same assignment from node A."""
     import meshnet_node.startup as startup_mod
 
     monkeypatch.setattr(
@@ -1381,6 +1424,53 @@ def test_second_node_downloads_same_shard_from_peer_without_huggingface(
         "detect_hardware",
         lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
     )
+    heartbeat_calls = []
+    monkeypatch.setattr(
+        startup_mod,
+        "_start_heartbeat",
+        lambda *args, **kwargs: heartbeat_calls.append((args, kwargs)),
+    )
+
+    tracker = TrackerServer(model_presets={"stub-model": {"layers_start": 0, "layers_end": 15}})
+    tracker_port = tracker.start()
+    tracker_url = f"http://127.0.0.1:{tracker_port}"
+    try:
+        node = run_startup(
+            tracker_url=tracker_url,
+            model="stub-model",
+            wallet_path=tmp_path / "wallet.json",
+            cache_dir=tmp_path / "shards",
+        )
+        try:
+            assert len(heartbeat_calls) == 1
+            args, kwargs = heartbeat_calls[0]
+            assert args[0] == tracker_url
+            assert args[2]["model"] == "stub-model"
+            assert kwargs["node_ref"] is node
+        finally:
+            node.stop()
+    finally:
+        tracker.stop()
+
+
+def test_real_model_startup_registers_downloaded_inventory_without_checksum(
+    tmp_path,
+    monkeypatch,
+    capsys,
+):
+    """Real model folders are reported as inventory without hashing their contents."""
+    import meshnet_node.startup as startup_mod
+
+    monkeypatch.setattr(
+        startup_mod,
+        "detect_hardware",
+        lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
+    )
+    monkeypatch.setattr(
+        startup_mod,
+        "compute_shard_checksum",
+        lambda _path: (_ for _ in ()).throw(AssertionError("real model startup must not hash model files")),
+    )
     hf_calls = []
 
     def fake_snapshot_download(repo_id, cache_dir, local_dir):
@@ -1404,36 +1494,75 @@ def test_second_node_downloads_same_shard_from_peer_without_huggingface(
     })
     tracker_port = tracker.start()
     tracker_url = f"http://127.0.0.1:{tracker_port}"
-    nodes = []
     try:
-        node_a = run_startup(
+        node = run_startup(
             tracker_url=tracker_url,
             model="tiny-llama",
-            wallet_path=tmp_path / "wallet-a.json",
+            wallet_path=tmp_path / "wallet.json",
-            cache_dir=tmp_path / "node-a-shards",
+            cache_dir=tmp_path / "node-shards",
         )
-        nodes.append(node_a)
+        try:
-        assert len(hf_calls) == 1
+            assert len(hf_calls) == 1
-
+            assert (tmp_path / "node-shards" / "tiny-llama" / "weights.json").exists()
-        node_b = run_startup(
+            output = capsys.readouterr().out
-            tracker_url=tracker_url,
+            assert "Cached at:" in output
-            model="tiny-llama",
+            network_map = _get_json(f"{tracker_url}/v1/network/map")
-            wallet_path=tmp_path / "wallet-b.json",
+            registered = network_map["nodes"][0]
-            cache_dir=tmp_path / "node-b-shards",
+            assert registered["downloaded_models"] == [{
-        )
+                "model": "tiny-llama",
-        nodes.append(node_b)
+                "shard_start": 0,
-
+                "shard_end": 15,
-        assert len(hf_calls) == 1
+                "path": str(tmp_path / "node-shards" / "tiny-llama"),
-        assert (tmp_path / "node-b-shards" / "tiny-llama" / "layers_0-15" / "weights.json").exists()
+                "file_count": 1,
-        output = capsys.readouterr().out
+                "total_bytes": (tmp_path / "node-shards" / "tiny-llama" / "weights.json").stat().st_size,
-        assert "download source: HuggingFace" in output
+                "hf_repo": "org/tiny-llama-shards",
-        assert "download source: peer" in output
+            }]
-    finally:
+        finally:
-        for node in reversed(nodes):
             node.stop()
+    finally:
         tracker.stop()
 
 
+def test_downloaded_model_inventory_reports_local_model_percentage(tmp_path):
+    import meshnet_node.startup as startup_mod
+
+    model_dir = tmp_path / "models" / "tiny-llama"
+    model_dir.mkdir(parents=True)
+    (model_dir / "config.json").write_bytes(b"{}")
+    (model_dir / "weights-a.safetensors").write_bytes(b"a" * 3)
+
+    inventory = startup_mod._downloaded_model_inventory(
+        "tiny-llama",
+        0,
+        1,
+        model_dir,
+        hf_repo="org/tiny-llama",
+        model_sources=[{
+            "full_files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
+            "file_sizes": {
+                "config.json": 2,
+                "weights-a.safetensors": 3,
+                "weights-b.safetensors": 5,
+            },
+        }],
+    )
+
+    assert inventory == [{
+        "model": "tiny-llama",
+        "shard_start": 0,
+        "shard_end": 1,
+        "path": str(model_dir),
+        "file_count": 2,
+        "total_bytes": 5,
+        "hf_repo": "org/tiny-llama",
+        "expected_file_count": 3,
+        "local_expected_file_count": 2,
+        "expected_bytes": 10,
+        "local_expected_bytes": 5,
+        "local_model_percentage": 50.0,
+    }]
+
+
 def test_network_assign_gap_found_field():
     """network/assign sets gap_found=True when a real gap exists, False when fully covered."""
     import json as _json
@@ -1626,6 +1755,30 @@ def test_startup_cpu_fallback(tmp_path, monkeypatch):
 # --------------------------------------------------- layer detection (US: composite configs)
 
 
+def test_detect_num_layers_prefers_flattened_local_model_config(tmp_path, monkeypatch):
+    import meshnet_node.startup as startup_mod
+
+    model_dir = tmp_path / "Qwen3.6-35B-A3B"
+    model_dir.mkdir()
+    (model_dir / "config.json").write_text("{}")
+    calls = []
+
+    class AutoConfigStub:
+        @staticmethod
+        def from_pretrained(model_id, cache_dir=None):
+            calls.append({"model_id": model_id, "cache_dir": cache_dir})
+            return types.SimpleNamespace(num_hidden_layers=37)
+
+    monkeypatch.setitem(
+        sys.modules,
+        "transformers",
+        types.SimpleNamespace(AutoConfig=AutoConfigStub),
+    )
+
+    assert startup_mod._detect_num_layers("unsloth/Qwen3.6-35B-A3B", cache_dir=tmp_path) == 37
+    assert calls == [{"model_id": str(model_dir), "cache_dir": None}]
+
+
 def test_layers_from_config_top_level():
     from meshnet_node.model_catalog import layers_from_config
 

tests/test_tracker_routing.py

@@ -75,6 +75,13 @@ def test_tracker_exposes_registered_model_metadata():
                     "activated_parameters": "32B",
                     "context_length": 256000,
                 },
+                "downloaded_models": [{
+                    "model": "Kimi-K2.7-Code",
+                    "hf_repo": "unsloth/Kimi-K2.7-Code",
+                    "shard_start": 0,
+                    "shard_end": 60,
+                    "local_model_percentage": 100.0,
+                }],
             },
         )
 
@@ -91,6 +98,7 @@ def test_tracker_exposes_registered_model_metadata():
     registered = network_map["nodes"][0]
     assert registered["num_layers"] == 61
     assert registered["model_metadata"]["context_length"] == 256000
+    assert registered["downloaded_models"][0]["local_model_percentage"] == 100.0
 
 
 def test_tracker_lists_recommended_kimi_before_nodes_register():
@@ -975,6 +983,30 @@ def test_tracker_route_rejects_non_extending_overlap():
         tracker.stop()
 
 
+def test_tracker_console_records_model_not_available():
+    tracker = TrackerServer()
+    tracker_port = tracker.start()
+    try:
+        try:
+            _post_json(
+                f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
+                {"model": "missing-model", "messages": [{"role": "user", "content": "hi"}]},
+            )
+            raise AssertionError("Expected 503")
+        except urllib.error.HTTPError as exc:
+            assert exc.code == 503
+
+        console = _get_json(f"http://127.0.0.1:{tracker_port}/v1/console")
+    finally:
+        tracker.stop()
+
+    assert any(
+        event["message"] == "no nodes available for model"
+        and event["fields"]["model"] == "missing-model"
+        for event in console["events"]
+    )
+
+
 def test_tracker_registration_rejects_invalid_payload():
     """Registration errors return a defined JSON 400 response."""
     tracker = TrackerServer()