Merge commit '47b243cd98fd94da7918cacf5725373b099208e5' into ralph/distributed-gguf-runtime

packages/node/meshnet_node/hardware.py

@@ -2,6 +2,7 @@
 
 import json
 import os
+import shutil
 import subprocess
 import time
 
@@ -183,6 +184,17 @@ def with_forced_cpu(hw: dict) -> dict:
     return forced
 
 
+def _with_model_drive(profile: dict) -> dict:
+    """Attach free space for the default model cache drive to tracker diagnostics."""
+    try:
+        cache_root = os.path.expanduser("~/.cache/meshnet/shards")
+        profile["model_drive_free_bytes"] = shutil.disk_usage(os.path.expanduser("~")).free
+        profile["model_drive_path"] = cache_root
+    except OSError:
+        pass
+    return profile
+
+
 def detect_hardware() -> dict:
     """Detect GPU model and available VRAM. Returns hardware profile dict."""
     ram_mb = _detect_ram_mb()
@@ -208,23 +220,23 @@ def detect_hardware() -> dict:
             }
             if torch_gpu is not None and torch_gpu.get("gcn_arch"):
                 profile["gcn_arch"] = torch_gpu["gcn_arch"]
-            return profile
+            return _with_model_drive(profile)
     except ImportError:
         pass
 
     torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb)
     if torch_inventory is not None:
-        return torch_inventory
+        return _with_model_drive(torch_inventory)
 
     nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb)
     if nvidia_gpu is not None:
-        return nvidia_gpu
+        return _with_model_drive(nvidia_gpu)
 
     windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb)
     if windows_gpu is not None:
-        return windows_gpu
+        return _with_model_drive(windows_gpu)
 
-    return {
+    return _with_model_drive({
         "device": "cpu",
         "gpu_name": None,
         "vram_mb": 0,
@@ -232,7 +244,7 @@ def detect_hardware() -> dict:
         "shared_vram_mb": 0,
         "ram_mb": ram_mb,
         "cuda_available": False,
-    }
+    })
 
 
 def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]:

packages/node/meshnet_node/startup.py

@@ -12,7 +12,7 @@ import urllib.error
 import urllib.parse
 import urllib.request
 from pathlib import Path
-from typing import Any
+from typing import Any, Callable
 
 from .admission import (
     AdmissionRequirement,
@@ -420,6 +420,7 @@ def _start_heartbeat(
     interval: float = _HEARTBEAT_INTERVAL_IDLE,
     node_ref: Any | None = None,
     start_time: float | None = None,
+    refresh_capability: Callable[[dict], dict | None] | None = None,
 ) -> threading.Thread:
     """Daemon thread: sends heartbeats and re-registers automatically after tracker restarts.
 
@@ -431,6 +432,7 @@ def _start_heartbeat(
     which is logged for now (hot-reload implemented in US-026).
     """
     _start_time = start_time or time.monotonic()
+    completed_directives: list[dict] = []
 
     def _current_requests_snapshot() -> list[dict]:
         if node_ref is None:
@@ -455,6 +457,8 @@ def _start_heartbeat(
         current_requests = _current_requests_snapshot()
         if current_requests:
             stats["current_requests"] = current_requests
+        if completed_directives:
+            stats["completed_directives"] = list(completed_directives)
         return stats
 
     def _sleep_interval() -> float:
@@ -462,9 +466,26 @@ def _start_heartbeat(
             return _HEARTBEAT_INTERVAL_BUSY
         return interval
 
+    def _refresh_proof(payload: dict) -> None:
+        """Re-prove the current shard so a re-registration never presents an aged proof.
+
+        The tracker refuses proofs older than its freshness budget: re-sending the
+        startup-time report after an outage would re-register the node unroutable.
+        """
+        if refresh_capability is None or "capability_report" not in payload:
+            return
+        try:
+            fresh = refresh_capability(payload)
+        except Exception as exc:
+            print(f"  [node] WARNING: capability re-validation failed: {exc}", flush=True)
+            return
+        if fresh:
+            payload["capability_report"] = fresh
+
     def _reregister() -> bool:
         nonlocal node_id
         try:
+            _refresh_proof(register_payload)
             resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload)
             node_id = resp.get("node_id", node_id)
             if node_ref is not None:
@@ -486,6 +507,7 @@ def _start_heartbeat(
             "managed_assignment": True,
         }
         try:
+            _refresh_proof(extra_payload)
             reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", extra_payload)
             print(
                 f"  [node] registered additional model — node ID: {reg_resp.get('node_id')}",
@@ -494,21 +516,26 @@ def _start_heartbeat(
         except Exception as exc:
             print(f"  [node] WARNING: additional model registration failed: {exc}", flush=True)
 
-    def _apply_directives(directives: list[dict]) -> None:
+    def _apply_directives(directives: list[dict]) -> dict | None:
         if not directives:
-            return
+            return None
         if node_ref is None or not hasattr(node_ref, "apply_tracker_directives"):
             print(f"  [node] tracker directives received: {directives}", flush=True)
-            return
+            return None
         try:
             applied = node_ref.apply_tracker_directives(directives)
         except Exception as exc:
             print(f"  [node] WARNING: failed to apply tracker directives: {exc}", flush=True)
-            return
+            return None
         if applied:
+            completed_directives.append(dict(applied))
             if applied.get("action") == "ADD_SHARD":
                 _register_additional_assignment(applied)
-                return
+                return applied
+            if applied.get("action") in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
+                # A release has no replacement range. It is not a failed
+                # heartbeat and must not re-register the released assignment.
+                return applied
             model_id = applied.get("model", register_payload.get("hf_repo") or register_payload.get("model"))
             register_payload["model"] = str(model_id).split("/")[-1]
             register_payload["hf_repo"] = model_id
@@ -516,6 +543,7 @@ def _start_heartbeat(
             register_payload["shard_end"] = applied["shard_end"]
             register_payload["quantization"] = applied.get("quantization", register_payload.get("quantization"))
             register_payload["tracker_mode"] = bool(applied.get("tracker_mode", False))
+        return applied
 
     def _loop() -> None:
         nonlocal node_id
@@ -543,7 +571,10 @@ def _start_heartbeat(
                 continue
 
             try:
-                resp = _post_json(hb_url, _get_stats())
+                heartbeat = _get_stats()
+                resp = _post_json(hb_url, heartbeat)
+                if heartbeat.get("completed_directives"):
+                    completed_directives.clear()
                 _apply_directives(resp.get("directives", []))
                 new_asgn = resp.get("new_assignment")
                 if new_asgn:
@@ -580,6 +611,7 @@ def _register_with_tracker(
     reg_payload: dict,
     node: Any,
     start_time: float,
+    refresh_capability: Callable[[dict], dict | None] | None = None,
 ) -> str | None:
     """Register with the tracker, or start background retries when it is unreachable."""
     try:
@@ -587,7 +619,14 @@ def _register_with_tracker(
         tracker_node_id = str(reg_resp.get("node_id") or "?")
         setattr(node, "tracker_node_id", tracker_node_id)
         print(f"  Registered with tracker — node ID: {tracker_node_id}", flush=True)
-        _start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=start_time)
+        _start_heartbeat(
+            tracker_url,
+            tracker_node_id,
+            reg_payload,
+            node_ref=node,
+            start_time=start_time,
+            refresh_capability=refresh_capability,
+        )
         return tracker_node_id
     except Exception as exc:
         setattr(node, "tracker_node_id", None)
@@ -599,6 +638,7 @@ def _register_with_tracker(
             reg_payload,
             node_ref=node,
             start_time=start_time,
+            refresh_capability=refresh_capability,
         )
         return None
 
@@ -748,6 +788,54 @@ def _admit_capability(
     return report
 
 
+def _capability_refresher(
+    node: Any,
+    *,
+    manifest: RecipeManifest,
+    recipe: Recipe,
+    detected_device: str,
+    cache_dir: Path | None,
+    force_cpu: bool,
+    validator: CapabilityValidator | None = None,
+) -> Callable[[dict], dict | None]:
+    """A fresh proof for what the node serves *now*, run at re-registration time.
+
+    The startup proof ages past the tracker's freshness budget, and directives
+    can move the node to a shard the startup proof never covered — so every
+    re-registration re-proves against the currently loaded backend rather than
+    replaying the report captured at boot.
+    """
+    def refresh(payload: dict) -> dict | None:
+        target_model = payload.get("hf_repo") or payload.get("model")
+        backend = None
+        accessor = getattr(node, "backend_for", None)
+        if callable(accessor) and target_model:
+            backend = accessor(str(target_model))
+        if backend is None:
+            backend = getattr(node, "backend", None)
+        if backend is None:
+            return None
+        context = CapabilityContext(
+            backend=backend,
+            selection=DoctorSelection(
+                model_id=str(getattr(backend, "model_id", target_model)),
+                shard_start=int(getattr(backend, "shard_start", 0) or 0),
+                shard_end=int(getattr(backend, "shard_end", 0) or 0),
+                quantization=str(getattr(backend, "quantization", None) or "auto"),
+                cache_dir=cache_dir,
+                force_cpu=force_cpu,
+            ),
+            recipe=recipe,
+            manifest=manifest,
+            device=_capability_device(backend, detected_device),
+        )
+        report = (validator or probe_capability)(context)
+        setattr(node, "capability_report", report)
+        return report.to_dict()
+
+    return refresh
+
+
 def run_startup(
     tracker_url: str,
     port: int = 0,
@@ -1079,6 +1167,15 @@ def run_startup(
         }
         tracker_node_id = _register_with_tracker(
             tracker_url, reg_payload, node, _node_start_time,
+            refresh_capability=_capability_refresher(
+                node,
+                manifest=manifest,
+                recipe=recipe,
+                detected_device=device,
+                cache_dir=cache_dir,
+                force_cpu=force_cpu,
+                validator=capability_validator,
+            ),
         )
 
         print(
@@ -1268,6 +1365,15 @@ def run_startup(
         }
         tracker_node_id = _register_with_tracker(
             tracker_url, auto_reg_payload, node, _node_start_time,
+            refresh_capability=_capability_refresher(
+                node,
+                manifest=manifest,
+                recipe=recipe,
+                detected_device=device,
+                cache_dir=cache_dir,
+                force_cpu=force_cpu,
+                validator=capability_validator,
+            ),
         )
         shard_label = _format_shard_label(
             proof_shard.start,
@@ -1477,6 +1583,15 @@ def run_startup(
         }
         tracker_node_id = _register_with_tracker(
             tracker_url, reg_payload, node, _node_start_time,
+            refresh_capability=_capability_refresher(
+                node,
+                manifest=manifest,
+                recipe=recipe,
+                detected_device=device,
+                cache_dir=cache_dir,
+                force_cpu=force_cpu,
+                validator=capability_validator,
+            ),
         )
         print(
             f"\n{'=' * 32}\n"
@@ -1564,7 +1679,22 @@ def run_startup(
         )
         node_id = str(reg_resp["node_id"])
         setattr(node, "tracker_node_id", node_id)
-        _start_heartbeat(tracker_url, node_id, reg_payload, node_ref=node, start_time=_node_start_time)
+        _start_heartbeat(
+            tracker_url,
+            node_id,
+            reg_payload,
+            node_ref=node,
+            start_time=_node_start_time,
+            refresh_capability=_capability_refresher(
+                node,
+                manifest=manifest,
+                recipe=recipe,
+                detected_device=device,
+                cache_dir=shard_path,
+                force_cpu=force_cpu,
+                validator=capability_validator,
+            ),
+        )
     except Exception:
         node.stop()
         raise

packages/node/meshnet_node/torch_server.py

@@ -1543,8 +1543,32 @@ class TorchNodeServer:
     def loaded_model_ids(self) -> list[str]:
         return list(self._backends.keys())
 
+    def backend_for(self, model_id: str) -> TorchModelShard | None:
+        """The loaded backend serving `model_id` — full repo id or short name."""
+        backend = self._backends.get(model_id)
+        if backend is not None:
+            return backend
+        short = model_id.split("/")[-1].lower()
+        for key, candidate in self._backends.items():
+            if key.split("/")[-1].lower() == short:
+                return candidate
+        return None
+
     def apply_tracker_directives(self, directives: list[dict]) -> dict | None:
         """Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more)."""
+        drop_all_directive = next(
+            (directive for directive in reversed(directives) if directive.get("action") == "DROP_ALL_SHARDS"),
+            None,
+        )
+        if drop_all_directive is not None:
+            self._backends.clear()
+            self._backend = None
+            self._tracker_mode = False
+            if self._server is not None:
+                self._server.backends = {}
+                self._server.backend = None
+                self._server.tracker_mode = False
+            return {"action": "DROP_ALL_SHARDS"}
         drop_directive = next(
             (directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"),
             None,

packages/tracker/meshnet_tracker/dashboard.html

@@ -44,12 +44,15 @@
   .empty { color:var(--dim); font-style:italic; }
   .pill { display:inline-block; padding:0 7px; border-radius:9px;
           border:1px solid var(--border); font-size:11px; }
-  input, button { font:inherit; color:var(--fg); background:var(--bg);
+  input, button, select { font:inherit; color:var(--fg); background:var(--bg);
           border:1px solid var(--border); border-radius:6px; padding:5px 8px; }
   input { width:100%; margin-bottom:6px; }
   button { cursor:pointer; color:var(--accent); }
   button:hover { border-color:var(--accent); }
   button.small { font-size:11px; padding:1px 7px; }
+  dialog { color:var(--fg); background:var(--panel); border:1px solid var(--border); border-radius:8px; min-width:min(420px,calc(100vw - 32px)); }
+  dialog::backdrop { background:rgba(0,0,0,.55); }
+  .placement-dialog-actions { display:flex; justify-content:flex-end; gap:8px; margin-top:12px; }
   .form-row { display:flex; gap:8px; }
   .form-row button { white-space:nowrap; }
   .error-msg { color:var(--bad); font-size:12px; min-height:16px; }
@@ -212,7 +215,7 @@
   .chat-compose button:disabled { opacity:.45; cursor:not-allowed; }
   .console {
     background:var(--bg); border:1px solid var(--border); border-radius:6px;
-    min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
+    min-height:160px; max-height:520px; overflow-y:auto; overflow-x:auto; padding:7px 9px;
     white-space:pre-wrap; word-break:break-word; font-size:11px;
   }
   .console-line { padding:1px 0; border-bottom:1px solid #161b22; }
@@ -296,6 +299,7 @@
   <section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
   <section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
   <section data-tab="admin" class="wide"><h2>Model placement</h2><div id="admin-model-placement-status" class="dim">Choose a model to load or release.</div><div id="admin-model-placement" class="empty">admin login required</div></section>
+  <section data-tab="admin" class="wide"><h2>Total node pool</h2><div id="admin-node-pool" class="empty">admin login required</div></section>
   <section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
   <section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section>
   <section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
@@ -323,6 +327,16 @@
     <div id="testing-log" class="console empty">no test output yet</div>
   </section>
 </main>
+<dialog id="model-placement-dialog">
+  <form method="dialog">
+    <div id="model-placement-dialog-title"></div>
+    <label for="model-placement-node">Node</label>
+    <select id="model-placement-node"></select>
+    <label id="model-placement-replace" style="display:none"><input type="checkbox" id="model-placement-replace-confirm"> Unload the currently loaded model before loading this one</label>
+    <div id="model-placement-replace-error" class="bad" style="display:none"></div>
+    <div class="placement-dialog-actions"><button value="cancel">Cancel</button><button type="button" id="model-placement-confirm">Confirm</button></div>
+  </form>
+</dialog>
 <script>
 "use strict";
 const $ = id => document.getElementById(id);
@@ -1082,6 +1096,7 @@ function renderBillingUsage(records) {
 }
 
 let consoleClearedAt = 0;
+const CONSOLE_MAX_LINES = 1000;
 
 function clearConsole() {
   consoleClearedAt = Date.now() / 1000;
@@ -1095,7 +1110,7 @@ function renderConsole(data) {
     $("console").innerHTML = '<div class="empty">no console events</div>';
     return;
   }
-  $("console").innerHTML = events.slice(-120).map(e => {
+  $("console").innerHTML = events.slice(-CONSOLE_MAX_LINES).map(e => {
     const level = String(e.level || "info");
     const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
     const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
@@ -1789,7 +1804,7 @@ async function requestSelectedModelLoad() {
   if (!selectedChatModel) return;
   const button = $("request-model-load");
   if (button) button.disabled = true;
-  const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel });
+  const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel, force: isAdmin });
   if (button) button.disabled = false;
   if (!result.ok) {
     alert(result.data.error || "model load request failed");
@@ -1799,27 +1814,73 @@ async function requestSelectedModelLoad() {
   $("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
 }
 
-async function requestAdminModelLoad(model) {
-  const result = await apiCall("/v1/models/load", "POST", { model, force: true });
+async function requestAdminModelLoad(model, nodeId, replacing) {
+  const result = await apiCall("/v1/models/load", "POST", { model, node_id: nodeId, force: replacing });
   if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
   const assignment = result.data.assignment || {};
   showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
   await refreshActiveTab(true);
 }
 
-async function releaseAdminModel(model) {
-  const result = await apiCall("/v1/models/release", "POST", { model });
+async function releaseAdminModel(model, nodeId) {
+  const result = await apiCall("/v1/models/release", "POST", { model, node_id: nodeId });
   if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model release request failed", true);
   showAdminModelPlacementStatus(`Release queued for ${result.data.released || 0} node(s) serving ${model}.`);
   await refreshActiveTab(true);
 }
 
+async function releaseAllNodeModels(nodeId) {
+  if (!confirm("Unload every model from this node?")) return;
+  const result = await apiCall("/v1/nodes/release-all", "POST", { node_id: nodeId });
+  if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "node unload failed", true);
+  showAdminModelPlacementStatus(`Unload queued for ${short(nodeId)}.`);
+  await refreshActiveTab(true);
+}
+
 function showAdminModelPlacementStatus(message, isError) {
   const status = $("admin-model-placement-status");
   status.textContent = message;
   status.className = isError ? "bad" : "ok";
 }
 
+function gib(bytes) { return bytes == null ? "not reported" : `${(Number(bytes) / 1073741824).toFixed(1)} GiB`; }
+
+function renderAdminNodePool(map) {
+  const groups = {};
+  for (const node of (map && map.nodes) || []) {
+    const account = node.wallet_address || "unbound account";
+    (groups[account] = groups[account] || []).push(node);
+  }
+  let html = "";
+  for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) {
+    html += `<div style="margin-top:10px"><b>${esc(short(account, 20))}</b> <span class="dim">${nodes.length} node(s)</span></div>`;
+    html += table(["node", "assignment", "state / slots", "model RAM", "RAM", "GPU / VRAM", "model drive", "action"], nodes.map(node => {
+      const hw = node.hardware_profile || {};
+      const cap = node.capacity || {};
+      // The network map keeps reported resource capacity under `capacity`.
+      node.ram_bytes = cap.ram_bytes ?? node.ram_bytes;
+      node.vram_bytes = cap.vram_bytes ?? node.vram_bytes;
+      const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
+      const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
+      const row = [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
+        esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`),
+        esc(gib(cap.loaded_model_bytes)),
+        esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))),
+        esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))];
+      return row.concat([
+        node.shard_start == null ? '<span class="dim">empty</span>' :
+          `<button class="small" data-admin-node-release="${esc(node.node_id)}">unload all</button>`,
+      ]);
+    }));
+  }
+  $("admin-node-pool").innerHTML = html || '<div class="empty">no nodes registered</div>';
+}
+
+$("admin-node-pool").addEventListener("click", event => {
+  const unload = event.target.closest("[data-admin-node-release]");
+  if (unload) void releaseAllNodeModels(unload.dataset.adminNodeRelease);
+});
+
 function renderAdminModelPlacement(models, map) {
   const nodes = (map && map.nodes) || [];
   const rows = ((models && models.data) || []).map(model => {
@@ -1827,7 +1888,8 @@ function renderAdminModelPlacement(models, map) {
     const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length;
     const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) ||
       (node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length;
-    const actions = `<button class="small" data-admin-model-load="${esc(model.id)}">load</button> ` +
+    const loadable = model.id !== "stub-model";
+    const actions = `<button class="small" data-admin-model-load="${esc(model.id)}"${loadable ? "" : " disabled"}>load</button> ` +
       `<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`;
     return [esc(model.name || model.id), String(serving), String(downloaded), actions];
   });
@@ -1839,10 +1901,49 @@ function renderAdminModelPlacement(models, map) {
 $("admin-model-placement").addEventListener("click", event => {
   const load = event.target.closest("[data-admin-model-load]");
   const release = event.target.closest("[data-admin-model-release]");
-  if (load) void requestAdminModelLoad(load.dataset.adminModelLoad);
-  if (release) void releaseAdminModel(release.dataset.adminModelRelease);
+  if (load) void chooseModelPlacementNode("load", load.dataset.adminModelLoad);
+  if (release) void chooseModelPlacementNode("release", release.dataset.adminModelRelease);
 });
 
+function chooseModelPlacementNode(action, model) {
+  const dialog = $("model-placement-dialog");
+  const select = $("model-placement-node");
+  const targetAlias = modelAliasKey(model);
+  const nodes = (lastNetworkMap?.nodes || []).filter(node => action === "load" ||
+    modelAliasKey(node.model) === targetAlias || modelAliasKey(node.hf_repo) === targetAlias);
+  if (!nodes.length) return showAdminModelPlacementStatus(`No node can ${action} ${model}.`, true);
+  $("model-placement-dialog-title").textContent = `${action === "load" ? "Load" : "Release"} ${model} on a node`;
+  select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
+  const replace = $("model-placement-replace");
+  const replaceConfirm = $("model-placement-replace-confirm");
+  const replaceError = $("model-placement-replace-error");
+  const confirmButton = $("model-placement-confirm");
+  const selectedNode = () => nodes.find(node => node.node_id === select.value);
+  const updateReplacementWarning = () => {
+    const node = selectedNode();
+    const occupied = action === "load" && node && node.shard_start != null && node.shard_end != null &&
+      modelAliasKey(node.hf_repo || node.model) !== targetAlias;
+    replace.style.display = occupied ? "" : "none";
+    replaceConfirm.checked = false;
+    replaceError.style.display = "none";
+  };
+  select.onchange = updateReplacementWarning;
+  updateReplacementWarning();
+  dialog.onclose = null;
+  confirmButton.onclick = () => {
+    const replacing = replace.style.display !== "none";
+    if (replacing && !replaceConfirm.checked) {
+      replaceError.textContent = "Tick the box to confirm that this will unload the current model.";
+      replaceError.style.display = "";
+      return;
+    }
+    dialog.close("confirm");
+    if (action === "load") void requestAdminModelLoad(model, select.value, replacing);
+    else void releaseAdminModel(model, select.value);
+  };
+  dialog.showModal();
+}
+
 function chatAuthToken() {
   if (accountApiKeys.length) return accountApiKeys[0];
   return null;
@@ -2485,11 +2586,13 @@ async function fetchAdminTab() {
   if (isAdmin) fetches.push(apiCall("/v1/admin/accounts"));
   const results = await Promise.all(fetches);
   const [raft, consoleData, summary, wallets, models, map, adminResp] = results;
+  if (map) lastNetworkMap = map;
   renderIfChanged("hive", raft, renderHive);
   renderIfChanged("console", consoleData, renderConsole);
   renderIfChanged("billing-summary", summary, data => renderBilling(data));
   renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary));
   renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map));
+  renderIfChanged("admin-node-pool", map, renderAdminNodePool);
   if (adminResp && adminResp.ok) {
     renderIfChanged("admin", adminResp.data.accounts || [], accounts => {
       const rows = accounts.map(a => {

packages/tracker/meshnet_tracker/server.py

@@ -87,7 +87,7 @@ from .model_files import files_for_layer_range, snapshot_dir_for_repo
 from .raft import RaftNode
 
 
-_CONSOLE_LIMIT = 300
+_CONSOLE_LIMIT = 1000
 _PROXY_PROGRESS_LOG_INTERVAL = 5.0
 _SESSION_COOKIE_NAME = "meshnet_session"
 
@@ -1121,12 +1121,15 @@ def _registration_quantization(body: dict, quantizations: list[str]) -> str | No
 
     An absent field predates the protocol adding it: it means "unknown", not
     "unsupported", so the node keeps the best precision it advertises and stays
-    routable. Anything the node states explicitly is taken at its word -- a null,
-    a non-string, or an unsupported name leaves it with no usable precision and
-    routing excludes it.
+    routable. An explicit "auto" means the same thing — the node's CLI default
+    delegates the choice, it does not refuse one. Anything else the node states
+    explicitly is taken at its word -- a null, a non-string, or an unsupported
+    name leaves it with no usable precision and routing excludes it.
     """
-    if "quantization" in body:
-        return _normalize_quantization(body["quantization"])
+    declared = body.get("quantization")
+    declared_auto = isinstance(declared, str) and declared.strip().lower() == "auto"
+    if "quantization" in body and not declared_auto:
+        return _normalize_quantization(declared)
     supported = [
         normalized for value in quantizations
         if (normalized := _normalize_quantization(value)) is not None
@@ -1245,6 +1248,7 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
         "quantization": node.quantization,
         "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
         "effective_throughput": round(_effective_throughput(node), 4),
+        "loaded_model_bytes": _assignment_memory_bytes(node, preset),
     }
     if preset is not None:
         summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
@@ -1514,7 +1518,9 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
             break
 
 
-def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
+def _request_model_load_locked(
+    server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
+) -> dict | None:
     """Queue an explicitly requested model on the best available joined node."""
     resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
     if preset is None or not preset.get("hf_repo"):
@@ -1530,6 +1536,8 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
             continue
         host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
         placeable = [node for node in host_nodes if _has_usable_quantization(node)]
+        if node_id is not None:
+            placeable = [node for node in placeable if node.node_id == node_id]
         if not placeable:
             continue
         anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec)
@@ -1548,21 +1556,26 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
     return None
 
 
-def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
+def _force_model_load_locked(
+    server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
+) -> dict | None:
     """Replace the fastest ready assignment after an explicit admin eviction."""
     resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
     if preset is None or not preset.get("hf_repo"):
         return None
     start, end = _preset_layer_bounds(preset)
-    candidates = [node for node in server.registry.values()
-                  if node.status == "ready" and node.pending_new_assignment is None
-                  and _has_usable_quantization(node)]
+    # An explicit admin eviction is permitted to recover a stuck/loading node
+    # and to use the preset default precision. It must only avoid a node that
+    # already has another assignment in flight.
+    candidates = [
+        node for node in server.registry.values()
+        if node.pending_new_assignment is None
+        and (node_id is None or node.node_id == node_id)
+    ]
     if not candidates:
         return None
     node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
-    shard_end = min(end, start + min(_node_layer_capacity(node, preset), end - start + 1) - 1)
-    if shard_end < start:
-        return None
+    shard_end = min(end, start + max(1, min(_node_layer_capacity(node, preset), end - start + 1)) - 1)
     quantization = _node_quantization(node, preset)
     directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization)
     replaced = node.hf_repo or node.model
@@ -1576,13 +1589,17 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
             "shard_start": start, "shard_end": shard_end, "replaced_model": replaced}
 
 
-def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
+def _release_model_locked(
+    server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
+) -> int:
     """Queue DROP_SHARD for every served shard and remove it from routing immediately."""
     resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
     if preset is None:
         return 0
     released = 0
     for node in server.registry.values():
+        if node_id is not None and node.node_id != node_id:
+            continue
         if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None:
             continue
         node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16"))
@@ -1591,6 +1608,16 @@ def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
     return released
 
 
+def _release_all_node_models_locked(server: "_TrackerHTTPServer", node_id: str) -> int:
+    """Queue removal of every loaded assignment on one node."""
+    node = server.registry.get(node_id)
+    if node is None or node.shard_start is None or node.shard_end is None:
+        return 0
+    node.pending_directives.append({"action": "DROP_ALL_SHARDS"})
+    node.status = "loading"
+    return 1
+
+
 def _preferred_node_quantization(
     node: _NodeEntry,
     preset: dict,
@@ -3109,6 +3136,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if self.path == "/v1/models/release":
             self._handle_model_release_request()
             return
+        if self.path == "/v1/nodes/release-all":
+            self._handle_node_release_all_request()
+            return
         if self.path == "/v1/models/vote":
             self._handle_model_coverage_vote()
             return
@@ -3285,7 +3315,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             node.hf_repo or node.model
             for node in alive
             if node.model is not None
-            and node.model not in server.model_presets
+            # The same model can be registered under its HF repository while
+            # the catalogue exposes its short preset id.  Do not emit a second
+            # repo-keyed entry when either node identifier resolves to a preset.
+            and _resolve_model_preset(
+                server.model_presets, node.hf_repo or node.model,
+            )[1] is None
             and node.shard_start is not None
             and node.shard_end is not None
             and node.num_layers is not None
@@ -3386,6 +3421,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                     "endpoint": node.endpoint,
                     "relay_addr": node.relay_addr,
                     "peer_id": node.peer_id,
+                    "wallet_address": node.wallet_address,
+                    "hardware_profile": dict(node.hardware_profile),
+                    "ram_bytes": node.ram_bytes,
+                    "vram_bytes": node.vram_bytes,
+                    "max_loaded_shards": node.max_loaded_shards,
                 }
                 for node in tracker_nodes
             ],
@@ -3409,12 +3449,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             memory_pool = _memory_pool_map(server)
 
         def capacity_for(node: _NodeEntry) -> dict:
-            preset = None
-            if node.model:
-                preset = server.model_presets.get(node.model)
-            if preset is None and node.hf_repo and node.num_layers:
-                preset = _hf_rebalance_preset([node])
-            return _node_capacity_summary(node, preset)
+            return _node_capacity_summary(node, _preset_for_node(server, node))
 
         def throughput_for(node: _NodeEntry) -> dict:
             if server.stats is None:
@@ -4832,6 +4867,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                 entry.uptime_seconds = float(body["uptime_seconds"])
             if "status" in body and body["status"] in ("ready", "loading"):
                 entry.status = body["status"]
+            completed_directives = body.get("completed_directives", [])
+            if isinstance(completed_directives, list):
+                for directive in completed_directives:
+                    if not isinstance(directive, dict) or directive.get("action") not in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
+                        continue
+                    # A node has confirmed the release.  Stop advertising its
+                    # old route immediately so the dashboard and routing state
+                    # agree with the runtime.
+                    entry.model = "stub-model"
+                    entry.hf_repo = None
+                    entry.shard_start = None
+                    entry.shard_end = None
+                    entry.tracker_mode = False
+                    entry.status = "ready"
             if "friendly_name" in body:
                 try:
                     entry.friendly_name = _normalize_friendly_name(body.get("friendly_name"))
@@ -4903,11 +4952,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if not isinstance(model, str) or not model.strip():
             self._send_json(400, {"error": "model is required"})
             return
+        node_id = body.get("node_id")
+        if node_id is not None and (not isinstance(node_id, str) or not node_id):
+            self._send_json(400, {"error": "node_id must be a non-empty string"})
+            return
+        _resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+        if preset is None or str(preset.get("hf_repo") or "").strip().lower() == "stub-model":
+            self._send_json(400, {"error": "stub-model is a local test backend and cannot be loaded onto a node"})
+            return
         with server.lock:
             self._purge_expired_nodes()
-            assignment = _request_model_load_locked(server, model)
+            assignment = _request_model_load_locked(server, model, node_id)
             if assignment is None and body.get("force") is True:
-                assignment = _force_model_load_locked(server, model)
+                assignment = _force_model_load_locked(server, model, node_id)
         if assignment is None:
             self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
             return
@@ -4924,14 +4981,39 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if not isinstance(model, str) or not model.strip():
             self._send_json(400, {"error": "model is required"})
             return
+        node_id = body.get("node_id")
+        if node_id is not None and (not isinstance(node_id, str) or not node_id):
+            self._send_json(400, {"error": "node_id must be a non-empty string"})
+            return
         with server.lock:
             self._purge_expired_nodes()
-            released = _release_model_locked(server, model)
+            released = _release_model_locked(server, model, node_id)
         if not released:
             self._send_json(404, {"error": "no served shards found for model"})
             return
         self._send_json(202, {"status": "release_queued", "released": released})
 
+    def _handle_node_release_all_request(self):
+        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
+        if not self._require_role("admin", "validator"):
+            return
+        body = self._read_json_body()
+        if body is None:
+            return
+        node_id = body.get("node_id")
+        if not isinstance(node_id, str) or not node_id:
+            self._send_json(400, {"error": "node_id must be a non-empty string"})
+            return
+        with server.lock:
+            self._purge_expired_nodes()
+            released = _release_all_node_models_locked(server, node_id)
+        if not released:
+            self._send_json(404, {"error": "no loaded models found for node"})
+            return
+        self._send_json(202, {
+            "status": "release_queued", "released": released, "node_id": node_id,
+        })
+
     def _handle_model_coverage_vote(self):
         """Record a rolling wish-list signal for an unavailable precision."""
         server: _TrackerHTTPServer = self.server  # type: ignore[assignment]