From 47b243cd98fd94da7918cacf5725373b099208e5 Mon Sep 17 00:00:00 2001
From: Dobromir Popov <dobromir.popov@gateway.one>
Date: Wed, 15 Jul 2026 13:55:38 +0200
Subject: [PATCH] model loading, dash

---
 packages/node/meshnet_node/startup.py         |  2 +-
 packages/node/meshnet_node/torch_server.py    | 13 +++++
 .../tracker/meshnet_tracker/dashboard.html    | 40 +++++++++++++---
 packages/tracker/meshnet_tracker/server.py    | 48 ++++++++++++++++---
 tests/test_dashboard.py                       |  7 +++
 5 files changed, 95 insertions(+), 15 deletions(-)

diff --git a/packages/node/meshnet_node/startup.py b/packages/node/meshnet_node/startup.py
index c0f6c73..dd8a4df 100644
--- a/packages/node/meshnet_node/startup.py
+++ b/packages/node/meshnet_node/startup.py
@@ -531,7 +531,7 @@ def _start_heartbeat(
             if applied.get("action") == "ADD_SHARD":
                 _register_additional_assignment(applied)
                 return applied
-            if applied.get("action") == "DROP_SHARD":
+            if applied.get("action") in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
                 # A release has no replacement range. It is not a failed
                 # heartbeat and must not re-register the released assignment.
                 return applied
diff --git a/packages/node/meshnet_node/torch_server.py b/packages/node/meshnet_node/torch_server.py
index 36fbeb9..d464561 100644
--- a/packages/node/meshnet_node/torch_server.py
+++ b/packages/node/meshnet_node/torch_server.py
@@ -1556,6 +1556,19 @@ class TorchNodeServer:
 
     def apply_tracker_directives(self, directives: list[dict]) -> dict | None:
         """Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more)."""
+        drop_all_directive = next(
+            (directive for directive in reversed(directives) if directive.get("action") == "DROP_ALL_SHARDS"),
+            None,
+        )
+        if drop_all_directive is not None:
+            self._backends.clear()
+            self._backend = None
+            self._tracker_mode = False
+            if self._server is not None:
+                self._server.backends = {}
+                self._server.backend = None
+                self._server.tracker_mode = False
+            return {"action": "DROP_ALL_SHARDS"}
         drop_directive = next(
             (directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"),
             None,
diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html
index f6615e1..0984262 100644
--- a/packages/tracker/meshnet_tracker/dashboard.html
+++ b/packages/tracker/meshnet_tracker/dashboard.html
@@ -333,7 +333,8 @@
     <label for="model-placement-node">Node</label>
     <select id="model-placement-node"></select>
     <label id="model-placement-replace" style="display:none"><input type="checkbox" id="model-placement-replace-confirm"> Unload the currently loaded model before loading this one</label>
-    <div class="placement-dialog-actions"><button value="cancel">Cancel</button><button value="confirm">Confirm</button></div>
+    <div id="model-placement-replace-error" class="bad" style="display:none"></div>
+    <div class="placement-dialog-actions"><button value="cancel">Cancel</button><button type="button" id="model-placement-confirm">Confirm</button></div>
   </form>
 </dialog>
 <script>
@@ -1828,6 +1829,14 @@ async function releaseAdminModel(model, nodeId) {
   await refreshActiveTab(true);
 }
 
+async function releaseAllNodeModels(nodeId) {
+  if (!confirm("Unload every model from this node?")) return;
+  const result = await apiCall("/v1/nodes/release-all", "POST", { node_id: nodeId });
+  if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "node unload failed", true);
+  showAdminModelPlacementStatus(`Unload queued for ${short(nodeId)}.`);
+  await refreshActiveTab(true);
+}
+
 function showAdminModelPlacementStatus(message, isError) {
   const status = $("admin-model-placement-status");
   status.textContent = message;
@@ -1845,7 +1854,7 @@ function renderAdminNodePool(map) {
   let html = "";
   for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) {
     html += `<div style="margin-top:10px"><b>${esc(short(account, 20))}</b> <span class="dim">${nodes.length} node(s)</span></div>`;
-    html += table(["node", "assignment", "state / slots", "RAM", "GPU / VRAM", "model drive"], nodes.map(node => {
+    html += table(["node", "assignment", "state / slots", "model RAM", "RAM", "GPU / VRAM", "model drive", "action"], nodes.map(node => {
       const hw = node.hardware_profile || {};
       const cap = node.capacity || {};
       // The network map keeps reported resource capacity under `capacity`.
@@ -1853,15 +1862,25 @@ function renderAdminNodePool(map) {
       node.vram_bytes = cap.vram_bytes ?? node.vram_bytes;
       const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
       const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
-      return [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
+      const row = [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
         esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`),
+        esc(gib(cap.loaded_model_bytes)),
         esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))),
         esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))];
+      return row.concat([
+        node.shard_start == null ? '<span class="dim">empty</span>' :
+          `<button class="small" data-admin-node-release="${esc(node.node_id)}">unload all</button>`,
+      ]);
     }));
   }
   $("admin-node-pool").innerHTML = html || '<div class="empty">no nodes registered</div>';
 }
 
+$("admin-node-pool").addEventListener("click", event => {
+  const unload = event.target.closest("[data-admin-node-release]");
+  if (unload) void releaseAllNodeModels(unload.dataset.adminNodeRelease);
+});
+
 function renderAdminModelPlacement(models, map) {
   const nodes = (map && map.nodes) || [];
   const rows = ((models && models.data) || []).map(model => {
@@ -1869,7 +1888,8 @@ function renderAdminModelPlacement(models, map) {
     const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length;
     const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) ||
       (node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length;
-    const actions = `<button class="small" data-admin-model-load="${esc(model.id)}">load</button> ` +
+    const loadable = model.id !== "stub-model";
+    const actions = `<button class="small" data-admin-model-load="${esc(model.id)}"${loadable ? "" : " disabled"}>load</button> ` +
       `<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`;
     return [esc(model.name || model.id), String(serving), String(downloaded), actions];
   });
@@ -1896,6 +1916,8 @@ function chooseModelPlacementNode(action, model) {
   select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
   const replace = $("model-placement-replace");
   const replaceConfirm = $("model-placement-replace-confirm");
+  const replaceError = $("model-placement-replace-error");
+  const confirmButton = $("model-placement-confirm");
   const selectedNode = () => nodes.find(node => node.node_id === select.value);
   const updateReplacementWarning = () => {
     const node = selectedNode();
@@ -1903,15 +1925,19 @@ function chooseModelPlacementNode(action, model) {
       modelAliasKey(node.hf_repo || node.model) !== targetAlias;
     replace.style.display = occupied ? "" : "none";
     replaceConfirm.checked = false;
+    replaceError.style.display = "none";
   };
   select.onchange = updateReplacementWarning;
   updateReplacementWarning();
-  dialog.onclose = () => {
-    if (dialog.returnValue !== "confirm") return;
+  dialog.onclose = null;
+  confirmButton.onclick = () => {
     const replacing = replace.style.display !== "none";
     if (replacing && !replaceConfirm.checked) {
-      return showAdminModelPlacementStatus("Confirm unloading the existing model before replacing it.", true);
+      replaceError.textContent = "Tick the box to confirm that this will unload the current model.";
+      replaceError.style.display = "";
+      return;
     }
+    dialog.close("confirm");
     if (action === "load") void requestAdminModelLoad(model, select.value, replacing);
     else void releaseAdminModel(model, select.value);
   };
diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py
index 2d5cb76..43e50ce 100644
--- a/packages/tracker/meshnet_tracker/server.py
+++ b/packages/tracker/meshnet_tracker/server.py
@@ -1228,6 +1228,7 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
         "quantization": node.quantization,
         "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
         "effective_throughput": round(_effective_throughput(node), 4),
+        "loaded_model_bytes": _assignment_memory_bytes(node, preset),
     }
     if preset is not None:
         summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
@@ -1587,6 +1588,16 @@ def _release_model_locked(
     return released
 
 
+def _release_all_node_models_locked(server: "_TrackerHTTPServer", node_id: str) -> int:
+    """Queue removal of every loaded assignment on one node."""
+    node = server.registry.get(node_id)
+    if node is None or node.shard_start is None or node.shard_end is None:
+        return 0
+    node.pending_directives.append({"action": "DROP_ALL_SHARDS"})
+    node.status = "loading"
+    return 1
+
+
 def _preferred_node_quantization(
     node: _NodeEntry,
     preset: dict,
@@ -3105,6 +3116,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if self.path == "/v1/models/release":
             self._handle_model_release_request()
             return
+        if self.path == "/v1/nodes/release-all":
+            self._handle_node_release_all_request()
+            return
         if self.path == "/v1/models/vote":
             self._handle_model_coverage_vote()
             return
@@ -3415,12 +3429,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             memory_pool = _memory_pool_map(server)
 
         def capacity_for(node: _NodeEntry) -> dict:
-            preset = None
-            if node.model:
-                preset = server.model_presets.get(node.model)
-            if preset is None and node.hf_repo and node.num_layers:
-                preset = _hf_rebalance_preset([node])
-            return _node_capacity_summary(node, preset)
+            return _node_capacity_summary(node, _preset_for_node(server, node))
 
         def throughput_for(node: _NodeEntry) -> dict:
             if server.stats is None:
@@ -4833,7 +4842,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             completed_directives = body.get("completed_directives", [])
             if isinstance(completed_directives, list):
                 for directive in completed_directives:
-                    if not isinstance(directive, dict) or directive.get("action") != "DROP_SHARD":
+                    if not isinstance(directive, dict) or directive.get("action") not in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
                         continue
                     # A node has confirmed the release.  Stop advertising its
                     # old route immediately so the dashboard and routing state
@@ -4919,6 +4928,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if node_id is not None and (not isinstance(node_id, str) or not node_id):
             self._send_json(400, {"error": "node_id must be a non-empty string"})
             return
+        _resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+        if preset is None or str(preset.get("hf_repo") or "").strip().lower() == "stub-model":
+            self._send_json(400, {"error": "stub-model is a local test backend and cannot be loaded onto a node"})
+            return
         with server.lock:
             self._purge_expired_nodes()
             assignment = _request_model_load_locked(server, model, node_id)
@@ -4952,6 +4965,27 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             return
         self._send_json(202, {"status": "release_queued", "released": released})
 
+    def _handle_node_release_all_request(self):
+        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
+        if not self._require_role("admin", "validator"):
+            return
+        body = self._read_json_body()
+        if body is None:
+            return
+        node_id = body.get("node_id")
+        if not isinstance(node_id, str) or not node_id:
+            self._send_json(400, {"error": "node_id must be a non-empty string"})
+            return
+        with server.lock:
+            self._purge_expired_nodes()
+            released = _release_all_node_models_locked(server, node_id)
+        if not released:
+            self._send_json(404, {"error": "no loaded models found for node"})
+            return
+        self._send_json(202, {
+            "status": "release_queued", "released": released, "node_id": node_id,
+        })
+
     def _handle_model_coverage_vote(self):
         """Record a rolling wish-list signal for an unavailable precision."""
         server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py
index 8a31040..b476412 100644
--- a/tests/test_dashboard.py
+++ b/tests/test_dashboard.py
@@ -128,7 +128,14 @@ def test_dashboard_exposes_admin_model_inventory_and_release_controls():
     assert "node_id: nodeId" in html
     assert "modelAliasKey(node.model)" in html
     assert 'id="model-placement-replace"' in html
+    assert 'id="model-placement-confirm"' in html
+    assert 'id="model-placement-replace-error"' in html
     assert "force: replacing" in html
+    assert "Tick the box to confirm" in html
+    assert "releaseAllNodeModels" in html
+    assert '"/v1/nodes/release-all"' in html
+    assert "model RAM" in html
+    assert "loaded_model_bytes" in html
 
 
 def test_network_map_includes_node_friendly_name():