popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

model loading, dash

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-15 13:55:38 +02:00
parent 2852b1f80b
commit 47b243cd98
5 changed files with 95 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -333,7 +333,8 @@
<label for="model-placement-node">Node</label>
<select id="model-placement-node"></select>
<label id="model-placement-replace" style="display:none"><input type="checkbox" id="model-placement-replace-confirm"> Unload the currently loaded model before loading this one</label>
<div class="placement-dialog-actions"><button value="cancel">Cancel</button><button value="confirm">Confirm</button></div>
<div id="model-placement-replace-error" class="bad" style="display:none"></div>
<div class="placement-dialog-actions"><button value="cancel">Cancel</button><button type="button" id="model-placement-confirm">Confirm</button></div>
</form>
</dialog>
<script>
@@ -1828,6 +1829,14 @@ async function releaseAdminModel(model, nodeId) {
await refreshActiveTab(true);
}
async function releaseAllNodeModels(nodeId) {
if (!confirm("Unload every model from this node?")) return;
const result = await apiCall("/v1/nodes/release-all", "POST", { node_id: nodeId });
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "node unload failed", true);
showAdminModelPlacementStatus(`Unload queued for ${short(nodeId)}.`);
await refreshActiveTab(true);
}
function showAdminModelPlacementStatus(message, isError) {
const status = $("admin-model-placement-status");
status.textContent = message;
@@ -1845,7 +1854,7 @@ function renderAdminNodePool(map) {
let html = "";
for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) {
html += `<div style="margin-top:10px"><b>${esc(short(account, 20))}</b> <span class="dim">${nodes.length} node(s)</span></div>`;
html += table(["node", "assignment", "state / slots", "RAM", "GPU / VRAM", "model drive"], nodes.map(node => {
html += table(["node", "assignment", "state / slots", "model RAM", "RAM", "GPU / VRAM", "model drive", "action"], nodes.map(node => {
const hw = node.hardware_profile || {};
const cap = node.capacity || {};
// The network map keeps reported resource capacity under `capacity`.
@@ -1853,15 +1862,25 @@ function renderAdminNodePool(map) {
node.vram_bytes = cap.vram_bytes ?? node.vram_bytes;
const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
return [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
const row = [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`),
esc(gib(cap.loaded_model_bytes)),
esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))),
esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))];
return row.concat([
node.shard_start == null ? '<span class="dim">empty</span>' :
`<button class="small" data-admin-node-release="${esc(node.node_id)}">unload all</button>`,
]);
}));
}
$("admin-node-pool").innerHTML = html || '<div class="empty">no nodes registered</div>';
}
$("admin-node-pool").addEventListener("click", event => {
const unload = event.target.closest("[data-admin-node-release]");
if (unload) void releaseAllNodeModels(unload.dataset.adminNodeRelease);
});
function renderAdminModelPlacement(models, map) {
const nodes = (map && map.nodes) || [];
const rows = ((models && models.data) || []).map(model => {
@@ -1869,7 +1888,8 @@ function renderAdminModelPlacement(models, map) {
const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length;
const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) ||
(node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length;
const actions = `<button class="small" data-admin-model-load="${esc(model.id)}">load</button> ` +
const loadable = model.id !== "stub-model";
const actions = `<button class="small" data-admin-model-load="${esc(model.id)}"${loadable ? "" : " disabled"}>load</button> ` +
`<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`;
return [esc(model.name || model.id), String(serving), String(downloaded), actions];
});
@@ -1896,6 +1916,8 @@ function chooseModelPlacementNode(action, model) {
select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
const replace = $("model-placement-replace");
const replaceConfirm = $("model-placement-replace-confirm");
const replaceError = $("model-placement-replace-error");
const confirmButton = $("model-placement-confirm");
const selectedNode = () => nodes.find(node => node.node_id === select.value);
const updateReplacementWarning = () => {
const node = selectedNode();
@@ -1903,15 +1925,19 @@ function chooseModelPlacementNode(action, model) {
modelAliasKey(node.hf_repo || node.model) !== targetAlias;
replace.style.display = occupied ? "" : "none";
replaceConfirm.checked = false;
replaceError.style.display = "none";
};
select.onchange = updateReplacementWarning;
updateReplacementWarning();
dialog.onclose = () => {
if (dialog.returnValue !== "confirm") return;
dialog.onclose = null;
confirmButton.onclick = () => {
const replacing = replace.style.display !== "none";
if (replacing && !replaceConfirm.checked) {
return showAdminModelPlacementStatus("Confirm unloading the existing model before replacing it.", true);
replaceError.textContent = "Tick the box to confirm that this will unload the current model.";
replaceError.style.display = "";
return;
}
dialog.close("confirm");
if (action === "load") void requestAdminModelLoad(model, select.value, replacing);
else void releaseAdminModel(model, select.value);
};

48
packages/tracker/meshnet_tracker/server.py
View File

@@ -1228,6 +1228,7 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
"quantization": node.quantization,
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
"effective_throughput": round(_effective_throughput(node), 4),
"loaded_model_bytes": _assignment_memory_bytes(node, preset),
}
if preset is not None:
summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
@@ -1587,6 +1588,16 @@ def _release_model_locked(
return released
def _release_all_node_models_locked(server: "_TrackerHTTPServer", node_id: str) -> int:
"""Queue removal of every loaded assignment on one node."""
node = server.registry.get(node_id)
if node is None or node.shard_start is None or node.shard_end is None:
return 0
node.pending_directives.append({"action": "DROP_ALL_SHARDS"})
node.status = "loading"
return 1
def _preferred_node_quantization(
node: _NodeEntry,
preset: dict,
@@ -3105,6 +3116,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if self.path == "/v1/models/release":
self._handle_model_release_request()
return
if self.path == "/v1/nodes/release-all":
self._handle_node_release_all_request()
return
if self.path == "/v1/models/vote":
self._handle_model_coverage_vote()
return
@@ -3415,12 +3429,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
memory_pool = _memory_pool_map(server)
def capacity_for(node: _NodeEntry) -> dict:
preset = None
if node.model:
preset = server.model_presets.get(node.model)
if preset is None and node.hf_repo and node.num_layers:
preset = _hf_rebalance_preset([node])
return _node_capacity_summary(node, preset)
return _node_capacity_summary(node, _preset_for_node(server, node))
def throughput_for(node: _NodeEntry) -> dict:
if server.stats is None:
@@ -4833,7 +4842,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
completed_directives = body.get("completed_directives", [])
if isinstance(completed_directives, list):
for directive in completed_directives:
if not isinstance(directive, dict) or directive.get("action") != "DROP_SHARD":
if not isinstance(directive, dict) or directive.get("action") not in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
continue
# A node has confirmed the release. Stop advertising its
# old route immediately so the dashboard and routing state
@@ -4919,6 +4928,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
_resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None or str(preset.get("hf_repo") or "").strip().lower() == "stub-model":
self._send_json(400, {"error": "stub-model is a local test backend and cannot be loaded onto a node"})
return
with server.lock:
self._purge_expired_nodes()
assignment = _request_model_load_locked(server, model, node_id)
@@ -4952,6 +4965,27 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return
self._send_json(202, {"status": "release_queued", "released": released})
def _handle_node_release_all_request(self):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if not self._require_role("admin", "validator"):
return
body = self._read_json_body()
if body is None:
return
node_id = body.get("node_id")
if not isinstance(node_id, str) or not node_id:
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock:
self._purge_expired_nodes()
released = _release_all_node_models_locked(server, node_id)
if not released:
self._send_json(404, {"error": "no loaded models found for node"})
return
self._send_json(202, {
"status": "release_queued", "released": released, "node_id": node_id,
})
def _handle_model_coverage_vote(self):
"""Record a rolling wish-list signal for an unavailable precision."""
server: _TrackerHTTPServer = self.server # type: ignore[assignment]