popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

node metrics

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-14 20:33:02 +02:00
parent b661590ac7
commit ba7c656364
6 changed files with 109 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -44,12 +44,15 @@
.empty { color:var(--dim); font-style:italic; }
.pill { display:inline-block; padding:0 7px; border-radius:9px;
border:1px solid var(--border); font-size:11px; }
input, button { font:inherit; color:var(--fg); background:var(--bg);
input, button, select { font:inherit; color:var(--fg); background:var(--bg);
border:1px solid var(--border); border-radius:6px; padding:5px 8px; }
input { width:100%; margin-bottom:6px; }
button { cursor:pointer; color:var(--accent); }
button:hover { border-color:var(--accent); }
button.small { font-size:11px; padding:1px 7px; }
dialog { color:var(--fg); background:var(--panel); border:1px solid var(--border); border-radius:8px; min-width:min(420px,calc(100vw - 32px)); }
dialog::backdrop { background:rgba(0,0,0,.55); }
.placement-dialog-actions { display:flex; justify-content:flex-end; gap:8px; margin-top:12px; }
.form-row { display:flex; gap:8px; }
.form-row button { white-space:nowrap; }
.error-msg { color:var(--bad); font-size:12px; min-height:16px; }
@@ -324,6 +327,14 @@
<div id="testing-log" class="console empty">no test output yet</div>
</section>
</main>
<dialog id="model-placement-dialog">
<form method="dialog">
<div id="model-placement-dialog-title"></div>
<label for="model-placement-node">Node</label>
<select id="model-placement-node"></select>
<div class="placement-dialog-actions"><button value="cancel">Cancel</button><button value="confirm">Confirm</button></div>
</form>
</dialog>
<script>
"use strict";
const $ = id => document.getElementById(id);
@@ -1801,16 +1812,16 @@ async function requestSelectedModelLoad() {
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
}
async function requestAdminModelLoad(model) {
const result = await apiCall("/v1/models/load", "POST", { model, force: true });
async function requestAdminModelLoad(model, nodeId) {
const result = await apiCall("/v1/models/load", "POST", { model, node_id: nodeId, force: true });
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
const assignment = result.data.assignment || {};
showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
await refreshActiveTab(true);
}
async function releaseAdminModel(model) {
const result = await apiCall("/v1/models/release", "POST", { model });
async function releaseAdminModel(model, nodeId) {
const result = await apiCall("/v1/models/release", "POST", { model, node_id: nodeId });
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model release request failed", true);
showAdminModelPlacementStatus(`Release queued for ${result.data.released || 0} node(s) serving ${model}.`);
await refreshActiveTab(true);
@@ -1836,6 +1847,9 @@ function renderAdminNodePool(map) {
html += table(["node", "assignment", "state / slots", "RAM", "GPU / VRAM", "model drive"], nodes.map(node => {
const hw = node.hardware_profile || {};
const cap = node.capacity || {};
// The network map keeps reported resource capacity under `capacity`.
node.ram_bytes = cap.ram_bytes ?? node.ram_bytes;
node.vram_bytes = cap.vram_bytes ?? node.vram_bytes;
const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
return [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
@@ -1866,10 +1880,26 @@ function renderAdminModelPlacement(models, map) {
$("admin-model-placement").addEventListener("click", event => {
const load = event.target.closest("[data-admin-model-load]");
const release = event.target.closest("[data-admin-model-release]");
if (load) void requestAdminModelLoad(load.dataset.adminModelLoad);
if (release) void releaseAdminModel(release.dataset.adminModelRelease);
if (load) void chooseModelPlacementNode("load", load.dataset.adminModelLoad);
if (release) void chooseModelPlacementNode("release", release.dataset.adminModelRelease);
});
function chooseModelPlacementNode(action, model) {
const dialog = $("model-placement-dialog");
const select = $("model-placement-node");
const aliases = new Set([model].filter(Boolean));
const nodes = (lastNetworkMap?.nodes || []).filter(node => action === "load" || aliases.has(node.model) || aliases.has(node.hf_repo));
if (!nodes.length) return showAdminModelPlacementStatus(`No node can ${action} ${model}.`, true);
$("model-placement-dialog-title").textContent = `${action === "load" ? "Load" : "Release"} ${model} on a node`;
select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
dialog.onclose = () => {
if (dialog.returnValue !== "confirm") return;
if (action === "load") void requestAdminModelLoad(model, select.value);
else void releaseAdminModel(model, select.value);
};
dialog.showModal();
}
function chatAuthToken() {
if (accountApiKeys.length) return accountApiKeys[0];
return null;
@@ -2512,6 +2542,7 @@ async function fetchAdminTab() {
if (isAdmin) fetches.push(apiCall("/v1/admin/accounts"));
const results = await Promise.all(fetches);
const [raft, consoleData, summary, wallets, models, map, adminResp] = results;
if (map) lastNetworkMap = map;
renderIfChanged("hive", raft, renderHive);
renderIfChanged("console", consoleData, renderConsole);
renderIfChanged("billing-summary", summary, data => renderBilling(data));

26
packages/tracker/meshnet_tracker/server.py
View File

@@ -1494,7 +1494,9 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
break
def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
def _request_model_load_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> dict | None:
"""Queue an explicitly requested model on the best available joined node."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None or not preset.get("hf_repo"):
@@ -1510,6 +1512,8 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
continue
host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
placeable = [node for node in host_nodes if _has_usable_quantization(node)]
if node_id is not None:
placeable = [node for node in placeable if node.node_id == node_id]
if not placeable:
continue
anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec)
@@ -1556,13 +1560,17 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
"shard_start": start, "shard_end": shard_end, "replaced_model": replaced}
def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
def _release_model_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> int:
"""Queue DROP_SHARD for every served shard and remove it from routing immediately."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None:
return 0
released = 0
for node in server.registry.values():
if node_id is not None and node.node_id != node_id:
continue
if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None:
continue
node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16"))
@@ -4880,10 +4888,14 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model, str) or not model.strip():
self._send_json(400, {"error": "model is required"})
return
node_id = body.get("node_id")
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock:
self._purge_expired_nodes()
assignment = _request_model_load_locked(server, model)
if assignment is None and body.get("force") is True:
assignment = _request_model_load_locked(server, model, node_id)
if assignment is None and node_id is None and body.get("force") is True:
assignment = _force_model_load_locked(server, model)
if assignment is None:
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
@@ -4901,9 +4913,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model, str) or not model.strip():
self._send_json(400, {"error": "model is required"})
return
node_id = body.get("node_id")
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock:
self._purge_expired_nodes()
released = _release_model_locked(server, model)
released = _release_model_locked(server, model, node_id)
if not released:
self._send_json(404, {"error": "no served shards found for model"})
return