popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dash - better model health

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-07 15:05:35 +02:00
parent 2469023083
commit 2a0d414593
3 changed files with 623 additions and 506 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -41,21 +41,21 @@
.error-msg { color:var(--bad); font-size:12px; min-height:16px; } .error-msg { color:var(--bad); font-size:12px; min-height:16px; }
.keybox { word-break:break-all; background:var(--bg); border:1px solid var(--border); .keybox { word-break:break-all; background:var(--bg); border:1px solid var(--border);
border-radius:6px; padding:4px 8px; margin:4px 0; font-size:11px; } border-radius:6px; padding:4px 8px; margin:4px 0; font-size:11px; }
.tabs { display:flex; gap:10px; margin-bottom:8px; } .tabs { display:flex; gap:10px; margin-bottom:8px; }
.tabs a { color:var(--dim); cursor:pointer; } .tabs a { color:var(--dim); cursor:pointer; }
.tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); } .tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
.wide { grid-column:1 / -1; } .wide { grid-column:1 / -1; }
.console { .console {
background:var(--bg); border:1px solid var(--border); border-radius:6px; background:var(--bg); border:1px solid var(--border); border-radius:6px;
min-height:160px; max-height:280px; overflow:auto; padding:7px 9px; min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
white-space:pre-wrap; word-break:break-word; font-size:11px; white-space:pre-wrap; word-break:break-word; font-size:11px;
} }
.console-line { padding:1px 0; border-bottom:1px solid #161b22; } .console-line { padding:1px 0; border-bottom:1px solid #161b22; }
.console-time { color:var(--dim); } .console-time { color:var(--dim); }
.console-level-info { color:var(--accent); } .console-level-info { color:var(--accent); }
.console-level-warn { color:var(--warn); } .console-level-warn { color:var(--warn); }
.console-level-error { color:var(--bad); } .console-level-error { color:var(--bad); }
</style> </style>
</head> </head>
<body> <body>
<header> <header>
@@ -72,11 +72,11 @@
<section><h2>Node pending payouts</h2><div id="pending" class="empty">loading…</div></section> <section><h2>Node pending payouts</h2><div id="pending" class="empty">loading…</div></section>
<section><h2>Settlement history</h2><div id="settlements" class="empty">loading…</div></section> <section><h2>Settlement history</h2><div id="settlements" class="empty">loading…</div></section>
<section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section> <section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section> <section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section> <section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section> <section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
<section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section> <section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section>
</main> </main>
<script> <script>
"use strict"; "use strict";
const $ = id => document.getElementById(id); const $ = id => document.getElementById(id);
@@ -84,6 +84,7 @@ const esc = s => String(s).replace(/[&<>"]/g,
c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;"}[c])); c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;"}[c]));
const usdt = v => (Math.round(v * 1e6) / 1e6).toFixed(6); const usdt = v => (Math.round(v * 1e6) / 1e6).toFixed(6);
const tps = v => (v === null || v === undefined) ? "?" : (Math.round(v * 10) / 10).toFixed(1); const tps = v => (v === null || v === undefined) ? "?" : (Math.round(v * 10) / 10).toFixed(1);
const copies = v => (v === null || v === undefined) ? "?" : Number(v).toFixed(2);
const short = (s, n=14) => { s = String(s); return s.length > n ? s.slice(0, 6) + "…" + s.slice(-5) : s; }; const short = (s, n=14) => { s = String(s); return s.length > n ? s.slice(0, 6) + "…" + s.slice(-5) : s; };
async function fetchJson(path) { async function fetchJson(path) {
@@ -129,16 +130,17 @@ function renderNodes(map) {
} }
let html = ""; let html = "";
for (const [model, group] of Object.entries(byModel)) { for (const [model, group] of Object.entries(byModel)) {
html += `<div><b>${esc(model)}</b> <span class="dim">(${group.length} node${group.length===1?"":"s"})</span></div>`; const supply = group.find(n => n.model_supply && n.model_supply.served_model_copies !== undefined);
html += table(["node", "shard", "tps (1h)", "queue", "health"], group.map(n => { const served = supply && supply.model_supply && supply.model_supply.served_model_copies;
html += `<div><b>${esc(model)}</b> <span class="dim">(${group.length} node${group.length===1?"":"s"} · ${esc(copies(served))} served)</span></div>`;
html += table(["node", "shard", "tps (1h)", "queue", "served"], group.map(n => {
const modelStats = (n.throughput && (n.throughput[n.hf_repo] || n.throughput[n.model])) || {}; const modelStats = (n.throughput && (n.throughput[n.hf_repo] || n.throughput[n.model])) || {};
return [ return [
esc(short(n.node_id || "?")), esc(short(n.node_id || "?")),
esc(`${n.shard_start ?? "?"}-${n.shard_end ?? "?"}`), esc(`${n.shard_start ?? "?"}-${n.shard_end ?? "?"}`),
`<span class="num">${esc(tps(modelStats.tokens_per_sec_last_hour))}</span>`, `<span class="num">${esc(tps(modelStats.tokens_per_sec_last_hour))}</span>`,
esc(String((n.stats && n.stats.queue_depth) ?? 0)), esc(String((n.stats && n.stats.queue_depth) ?? 0)),
(n.stats && (n.stats.alive === false || n.stats.healthy === false)) `<span class="num">${esc(copies(n.model_supply && n.model_supply.served_model_copies))}</span>`,
? '<span class="bad">down</span>' : '<span class="ok">up</span>',
]; })); ]; }));
} }
$("nodes").innerHTML = html; $("nodes").innerHTML = html;
@@ -209,10 +211,10 @@ function renderStats(stats) {
$("stats").innerHTML = table(["model", "rpm (1h)", "rpm (24h)", "rpm (30d)"], rows); $("stats").innerHTML = table(["model", "rpm (1h)", "rpm (24h)", "rpm (30d)"], rows);
} }
function renderThroughput(stats) { function renderThroughput(stats) {
const nodes = (stats && stats.nodes) || {}; const nodes = (stats && stats.nodes) || {};
const rows = []; const rows = [];
for (const [nodeId, nodeStats] of Object.entries(nodes)) { for (const [nodeId, nodeStats] of Object.entries(nodes)) {
for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) { for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) {
rows.push([ rows.push([
esc(short(nodeId)), esc(short(nodeId)),
@@ -221,66 +223,66 @@ function renderThroughput(stats) {
`<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`, `<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`,
]); ]);
} }
} }
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows); $("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
} }
function renderInferenceHistory(data) { function renderInferenceHistory(data) {
const events = (data && data.events) || []; const events = (data && data.events) || [];
const started = new Map(); const started = new Map();
const completed = []; const completed = [];
for (const e of events) { for (const e of events) {
const f = e.fields || {}; const f = e.fields || {};
const id = f.request_id; const id = f.request_id;
if (!id) continue; if (!id) continue;
if (e.message === "proxy route selected") { if (e.message === "proxy route selected") {
started.set(id, e); started.set(id, e);
} else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") { } else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") {
completed.push(e); completed.push(e);
started.delete(id); started.delete(id);
} }
} }
const activeByModel = {}; const activeByModel = {};
for (const e of started.values()) { for (const e of started.values()) {
const f = e.fields || {}; const f = e.fields || {};
const model = f.model || f.route_model || "?"; const model = f.model || f.route_model || "?";
activeByModel[model] = (activeByModel[model] || 0) + 1; activeByModel[model] = (activeByModel[model] || 0) + 1;
} }
const active = Object.entries(activeByModel) const active = Object.entries(activeByModel)
.map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`) .map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`)
.join(" &middot; "); .join(" &middot; ");
const rows = completed.slice(-20).reverse().map(e => { const rows = completed.slice(-20).reverse().map(e => {
const f = e.fields || {}; const f = e.fields || {};
return [ return [
new Date((e.ts || 0) * 1000).toLocaleTimeString(), new Date((e.ts || 0) * 1000).toLocaleTimeString(),
esc(short(f.model || f.route_model || "?", 28)), esc(short(f.model || f.route_model || "?", 28)),
esc(short(f.request_id || "?", 18)), esc(short(f.request_id || "?", 18)),
`<span class="num">${esc(tps(f.tokens_per_sec))}</span>`, `<span class="num">${esc(tps(f.tokens_per_sec))}</span>`,
`<span class="num">${esc(String(f.tokens ?? "?"))}</span>`, `<span class="num">${esc(String(f.tokens ?? "?"))}</span>`,
`<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`, `<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`,
f.stream ? "stream" : "json", f.stream ? "stream" : "json",
]; ];
}); });
$("inference-history").innerHTML = $("inference-history").innerHTML =
`<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` + `<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` +
(rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows) (rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows)
: '<div class="empty">no completed inference requests</div>'); : '<div class="empty">no completed inference requests</div>');
} }
function renderConsole(data) { function renderConsole(data) {
const events = (data && data.events) || []; const events = (data && data.events) || [];
if (!events.length) { if (!events.length) {
$("console").innerHTML = '<div class="empty">no console events</div>'; $("console").innerHTML = '<div class="empty">no console events</div>';
return; return;
} }
$("console").innerHTML = events.slice(-120).map(e => { $("console").innerHTML = events.slice(-120).map(e => {
const level = String(e.level || "info"); const level = String(e.level || "info");
const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info"; const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : ""; const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` + return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` +
`<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`; `<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`;
}).join(""); }).join("");
} }
// ---- account panel (registration / login / balance / usage / API keys) ---- // ---- account panel (registration / login / balance / usage / API keys) ----
@@ -436,24 +438,24 @@ async function renderAdminPanel() {
async function refresh() { async function refresh() {
$("self-url").textContent = location.host; $("self-url").textContent = location.host;
const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([ const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
fetchJson("/v1/raft/status"), fetchJson("/v1/raft/status"),
fetchJson("/v1/network/map"), fetchJson("/v1/network/map"),
fetchJson("/v1/billing/summary"), fetchJson("/v1/billing/summary"),
fetchJson("/v1/billing/settlements"), fetchJson("/v1/billing/settlements"),
fetchJson("/v1/registry/wallets"), fetchJson("/v1/registry/wallets"),
fetchJson("/v1/stats"), fetchJson("/v1/stats"),
fetchJson("/v1/console"), fetchJson("/v1/console"),
]); ]);
renderHive(raft); renderHive(raft);
renderNodes(map); renderNodes(map);
renderBilling(summary); renderBilling(summary);
renderSettlements(settlements); renderSettlements(settlements);
renderFraud(wallets, summary); renderFraud(wallets, summary);
renderStats(stats); renderStats(stats);
renderThroughput(stats); renderThroughput(stats);
renderInferenceHistory(consoleData); renderInferenceHistory(consoleData);
renderConsole(consoleData); renderConsole(consoleData);
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString(); $("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
} }
refresh(); refresh();

870
packages/tracker/meshnet_tracker/server.py
View File

File diff suppressed because it is too large Load Diff

53
tests/test_tracker_routing.py
View File

@@ -162,6 +162,59 @@ def test_network_map_exposes_pool_size_and_speed_summary():
assert pool["total_effective_throughput"] == 10.0 assert pool["total_effective_throughput"] == 10.0
def test_network_map_exposes_served_model_copy_count():
tracker = TrackerServer()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_post_json(
f"{url}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:7201",
"model": "copy-count-test",
"hf_repo": "example/copy-count-test",
"num_layers": 37,
"shard_start": 0,
"shard_end": 21,
"hardware_profile": {},
},
)
network_map = _get_json(f"{url}/v1/network/map")
assert network_map["nodes"][0]["model_supply"]["served_model_copies"] == 0.59
_post_json(
f"{url}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:7202",
"model": "copy-count-test",
"hf_repo": "example/copy-count-test",
"num_layers": 37,
"shard_start": 22,
"shard_end": 36,
"hardware_profile": {},
},
)
network_map = _get_json(f"{url}/v1/network/map")
assert network_map["nodes"][0]["model_supply"]["served_model_copies"] == 1.0
_post_json(
f"{url}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:7203",
"model": "copy-count-test",
"hf_repo": "example/copy-count-test",
"num_layers": 37,
"shard_start": 0,
"shard_end": 36,
"hardware_profile": {},
},
)
network_map = _get_json(f"{url}/v1/network/map")
assert network_map["nodes"][0]["model_supply"]["served_model_copies"] == 2.0
finally:
tracker.stop()
def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough(): def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
tracker = TrackerServer() tracker = TrackerServer()
port = tracker.start() port = tracker.start()