dash - better model health
This commit is contained in:
Dobromir Popov
@@ -41,21 +41,21 @@
|
||||
.error-msg { color:var(--bad); font-size:12px; min-height:16px; }
|
||||
.keybox { word-break:break-all; background:var(--bg); border:1px solid var(--border);
|
||||
border-radius:6px; padding:4px 8px; margin:4px 0; font-size:11px; }
|
||||
.tabs { display:flex; gap:10px; margin-bottom:8px; }
|
||||
.tabs a { color:var(--dim); cursor:pointer; }
|
||||
.tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
|
||||
.wide { grid-column:1 / -1; }
|
||||
.console {
|
||||
background:var(--bg); border:1px solid var(--border); border-radius:6px;
|
||||
min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
|
||||
white-space:pre-wrap; word-break:break-word; font-size:11px;
|
||||
}
|
||||
.console-line { padding:1px 0; border-bottom:1px solid #161b22; }
|
||||
.console-time { color:var(--dim); }
|
||||
.console-level-info { color:var(--accent); }
|
||||
.console-level-warn { color:var(--warn); }
|
||||
.console-level-error { color:var(--bad); }
|
||||
</style>
|
||||
.tabs { display:flex; gap:10px; margin-bottom:8px; }
|
||||
.tabs a { color:var(--dim); cursor:pointer; }
|
||||
.tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
|
||||
.wide { grid-column:1 / -1; }
|
||||
.console {
|
||||
background:var(--bg); border:1px solid var(--border); border-radius:6px;
|
||||
min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
|
||||
white-space:pre-wrap; word-break:break-word; font-size:11px;
|
||||
}
|
||||
.console-line { padding:1px 0; border-bottom:1px solid #161b22; }
|
||||
.console-time { color:var(--dim); }
|
||||
.console-level-info { color:var(--accent); }
|
||||
.console-level-warn { color:var(--warn); }
|
||||
.console-level-error { color:var(--bad); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
@@ -72,11 +72,11 @@
|
||||
<section><h2>Node pending payouts</h2><div id="pending" class="empty">loading…</div></section>
|
||||
<section><h2>Settlement history</h2><div id="settlements" class="empty">loading…</div></section>
|
||||
<section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
|
||||
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
||||
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
|
||||
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
|
||||
<section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section>
|
||||
</main>
|
||||
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
||||
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
|
||||
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
|
||||
<section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section>
|
||||
</main>
|
||||
<script>
|
||||
"use strict";
|
||||
const $ = id => document.getElementById(id);
|
||||
@@ -84,6 +84,7 @@ const esc = s => String(s).replace(/[&<>"]/g,
|
||||
c => ({"&":"&","<":"<",">":">",'"':"""}[c]));
|
||||
const usdt = v => (Math.round(v * 1e6) / 1e6).toFixed(6);
|
||||
const tps = v => (v === null || v === undefined) ? "?" : (Math.round(v * 10) / 10).toFixed(1);
|
||||
const copies = v => (v === null || v === undefined) ? "?" : Number(v).toFixed(2);
|
||||
const short = (s, n=14) => { s = String(s); return s.length > n ? s.slice(0, 6) + "…" + s.slice(-5) : s; };
|
||||
|
||||
async function fetchJson(path) {
|
||||
@@ -129,16 +130,17 @@ function renderNodes(map) {
|
||||
}
|
||||
let html = "";
|
||||
for (const [model, group] of Object.entries(byModel)) {
|
||||
html += `<div><b>${esc(model)}</b> <span class="dim">(${group.length} node${group.length===1?"":"s"})</span></div>`;
|
||||
html += table(["node", "shard", "tps (1h)", "queue", "health"], group.map(n => {
|
||||
const supply = group.find(n => n.model_supply && n.model_supply.served_model_copies !== undefined);
|
||||
const served = supply && supply.model_supply && supply.model_supply.served_model_copies;
|
||||
html += `<div><b>${esc(model)}</b> <span class="dim">(${group.length} node${group.length===1?"":"s"} · ${esc(copies(served))} served)</span></div>`;
|
||||
html += table(["node", "shard", "tps (1h)", "queue", "served"], group.map(n => {
|
||||
const modelStats = (n.throughput && (n.throughput[n.hf_repo] || n.throughput[n.model])) || {};
|
||||
return [
|
||||
esc(short(n.node_id || "?")),
|
||||
esc(`${n.shard_start ?? "?"}-${n.shard_end ?? "?"}`),
|
||||
`<span class="num">${esc(tps(modelStats.tokens_per_sec_last_hour))}</span>`,
|
||||
esc(String((n.stats && n.stats.queue_depth) ?? 0)),
|
||||
(n.stats && (n.stats.alive === false || n.stats.healthy === false))
|
||||
? '<span class="bad">down</span>' : '<span class="ok">up</span>',
|
||||
`<span class="num">${esc(copies(n.model_supply && n.model_supply.served_model_copies))}</span>`,
|
||||
]; }));
|
||||
}
|
||||
$("nodes").innerHTML = html;
|
||||
@@ -209,10 +211,10 @@ function renderStats(stats) {
|
||||
$("stats").innerHTML = table(["model", "rpm (1h)", "rpm (24h)", "rpm (30d)"], rows);
|
||||
}
|
||||
|
||||
function renderThroughput(stats) {
|
||||
const nodes = (stats && stats.nodes) || {};
|
||||
const rows = [];
|
||||
for (const [nodeId, nodeStats] of Object.entries(nodes)) {
|
||||
function renderThroughput(stats) {
|
||||
const nodes = (stats && stats.nodes) || {};
|
||||
const rows = [];
|
||||
for (const [nodeId, nodeStats] of Object.entries(nodes)) {
|
||||
for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) {
|
||||
rows.push([
|
||||
esc(short(nodeId)),
|
||||
@@ -221,66 +223,66 @@ function renderThroughput(stats) {
|
||||
`<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`,
|
||||
]);
|
||||
}
|
||||
}
|
||||
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
|
||||
}
|
||||
|
||||
function renderInferenceHistory(data) {
|
||||
const events = (data && data.events) || [];
|
||||
const started = new Map();
|
||||
const completed = [];
|
||||
for (const e of events) {
|
||||
const f = e.fields || {};
|
||||
const id = f.request_id;
|
||||
if (!id) continue;
|
||||
if (e.message === "proxy route selected") {
|
||||
started.set(id, e);
|
||||
} else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") {
|
||||
completed.push(e);
|
||||
started.delete(id);
|
||||
}
|
||||
}
|
||||
const activeByModel = {};
|
||||
for (const e of started.values()) {
|
||||
const f = e.fields || {};
|
||||
const model = f.model || f.route_model || "?";
|
||||
activeByModel[model] = (activeByModel[model] || 0) + 1;
|
||||
}
|
||||
const active = Object.entries(activeByModel)
|
||||
.map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`)
|
||||
.join(" · ");
|
||||
const rows = completed.slice(-20).reverse().map(e => {
|
||||
const f = e.fields || {};
|
||||
return [
|
||||
new Date((e.ts || 0) * 1000).toLocaleTimeString(),
|
||||
esc(short(f.model || f.route_model || "?", 28)),
|
||||
esc(short(f.request_id || "?", 18)),
|
||||
`<span class="num">${esc(tps(f.tokens_per_sec))}</span>`,
|
||||
`<span class="num">${esc(String(f.tokens ?? "?"))}</span>`,
|
||||
`<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`,
|
||||
f.stream ? "stream" : "json",
|
||||
];
|
||||
});
|
||||
$("inference-history").innerHTML =
|
||||
`<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` +
|
||||
(rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows)
|
||||
: '<div class="empty">no completed inference requests</div>');
|
||||
}
|
||||
|
||||
function renderConsole(data) {
|
||||
const events = (data && data.events) || [];
|
||||
if (!events.length) {
|
||||
$("console").innerHTML = '<div class="empty">no console events</div>';
|
||||
return;
|
||||
}
|
||||
$("console").innerHTML = events.slice(-120).map(e => {
|
||||
const level = String(e.level || "info");
|
||||
const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
|
||||
const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
|
||||
return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` +
|
||||
`<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`;
|
||||
}).join("");
|
||||
}
|
||||
}
|
||||
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
|
||||
}
|
||||
|
||||
function renderInferenceHistory(data) {
|
||||
const events = (data && data.events) || [];
|
||||
const started = new Map();
|
||||
const completed = [];
|
||||
for (const e of events) {
|
||||
const f = e.fields || {};
|
||||
const id = f.request_id;
|
||||
if (!id) continue;
|
||||
if (e.message === "proxy route selected") {
|
||||
started.set(id, e);
|
||||
} else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") {
|
||||
completed.push(e);
|
||||
started.delete(id);
|
||||
}
|
||||
}
|
||||
const activeByModel = {};
|
||||
for (const e of started.values()) {
|
||||
const f = e.fields || {};
|
||||
const model = f.model || f.route_model || "?";
|
||||
activeByModel[model] = (activeByModel[model] || 0) + 1;
|
||||
}
|
||||
const active = Object.entries(activeByModel)
|
||||
.map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`)
|
||||
.join(" · ");
|
||||
const rows = completed.slice(-20).reverse().map(e => {
|
||||
const f = e.fields || {};
|
||||
return [
|
||||
new Date((e.ts || 0) * 1000).toLocaleTimeString(),
|
||||
esc(short(f.model || f.route_model || "?", 28)),
|
||||
esc(short(f.request_id || "?", 18)),
|
||||
`<span class="num">${esc(tps(f.tokens_per_sec))}</span>`,
|
||||
`<span class="num">${esc(String(f.tokens ?? "?"))}</span>`,
|
||||
`<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`,
|
||||
f.stream ? "stream" : "json",
|
||||
];
|
||||
});
|
||||
$("inference-history").innerHTML =
|
||||
`<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` +
|
||||
(rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows)
|
||||
: '<div class="empty">no completed inference requests</div>');
|
||||
}
|
||||
|
||||
function renderConsole(data) {
|
||||
const events = (data && data.events) || [];
|
||||
if (!events.length) {
|
||||
$("console").innerHTML = '<div class="empty">no console events</div>';
|
||||
return;
|
||||
}
|
||||
$("console").innerHTML = events.slice(-120).map(e => {
|
||||
const level = String(e.level || "info");
|
||||
const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
|
||||
const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
|
||||
return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` +
|
||||
`<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`;
|
||||
}).join("");
|
||||
}
|
||||
|
||||
// ---- account panel (registration / login / balance / usage / API keys) ----
|
||||
|
||||
@@ -436,24 +438,24 @@ async function renderAdminPanel() {
|
||||
|
||||
async function refresh() {
|
||||
$("self-url").textContent = location.host;
|
||||
const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
|
||||
fetchJson("/v1/raft/status"),
|
||||
fetchJson("/v1/network/map"),
|
||||
fetchJson("/v1/billing/summary"),
|
||||
fetchJson("/v1/billing/settlements"),
|
||||
fetchJson("/v1/registry/wallets"),
|
||||
fetchJson("/v1/stats"),
|
||||
fetchJson("/v1/console"),
|
||||
]);
|
||||
const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
|
||||
fetchJson("/v1/raft/status"),
|
||||
fetchJson("/v1/network/map"),
|
||||
fetchJson("/v1/billing/summary"),
|
||||
fetchJson("/v1/billing/settlements"),
|
||||
fetchJson("/v1/registry/wallets"),
|
||||
fetchJson("/v1/stats"),
|
||||
fetchJson("/v1/console"),
|
||||
]);
|
||||
renderHive(raft);
|
||||
renderNodes(map);
|
||||
renderBilling(summary);
|
||||
renderSettlements(settlements);
|
||||
renderFraud(wallets, summary);
|
||||
renderStats(stats);
|
||||
renderThroughput(stats);
|
||||
renderInferenceHistory(consoleData);
|
||||
renderConsole(consoleData);
|
||||
renderFraud(wallets, summary);
|
||||
renderStats(stats);
|
||||
renderThroughput(stats);
|
||||
renderInferenceHistory(consoleData);
|
||||
renderConsole(consoleData);
|
||||
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
|
||||
}
|
||||
refresh();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user