dash - better model health

2026-07-07 15:05:35 +02:00
parent 2469023083
commit 2a0d414593
3 changed files with 623 additions and 506 deletions
--- a/packages/tracker/meshnet_tracker/dashboard.html
+++ b/packages/tracker/meshnet_tracker/dashboard.html
@@ -84,6 +84,7 @@ const esc = s => String(s).replace(/[&<>"]/g,
  c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;"}[c]));
 const usdt = v => (Math.round(v * 1e6) / 1e6).toFixed(6);
 const tps = v => (v === null || v === undefined) ? "?" : (Math.round(v * 10) / 10).toFixed(1);
+const copies = v => (v === null || v === undefined) ? "?" : Number(v).toFixed(2);
 const short = (s, n=14) => { s = String(s); return s.length > n ? s.slice(0, 6) + "…" + s.slice(-5) : s; };

 async function fetchJson(path) {
@@ -129,16 +130,17 @@ function renderNodes(map) {
  }
  let html = "";
  for (const [model, group] of Object.entries(byModel)) {
-    html += `<div><b>${esc(model)}</b> <span class="dim">(${group.length} node${group.length===1?"":"s"})</span></div>`;
-    html += table(["node", "shard", "tps (1h)", "queue", "health"], group.map(n => {
+    const supply = group.find(n => n.model_supply && n.model_supply.served_model_copies !== undefined);
+    const served = supply && supply.model_supply && supply.model_supply.served_model_copies;
+    html += `<div><b>${esc(model)}</b> <span class="dim">(${group.length} node${group.length===1?"":"s"} · ${esc(copies(served))} served)</span></div>`;
+    html += table(["node", "shard", "tps (1h)", "queue", "served"], group.map(n => {
      const modelStats = (n.throughput && (n.throughput[n.hf_repo] || n.throughput[n.model])) || {};
      return [
      esc(short(n.node_id || "?")),
      esc(`${n.shard_start ?? "?"}-${n.shard_end ?? "?"}`),
      `<span class="num">${esc(tps(modelStats.tokens_per_sec_last_hour))}</span>`,
      esc(String((n.stats && n.stats.queue_depth) ?? 0)),
-      (n.stats && (n.stats.alive === false || n.stats.healthy === false))
-        ? '<span class="bad">down</span>' : '<span class="ok">up</span>',
+      `<span class="num">${esc(copies(n.model_supply && n.model_supply.served_model_copies))}</span>`,
    ]; }));
  }
  $("nodes").innerHTML = html;
--- a/packages/tracker/meshnet_tracker/server.py
+++ b/packages/tracker/meshnet_tracker/server.py
@@ -728,6 +728,33 @@ def _coverage_percentage(
    return round((covered / required_layers) * 100, 2)


+def _served_model_copies(
+    nodes: list[_NodeEntry],
+    required_start: int,
+    required_end: int,
+) -> float:
+    required_layers = required_end - required_start + 1
+    if required_layers <= 0:
+        return 0.0
+
+    layer_counts = []
+    for layer in range(required_start, required_end + 1):
+        count = 0
+        for node in nodes:
+            if node.shard_start is None or node.shard_end is None:
+                continue
+            if node.shard_start <= layer <= node.shard_end:
+                count += 1
+        layer_counts.append(count)
+
+    if not layer_counts:
+        return 0.0
+
+    complete_copies = min(layer_counts)
+    residual_layers = sum(1 for count in layer_counts if count > complete_copies)
+    return round(complete_copies + (residual_layers / required_layers), 2)
+
+
 def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
    start = int(preset.get("layers_start", 0))
    if "layers_end" in preset:
@@ -2143,6 +2170,40 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                result[model] = server.stats.get_node_model_stats(node.node_id, model)
            return result

+        def model_supply_for(node: _NodeEntry) -> dict:
+            model = node.model or node.hf_repo
+            if not model:
+                return {}
+            resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+            if preset is not None:
+                required_start, required_end = _preset_layer_bounds(preset)
+                model_nodes = [
+                    n for n in nodes
+                    if _node_matches_preset(n, resolved_name, preset)  # type: ignore[arg-type]
+                ]
+            else:
+                model_nodes = [
+                    n for n in nodes
+                    if _node_matches_model(n, model)
+                    and n.shard_start is not None
+                    and n.shard_end is not None
+                    and n.num_layers is not None
+                ]
+                if not model_nodes:
+                    return {}
+                required_start = 0
+                required_end = max(n.num_layers for n in model_nodes if n.num_layers is not None) - 1
+            return {
+                "model": resolved_name or model,
+                "required_start": required_start,
+                "required_end": required_end,
+                "served_model_copies": _served_model_copies(
+                    model_nodes,
+                    required_start,
+                    required_end,
+                ),
+            }
+
        self._send_json(200, {
            "relay_url": server.relay_url,
            "pool": _pool_summary(nodes),
@@ -2173,6 +2234,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                    "tracker_mode": node.tracker_mode,
                    "last_heartbeat": node.last_heartbeat,
                    "capacity": capacity_for(node),
+                    "model_supply": model_supply_for(node),
                    "throughput": throughput_for(node),
                    "stats": _node_health(node, server.heartbeat_timeout),
                }
--- a/tests/test_tracker_routing.py
+++ b/tests/test_tracker_routing.py
@@ -162,6 +162,59 @@ def test_network_map_exposes_pool_size_and_speed_summary():
    assert pool["total_effective_throughput"] == 10.0


+def test_network_map_exposes_served_model_copy_count():
+    tracker = TrackerServer()
+    port = tracker.start()
+    url = f"http://127.0.0.1:{port}"
+    try:
+        _post_json(
+            f"{url}/v1/nodes/register",
+            {
+                "endpoint": "http://127.0.0.1:7201",
+                "model": "copy-count-test",
+                "hf_repo": "example/copy-count-test",
+                "num_layers": 37,
+                "shard_start": 0,
+                "shard_end": 21,
+                "hardware_profile": {},
+            },
+        )
+        network_map = _get_json(f"{url}/v1/network/map")
+        assert network_map["nodes"][0]["model_supply"]["served_model_copies"] == 0.59
+
+        _post_json(
+            f"{url}/v1/nodes/register",
+            {
+                "endpoint": "http://127.0.0.1:7202",
+                "model": "copy-count-test",
+                "hf_repo": "example/copy-count-test",
+                "num_layers": 37,
+                "shard_start": 22,
+                "shard_end": 36,
+                "hardware_profile": {},
+            },
+        )
+        network_map = _get_json(f"{url}/v1/network/map")
+        assert network_map["nodes"][0]["model_supply"]["served_model_copies"] == 1.0
+
+        _post_json(
+            f"{url}/v1/nodes/register",
+            {
+                "endpoint": "http://127.0.0.1:7203",
+                "model": "copy-count-test",
+                "hf_repo": "example/copy-count-test",
+                "num_layers": 37,
+                "shard_start": 0,
+                "shard_end": 36,
+                "hardware_profile": {},
+            },
+        )
+        network_map = _get_json(f"{url}/v1/network/map")
+        assert network_map["nodes"][0]["model_supply"]["served_model_copies"] == 2.0
+    finally:
+        tracker.stop()
+
+
 def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
    tracker = TrackerServer()
    port = tracker.start()