popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

routing tests, launch.configs, redirect, stats and route statistics

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-11 11:39:47 +03:00
parent f54ea100fb
commit 11bf460027
12 changed files with 546 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
packages/tracker/meshnet_tracker/cli.py
View File

@@ -2,6 +2,7 @@
import argparse
import os
import socket
import sys
import time
from pathlib import Path
@@ -53,7 +54,10 @@ def _load_env_file(path: Path) -> None:
def _load_env_defaults() -> None:
"""Load local and user-level tracker env defaults before parsing arguments."""
"""Load machine-specific, local, and user-level tracker env defaults."""
machine = socket.gethostname().strip()
if machine:
_load_env_file(Path.cwd() / f".env.{machine}")
_load_env_file(Path.cwd() / ".env")
_load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")

27
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -240,6 +240,7 @@
<section data-tab="overview"><h2>Nodes &amp; coverage</h2><div id="nodes" class="empty">loading…</div></section>
<section data-tab="overview"><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
<section data-tab="overview" class="wide"><h2>Model inference speed</h2><div id="model-speed-chart" class="empty">loading…</div></section>
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
<section data-tab="chat" class="wide chat-section">
<h2 style="display:none">Chat / inference</h2>
@@ -752,6 +753,27 @@ function renderRouting(routing) {
el.innerHTML = html;
}
function renderModelSpeed(reports) {
const el = $("model-speed-chart");
if (!el) return;
const models = Object.entries(reports || {});
if (!models.length) { el.innerHTML = '<div class="empty">no observed route samples yet</div>'; return; }
let html = '';
for (const [model, report] of models) {
const routes = report.routes || [];
html += `<div style="margin:8px 0"><b>${esc(model)}</b>`;
html += table(["hops", "devices", "latency", "hop penalty", "speed", "samples", "route drill"], routes.map(route => [
`<span class="num">${esc(String(route.hop_count))}</span>`, esc(route.device_mix || "?"),
`<span class="num">${esc(route.latency_ms == null ? "—" : route.latency_ms.toFixed(2) + " ms")}</span>`,
`<span class="num">${esc(route.latency_penalty_ms == null ? "—" : route.latency_penalty_ms.toFixed(2) + " ms")}</span>`,
`<span class="num">${esc(tps(route.tps))}</span>`, `<span class="num">${esc(String(route.samples || 0))}</span>`,
`<details><summary>nodes</summary><span class="dim">${esc((route.node_ids || []).join(" → "))}</span></details>`,
]));
html += '</div>';
}
el.innerHTML = html;
}
function renderCallWall(consoleData, stats, map) {
const events = (consoleData && consoleData.events) || [];
const nowSec = Date.now() / 1000;
@@ -1954,6 +1976,11 @@ async function fetchOverviewTab() {
renderIfChanged("nodes", map, renderNodes);
renderIfChanged("stats", stats, renderStats);
renderIfChanged("routing", routing, renderRouting);
const speedModels = Object.keys((routing && routing.models) || {});
const reports = Object.fromEntries(await Promise.all(speedModels.map(async model => [
model, await fetchJson("/v1/model-speed?model=" + encodeURIComponent(model)),
])));
renderIfChanged("model-speed", reports, renderModelSpeed);
renderIfChanged("call-wall", { consoleData, stats, map }, data => renderCallWall(data.consoleData, data.stats, data.map));
}

64
packages/tracker/meshnet_tracker/routing_stats.py
View File

@@ -23,6 +23,7 @@ with a different shard produces a new arm automatically.
from __future__ import annotations
import random
import sqlite3
import threading
import time
from dataclasses import dataclass, field
@@ -45,6 +46,7 @@ class RoutingConfig:
@dataclass
class RouteStat:
ewma_tps: float = 0.0
ewma_latency_ms: float = 0.0
weight: float = 0.0 # decayed effective sample mass
last_sample_ts: float = 0.0
epoch: int = 0
@@ -75,11 +77,15 @@ def route_signature(model_key: str, nodes: Iterable[Any]) -> str:
class RouteStatsStore:
"""Thread-safe per-route decayed throughput statistics."""
def __init__(self, config: RoutingConfig | None = None) -> None:
def __init__(self, config: RoutingConfig | None = None, db_path: str | None = None) -> None:
self.config = config or RoutingConfig()
self._lock = threading.Lock()
self._stats: dict[str, RouteStat] = {}
self._epochs: dict[str, int] = {}
self._db_path = db_path
if db_path:
self._init_db()
self._load_from_db()
def epoch(self, model_key: str) -> int:
with self._lock:
@@ -119,6 +125,7 @@ class RouteStatsStore:
carried = stat.decayed_weight(ts, cfg.stats_half_life_seconds)
total = carried + 1.0
stat.ewma_tps = (stat.ewma_tps * carried + tps) / total
stat.ewma_latency_ms = (stat.ewma_latency_ms * carried + elapsed_seconds * 1000.0) / total
stat.weight = total
stat.last_sample_ts = ts
stat.epoch = self._epochs.get(model_key, 0)
@@ -143,11 +150,64 @@ class RouteStatsStore:
status = "proven"
return {
"tps": round(stat.ewma_tps, 4) if stat.samples else None,
"latency_ms": round(stat.ewma_latency_ms, 3) if stat.samples else None,
"weight": round(weight, 4),
"samples": stat.samples,
"status": status,
}
def model_rows(self, model_key: str, now: float | None = None) -> list[dict]:
"""All measured route samples, including pinned experiment routes."""
prefix = f"{model_key}|"
with self._lock:
signatures = [signature for signature in self._stats if signature.startswith(prefix)]
rows = [
{
"signature": signature,
"hop_count": signature.count("->") + 1,
**self.snapshot(signature, model_key, now=now),
}
for signature in signatures
]
return sorted(rows, key=lambda row: (row["hop_count"], row["signature"]))
def _init_db(self) -> None:
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
con.execute(
"CREATE TABLE IF NOT EXISTS route_stats "
"(signature TEXT PRIMARY KEY, ewma_tps REAL NOT NULL, ewma_latency_ms REAL NOT NULL, "
"weight REAL NOT NULL, last_sample_ts REAL NOT NULL, epoch INTEGER NOT NULL, samples INTEGER NOT NULL)"
)
con.execute("CREATE TABLE IF NOT EXISTS route_stat_epochs (model_key TEXT PRIMARY KEY, epoch INTEGER NOT NULL)")
con.commit()
con.close()
def save_to_db(self) -> None:
if not self._db_path:
return
with self._lock:
rows = [
(signature, stat.ewma_tps, stat.ewma_latency_ms, stat.weight, stat.last_sample_ts, stat.epoch, stat.samples)
for signature, stat in self._stats.items()
]
epochs = list(self._epochs.items())
con = sqlite3.connect(self._db_path)
con.executemany("INSERT OR REPLACE INTO route_stats VALUES (?,?,?,?,?,?,?)", rows)
con.executemany("INSERT OR REPLACE INTO route_stat_epochs VALUES (?,?)", epochs)
con.commit()
con.close()
def _load_from_db(self) -> None:
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
rows = con.execute("SELECT signature, ewma_tps, ewma_latency_ms, weight, last_sample_ts, epoch, samples FROM route_stats").fetchall()
epochs = con.execute("SELECT model_key, epoch FROM route_stat_epochs").fetchall()
con.close()
self._stats = {
signature: RouteStat(float(tps), float(latency), float(weight), float(last_sample_ts), int(epoch), int(samples))
for signature, tps, latency, weight, last_sample_ts, epoch, samples in rows
}
self._epochs = {str(model_key): int(epoch) for model_key, epoch in epochs}
def prune(self, now: float | None = None) -> int:
"""Drop routes with no samples for `prune_after_seconds`."""
ts = time.time() if now is None else now
@@ -246,6 +306,8 @@ def route_table(
for n in cand.nodes
],
"tps": r["tps"],
"latency_ms": r["latency_ms"],
"hop_count": len(cand.nodes),
"coefficient": coefficient,
"expected_share": round(share, 4),
"samples": r["samples"],

50
packages/tracker/meshnet_tracker/server.py
View File

@@ -2699,12 +2699,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
parsed = urllib.parse.urlparse(self.path)
if parsed.path == "/v1/route":
if parsed.path == "/":
self.send_response(302)
self.send_header("Location", "/dashboard")
self.end_headers()
elif parsed.path == "/v1/route":
self._handle_route(parsed)
elif parsed.path == "/v1/routes":
self._handle_routes(parsed)
elif parsed.path == "/v1/routing":
self._handle_routing(parsed)
elif parsed.path == "/v1/model-speed":
self._handle_model_speed(parsed)
elif parsed.path == "/v1/nodes/assign":
self._handle_assign(parsed)
elif parsed.path == "/v1/network/assign":
@@ -5725,6 +5731,44 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"models": out,
})
def _handle_model_speed(self, parsed: urllib.parse.ParseResult):
"""Drill into observed model speed and the latency cost of each route."""
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
model = (urllib.parse.parse_qs(parsed.query).get("model") or [None])[0]
if not model:
self._send_json(400, {"error": "missing 'model' query parameter"})
return
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
stats_key = resolved_name or model
with server.lock:
self._purge_expired_nodes()
nodes_by_id = dict(server.registry)
routes = server.route_stats.model_rows(stats_key)
one_hop = [r["latency_ms"] for r in routes if r["hop_count"] == 1 and r["latency_ms"] is not None]
baseline_ms = min(one_hop) if one_hop else None
for route in routes:
node_ids = [part.split("[", 1)[0] for part in route["signature"].split("|", 1)[1].split("->")]
devices = []
for node_id in node_ids:
node = nodes_by_id.get(node_id)
profile = node.hardware_profile if node is not None else {}
devices.append("gpu" if node is not None and (node.vram_bytes > 0 or profile.get("device") == "cuda") else "cpu")
route["node_ids"] = node_ids
route["device_mix"] = "-".join(dict.fromkeys(devices))
route["latency_penalty_ms"] = round(route["latency_ms"] - baseline_ms, 3) if baseline_ms is not None else None
node_rows = []
for node in nodes_by_id.values():
if not _node_matches_model(node, model) and not (preset and _node_matches_preset(node, resolved_name or model, preset)):
continue
sample = server.stats.get_node_model_stats(node.node_id, stats_key) if server.stats else {}
node_rows.append({
"node_id": node.node_id,
"device": "gpu" if node.vram_bytes > 0 or node.hardware_profile.get("device") == "cuda" else "cpu",
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
**sample,
})
self._send_json(200, {"model": stats_key, "baseline_one_hop_latency_ms": baseline_ms, "routes": routes, "nodes": node_rows})
def _handle_routes(self, parsed: urllib.parse.ParseResult):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
params = urllib.parse.parse_qs(parsed.query)
@@ -5967,7 +6011,7 @@ class TrackerServer:
os.environ.get("MESHNET_ROUTE_MIN_SAMPLE_TOKENS", RoutingConfig.min_sample_tokens)
),
)
self._route_stats = RouteStatsStore(routing_config)
self._route_stats = RouteStatsStore(routing_config, db_path=stats_db)
self.port: int | None = None
def _start_embedded_relay(self) -> dict:
@@ -6322,6 +6366,7 @@ class TrackerServer:
while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
if self._stats is not None:
self._stats.save_to_db()
self._route_stats.save_to_db()
if self._billing is not None:
self._billing.save_to_db()
if self._accounts is not None:
@@ -6382,6 +6427,7 @@ class TrackerServer:
self._hf_pricing_stop.set()
if self._stats is not None:
self._stats.save_to_db()
self._route_stats.save_to_db()
if self._billing is not None:
self._billing.save_to_db()
if self._accounts is not None: