dash QOL

packages/node/meshnet_node/downloader.py

@@ -1,4 +1,4 @@
-"""Shard downloader — fetches model shards from peers or HuggingFace Hub.
+"""Shard downloader — fetches model files from peers, tracker sources, or HuggingFace.
 
 Cache layout: ~/.cache/meshnet/shards/<model>/
 

packages/node/meshnet_node/model_backend.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import base64
 from dataclasses import dataclass
+import json
 from pathlib import Path
 from typing import Any, Literal
 
@@ -22,6 +23,10 @@ class InsufficientVRAMError(ModelBackendError):
     """Raised when a requested shard cannot fit in available CUDA memory."""
 
 
+class PartialModelLoadUnsupported(ModelBackendError):
+    """Raised when a shard cannot be materialized from a local snapshot subset."""
+
+
 @dataclass(frozen=True)
 class TensorPayload:
     body: bytes
@@ -94,20 +99,39 @@ class TorchModelShard:
             None if load_source != model_id else cache_dir,
         )
         try:
-            load_kwargs = {
-                "device_map": "auto" if uses_quantized_weights else None,
-                "dtype": dtype,
-                "low_cpu_mem_usage": True,
-                "cache_dir": str(cache_dir) if cache_dir is not None and load_source == model_id else None,
-            }
-            if quant_config is not None:
-                load_kwargs["quantization_config"] = quant_config
-            self.model = AutoModelForCausalLM.from_pretrained(
+            total_layers_hint = _total_layers_for_local_snapshot(AutoConfig, load_source)
+            if _should_partial_materialize_shard(
                 load_source,
-                **load_kwargs,
-            )
-            if not uses_quantized_weights:
-                self.model.to(self.device)
+                shard_start,
+                shard_end,
+                total_layers_hint=total_layers_hint,
+                uses_quantized_weights=uses_quantized_weights,
+            ):
+                self.model = _load_partial_model_from_snapshot(
+                    AutoConfig,
+                    AutoModelForCausalLM,
+                    torch,
+                    load_source,
+                    shard_start,
+                    shard_end,
+                    dtype,
+                    self.device,
+                )
+            else:
+                load_kwargs = {
+                    "device_map": "auto" if uses_quantized_weights else None,
+                    "dtype": dtype,
+                    "low_cpu_mem_usage": True,
+                    "cache_dir": str(cache_dir) if cache_dir is not None and load_source == model_id else None,
+                }
+                if quant_config is not None:
+                    load_kwargs["quantization_config"] = quant_config
+                self.model = AutoModelForCausalLM.from_pretrained(
+                    load_source,
+                    **load_kwargs,
+                )
+                if not uses_quantized_weights:
+                    self.model.to(self.device)
         except Exception as exc:
             if _looks_like_oom(exc):
                 raise InsufficientVRAMError(
@@ -357,6 +381,135 @@ def load_torch_shard(
     return TorchModelShard(model_id, shard_start, shard_end, quantization, cache_dir)
 
 
+def _total_layers_for_local_snapshot(auto_config: Any, load_source: str) -> int | None:
+    snapshot_dir = Path(load_source)
+    if not (snapshot_dir / "config.json").exists():
+        return None
+    from .model_catalog import layers_from_config
+
+    try:
+        cfg = auto_config.from_pretrained(str(snapshot_dir))
+    except Exception:
+        return None
+    return layers_from_config(cfg)
+
+
+def _should_partial_materialize_shard(
+    load_source: str,
+    shard_start: int,
+    shard_end: int,
+    *,
+    total_layers_hint: int | None,
+    uses_quantized_weights: bool,
+) -> bool:
+    if uses_quantized_weights:
+        return False
+    snapshot_dir = Path(load_source)
+    if not snapshot_dir.exists() or not (snapshot_dir / "config.json").exists():
+        return False
+    if not (snapshot_dir / "model.safetensors.index.json").exists():
+        return False
+    if total_layers_hint is None:
+        return False
+    return not (shard_start == 0 and shard_end >= total_layers_hint - 1)
+
+
+def _load_partial_model_from_snapshot(
+    auto_config: Any,
+    auto_model_for_causal_lm: Any,
+    torch: Any,
+    load_source: str,
+    shard_start: int,
+    shard_end: int,
+    dtype: Any,
+    device: Any,
+    *,
+    init_empty_weights_fn: Any | None = None,
+    set_tensor_fn: Any | None = None,
+    safe_open_fn: Any | None = None,
+) -> Any:
+    from .model_catalog import layers_from_config
+    from .safetensors_selection import (
+        INDEX_FILENAME,
+        select_tensor_names_for_layers_from_index,
+    )
+
+    if init_empty_weights_fn is None:
+        from accelerate import init_empty_weights as init_empty_weights_fn
+    if set_tensor_fn is None:
+        from accelerate.utils import set_module_tensor_to_device as set_tensor_fn
+    if safe_open_fn is None:
+        from safetensors import safe_open as safe_open_fn
+
+    snapshot_dir = Path(load_source)
+    cfg = auto_config.from_pretrained(str(snapshot_dir))
+    total_layers = layers_from_config(cfg)
+    if total_layers is None:
+        raise PartialModelLoadUnsupported(
+            f"could not determine num_hidden_layers for local snapshot {snapshot_dir}"
+        )
+    if shard_end >= total_layers:
+        raise ValueError(
+            f"shard_end {shard_end} exceeds last layer index {total_layers - 1}"
+        )
+
+    index_path = snapshot_dir / INDEX_FILENAME
+    try:
+        index = json.loads(index_path.read_text(encoding="utf-8"))
+    except FileNotFoundError as exc:
+        raise PartialModelLoadUnsupported(
+            f"missing SafeTensors index for partial load: {index_path}"
+        ) from exc
+    weight_map = index.get("weight_map")
+    if not isinstance(weight_map, dict):
+        raise PartialModelLoadUnsupported(f"{INDEX_FILENAME} must contain a weight_map object")
+
+    tensor_names = select_tensor_names_for_layers_from_index(
+        weight_map,
+        shard_start,
+        shard_end,
+        total_layers=total_layers,
+    )
+    if not tensor_names:
+        raise PartialModelLoadUnsupported(
+            f"no checkpoint tensors matched layers {shard_start}-{shard_end} in {snapshot_dir}"
+        )
+
+    with init_empty_weights_fn():
+        model = auto_model_for_causal_lm.from_config(cfg, torch_dtype=dtype)
+    tie_weights = getattr(model, "tie_weights", None)
+    if callable(tie_weights):
+        tie_weights()
+
+    tensors_by_file: dict[str, list[str]] = {}
+    for tensor_name in sorted(tensor_names):
+        rel_file = weight_map.get(tensor_name)
+        if not isinstance(rel_file, str):
+            continue
+        tensors_by_file.setdefault(rel_file, []).append(tensor_name)
+
+    for rel_file, names in tensors_by_file.items():
+        checkpoint_file = snapshot_dir / rel_file
+        if not checkpoint_file.exists():
+            raise PartialModelLoadUnsupported(
+                f"checkpoint file advertised in {INDEX_FILENAME} is missing: {checkpoint_file}"
+            )
+        with safe_open_fn(str(checkpoint_file), framework="pt", device="cpu") as handle:
+            for tensor_name in names:
+                set_tensor_fn(
+                    model,
+                    tensor_name,
+                    device,
+                    value=handle.get_tensor(tensor_name),
+                    dtype=dtype,
+                )
+
+    for module in _active_modules_for_shard(model, shard_start, shard_end):
+        if hasattr(module, "to"):
+            module.to(device)
+    return model
+
+
 def _model_load_plan(
     auto_config: Any,
     model_id: str,
@@ -442,6 +595,37 @@ def _position_embeddings(model: Any) -> Any | None:
     return None
 
 
+def _rotary_embedding_module(model: Any) -> Any | None:
+    if hasattr(model, "model") and hasattr(model.model, "rotary_emb"):
+        return model.model.rotary_emb
+    if hasattr(model, "transformer") and hasattr(model.transformer, "rotary_emb"):
+        return model.transformer.rotary_emb
+    return None
+
+
+def _active_modules_for_shard(model: Any, shard_start: int, shard_end: int) -> list[Any]:
+    active: list[Any] = []
+
+    def add(module: Any | None) -> None:
+        if module is None:
+            return
+        if any(existing is module for existing in active):
+            return
+        active.append(module)
+
+    if shard_start == 0:
+        add(_embed_tokens(model))
+        add(_position_embeddings(model))
+    add(_rotary_embedding_module(model))
+    for layer in _model_layers(model)[shard_start:shard_end + 1]:
+        add(layer)
+    total_layers = len(_model_layers(model))
+    if shard_end >= total_layers - 1:
+        add(_final_norm(model))
+        add(getattr(model, "lm_head", None))
+    return active
+
+
 def _final_norm(model: Any) -> Any | None:
     if hasattr(model, "model") and hasattr(model.model, "norm"):
         return model.model.norm
@@ -485,11 +669,7 @@ def _rotary_position_embeddings(model: Any, hidden_states: Any, position_ids: An
     """Return model-level rotary embeddings required by newer HF decoder layers."""
     if position_ids is None:
         return None
-    rotary = None
-    if hasattr(model, "model") and hasattr(model.model, "rotary_emb"):
-        rotary = model.model.rotary_emb
-    elif hasattr(model, "transformer") and hasattr(model.transformer, "rotary_emb"):
-        rotary = model.transformer.rotary_emb
+    rotary = _rotary_embedding_module(model)
     if rotary is None:
         return None
     return rotary(hidden_states, position_ids)

packages/node/meshnet_node/safetensors_selection.py

@@ -118,6 +118,23 @@ def select_files_for_layers_from_index(
     return selected
 
 
+def select_tensor_names_for_layers_from_index(
+    weight_map: dict[str, str],
+    start_layer: int,
+    end_layer: int,
+    *,
+    total_layers: int | None = None,
+) -> set[str]:
+    """Pure variant that returns checkpoint tensor names instead of file paths."""
+    selected: set[str] = set()
+    for tensor_name, rel_file in weight_map.items():
+        if not isinstance(tensor_name, str) or not isinstance(rel_file, str):
+            continue
+        if _tensor_belongs_to_range(tensor_name, start_layer, end_layer, total_layers):
+            selected.add(tensor_name)
+    return selected
+
+
 def _tensor_belongs_to_range(
     tensor_name: str,
     start_layer: int,

packages/tracker/meshnet_tracker/billing.py

@@ -453,13 +453,13 @@ class BillingLedger:
         with self._lock:
             return self._node_pending.get(wallet, 0.0)
 
-    def usage_for(self, api_keys: list[str], *, recent_limit: int = 20) -> dict:
+    def usage_for(self, api_keys: list[str], *, recent_limit: int | None = None) -> dict:
         """Aggregate charge history for a set of API keys (dashboard view)."""
         keys = set(api_keys)
         requests = 0
         total_tokens = 0
         total_cost = 0.0
-        recent: list[dict] = []
+        records: list[dict] = []
         with self._lock:
             for event in self._event_log:
                 if event.get("type") != "charge" or event.get("api_key") not in keys:
@@ -467,18 +467,20 @@ class BillingLedger:
                 requests += 1
                 total_tokens += int(event.get("total_tokens", 0))
                 total_cost += float(event.get("cost", 0.0))
-                recent.append({
+                records.append({
                     "api_key": event["api_key"],
                     "model": event.get("model"),
                     "total_tokens": event.get("total_tokens", 0),
                     "cost": event.get("cost", 0.0),
                     "ts": event.get("ts", 0.0),
                 })
+        recent = records[-recent_limit:] if recent_limit is not None else records
         return {
             "requests": requests,
             "total_tokens": total_tokens,
             "total_cost": total_cost,
-            "recent": recent[-recent_limit:],
+            "records": records,
+            "recent": recent,
         }
 
     def snapshot(self) -> dict:

packages/tracker/meshnet_tracker/dashboard.html

@@ -39,12 +39,41 @@
   .form-row { display:flex; gap:8px; }
   .form-row button { white-space:nowrap; }
   .error-msg { color:var(--bad); font-size:12px; min-height:16px; }
-  .keybox { word-break:break-all; background:var(--bg); border:1px solid var(--border);
+  .keybox { display:flex; flex-wrap:wrap; align-items:center; gap:6px;
+          position:relative;
+          word-break:break-all; background:var(--bg); border:1px solid var(--border);
           border-radius:6px; padding:4px 8px; margin:4px 0; font-size:11px; }
+  .key-text { cursor:text; flex:1 1 auto; min-width:12rem; }
+  .copy-tooltip {
+    position:absolute; right:8px; top:-26px;
+    background:var(--panel); border:1px solid var(--border); color:var(--ok);
+    padding:2px 8px; border-radius:4px; font-size:11px;
+    pointer-events:none; z-index:1; white-space:nowrap;
+  }
   .tabs { display:flex; gap:10px; margin-bottom:8px; }
   .tabs a { color:var(--dim); cursor:pointer; }
   .tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
+  .dashboard-tabs { display:flex; gap:10px; padding:10px 20px 0; border-bottom:1px solid var(--border); }
+  .dashboard-tabs button { border:0; border-bottom:1px solid transparent; border-radius:0;
+          background:transparent; color:var(--dim); padding:5px 0 8px; }
+  .dashboard-tabs button.active { color:var(--accent); border-bottom-color:var(--accent); }
   .wide { grid-column:1 / -1; }
+  section[hidden] { display:none !important; }
+  .chat-shell { display:grid; grid-template-columns:minmax(0, 1.35fr) minmax(320px, 0.65fr); gap:12px; }
+  .chat-pane { display:flex; flex-direction:column; gap:10px; min-width:0; }
+  .chat-panel { background:var(--bg); border:1px solid var(--border); border-radius:6px; padding:10px; }
+  .chat-controls { display:flex; gap:10px; align-items:end; flex-wrap:wrap; }
+  .chat-controls label { display:flex; flex-direction:column; gap:4px; color:var(--dim); }
+  .chat-controls select { min-width:220px; }
+  .chat-history { display:flex; flex-direction:column; gap:8px; min-height:220px; max-height:420px; overflow:auto; }
+  .chat-message { border:1px solid #21262d; border-radius:6px; padding:8px 10px; background:#10151d; }
+  .chat-role { color:var(--dim); font-size:11px; text-transform:uppercase; letter-spacing:.06em; margin-bottom:4px; }
+  .chat-role-user { color:var(--accent); }
+  .chat-role-assistant { color:var(--ok); }
+  .chat-role-error { color:var(--bad); }
+  .chat-compose { display:flex; flex-direction:column; gap:8px; }
+  .chat-compose textarea { min-height:112px; resize:vertical; width:100%; }
+  .chat-status { color:var(--dim); font-size:12px; }
   .console {
     background:var(--bg); border:1px solid var(--border); border-radius:6px;
     min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
@@ -55,6 +84,10 @@
   .console-level-info { color:var(--accent); }
   .console-level-warn { color:var(--warn); }
   .console-level-error { color:var(--bad); }
+  .status-pending { color:var(--warn); }
+  .status-processing { color:var(--accent); }
+  .status-failed { color:var(--bad); }
+  .status-complete { color:var(--ok); }
 </style>
 </head>
 <body>
@@ -63,19 +96,50 @@
   <span class="meta" id="self-url"></span>
   <span class="meta" id="refreshed"></span>
 </header>
+<nav class="dashboard-tabs" aria-label="Dashboard sections">
+  <button id="tab-overview" class="active" onclick="switchDashboardTab('overview')">Overview</button>
+  <button id="tab-chat" onclick="switchDashboardTab('chat')">Chat</button>
+  <button id="tab-billing" style="display:none" onclick="switchDashboardTab('billing')">Billing</button>
+  <button id="tab-admin" style="display:none" onclick="switchDashboardTab('admin')">Admin</button>
+</nav>
 <main>
-  <section id="account-section"><h2>Account</h2><div id="account">loading…</div></section>
-  <section id="admin-section" style="display:none"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
-  <section><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
-  <section><h2>Nodes &amp; coverage</h2><div id="nodes" class="empty">loading…</div></section>
-  <section><h2>Client balances</h2><div id="clients" class="empty">loading…</div></section>
-  <section><h2>Node pending payouts</h2><div id="pending" class="empty">loading…</div></section>
-  <section><h2>Settlement history</h2><div id="settlements" class="empty">loading…</div></section>
-  <section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
-  <section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
-  <section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
-  <section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
-  <section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section>
+  <section data-tab="overview" id="account-section"><h2>Account</h2><div id="account">loading…</div></section>
+  <section data-tab="overview"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
+  <section data-tab="overview"><h2>Nodes &amp; coverage</h2><div id="nodes" class="empty">loading…</div></section>
+  <section data-tab="overview"><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
+  <section data-tab="overview"><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
+  <section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
+  <section data-tab="chat" class="wide">
+    <h2>Chat / inference</h2>
+    <div class="chat-shell">
+      <div class="chat-pane">
+        <div class="chat-panel chat-controls">
+          <label>Model
+            <select id="chat-model" onchange="selectChatModel(this.value)"></select>
+          </label>
+          <button class="small" onclick="clearChatHistory()">clear history</button>
+        </div>
+        <div class="chat-panel chat-compose">
+          <textarea id="chat-prompt" placeholder="Ask a question or describe the task"></textarea>
+          <div class="form-row">
+            <button onclick="sendChat()" id="chat-send">Send</button>
+          </div>
+        </div>
+      </div>
+      <div class="chat-pane">
+        <div class="chat-panel">
+          <div id="chat-status" class="chat-status">select a model to start</div>
+          <div id="chat-history" class="chat-history empty">no messages yet</div>
+        </div>
+      </div>
+    </div>
+  </section>
+  <section data-tab="billing"><h2>Request history</h2><div id="billing-usage" class="empty">login required</div></section>
+  <section data-tab="billing" data-admin-only><h2>Node pending payouts</h2><div id="pending" class="empty">admin login required</div></section>
+  <section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
+  <section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
+  <section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
+  <section data-tab="admin" class="wide"><h2>Console output</h2><div id="console" class="console empty">admin login required</div></section>
 </main>
 <script>
 "use strict";
@@ -211,7 +275,7 @@ function renderStats(stats) {
   $("stats").innerHTML = table(["model", "rpm (1h)", "rpm (24h)", "rpm (30d)"], rows);
 }
 
-function renderThroughput(stats) {
+function renderThroughputHtml(stats) {
   const nodes = (stats && stats.nodes) || {};
   const rows = [];
   for (const [nodeId, nodeStats] of Object.entries(nodes)) {
@@ -224,49 +288,187 @@ function renderThroughput(stats) {
       ]);
     }
   }
-  $("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
+  if (!rows.length) return '<div class="empty">no throughput samples yet</div>';
+  return table(["node", "model", "tps (1h)", "samples"], rows);
 }
 
-function renderInferenceHistory(data) {
-  const events = (data && data.events) || [];
-  const started = new Map();
-  const completed = [];
+function hiveThroughputSummary(stats) {
+  const nodes = (stats && stats.nodes) || {};
+  let totalTps = 0;
+  let samples = 0;
+  for (const nodeStats of Object.values(nodes)) {
+    for (const s of Object.values((nodeStats && nodeStats.models) || {})) {
+      const t = Number(s.tokens_per_sec_last_hour);
+      if (Number.isFinite(t)) totalTps += t;
+      samples += Number(s.sample_count_last_hour || 0);
+    }
+  }
+  return { totalTps, samples };
+}
+
+function buildCallWallStates(events) {
+  const byId = new Map();
   for (const e of events) {
     const f = e.fields || {};
     const id = f.request_id;
     if (!id) continue;
-    if (e.message === "proxy route selected") {
-      started.set(id, e);
-    } else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") {
-      completed.push(e);
-      started.delete(id);
+    let rec = byId.get(id);
+    if (!rec) {
+      rec = { id, events: [] };
+      byId.set(id, rec);
+    }
+    rec.events.push(e);
+    const msg = e.message;
+    if (msg === "proxy route selected") {
+      rec.status = "pending";
+      rec.started = e.ts;
+      rec.model = f.model || f.route_model || "?";
+      rec.route = f.route || f.nodes;
+      rec.stream = f.stream;
+    } else if (msg === "proxy via relay" || msg === "proxy connected") {
+      rec.status = "processing";
+      if (!rec.started) rec.started = e.ts;
+      rec.model = rec.model || f.model || f.route_model || "?";
+    } else if (msg === "proxy progress") {
+      rec.status = "processing";
+      rec.model = rec.model || f.model || f.route_model || "?";
+      rec.tokens = f.tokens;
+      rec.tps = f.tokens_per_sec;
+      rec.elapsed = f.elapsed_seconds;
+      rec.stream = f.stream;
+    } else if (msg === "relay proxy failed, trying direct") {
+      rec.status = "processing";
+      rec.warn = "relay failed, trying direct";
+    } else if (msg === "proxy complete") {
+      rec.status = "complete";
+      rec.model = rec.model || f.model || f.route_model || "?";
+      rec.tokens = f.tokens;
+      rec.tps = f.tokens_per_sec;
+      rec.elapsed = f.elapsed_seconds;
+      rec.stream = f.stream;
+      rec.terminal = e;
+    } else if (msg === "proxy failed" || msg === "direct proxy failed after relay") {
+      rec.status = "failed";
+      rec.model = rec.model || f.model || f.route_model || "?";
+      rec.error = f.error || msg;
+      rec.terminal = e;
     }
   }
-  const activeByModel = {};
-  for (const e of started.values()) {
-    const f = e.fields || {};
-    const model = f.model || f.route_model || "?";
-    activeByModel[model] = (activeByModel[model] || 0) + 1;
+  return byId;
+}
+
+function callWallAgeSeconds(rec, nowSec) {
+  const start = rec.started || (rec.events[0] && rec.events[0].ts) || nowSec;
+  return Math.max(0, nowSec - start);
+}
+
+function renderCallWall(consoleData, stats) {
+  const events = (consoleData && consoleData.events) || [];
+  const nowSec = Date.now() / 1000;
+  const states = buildCallWallStates(events);
+  const active = [];
+  const terminal = [];
+  for (const rec of states.values()) {
+    if (rec.status === "pending" || rec.status === "processing") active.push(rec);
+    else if (rec.status === "complete" || rec.status === "failed") terminal.push(rec);
   }
-  const active = Object.entries(activeByModel)
-    .map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`)
-    .join(" &middot; ");
-  const rows = completed.slice(-20).reverse().map(e => {
+  active.sort((a, b) => (a.started || 0) - (b.started || 0));
+  terminal.sort((a, b) => (b.terminal && b.terminal.ts) - (a.terminal && a.terminal.ts));
+
+  const hive = hiveThroughputSummary(stats);
+  const pending = active.filter(r => r.status === "pending").length;
+  const processing = active.filter(r => r.status === "processing").length;
+  const failedRecent = terminal.filter(r => r.status === "failed").length;
+
+  let html =
+    `<div class="dim" style="margin-bottom:6px">` +
+    `hive tps (1h): <b>${esc(tps(hive.totalTps))}</b> · samples: <b>${hive.samples}</b> · ` +
+    `active: <span class="status-processing">${processing}</span> processing · ` +
+    `<span class="status-pending">${pending}</span> pending` +
+    (failedRecent ? ` · <span class="status-failed">${failedRecent} recent failures</span>` : "") +
+    `</div>`;
+
+  if (active.length) {
+    html += table(["status", "age", "model", "request", "tps", "tokens", "route / note"], active.map(rec => {
+      const statusCls = rec.status === "pending" ? "status-pending" : "status-processing";
+      const note = rec.warn || (rec.route ? short(String(rec.route), 28) : "");
+      return [
+        `<span class="${statusCls}">${esc(rec.status)}</span>`,
+        `<span class="num">${esc(callWallAgeSeconds(rec, nowSec).toFixed(1))}s</span>`,
+        esc(short(rec.model || "?", 28)),
+        esc(short(rec.id, 18)),
+        `<span class="num">${esc(tps(rec.tps))}</span>`,
+        `<span class="num">${esc(String(rec.tokens ?? "—"))}</span>`,
+        esc(note),
+      ];
+    }));
+  } else {
+    html += '<div class="empty">no in-flight requests</div>';
+  }
+
+  const historyRows = terminal.slice(0, 40).map(rec => {
+    const e = rec.terminal || {};
     const f = e.fields || {};
+    const statusCls = rec.status === "failed" ? "status-failed" : "status-complete";
+    const detail = rec.status === "failed"
+      ? esc(short(rec.error || "?", 40))
+      : (f.stream ? "stream" : "json");
     return [
       new Date((e.ts || 0) * 1000).toLocaleTimeString(),
-      esc(short(f.model || f.route_model || "?", 28)),
-      esc(short(f.request_id || "?", 18)),
-      `<span class="num">${esc(tps(f.tokens_per_sec))}</span>`,
-      `<span class="num">${esc(String(f.tokens ?? "?"))}</span>`,
-      `<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`,
-      f.stream ? "stream" : "json",
+      `<span class="${statusCls}">${esc(rec.status)}</span>`,
+      esc(short(rec.model || "?", 28)),
+      esc(short(rec.id, 18)),
+      `<span class="num">${esc(tps(rec.tps ?? f.tokens_per_sec))}</span>`,
+      `<span class="num">${esc(String(rec.tokens ?? f.tokens ?? "?"))}</span>`,
+      `<span class="num">${esc(String(rec.elapsed ?? f.elapsed_seconds ?? "?"))}</span>`,
+      detail,
     ];
   });
-  $("inference-history").innerHTML =
-    `<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` +
-    (rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows)
-                 : '<div class="empty">no completed inference requests</div>');
+  html += '<div style="margin-top:8px"><b class="dim">recent completed / failed</b></div>';
+  html += historyRows.length
+    ? table(["time", "status", "model", "request", "tps", "tokens", "sec", "detail"], historyRows)
+    : '<div class="empty">no completed requests yet</div>';
+  $("call-wall").innerHTML = html;
+}
+
+function groupUsageByDayModel(records) {
+  const groups = new Map();
+  for (const u of records) {
+    const day = new Date((u.ts || 0) * 1000).toLocaleDateString();
+    const model = u.model || "?";
+    const key = day + "\0" + model;
+    const g = groups.get(key) || { day, model, requests: 0, tokens: 0, cost: 0 };
+    g.requests += 1;
+    g.tokens += Number(u.total_tokens || 0);
+    g.cost += Number(u.cost || 0);
+    groups.set(key, g);
+  }
+  return Array.from(groups.values()).sort((a, b) => {
+    const dayCmp = b.day.localeCompare(a.day);
+    return dayCmp !== 0 ? dayCmp : a.model.localeCompare(b.model);
+  });
+}
+
+function renderBillingUsage(records) {
+  const el = $("billing-usage");
+  if (!el) return;
+  if (!sessionToken) {
+    el.innerHTML = '<div class="empty">login required</div>';
+    return;
+  }
+  if (!records.length) {
+    el.innerHTML = '<div class="empty">no billed requests yet</div>';
+    return;
+  }
+  const rows = records.slice().reverse().map(u => [
+    new Date((u.ts || 0) * 1000).toLocaleString(),
+    esc(short(u.model || "?", 28)),
+    esc(short(u.api_key || "?", 14)),
+    `<span class="num">${esc(String(u.total_tokens))}</span>`,
+    `<span class="num">${usdt(u.cost)}</span>`,
+  ]);
+  el.innerHTML = `<div class="dim" style="margin-bottom:6px">${records.length} request${records.length === 1 ? "" : "s"}</div>` +
+    table(["time", "model", "api key", "tokens", "cost (USDT)"], rows);
 }
 
 function renderConsole(data) {
@@ -288,10 +490,123 @@ function renderConsole(data) {
 
 let sessionToken = localStorage.getItem("meshnet_session") || null;
 let authTab = "login";
+let dashboardTab = "overview";
+let isAdmin = false;
+let isLoggedIn = false;
+let accountApiKeys = [];
+let accountUsageRecords = [];
+let lastStats = null;
+let availableModels = [];
+let chatHistory = [];
+let chatBusy = false;
+let selectedChatModel = localStorage.getItem("meshnet_chat_model") || "";
 
-async function apiCall(path, method, body) {
+function switchDashboardTab(name) {
+  if (name === "admin" && !isAdmin) name = "overview";
+  if (name === "billing" && !isLoggedIn) name = "overview";
+  dashboardTab = name;
+  updateSectionVisibility();
+  for (const tabName of ["overview", "chat", "billing", "admin"]) {
+    const button = $("tab-" + tabName);
+    if (button) button.classList.toggle("active", tabName === dashboardTab);
+  }
+}
+
+function updateSectionVisibility() {
+  for (const section of document.querySelectorAll("main section[data-tab]")) {
+    const onTab = section.dataset.tab === dashboardTab;
+    const adminOnly = section.hasAttribute("data-admin-only");
+    section.hidden = !onTab || (adminOnly && !isAdmin);
+  }
+}
+
+function renderChatStatus(text) {
+  $("chat-status").textContent = text;
+}
+
+function renderChatHistory() {
+  const history = $("chat-history");
+  if (!chatHistory.length) {
+    history.classList.add("empty");
+    history.innerHTML = "no messages yet";
+    return;
+  }
+  history.classList.remove("empty");
+  history.innerHTML = chatHistory.map(msg => {
+    const roleClass = msg.role === "user" ? "chat-role-user" : msg.role === "assistant" ? "chat-role-assistant" : "chat-role-error";
+    const label = msg.role === "user" ? "user" : msg.role === "assistant" ? "assistant" : "error";
+    const meta = msg.model ? ` <span class="dim">· ${esc(short(msg.model, 24))}</span>` : "";
+    return `<div class="chat-message"><div class="chat-role ${roleClass}">${label}${meta}</div><div>${esc(msg.content)}</div></div>`;
+  }).join("");
+  history.scrollTop = history.scrollHeight;
+}
+
+function renderChatModels() {
+  const select = $("chat-model");
+  if (!select) return;
+  const models = availableModels.slice();
+  if (!models.length) {
+    select.innerHTML = '<option value="">no models available</option>';
+    select.disabled = true;
+    return;
+  }
+  select.disabled = false;
+  const preferred = models.find(m => m.id === selectedChatModel)
+    || models[0];
+  selectedChatModel = preferred.id;
+  localStorage.setItem("meshnet_chat_model", selectedChatModel);
+  select.innerHTML = models.map(model => {
+    const label = model.name && model.name !== model.id
+      ? `${model.name} (${model.id})`
+      : model.id;
+    const suffix = model.recommended ? " [recommended]" : "";
+    return `<option value="${esc(model.id)}"${model.id === selectedChatModel ? " selected" : ""}>${esc(label + suffix)}</option>`;
+  }).join("");
+  select.value = selectedChatModel;
+}
+
+function selectChatModel(value) {
+  selectedChatModel = value || "";
+  localStorage.setItem("meshnet_chat_model", selectedChatModel);
+}
+
+function clearChatHistory() {
+  chatHistory = [];
+  renderChatHistory();
+  renderChatStatus("history cleared");
+}
+
+function chatAuthToken() {
+  if (accountApiKeys.length) return accountApiKeys[0];
+  return null;
+}
+
+function setAdminMode(enabled) {
+  isAdmin = enabled;
+  $("tab-admin").style.display = enabled ? "" : "none";
+  if (!enabled && dashboardTab === "admin") {
+    switchDashboardTab("overview");
+  } else {
+    updateSectionVisibility();
+  }
+}
+
+function setLoggedInMode(enabled) {
+  isLoggedIn = enabled;
+  $("tab-billing").style.display = enabled ? "" : "none";
+  if (!enabled) {
+    accountUsageRecords = [];
+    renderBillingUsage([]);
+    if (dashboardTab === "billing") switchDashboardTab("overview");
+  } else {
+    updateSectionVisibility();
+  }
+}
+
+async function apiCall(path, method, body, bearerToken) {
   const headers = { "Content-Type": "application/json" };
-  if (sessionToken) headers["Authorization"] = "Bearer " + sessionToken;
+  const token = bearerToken === undefined ? sessionToken : bearerToken;
+  if (token) headers["Authorization"] = "Bearer " + token;
   try {
     const r = await fetch(path, {
       method: method || "GET",
@@ -327,7 +642,10 @@ function renderAuthForms(errorMsg) {
   $("account").innerHTML =
     `<div class="tabs">${tab("login", "Log in")}${tab("register", "Register")}</div>` +
     form + `<div class="error-msg">${errorMsg ? esc(errorMsg) : ""}</div>`;
-  $("admin-section").style.display = "none";
+  accountApiKeys = [];
+  renderChatAuthHint();
+  setLoggedInMode(false);
+  setAdminMode(false);
 }
 
 function switchAuthTab(name) { authTab = name; renderAuthForms(); }
@@ -375,15 +693,82 @@ async function topupKey(key) {
   await renderAccountPanel();
 }
 
+const COPY_TOOLTIP_MS = 2000;
+
+function showCopiedTooltip(anchor) {
+  const box = (anchor && anchor.closest && anchor.closest(".keybox")) || anchor;
+  if (!box) return;
+  const existing = box.querySelector(".copy-tooltip");
+  if (existing) existing.remove();
+  const tip = document.createElement("span");
+  tip.className = "copy-tooltip";
+  tip.textContent = "Copied!";
+  tip.setAttribute("role", "status");
+  box.appendChild(tip);
+  setTimeout(() => tip.remove(), COPY_TOOLTIP_MS);
+}
+
+async function copyApiKeyText(text, anchor) {
+  try {
+    await navigator.clipboard.writeText(text);
+  } catch {
+    const ta = document.createElement("textarea");
+    ta.value = text;
+    ta.style.position = "fixed";
+    ta.style.left = "-9999px";
+    document.body.appendChild(ta);
+    ta.select();
+    try { document.execCommand("copy"); } catch { /* ignore */ }
+    document.body.removeChild(ta);
+  }
+  if (anchor) showCopiedTooltip(anchor);
+}
+
+function selectApiKeyText(el) {
+  const range = document.createRange();
+  range.selectNodeContents(el);
+  const sel = window.getSelection();
+  if (!sel) return;
+  sel.removeAllRanges();
+  sel.addRange(range);
+}
+
+function copyApiKeyFromTextEl(el) {
+  const key = el.dataset.key || el.textContent || "";
+  return copyApiKeyText(key, el);
+}
+
+function copyApiKeyFromButton(button) {
+  const el = button.closest(".keybox") && button.closest(".keybox").querySelector(".key-text");
+  const key = (el && el.dataset.key) || "";
+  return copyApiKeyText(key, button);
+}
+
+function renderChatAuthHint() {
+  if (chatAuthToken()) {
+    renderChatStatus("ready to send with your active API key");
+  } else if (sessionToken) {
+    renderChatStatus("create an API key in Account to use chat on a billing-enabled tracker");
+  } else {
+    renderChatStatus("log in if this tracker requires an API key");
+  }
+}
+
 async function renderAccountPanel() {
   const r = await apiCall("/v1/account");
   if (r.status === 404) {  // accounts disabled on this tracker
     $("account-section").style.display = "none";
-    $("admin-section").style.display = "none";
+    accountApiKeys = [];
+    accountUsageRecords = [];
+    renderChatAuthHint();
+    setLoggedInMode(false);
+    setAdminMode(false);
     return;
   }
   if (!r.ok) { setSession(null); renderAuthForms(); return; }
   const { account, api_keys, balances, total_balance, usage, topup_amount } = r.data;
+  accountApiKeys = Array.isArray(api_keys) ? api_keys.slice() : [];
+  accountUsageRecords = (usage && (usage.records || usage.recent)) || [];
   const who = account.email || account.wallet || account.account_id;
   let html =
     `<div><b>${esc(who)}</b> <span class="pill">${esc(account.role)}</span> ` +
@@ -395,8 +780,10 @@ async function renderAccountPanel() {
     '<button class="small" onclick="createKey()">+ new key</button></div>';
   if (api_keys.length) {
     for (const key of api_keys) {
-      html += `<div class="keybox">${esc(key)}` +
-        ` <span class="dim">(${usdt(balances[key] ?? 0)} USDT)</span>` +
+      html += `<div class="keybox">` +
+        `<span class="key-text" data-key="${esc(key)}" onclick="selectApiKeyText(this)" ondblclick="copyApiKeyFromTextEl(this)">${esc(key)}</span>` +
+        `<span class="dim">(${usdt(balances[key] ?? 0)} USDT)</span>` +
+        `<button class="small" type="button" onclick="copyApiKeyFromButton(this)">copy</button>` +
         (topup_amount > 0
           ? ` <button class="small" onclick="topupKey('${esc(key)}')">+${usdt(topup_amount)} (devnet)</button>`
           : "") +
@@ -405,24 +792,86 @@ async function renderAccountPanel() {
   } else {
     html += '<div class="empty">no active keys</div>';
   }
-  if (usage.recent && usage.recent.length) {
-    html += '<div style="margin-top:6px"><b class="dim">recent usage</b></div>' +
-      table(["time", "model", "tokens", "cost"], usage.recent.slice().reverse().map(u => [
-        new Date(u.ts * 1000).toLocaleTimeString(),
-        esc(short(u.model || "?", 24)),
-        `<span class="num">${esc(String(u.total_tokens))}</span>`,
-        `<span class="num">${usdt(u.cost)}</span>`,
+  html += '<div style="margin-top:8px"><b class="dim">node throughput</b></div>' +
+    `<div id="account-throughput">${renderThroughputHtml(lastStats)}</div>`;
+  const grouped = groupUsageByDayModel(accountUsageRecords);
+  if (grouped.length) {
+    html += '<div style="margin-top:8px"><b class="dim">usage by day / model</b> ' +
+      '<span class="dim">(full request list on Billing tab)</span></div>' +
+      table(["day", "model", "requests", "tokens", "cost (USDT)"], grouped.map(g => [
+        esc(g.day),
+        esc(short(g.model, 28)),
+        `<span class="num">${g.requests}</span>`,
+        `<span class="num">${esc(String(g.tokens))}</span>`,
+        `<span class="num">${usdt(g.cost)}</span>`,
       ]));
   }
   $("account").innerHTML = html;
+  renderBillingUsage(accountUsageRecords);
+  renderChatAuthHint();
+  renderChatModels();
+  renderChatHistory();
+  setLoggedInMode(true);
+  setAdminMode(account.role === "admin");
   if (account.role === "admin") await renderAdminPanel();
-  else $("admin-section").style.display = "none";
+}
+
+async function sendChat() {
+  const promptEl = $("chat-prompt");
+  const prompt = promptEl.value.trim();
+  if (!prompt || chatBusy) return;
+  if (!selectedChatModel) {
+    renderChatStatus("select a model first");
+    return;
+  }
+  const bearerToken = chatAuthToken();
+  const body = {
+    model: selectedChatModel,
+    messages: [
+      ...chatHistory
+        .filter(msg => msg.role === "user" || msg.role === "assistant")
+        .map(msg => ({ role: msg.role, content: msg.content })),
+      { role: "user", content: prompt },
+    ],
+    stream: false,
+    max_tokens: 256,
+  };
+  chatBusy = true;
+  $("chat-send").disabled = true;
+  promptEl.value = "";
+  chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
+  renderChatHistory();
+  renderChatStatus("sending request…");
+  const r = await apiCall("/v1/chat/completions", "POST", body, bearerToken);
+  chatBusy = false;
+  $("chat-send").disabled = false;
+  if (!r.ok) {
+    const error = r.data && r.data.error
+      ? (typeof r.data.error === "string" ? r.data.error : r.data.error.message || "request failed")
+      : "request failed";
+    chatHistory.push({ role: "error", content: error, model: selectedChatModel });
+    renderChatHistory();
+    renderChatStatus(error);
+    promptEl.focus();
+    return;
+  }
+  const reply = (r.data && r.data.choices && r.data.choices[0] && r.data.choices[0].message && r.data.choices[0].message.content) || "";
+  const usage = r.data && r.data.usage;
+  chatHistory.push({
+    role: "assistant",
+    content: reply || "(empty response)",
+    model: selectedChatModel,
+  });
+  renderChatHistory();
+  renderChatStatus(usage
+    ? `done: ${usage.total_tokens ?? "?"} tokens`
+    : "done");
+  promptEl.focus();
 }
 
 async function renderAdminPanel() {
   const r = await apiCall("/v1/admin/accounts");
-  if (!r.ok) { $("admin-section").style.display = "none"; return; }
-  $("admin-section").style.display = "";
+  if (!r.ok) { setAdminMode(false); return; }
   const rows = (r.data.accounts || []).map(a => {
     const balance = Object.values(a.balances || {}).reduce((x, y) => x + y, 0);
     return [
@@ -438,28 +887,45 @@ async function renderAdminPanel() {
 
 async function refresh() {
   $("self-url").textContent = location.host;
-  const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
+  const [raft, map, stats, models, consoleData, adminData] = await Promise.all([
     fetchJson("/v1/raft/status"),
     fetchJson("/v1/network/map"),
-    fetchJson("/v1/billing/summary"),
-    fetchJson("/v1/billing/settlements"),
-    fetchJson("/v1/registry/wallets"),
     fetchJson("/v1/stats"),
+    fetchJson("/v1/models"),
     fetchJson("/v1/console"),
+    isAdmin ? Promise.all([
+      fetchJson("/v1/billing/summary"),
+      fetchJson("/v1/billing/settlements"),
+      fetchJson("/v1/registry/wallets"),
+    ]) : Promise.resolve([null, null, null]),
   ]);
+  const [summary, settlements, wallets] = adminData;
+  lastStats = stats;
+  availableModels = ((models && models.data) || []).map(model => ({
+    id: model.id,
+    name: model.name || model.id,
+    recommended: Boolean(model.recommended),
+    aliases: model.aliases || [],
+  })).filter(model => model.id);
   renderHive(raft);
   renderNodes(map);
   renderBilling(summary);
   renderSettlements(settlements);
   renderFraud(wallets, summary);
   renderStats(stats);
-  renderThroughput(stats);
-  renderInferenceHistory(consoleData);
+  renderCallWall(consoleData, stats);
   renderConsole(consoleData);
+  const throughputEl = $("account-throughput");
+  if (throughputEl) throughputEl.innerHTML = renderThroughputHtml(stats);
+  renderChatModels();
+  renderChatHistory();
   $("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
 }
 refresh();
 renderAccountPanel();
+renderChatModels();
+renderChatHistory();
+renderChatAuthHint();
 setInterval(refresh, 4000);
 setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000);
 </script>

packages/tracker/meshnet_tracker/server.py

@@ -142,18 +142,18 @@ DEFAULT_CALLER_CREDIT_USDT = 1.0
 DEFAULT_DEVNET_TOPUP_USDT = 1.0
 
 
-def _model_aliases(model: str | None) -> set[str]:
-    """Return stable lookup aliases for a model repo or display name."""
-    if not model:
-        return set()
-    normalized = model.strip()
-    if not normalized:
-        return set()
-    aliases = {normalized}
-    short = normalized.rsplit("/", 1)[-1]
-    aliases.add(short)
-    lowered = short.lower()
-    aliases.add(lowered)
+def _model_aliases(model: str | None) -> set[str]:
+    """Return stable lookup aliases for a model repo or display name."""
+    if not model:
+        return set()
+    normalized = model.strip()
+    if not normalized:
+        return set()
+    aliases = {normalized}
+    short = normalized.rsplit("/", 1)[-1]
+    aliases.add(short)
+    lowered = short.lower()
+    aliases.add(lowered)
     if lowered.endswith("-instruct"):
         aliases.add(lowered.removesuffix("-instruct"))
     return aliases
@@ -542,8 +542,8 @@ class _NodeEntry:
         "model_tokens_per_sec",
         "pending_directives", "last_heartbeat", "tracker_mode",
         "relay_addr", "cert_fingerprint", "peer_id",
-        # heartbeat stats (reported by node, cumulative)
-        "total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
+        # heartbeat stats (reported by node, cumulative)
+        "total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
         "status",  # "ready" | "loading"
         "heartbeats_expected", "heartbeats_received",
         # dynamic reassignment queued by the tracker
@@ -604,18 +604,18 @@ class _NodeEntry:
         self.peer_id = peer_id
         self.pending_directives: list[dict] = []
         self.last_heartbeat: float = time.monotonic()
-        self.total_requests: int = 0
-        self.failed_requests: int = 0
-        self.queue_depth: int = 0
-        self.proxy_inflight: int = 0
-        self.uptime_seconds: float = 0.0
+        self.total_requests: int = 0
+        self.failed_requests: int = 0
+        self.queue_depth: int = 0
+        self.proxy_inflight: int = 0
+        self.uptime_seconds: float = 0.0
         self.status: str = "ready"
         self.heartbeats_expected: int = 0
         self.heartbeats_received: int = 0
         self.pending_new_assignment: dict | None = None
 
 
-def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
+def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
     """Effective tokens/s accounting for current queue depth."""
     observed = None
     if model:
@@ -626,22 +626,22 @@ def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float
                 if observed is not None:
                     break
     base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
-    return base / (_effective_queue_depth(node) + 1)
-
-
-def _effective_queue_depth(node: "_NodeEntry") -> int:
-    """Best current load estimate: heartbeat queue or tracker-routed in-flight."""
-    return max(node.queue_depth, node.proxy_inflight)
-
-
-def _record_proxy_inflight(
-    server: "_TrackerHTTPServer",
-    nodes: list["_NodeEntry"],
-    delta: int,
-) -> None:
-    with server.lock:
-        for node in nodes:
-            node.proxy_inflight = max(0, node.proxy_inflight + delta)
+    return base / (_effective_queue_depth(node) + 1)
+
+
+def _effective_queue_depth(node: "_NodeEntry") -> int:
+    """Best current load estimate: heartbeat queue or tracker-routed in-flight."""
+    return max(node.queue_depth, node.proxy_inflight)
+
+
+def _record_proxy_inflight(
+    server: "_TrackerHTTPServer",
+    nodes: list["_NodeEntry"],
+    delta: int,
+) -> None:
+    with server.lock:
+        for node in nodes:
+            node.proxy_inflight = max(0, node.proxy_inflight + delta)
 
 
 def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
@@ -707,12 +707,12 @@ def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
             "model": node.model,
             "hf_repo": node.hf_repo,
             "endpoint": node.endpoint,
-            "shard": f"{node.shard_start}-{node.shard_end}",
-            "num_layers": node.num_layers,
-            "queue_depth": _effective_queue_depth(node),
-            "heartbeat_queue_depth": node.queue_depth,
-            "proxy_inflight": node.proxy_inflight,
-        }
+            "shard": f"{node.shard_start}-{node.shard_end}",
+            "num_layers": node.num_layers,
+            "queue_depth": _effective_queue_depth(node),
+            "heartbeat_queue_depth": node.queue_depth,
+            "proxy_inflight": node.proxy_inflight,
+        }
         for node in nodes
     ]
 
@@ -749,11 +749,11 @@ def _coverage_percentage(
     return round((covered / required_layers) * 100, 2)
 
 
-def _served_model_copies(
-    nodes: list[_NodeEntry],
-    required_start: int,
-    required_end: int,
-) -> float:
+def _served_model_copies(
+    nodes: list[_NodeEntry],
+    required_start: int,
+    required_end: int,
+) -> float:
     required_layers = required_end - required_start + 1
     if required_layers <= 0:
         return 0.0
@@ -772,60 +772,60 @@ def _served_model_copies(
         return 0.0
 
     complete_copies = min(layer_counts)
-    residual_layers = sum(1 for count in layer_counts if count > complete_copies)
-    return round(complete_copies + (residual_layers / required_layers), 2)
-
-
-def _model_health_summary(
-    server: "_TrackerHTTPServer",
-    model: str | None,
-    hf_repo: str | None = None,
-) -> dict:
-    lookup = hf_repo or model
-    if not lookup:
-        return {}
-
-    resolved_name, preset = _resolve_model_preset(server.model_presets, lookup)
-    if preset is not None:
-        required_start, required_end = _preset_layer_bounds(preset)
-        model_nodes = [
-            node for node in server.registry.values()
-            if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
-        ]
-        model_id = resolved_name or lookup
-    else:
-        model_nodes = [
-            node for node in server.registry.values()
-            if _node_matches_model(node, lookup)
-            and node.shard_start is not None
-            and node.shard_end is not None
-            and node.num_layers is not None
-        ]
-        if not model_nodes:
-            return {
-                "model": lookup,
-                "served_model_copies": 0.0,
-                "coverage_percentage": 0.0,
-                "node_count": 0,
-            }
-        required_start = 0
-        required_end = max(node.num_layers for node in model_nodes if node.num_layers is not None) - 1
-        model_id = hf_repo or model or lookup
-
-    return {
-        "model": model_id,
-        "required_start": required_start,
-        "required_end": required_end,
-        "served_model_copies": _served_model_copies(model_nodes, required_start, required_end),
-        "coverage_percentage": _coverage_percentage(model_nodes, required_start, required_end),
-        "node_count": len(model_nodes),
-    }
-
-
-def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
-    start = int(preset.get("layers_start", 0))
-    if "layers_end" in preset:
-        return start, int(preset["layers_end"])
+    residual_layers = sum(1 for count in layer_counts if count > complete_copies)
+    return round(complete_copies + (residual_layers / required_layers), 2)
+
+
+def _model_health_summary(
+    server: "_TrackerHTTPServer",
+    model: str | None,
+    hf_repo: str | None = None,
+) -> dict:
+    lookup = hf_repo or model
+    if not lookup:
+        return {}
+
+    resolved_name, preset = _resolve_model_preset(server.model_presets, lookup)
+    if preset is not None:
+        required_start, required_end = _preset_layer_bounds(preset)
+        model_nodes = [
+            node for node in server.registry.values()
+            if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
+        ]
+        model_id = resolved_name or lookup
+    else:
+        model_nodes = [
+            node for node in server.registry.values()
+            if _node_matches_model(node, lookup)
+            and node.shard_start is not None
+            and node.shard_end is not None
+            and node.num_layers is not None
+        ]
+        if not model_nodes:
+            return {
+                "model": lookup,
+                "served_model_copies": 0.0,
+                "coverage_percentage": 0.0,
+                "node_count": 0,
+            }
+        required_start = 0
+        required_end = max(node.num_layers for node in model_nodes if node.num_layers is not None) - 1
+        model_id = hf_repo or model or lookup
+
+    return {
+        "model": model_id,
+        "required_start": required_start,
+        "required_end": required_end,
+        "served_model_copies": _served_model_copies(model_nodes, required_start, required_end),
+        "coverage_percentage": _coverage_percentage(model_nodes, required_start, required_end),
+        "node_count": len(model_nodes),
+    }
+
+
+def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
+    start = int(preset.get("layers_start", 0))
+    if "layers_end" in preset:
+        return start, int(preset["layers_end"])
     return start, start + int(preset["total_layers"]) - 1
 
 
@@ -986,13 +986,13 @@ def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
     return {
         "node_id": node.node_id,
         "endpoint": node.endpoint,
-        "alive": alive,
-        "last_seen_seconds_ago": round(age, 1),
-        "status": node.status,
-        "queue_depth": _effective_queue_depth(node),
-        "heartbeat_queue_depth": node.queue_depth,
-        "proxy_inflight": node.proxy_inflight,
-        "total_requests": node.total_requests,
+        "alive": alive,
+        "last_seen_seconds_ago": round(age, 1),
+        "status": node.status,
+        "queue_depth": _effective_queue_depth(node),
+        "heartbeat_queue_depth": node.queue_depth,
+        "proxy_inflight": node.proxy_inflight,
+        "total_requests": node.total_requests,
         "heartbeat_success_rate": hb_rate,
         "inference_success_rate": inf_rate,
         "capacity": _node_capacity_summary(node),
@@ -1357,32 +1357,32 @@ def _drop_directive(node: _NodeEntry, model: str, start: int, end: int, quantiza
     }
 
 
-def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
-    now = time.monotonic()
-    expired_ids = [
-        node_id for node_id, entry in server.registry.items()
-        if (now - entry.last_heartbeat) > server.heartbeat_timeout
-    ]
-    expired_entries: list[tuple[str, _NodeEntry]] = []
-    for node_id in expired_ids:
-        entry = server.registry.pop(node_id)
-        expired_entries.append((node_id, entry))
-    if expired_ids:
-        _rebalance_all_locked(server)
-    for node_id, entry in expired_entries:
-        _tracker_log(
-            server,
-            "warn",
-            "node expired",
-            node_id=node_id,
-            endpoint=entry.endpoint,
-            model=entry.model,
-            hf_repo=entry.hf_repo,
-            shard=f"{entry.shard_start}-{entry.shard_end}",
-            heartbeat_timeout_seconds=server.heartbeat_timeout,
-            model_health=_model_health_summary(server, entry.model, entry.hf_repo),
-        )
-    return expired_ids
+def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
+    now = time.monotonic()
+    expired_ids = [
+        node_id for node_id, entry in server.registry.items()
+        if (now - entry.last_heartbeat) > server.heartbeat_timeout
+    ]
+    expired_entries: list[tuple[str, _NodeEntry]] = []
+    for node_id in expired_ids:
+        entry = server.registry.pop(node_id)
+        expired_entries.append((node_id, entry))
+    if expired_ids:
+        _rebalance_all_locked(server)
+    for node_id, entry in expired_entries:
+        _tracker_log(
+            server,
+            "warn",
+            "node expired",
+            node_id=node_id,
+            endpoint=entry.endpoint,
+            model=entry.model,
+            hf_repo=entry.hf_repo,
+            shard=f"{entry.shard_start}-{entry.shard_end}",
+            heartbeat_timeout_seconds=server.heartbeat_timeout,
+            model_health=_model_health_summary(server, entry.model, entry.hf_repo),
+        )
+    return expired_ids
 
 
 def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
@@ -1745,7 +1745,7 @@ def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -
     return None
 
 
-def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
+def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
     event = {
         "ts": time.time(),
         "level": level,
@@ -1759,40 +1759,40 @@ def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **field
     with server.console_lock:
         server.console_events.append(event)
     extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
-    suffix = f"  {extras}" if extras else ""
-    print(f"[tracker] {level}: {message}{suffix}", flush=True)
-
-
-def _tracker_log_proxy_progress(
-    server: "_TrackerHTTPServer",
-    *,
-    request_id: str,
-    model: str,
-    route_model: str,
-    tokens: int,
-    started: float,
-    route_nodes: list["_NodeEntry"],
-    stream: bool = True,
-    relay: bool = False,
-) -> None:
-    elapsed = time.monotonic() - started
-    _tracker_log(
-        server,
-        "info",
-        "proxy progress",
-        request_id=request_id,
-        model=model,
-        route_model=route_model,
-        stream=stream,
-        relay=relay or None,
-        tokens=tokens,
-        elapsed_seconds=round(elapsed, 4),
-        tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
-        route=_node_route_summary(route_nodes),
-    )
-
-
-def _node_id_for_registration(
+    suffix = f"  {extras}" if extras else ""
+    print(f"[tracker] {level}: {message}{suffix}", flush=True)
+
+
+def _tracker_log_proxy_progress(
+    server: "_TrackerHTTPServer",
+    *,
+    request_id: str,
+    model: str,
+    route_model: str,
+    tokens: int,
+    started: float,
+    route_nodes: list["_NodeEntry"],
+    stream: bool = True,
+    relay: bool = False,
+) -> None:
+    elapsed = time.monotonic() - started
+    _tracker_log(
+        server,
+        "info",
+        "proxy progress",
+        request_id=request_id,
+        model=model,
+        route_model=route_model,
+        stream=stream,
+        relay=relay or None,
+        tokens=tokens,
+        elapsed_seconds=round(elapsed, 4),
+        tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
+        route=_node_route_summary(route_nodes),
+    )
+
+
+def _node_id_for_registration(
     endpoint: str,
     model: str,
     wallet_address: str | None,
@@ -2273,8 +2273,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                 result[model] = server.stats.get_node_model_stats(node.node_id, model)
             return result
 
-        def model_supply_for(node: _NodeEntry) -> dict:
-            return _model_health_summary(server, node.model, node.hf_repo)
+        def model_supply_for(node: _NodeEntry) -> dict:
+            return _model_health_summary(server, node.model, node.hf_repo)
 
         self._send_json(200, {
             "relay_url": server.relay_url,
@@ -2537,23 +2537,23 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         # Strip the first-shard node we're about to proxy to — it's already handling the request.
         downstream_hops = [h for h in route_hops if h["endpoint"].rstrip("/") != node.endpoint.rstrip("/")]
         downstream_urls = json.dumps(downstream_hops)
-        route_debug = " -> ".join(
-            f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
-            for n in route_nodes
-        )
-        inflight_nodes = route_nodes or [node]
-        inflight_recorded = True
-        _record_proxy_inflight(server, inflight_nodes, 1)
-
-        def finish_proxy_inflight() -> None:
-            nonlocal inflight_recorded
-            if inflight_recorded:
-                _record_proxy_inflight(server, inflight_nodes, -1)
-                inflight_recorded = False
-
-        _tracker_log(
-            server,
-            "info",
+        route_debug = " -> ".join(
+            f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
+            for n in route_nodes
+        )
+        inflight_nodes = route_nodes or [node]
+        inflight_recorded = True
+        _record_proxy_inflight(server, inflight_nodes, 1)
+
+        def finish_proxy_inflight() -> None:
+            nonlocal inflight_recorded
+            if inflight_recorded:
+                _record_proxy_inflight(server, inflight_nodes, -1)
+                inflight_recorded = False
+
+        _tracker_log(
+            server,
+            "info",
             "proxy route selected",
             request_id=request_id,
             model=model,
@@ -2604,15 +2604,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             if first is not None and first.get("stream"):
                 # Streamed response (US-036): forward SSE chunks as they arrive
                 # and run the same token accounting as the direct stream path.
-                self._stream_relayed_frames(
-                    first, frames, started,
-                    model, route_model, route_nodes, api_key, node_work,
-                    request_body=body,
-                )
-                finish_proxy_inflight()
-                return
-            if first is not None:
-                elapsed = time.monotonic() - started
+                self._stream_relayed_frames(
+                    first, frames, started,
+                    model, route_model, route_nodes, api_key, node_work,
+                    request_body=body,
+                )
+                finish_proxy_inflight()
+                return
+            if first is not None:
+                elapsed = time.monotonic() - started
                 self._send_relayed_response(first)
                 if int(first.get("status", 503)) < 400:
                     body_text = first.get("body") or ""
@@ -2635,12 +2635,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                         tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
                         route=_node_route_summary(route_nodes),
                     )
-                    self._bill_completed(
-                        api_key, model, tokens, node_work,
-                        input_tokens=in_tokens, output_tokens=out_tokens,
-                    )
-                finish_proxy_inflight()
-                return
+                    self._bill_completed(
+                        api_key, model, tokens, node_work,
+                        input_tokens=in_tokens, output_tokens=out_tokens,
+                    )
+                finish_proxy_inflight()
+                return
             _tracker_log(
                 server,
                 "warn",
@@ -2661,12 +2661,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             self.send_header("Content-Type", "application/json")
             self.send_header("Content-Length", str(len(err_body)))
             self.end_headers()
-            try:
-                self.wfile.write(err_body)
-            except BrokenPipeError:
-                pass
-            finish_proxy_inflight()
-            return
+            try:
+                self.wfile.write(err_body)
+            except BrokenPipeError:
+                pass
+            finish_proxy_inflight()
+            return
         except Exception as exc:
             if node.relay_addr:
                 _tracker_log(
@@ -2687,13 +2687,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                     target_url=target_url,
                     error=repr(exc),
                 )
-            self._send_json(503, {"error": {
-                "message": f"upstream node unreachable: {exc}",
-                "type": "service_unavailable",
-                "code": "upstream_error",
-            }})
-            finish_proxy_inflight()
-            return
+            self._send_json(503, {"error": {
+                "message": f"upstream node unreachable: {exc}",
+                "type": "service_unavailable",
+                "code": "upstream_error",
+            }})
+            finish_proxy_inflight()
+            return
 
         with upstream:
             content_type = upstream.headers.get("Content-Type", "application/json")
@@ -2784,13 +2784,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
                 self.send_header("Content-Type", content_type)
                 self.send_header("Content-Length", str(len(resp_body)))
                 self.end_headers()
-                try:
-                    self.wfile.write(resp_body)
-                except BrokenPipeError:
-                    pass
-        finish_proxy_inflight()
-
-    def _record_observed_throughput(
+                try:
+                    self.wfile.write(resp_body)
+                except BrokenPipeError:
+                    pass
+        finish_proxy_inflight()
+
+    def _record_observed_throughput(
         self,
         requested_model: str,
         route_model: str,
@@ -3175,50 +3175,50 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         with server.lock:
             self._purge_expired_nodes()
             # Dedup: if this endpoint is already registered, remove the old entry first.
-            stale_ids = [
-                eid for eid, e in server.registry.items()
-                if e.endpoint == entry.endpoint.rstrip("/")
-            ]
-            stale_entries: list[tuple[str, _NodeEntry]] = []
-            for eid in stale_ids:
-                old = server.registry.pop(eid)
-                stale_entries.append((eid, old))
-            server.registry[node_id] = entry
-            if entry.managed_assignment:
-                if entry.hf_repo:
-                    _rebalance_hf_model_locked(server, entry.hf_repo)
-                else:
+            stale_ids = [
+                eid for eid, e in server.registry.items()
+                if e.endpoint == entry.endpoint.rstrip("/")
+            ]
+            stale_entries: list[tuple[str, _NodeEntry]] = []
+            for eid in stale_ids:
+                old = server.registry.pop(eid)
+                stale_entries.append((eid, old))
+            server.registry[node_id] = entry
+            if entry.managed_assignment:
+                if entry.hf_repo:
+                    _rebalance_hf_model_locked(server, entry.hf_repo)
+                else:
                     _rebalance_model_locked(server, model)
             assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
-            if assignment_directive is not None:
-                entry.pending_directives.clear()
-
-            model_health = _model_health_summary(server, entry.model, entry.hf_repo)
-            for eid, old in stale_entries:
-                _tracker_log(
-                    server,
-                    "info",
-                    "node re-registered",
-                    old_node_id=eid,
-                    node_id=node_id,
-                    endpoint=old.endpoint,
-                    old_model=old.model,
-                    old_hf_repo=old.hf_repo,
-                    old_model_health=_model_health_summary(server, old.model, old.hf_repo),
-                    model_health=model_health,
-                )
-            _tracker_log(
-                server,
-                "info",
-                "node registered",
-                node_id=node_id,
+            if assignment_directive is not None:
+                entry.pending_directives.clear()
+
+            model_health = _model_health_summary(server, entry.model, entry.hf_repo)
+            for eid, old in stale_entries:
+                _tracker_log(
+                    server,
+                    "info",
+                    "node re-registered",
+                    old_node_id=eid,
+                    node_id=node_id,
+                    endpoint=old.endpoint,
+                    old_model=old.model,
+                    old_hf_repo=old.hf_repo,
+                    old_model_health=_model_health_summary(server, old.model, old.hf_repo),
+                    model_health=model_health,
+                )
+            _tracker_log(
+                server,
+                "info",
+                "node registered",
+                node_id=node_id,
                 endpoint=entry.endpoint,
                 model=entry.model,
-                hf_repo=entry.hf_repo,
-                shard=f"{entry.shard_start}-{entry.shard_end}",
-                tracker_mode=entry.tracker_mode,
-                model_health=model_health,
-            )
+                hf_repo=entry.hf_repo,
+                shard=f"{entry.shard_start}-{entry.shard_end}",
+                tracker_mode=entry.tracker_mode,
+                model_health=model_health,
+            )
 
         shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
         repo_info = f" [{hf_repo}]" if hf_repo else ""
@@ -3560,7 +3560,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             return
         keys = server.accounts.keys_for(account["account_id"])  # type: ignore[union-attr]
         balances = {}
-        usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": []}
+        usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": [], "records": []}
         if server.billing is not None:
             balances = {key: server.billing.get_client_balance(key) for key in keys}
             usage = server.billing.usage_for(keys)