popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge origin/master: streaming progress, dashboard call wall, and heartbeat scaffolding.

Browse Source 
Resolve conflicts in dashboard.html (Call wall + live TPS/queue from remote) and server.py (proxy progress logging, request id forwarding, current_requests on node entries).

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Dobromir Popov
2026-07-07 17:44:18 +03:00
parent e81d989f39 3eb7c6b93e
commit 08e9c22ccf
5 changed files with 199 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
packages/tracker/meshnet_tracker/server.py
View File

@@ -544,6 +544,7 @@ class _NodeEntry:
"relay_addr", "cert_fingerprint", "peer_id",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
"current_requests",
"status", # "ready" | "loading"
"heartbeats_expected", "heartbeats_received",
# dynamic reassignment queued by the tracker
@@ -608,6 +609,7 @@ class _NodeEntry:
self.failed_requests: int = 0
self.queue_depth: int = 0
self.proxy_inflight: int = 0
self.current_requests: list[dict] = []
self.uptime_seconds: float = 0.0
self.status: str = "ready"
self.heartbeats_expected: int = 0
@@ -634,6 +636,40 @@ def _effective_queue_depth(node: "_NodeEntry") -> int:
return max(node.queue_depth, node.proxy_inflight)
_CURRENT_REQUEST_FIELDS = frozenset({
"request_id", "model", "kind", "tokens", "tokens_per_sec",
"elapsed_seconds", "routing_complete",
})
def _normalize_current_requests(items: object, *, limit: int = 32) -> list[dict]:
"""Sanitize node-reported in-flight request snapshots from heartbeats."""
if not isinstance(items, list):
return []
out: list[dict] = []
for item in items[:limit]:
if not isinstance(item, dict):
continue
request_id = item.get("request_id")
if not request_id:
continue
rec: dict = {"request_id": str(request_id)}
for key in _CURRENT_REQUEST_FIELDS:
if key == "request_id" or key not in item:
continue
value = item[key]
if key in {"tokens"}:
rec[key] = int(value)
elif key in {"tokens_per_sec", "elapsed_seconds"}:
rec[key] = float(value)
elif key == "routing_complete":
rec[key] = bool(value)
else:
rec[key] = str(value)
out.append(rec)
return out
def _record_proxy_inflight(
server: "_TrackerHTTPServer",
nodes: list["_NodeEntry"],
@@ -712,6 +748,7 @@ def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
"current_requests": list(node.current_requests),
}
for node in nodes
]
@@ -992,6 +1029,7 @@ def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
"current_requests": list(node.current_requests),
"total_requests": node.total_requests,
"heartbeat_success_rate": hb_rate,
"inference_success_rate": inf_rate,
@@ -1194,10 +1232,11 @@ def _billable_non_stream_split(payload: dict, request_body: dict) -> tuple[int,
Completion stays capped by the request's max-tokens bound, as before.
"""
usage = _usage_split(payload)
prompt_estimate = _estimate_prompt_tokens(request_body) or 0
prompt = (usage or {}).get("prompt")
completion = (usage or {}).get("completion")
if prompt is None:
prompt = _estimate_prompt_tokens(request_body) or 0
prompt = prompt_estimate
if completion is None:
total = (usage or {}).get("total")
if total is not None:
@@ -1205,8 +1244,9 @@ def _billable_non_stream_split(payload: dict, request_body: dict) -> tuple[int,
else:
completion = _observed_non_stream_completion_tokens(payload)
limit = _requested_completion_token_limit(request_body)
if limit is not None:
if limit is not None and completion > limit:
completion = min(completion, limit)
prompt = max(prompt, prompt_estimate)
return max(0, prompt), max(0, completion)
@@ -1776,6 +1816,7 @@ def _tracker_log_proxy_progress(
relay: bool = False,
) -> None:
elapsed = time.monotonic() - started
effective_elapsed = max(elapsed, 1e-6)
_tracker_log(
server,
"info",
@@ -1787,7 +1828,7 @@ def _tracker_log_proxy_progress(
relay=relay or None,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
tokens_per_sec=round(tokens / effective_elapsed, 4) if tokens > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
@@ -2476,6 +2517,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
node = max(candidates, key=lambda n: _effective_throughput(n, model))
target_url = f"{node.endpoint}/v1/chat/completions"
request_id = str(body.get("id") or f"req-{time.time_ns():x}")
body["id"] = request_id
raw_body = json.dumps(body).encode()
# Pre-resolve the downstream route so the first-shard node skips its own
# tracker query. We already hold the full registry picture — no need for
@@ -2608,6 +2651,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
first, frames, started,
model, route_model, route_nodes, api_key, node_work,
request_body=body,
request_id=request_id,
)
finish_proxy_inflight()
return
@@ -2705,18 +2749,34 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.end_headers()
stream_usage: dict | None = None
observed_stream_tokens = 0
client_gone = False
try:
while True:
line = upstream.readline()
if not line:
break
self.wfile.write(line)
self.wfile.flush()
if not client_gone:
try:
self.wfile.write(line)
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
# Keep draining upstream so completed node work is still billed.
client_gone = True
observed, usage = _stream_line_tokens(line)
observed_stream_tokens += observed
if observed:
_tracker_log_proxy_progress(
server,
request_id=request_id,
model=model,
route_model=route_model,
tokens=observed_stream_tokens,
started=started,
route_nodes=route_nodes,
)
if usage is not None:
stream_usage = usage
except BrokenPipeError:
except (BrokenPipeError, ConnectionResetError):
pass
elapsed = time.monotonic() - started
# Bill even on client disconnect — the nodes did the work.
@@ -2786,7 +2846,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.end_headers()
try:
self.wfile.write(resp_body)
except BrokenPipeError:
except (BrokenPipeError, ConnectionResetError):
pass
finish_proxy_inflight()
@@ -2805,8 +2865,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
can refine this later without changing the external stats shape.
"""
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if server.stats is None or total_tokens <= 0 or elapsed_seconds <= 0:
if server.stats is None or total_tokens <= 0:
return
elapsed_seconds = max(elapsed_seconds, 1e-6)
models = [m for m in (requested_model, route_model) if m]
if len(models) == 2 and models[0] == models[1]:
models = [models[0]]
@@ -2917,6 +2978,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
api_key: str | None,
node_work: list,
request_body: dict,
request_id: str,
) -> None:
"""Forward a streamed relay response (US-036) to the client as SSE,
billing with the same accounting as the direct stream path."""
@@ -2925,6 +2987,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", headers.get("Content-Type", "text/event-stream; charset=utf-8"))
self.send_header("Cache-Control", "no-cache")
self.end_headers()
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
stream_usage: dict | None = None
observed_stream_tokens = 0
client_gone = False
@@ -2943,6 +3006,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
for line in data.splitlines():
observed, usage = _stream_line_tokens(line)
observed_stream_tokens += observed
if observed:
_tracker_log_proxy_progress(
server,
request_id=request_id,
model=model,
route_model=route_model,
tokens=observed_stream_tokens,
started=started,
route_nodes=route_nodes,
relay=True,
)
if usage is not None:
stream_usage = usage
elapsed = time.monotonic() - started
@@ -2953,12 +3027,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
)
tokens = in_tokens + out_tokens
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
_tracker_log(
server,
"info",
"proxy complete",
request_id=session_id,
request_id=request_id,
model=model,
route_model=route_model,
status=200,
@@ -3279,6 +3352,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
entry.failed_requests = int(body["failed_requests"])
if "queue_depth" in body:
entry.queue_depth = int(body["queue_depth"])
if "current_requests" in body:
entry.current_requests = _normalize_current_requests(body["current_requests"])
if "uptime_seconds" in body:
entry.uptime_seconds = float(body["uptime_seconds"])
if "status" in body and body["status"] in ("ready", "loading"):