popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better dash and inference api QOL

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-07 15:51:27 +02:00
parent 938a0a721b
commit 640ef78711
4 changed files with 404 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
packages/tracker/meshnet_tracker/server.py
View File

@@ -142,15 +142,18 @@ DEFAULT_CALLER_CREDIT_USDT = 1.0
DEFAULT_DEVNET_TOPUP_USDT = 1.0
def _model_aliases(model: str | None) -> set[str]:
"""Return stable lookup aliases for a model repo or display name."""
if not model:
return set()
aliases = {model}
short = model.rsplit("/", 1)[-1]
aliases.add(short)
lowered = short.lower()
aliases.add(lowered)
def _model_aliases(model: str | None) -> set[str]:
"""Return stable lookup aliases for a model repo or display name."""
if not model:
return set()
normalized = model.strip()
if not normalized:
return set()
aliases = {normalized}
short = normalized.rsplit("/", 1)[-1]
aliases.add(short)
lowered = short.lower()
aliases.add(lowered)
if lowered.endswith("-instruct"):
aliases.add(lowered.removesuffix("-instruct"))
return aliases
@@ -539,8 +542,8 @@ class _NodeEntry:
"model_tokens_per_sec",
"pending_directives", "last_heartbeat", "tracker_mode",
"relay_addr", "cert_fingerprint", "peer_id",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "uptime_seconds",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
"status", # "ready" | "loading"
"heartbeats_expected", "heartbeats_received",
# dynamic reassignment queued by the tracker
@@ -601,17 +604,18 @@ class _NodeEntry:
self.peer_id = peer_id
self.pending_directives: list[dict] = []
self.last_heartbeat: float = time.monotonic()
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
self.uptime_seconds: float = 0.0
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
self.proxy_inflight: int = 0
self.uptime_seconds: float = 0.0
self.status: str = "ready"
self.heartbeats_expected: int = 0
self.heartbeats_received: int = 0
self.pending_new_assignment: dict | None = None
def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
"""Effective tokens/s accounting for current queue depth."""
observed = None
if model:
@@ -622,7 +626,22 @@ def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float
if observed is not None:
break
base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
return base / (node.queue_depth + 1)
return base / (_effective_queue_depth(node) + 1)
def _effective_queue_depth(node: "_NodeEntry") -> int:
"""Best current load estimate: heartbeat queue or tracker-routed in-flight."""
return max(node.queue_depth, node.proxy_inflight)
def _record_proxy_inflight(
server: "_TrackerHTTPServer",
nodes: list["_NodeEntry"],
delta: int,
) -> None:
with server.lock:
for node in nodes:
node.proxy_inflight = max(0, node.proxy_inflight + delta)
def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
@@ -688,10 +707,12 @@ def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
"model": node.model,
"hf_repo": node.hf_repo,
"endpoint": node.endpoint,
"shard": f"{node.shard_start}-{node.shard_end}",
"num_layers": node.num_layers,
"queue_depth": node.queue_depth,
}
"shard": f"{node.shard_start}-{node.shard_end}",
"num_layers": node.num_layers,
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
}
for node in nodes
]
@@ -965,11 +986,13 @@ def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
return {
"node_id": node.node_id,
"endpoint": node.endpoint,
"alive": alive,
"last_seen_seconds_ago": round(age, 1),
"status": node.status,
"queue_depth": node.queue_depth,
"total_requests": node.total_requests,
"alive": alive,
"last_seen_seconds_ago": round(age, 1),
"status": node.status,
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
"total_requests": node.total_requests,
"heartbeat_success_rate": hb_rate,
"inference_success_rate": inf_rate,
"capacity": _node_capacity_summary(node),
@@ -1722,7 +1745,7 @@ def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -
return None
def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
event = {
"ts": time.time(),
"level": level,
@@ -1736,11 +1759,40 @@ def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **field
with server.console_lock:
server.console_events.append(event)
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
suffix = f" {extras}" if extras else ""
print(f"[tracker] {level}: {message}{suffix}", flush=True)
def _node_id_for_registration(
suffix = f" {extras}" if extras else ""
print(f"[tracker] {level}: {message}{suffix}", flush=True)
def _tracker_log_proxy_progress(
server: "_TrackerHTTPServer",
*,
request_id: str,
model: str,
route_model: str,
tokens: int,
started: float,
route_nodes: list["_NodeEntry"],
stream: bool = True,
relay: bool = False,
) -> None:
elapsed = time.monotonic() - started
_tracker_log(
server,
"info",
"proxy progress",
request_id=request_id,
model=model,
route_model=route_model,
stream=stream,
relay=relay or None,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
def _node_id_for_registration(
endpoint: str,
model: str,
wallet_address: str | None,
@@ -2485,13 +2537,23 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Strip the first-shard node we're about to proxy to — it's already handling the request.
downstream_hops = [h for h in route_hops if h["endpoint"].rstrip("/") != node.endpoint.rstrip("/")]
downstream_urls = json.dumps(downstream_hops)
route_debug = " -> ".join(
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
for n in route_nodes
)
_tracker_log(
server,
"info",
route_debug = " -> ".join(
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
for n in route_nodes
)
inflight_nodes = route_nodes or [node]
inflight_recorded = True
_record_proxy_inflight(server, inflight_nodes, 1)
def finish_proxy_inflight() -> None:
nonlocal inflight_recorded
if inflight_recorded:
_record_proxy_inflight(server, inflight_nodes, -1)
inflight_recorded = False
_tracker_log(
server,
"info",
"proxy route selected",
request_id=request_id,
model=model,
@@ -2542,14 +2604,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if first is not None and first.get("stream"):
# Streamed response (US-036): forward SSE chunks as they arrive
# and run the same token accounting as the direct stream path.
self._stream_relayed_frames(
first, frames, started,
model, route_model, route_nodes, api_key, node_work,
request_body=body,
)
return
if first is not None:
elapsed = time.monotonic() - started
self._stream_relayed_frames(
first, frames, started,
model, route_model, route_nodes, api_key, node_work,
request_body=body,
)
finish_proxy_inflight()
return
if first is not None:
elapsed = time.monotonic() - started
self._send_relayed_response(first)
if int(first.get("status", 503)) < 400:
body_text = first.get("body") or ""
@@ -2572,11 +2635,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
return
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
finish_proxy_inflight()
return
_tracker_log(
server,
"warn",
@@ -2597,11 +2661,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(err_body)))
self.end_headers()
try:
self.wfile.write(err_body)
except BrokenPipeError:
pass
return
try:
self.wfile.write(err_body)
except BrokenPipeError:
pass
finish_proxy_inflight()
return
except Exception as exc:
if node.relay_addr:
_tracker_log(
@@ -2622,12 +2687,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
target_url=target_url,
error=repr(exc),
)
self._send_json(503, {"error": {
"message": f"upstream node unreachable: {exc}",
"type": "service_unavailable",
"code": "upstream_error",
}})
return
self._send_json(503, {"error": {
"message": f"upstream node unreachable: {exc}",
"type": "service_unavailable",
"code": "upstream_error",
}})
finish_proxy_inflight()
return
with upstream:
content_type = upstream.headers.get("Content-Type", "application/json")
@@ -2718,12 +2784,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(resp_body)))
self.end_headers()
try:
self.wfile.write(resp_body)
except BrokenPipeError:
pass
def _record_observed_throughput(
try:
self.wfile.write(resp_body)
except BrokenPipeError:
pass
finish_proxy_inflight()
def _record_observed_throughput(
self,
requested_model: str,
route_model: str,