popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dash QOL

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-07 17:37:38 +03:00
parent 640ef78711
commit e81d989f39
12 changed files with 1392 additions and 358 deletions
Download Patch File Download Diff File Expand all files Collapse all files

528
packages/tracker/meshnet_tracker/server.py
View File

@@ -142,18 +142,18 @@ DEFAULT_CALLER_CREDIT_USDT = 1.0
DEFAULT_DEVNET_TOPUP_USDT = 1.0
def _model_aliases(model: str | None) -> set[str]:
"""Return stable lookup aliases for a model repo or display name."""
if not model:
return set()
normalized = model.strip()
if not normalized:
return set()
aliases = {normalized}
short = normalized.rsplit("/", 1)[-1]
aliases.add(short)
lowered = short.lower()
aliases.add(lowered)
def _model_aliases(model: str | None) -> set[str]:
"""Return stable lookup aliases for a model repo or display name."""
if not model:
return set()
normalized = model.strip()
if not normalized:
return set()
aliases = {normalized}
short = normalized.rsplit("/", 1)[-1]
aliases.add(short)
lowered = short.lower()
aliases.add(lowered)
if lowered.endswith("-instruct"):
aliases.add(lowered.removesuffix("-instruct"))
return aliases
@@ -542,8 +542,8 @@ class _NodeEntry:
"model_tokens_per_sec",
"pending_directives", "last_heartbeat", "tracker_mode",
"relay_addr", "cert_fingerprint", "peer_id",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
"status", # "ready" | "loading"
"heartbeats_expected", "heartbeats_received",
# dynamic reassignment queued by the tracker
@@ -604,18 +604,18 @@ class _NodeEntry:
self.peer_id = peer_id
self.pending_directives: list[dict] = []
self.last_heartbeat: float = time.monotonic()
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
self.proxy_inflight: int = 0
self.uptime_seconds: float = 0.0
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
self.proxy_inflight: int = 0
self.uptime_seconds: float = 0.0
self.status: str = "ready"
self.heartbeats_expected: int = 0
self.heartbeats_received: int = 0
self.pending_new_assignment: dict | None = None
def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
"""Effective tokens/s accounting for current queue depth."""
observed = None
if model:
@@ -626,22 +626,22 @@ def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float
if observed is not None:
break
base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
return base / (_effective_queue_depth(node) + 1)
def _effective_queue_depth(node: "_NodeEntry") -> int:
"""Best current load estimate: heartbeat queue or tracker-routed in-flight."""
return max(node.queue_depth, node.proxy_inflight)
def _record_proxy_inflight(
server: "_TrackerHTTPServer",
nodes: list["_NodeEntry"],
delta: int,
) -> None:
with server.lock:
for node in nodes:
node.proxy_inflight = max(0, node.proxy_inflight + delta)
return base / (_effective_queue_depth(node) + 1)
def _effective_queue_depth(node: "_NodeEntry") -> int:
"""Best current load estimate: heartbeat queue or tracker-routed in-flight."""
return max(node.queue_depth, node.proxy_inflight)
def _record_proxy_inflight(
server: "_TrackerHTTPServer",
nodes: list["_NodeEntry"],
delta: int,
) -> None:
with server.lock:
for node in nodes:
node.proxy_inflight = max(0, node.proxy_inflight + delta)
def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
@@ -707,12 +707,12 @@ def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
"model": node.model,
"hf_repo": node.hf_repo,
"endpoint": node.endpoint,
"shard": f"{node.shard_start}-{node.shard_end}",
"num_layers": node.num_layers,
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
}
"shard": f"{node.shard_start}-{node.shard_end}",
"num_layers": node.num_layers,
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
}
for node in nodes
]
@@ -749,11 +749,11 @@ def _coverage_percentage(
return round((covered / required_layers) * 100, 2)
def _served_model_copies(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
) -> float:
def _served_model_copies(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
) -> float:
required_layers = required_end - required_start + 1
if required_layers <= 0:
return 0.0
@@ -772,60 +772,60 @@ def _served_model_copies(
return 0.0
complete_copies = min(layer_counts)
residual_layers = sum(1 for count in layer_counts if count > complete_copies)
return round(complete_copies + (residual_layers / required_layers), 2)
def _model_health_summary(
server: "_TrackerHTTPServer",
model: str | None,
hf_repo: str | None = None,
) -> dict:
lookup = hf_repo or model
if not lookup:
return {}
resolved_name, preset = _resolve_model_preset(server.model_presets, lookup)
if preset is not None:
required_start, required_end = _preset_layer_bounds(preset)
model_nodes = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
model_id = resolved_name or lookup
else:
model_nodes = [
node for node in server.registry.values()
if _node_matches_model(node, lookup)
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None
]
if not model_nodes:
return {
"model": lookup,
"served_model_copies": 0.0,
"coverage_percentage": 0.0,
"node_count": 0,
}
required_start = 0
required_end = max(node.num_layers for node in model_nodes if node.num_layers is not None) - 1
model_id = hf_repo or model or lookup
return {
"model": model_id,
"required_start": required_start,
"required_end": required_end,
"served_model_copies": _served_model_copies(model_nodes, required_start, required_end),
"coverage_percentage": _coverage_percentage(model_nodes, required_start, required_end),
"node_count": len(model_nodes),
}
def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
start = int(preset.get("layers_start", 0))
if "layers_end" in preset:
return start, int(preset["layers_end"])
residual_layers = sum(1 for count in layer_counts if count > complete_copies)
return round(complete_copies + (residual_layers / required_layers), 2)
def _model_health_summary(
server: "_TrackerHTTPServer",
model: str | None,
hf_repo: str | None = None,
) -> dict:
lookup = hf_repo or model
if not lookup:
return {}
resolved_name, preset = _resolve_model_preset(server.model_presets, lookup)
if preset is not None:
required_start, required_end = _preset_layer_bounds(preset)
model_nodes = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
model_id = resolved_name or lookup
else:
model_nodes = [
node for node in server.registry.values()
if _node_matches_model(node, lookup)
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None
]
if not model_nodes:
return {
"model": lookup,
"served_model_copies": 0.0,
"coverage_percentage": 0.0,
"node_count": 0,
}
required_start = 0
required_end = max(node.num_layers for node in model_nodes if node.num_layers is not None) - 1
model_id = hf_repo or model or lookup
return {
"model": model_id,
"required_start": required_start,
"required_end": required_end,
"served_model_copies": _served_model_copies(model_nodes, required_start, required_end),
"coverage_percentage": _coverage_percentage(model_nodes, required_start, required_end),
"node_count": len(model_nodes),
}
def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
start = int(preset.get("layers_start", 0))
if "layers_end" in preset:
return start, int(preset["layers_end"])
return start, start + int(preset["total_layers"]) - 1
@@ -986,13 +986,13 @@ def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
return {
"node_id": node.node_id,
"endpoint": node.endpoint,
"alive": alive,
"last_seen_seconds_ago": round(age, 1),
"status": node.status,
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
"total_requests": node.total_requests,
"alive": alive,
"last_seen_seconds_ago": round(age, 1),
"status": node.status,
"queue_depth": _effective_queue_depth(node),
"heartbeat_queue_depth": node.queue_depth,
"proxy_inflight": node.proxy_inflight,
"total_requests": node.total_requests,
"heartbeat_success_rate": hb_rate,
"inference_success_rate": inf_rate,
"capacity": _node_capacity_summary(node),
@@ -1357,32 +1357,32 @@ def _drop_directive(node: _NodeEntry, model: str, start: int, end: int, quantiza
}
def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
now = time.monotonic()
expired_ids = [
node_id for node_id, entry in server.registry.items()
if (now - entry.last_heartbeat) > server.heartbeat_timeout
]
expired_entries: list[tuple[str, _NodeEntry]] = []
for node_id in expired_ids:
entry = server.registry.pop(node_id)
expired_entries.append((node_id, entry))
if expired_ids:
_rebalance_all_locked(server)
for node_id, entry in expired_entries:
_tracker_log(
server,
"warn",
"node expired",
node_id=node_id,
endpoint=entry.endpoint,
model=entry.model,
hf_repo=entry.hf_repo,
shard=f"{entry.shard_start}-{entry.shard_end}",
heartbeat_timeout_seconds=server.heartbeat_timeout,
model_health=_model_health_summary(server, entry.model, entry.hf_repo),
)
return expired_ids
def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
now = time.monotonic()
expired_ids = [
node_id for node_id, entry in server.registry.items()
if (now - entry.last_heartbeat) > server.heartbeat_timeout
]
expired_entries: list[tuple[str, _NodeEntry]] = []
for node_id in expired_ids:
entry = server.registry.pop(node_id)
expired_entries.append((node_id, entry))
if expired_ids:
_rebalance_all_locked(server)
for node_id, entry in expired_entries:
_tracker_log(
server,
"warn",
"node expired",
node_id=node_id,
endpoint=entry.endpoint,
model=entry.model,
hf_repo=entry.hf_repo,
shard=f"{entry.shard_start}-{entry.shard_end}",
heartbeat_timeout_seconds=server.heartbeat_timeout,
model_health=_model_health_summary(server, entry.model, entry.hf_repo),
)
return expired_ids
def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
@@ -1745,7 +1745,7 @@ def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -
return None
def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
event = {
"ts": time.time(),
"level": level,
@@ -1759,40 +1759,40 @@ def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **field
with server.console_lock:
server.console_events.append(event)
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
suffix = f" {extras}" if extras else ""
print(f"[tracker] {level}: {message}{suffix}", flush=True)
def _tracker_log_proxy_progress(
server: "_TrackerHTTPServer",
*,
request_id: str,
model: str,
route_model: str,
tokens: int,
started: float,
route_nodes: list["_NodeEntry"],
stream: bool = True,
relay: bool = False,
) -> None:
elapsed = time.monotonic() - started
_tracker_log(
server,
"info",
"proxy progress",
request_id=request_id,
model=model,
route_model=route_model,
stream=stream,
relay=relay or None,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
def _node_id_for_registration(
suffix = f" {extras}" if extras else ""
print(f"[tracker] {level}: {message}{suffix}", flush=True)
def _tracker_log_proxy_progress(
server: "_TrackerHTTPServer",
*,
request_id: str,
model: str,
route_model: str,
tokens: int,
started: float,
route_nodes: list["_NodeEntry"],
stream: bool = True,
relay: bool = False,
) -> None:
elapsed = time.monotonic() - started
_tracker_log(
server,
"info",
"proxy progress",
request_id=request_id,
model=model,
route_model=route_model,
stream=stream,
relay=relay or None,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
def _node_id_for_registration(
endpoint: str,
model: str,
wallet_address: str | None,
@@ -2273,8 +2273,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
result[model] = server.stats.get_node_model_stats(node.node_id, model)
return result
def model_supply_for(node: _NodeEntry) -> dict:
return _model_health_summary(server, node.model, node.hf_repo)
def model_supply_for(node: _NodeEntry) -> dict:
return _model_health_summary(server, node.model, node.hf_repo)
self._send_json(200, {
"relay_url": server.relay_url,
@@ -2537,23 +2537,23 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Strip the first-shard node we're about to proxy to — it's already handling the request.
downstream_hops = [h for h in route_hops if h["endpoint"].rstrip("/") != node.endpoint.rstrip("/")]
downstream_urls = json.dumps(downstream_hops)
route_debug = " -> ".join(
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
for n in route_nodes
)
inflight_nodes = route_nodes or [node]
inflight_recorded = True
_record_proxy_inflight(server, inflight_nodes, 1)
def finish_proxy_inflight() -> None:
nonlocal inflight_recorded
if inflight_recorded:
_record_proxy_inflight(server, inflight_nodes, -1)
inflight_recorded = False
_tracker_log(
server,
"info",
route_debug = " -> ".join(
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
for n in route_nodes
)
inflight_nodes = route_nodes or [node]
inflight_recorded = True
_record_proxy_inflight(server, inflight_nodes, 1)
def finish_proxy_inflight() -> None:
nonlocal inflight_recorded
if inflight_recorded:
_record_proxy_inflight(server, inflight_nodes, -1)
inflight_recorded = False
_tracker_log(
server,
"info",
"proxy route selected",
request_id=request_id,
model=model,
@@ -2604,15 +2604,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if first is not None and first.get("stream"):
# Streamed response (US-036): forward SSE chunks as they arrive
# and run the same token accounting as the direct stream path.
self._stream_relayed_frames(
first, frames, started,
model, route_model, route_nodes, api_key, node_work,
request_body=body,
)
finish_proxy_inflight()
return
if first is not None:
elapsed = time.monotonic() - started
self._stream_relayed_frames(
first, frames, started,
model, route_model, route_nodes, api_key, node_work,
request_body=body,
)
finish_proxy_inflight()
return
if first is not None:
elapsed = time.monotonic() - started
self._send_relayed_response(first)
if int(first.get("status", 503)) < 400:
body_text = first.get("body") or ""
@@ -2635,12 +2635,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
finish_proxy_inflight()
return
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
finish_proxy_inflight()
return
_tracker_log(
server,
"warn",
@@ -2661,12 +2661,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(err_body)))
self.end_headers()
try:
self.wfile.write(err_body)
except BrokenPipeError:
pass
finish_proxy_inflight()
return
try:
self.wfile.write(err_body)
except BrokenPipeError:
pass
finish_proxy_inflight()
return
except Exception as exc:
if node.relay_addr:
_tracker_log(
@@ -2687,13 +2687,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
target_url=target_url,
error=repr(exc),
)
self._send_json(503, {"error": {
"message": f"upstream node unreachable: {exc}",
"type": "service_unavailable",
"code": "upstream_error",
}})
finish_proxy_inflight()
return
self._send_json(503, {"error": {
"message": f"upstream node unreachable: {exc}",
"type": "service_unavailable",
"code": "upstream_error",
}})
finish_proxy_inflight()
return
with upstream:
content_type = upstream.headers.get("Content-Type", "application/json")
@@ -2784,13 +2784,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(resp_body)))
self.end_headers()
try:
self.wfile.write(resp_body)
except BrokenPipeError:
pass
finish_proxy_inflight()
def _record_observed_throughput(
try:
self.wfile.write(resp_body)
except BrokenPipeError:
pass
finish_proxy_inflight()
def _record_observed_throughput(
self,
requested_model: str,
route_model: str,
@@ -3175,50 +3175,50 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
self._purge_expired_nodes()
# Dedup: if this endpoint is already registered, remove the old entry first.
stale_ids = [
eid for eid, e in server.registry.items()
if e.endpoint == entry.endpoint.rstrip("/")
]
stale_entries: list[tuple[str, _NodeEntry]] = []
for eid in stale_ids:
old = server.registry.pop(eid)
stale_entries.append((eid, old))
server.registry[node_id] = entry
if entry.managed_assignment:
if entry.hf_repo:
_rebalance_hf_model_locked(server, entry.hf_repo)
else:
stale_ids = [
eid for eid, e in server.registry.items()
if e.endpoint == entry.endpoint.rstrip("/")
]
stale_entries: list[tuple[str, _NodeEntry]] = []
for eid in stale_ids:
old = server.registry.pop(eid)
stale_entries.append((eid, old))
server.registry[node_id] = entry
if entry.managed_assignment:
if entry.hf_repo:
_rebalance_hf_model_locked(server, entry.hf_repo)
else:
_rebalance_model_locked(server, model)
assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
if assignment_directive is not None:
entry.pending_directives.clear()
model_health = _model_health_summary(server, entry.model, entry.hf_repo)
for eid, old in stale_entries:
_tracker_log(
server,
"info",
"node re-registered",
old_node_id=eid,
node_id=node_id,
endpoint=old.endpoint,
old_model=old.model,
old_hf_repo=old.hf_repo,
old_model_health=_model_health_summary(server, old.model, old.hf_repo),
model_health=model_health,
)
_tracker_log(
server,
"info",
"node registered",
node_id=node_id,
if assignment_directive is not None:
entry.pending_directives.clear()
model_health = _model_health_summary(server, entry.model, entry.hf_repo)
for eid, old in stale_entries:
_tracker_log(
server,
"info",
"node re-registered",
old_node_id=eid,
node_id=node_id,
endpoint=old.endpoint,
old_model=old.model,
old_hf_repo=old.hf_repo,
old_model_health=_model_health_summary(server, old.model, old.hf_repo),
model_health=model_health,
)
_tracker_log(
server,
"info",
"node registered",
node_id=node_id,
endpoint=entry.endpoint,
model=entry.model,
hf_repo=entry.hf_repo,
shard=f"{entry.shard_start}-{entry.shard_end}",
tracker_mode=entry.tracker_mode,
model_health=model_health,
)
hf_repo=entry.hf_repo,
shard=f"{entry.shard_start}-{entry.shard_end}",
tracker_mode=entry.tracker_mode,
model_health=model_health,
)
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
repo_info = f" [{hf_repo}]" if hf_repo else ""
@@ -3560,7 +3560,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return
keys = server.accounts.keys_for(account["account_id"]) # type: ignore[union-attr]
balances = {}
usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": []}
usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": [], "records": []}
if server.billing is not None:
balances = {key: server.billing.get_client_balance(key) for key in keys}
usage = server.billing.usage_for(keys)