popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

routing and dashboard fixes

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-07 14:53:37 +02:00
parent 339577a26c
commit 80bd83ae0a
5 changed files with 385 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

259
packages/tracker/meshnet_tracker/server.py
View File

@@ -636,7 +636,7 @@ def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
return 0.5 + 0.5 * reputation
def _select_route(
def _select_route(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
@@ -678,10 +678,25 @@ def _select_route(
covered_up_to = best.shard_end
candidates = [n for n in candidates if n is not best]
return route, ""
def _coverage_percentage(
return route, ""
def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
return [
{
"node_id": node.node_id,
"model": node.model,
"hf_repo": node.hf_repo,
"endpoint": node.endpoint,
"shard": f"{node.shard_start}-{node.shard_end}",
"num_layers": node.num_layers,
"queue_depth": node.queue_depth,
}
for node in nodes
]
def _coverage_percentage(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
@@ -746,14 +761,14 @@ def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
return DEFAULT_RAM_BYTES, "ram-default"
def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
bytes_per_layer = _preset_bytes_per_layer(preset)
quantization = _node_quantization(node, preset)
layer_bytes = bytes_per_layer[quantization]
if layer_bytes <= 0:
return 0
memory_budget_bytes, _ = _node_memory_budget_bytes(node)
return int((memory_budget_bytes * 0.8) // layer_bytes)
def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
bytes_per_layer = _preset_bytes_per_layer(preset)
quantization = _node_quantization(node, preset)
layer_bytes = bytes_per_layer[quantization]
if layer_bytes <= 0:
return 0
memory_budget_bytes, _ = _node_memory_budget_bytes(node)
return int((memory_budget_bytes * 0.8) // layer_bytes)
def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
@@ -817,16 +832,17 @@ def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
}
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
if total_layers <= 0:
return 0
if memory_mb <= 0:
return max(1, total_layers // 2)
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
return min(
total_layers,
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
)
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
if total_layers <= 0:
return 0
if memory_mb <= 0:
return max(1, total_layers // 2)
memory_bytes = memory_mb * 1024 * 1024
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
return min(
total_layers,
max(1, int((memory_bytes * 0.8) // bytes_per_layer)),
)
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
@@ -2330,17 +2346,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Pre-resolve the downstream route so the first-shard node skips its own
# tracker query. We already hold the full registry picture — no need for
# a second round-trip.
route_model = node.hf_repo or node.model or model
with server.lock:
if route_model in server.model_presets:
preset = server.model_presets[route_model]
rs, re = _preset_layer_bounds(preset)
all_nodes: list = [n for n in server.registry.values() if n.model == route_model]
else:
all_nodes = [
n for n in server.registry.values()
if _node_matches_model(n, route_model)
and n.shard_start is not None and n.num_layers is not None
route_model = node.hf_repo or node.model or model
with server.lock:
resolved_route_model, route_preset = _resolve_model_preset(server.model_presets, route_model)
if route_preset is not None:
route_model = resolved_route_model or route_model
preset = route_preset
rs, re = _preset_layer_bounds(preset)
all_nodes: list = [
n for n in server.registry.values()
if _node_matches_preset(n, route_model, preset)
and n.shard_start is not None
and n.shard_end is not None
]
else:
all_nodes = [
n for n in server.registry.values()
if _node_matches_model(n, route_model)
and n.shard_start is not None and n.num_layers is not None
]
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
if pinned_nodes is not None:
@@ -2356,6 +2379,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
route_model=route_model,
error=route_error,
candidate_count=len(all_nodes),
candidates=_node_route_summary(all_nodes),
)
self._send_json(503, {"error": {
"message": route_error,
@@ -2393,6 +2417,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
head_endpoint=node.endpoint,
downstream=downstream_urls,
route=route_debug or "<empty>",
nodes=_node_route_summary(route_nodes),
)
req = urllib.request.Request(
@@ -2450,12 +2475,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body)
except (json.JSONDecodeError, TypeError):
in_tokens, out_tokens = 0, 0
tokens = in_tokens + out_tokens
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
tokens = in_tokens + out_tokens
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
_tracker_log(
server,
"info",
"proxy complete",
request_id=request_id,
model=model,
route_model=route_model,
status=int(first.get("status", 503)),
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
return
_tracker_log(
server,
@@ -2539,35 +2577,57 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
in_tokens, out_tokens = _stream_billable_split(
observed_stream_tokens, stream_usage, body
)
self._record_observed_throughput(
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
)
self._bill_completed(
api_key, model, in_tokens + out_tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
else:
# Non-streaming: buffer and relay
resp_body = upstream.read()
elapsed = time.monotonic() - started
self._record_observed_throughput(
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
)
tokens = in_tokens + out_tokens
_tracker_log(
server,
"info",
"proxy complete",
request_id=request_id,
target_url=target_url,
bytes=len(resp_body),
model=model,
route_model=route_model,
status=200,
stream=True,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
observed_output = ""
try:
response_payload = json.loads(resp_body)
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
observed_output = _observed_output_from_non_stream_payload(response_payload)
except json.JSONDecodeError:
in_tokens, out_tokens = 0, 0
tokens = in_tokens + out_tokens
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
else:
# Non-streaming: buffer and relay
resp_body = upstream.read()
elapsed = time.monotonic() - started
observed_output = ""
try:
response_payload = json.loads(resp_body)
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
observed_output = _observed_output_from_non_stream_payload(response_payload)
except json.JSONDecodeError:
in_tokens, out_tokens = 0, 0
tokens = in_tokens + out_tokens
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
_tracker_log(
server,
"info",
"proxy complete",
request_id=request_id,
model=model,
route_model=route_model,
target_url=target_url,
status=200,
bytes=len(resp_body),
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
@@ -2740,13 +2800,30 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
in_tokens, out_tokens = _stream_billable_split(
observed_stream_tokens, stream_usage, request_body
)
self._record_observed_throughput(
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
)
self._bill_completed(
api_key, model, in_tokens + out_tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
self._record_observed_throughput(
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
)
tokens = in_tokens + out_tokens
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
_tracker_log(
server,
"info",
"proxy complete",
request_id=session_id,
model=model,
route_model=route_model,
status=200,
stream=True,
relay=True,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
def _send_relayed_response(self, response: dict) -> None:
status = int(response.get("status", 503))
@@ -4140,11 +4217,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"shard_end": shard_end,
"num_layers": total_l,
"gap_found": True,
"price_per_token": 0.0,
"deployment": _deployment_summary(pool_with_caller, preset),
"model_sources": self._model_sources(
str(resolved_name),
preset,
"price_per_token": 0.0,
"bytes_per_layer": _preset_bytes_per_layer(preset),
"deployment": _deployment_summary(pool_with_caller, preset),
"model_sources": self._model_sources(
str(resolved_name),
preset,
shard_start,
shard_end,
),
@@ -4236,12 +4314,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"shard_start": shard_start,
"shard_end": shard_end,
"num_layers": total_l,
"gap_found": gap_found,
"price_per_token": 0.0,
"model_sources": self._model_sources(
str(resolved_name or best_repo),
best_preset or {"hf_repo": best_repo},
shard_start,
"gap_found": gap_found,
"price_per_token": 0.0,
"bytes_per_layer": _preset_bytes_per_layer(best_preset) if best_preset is not None else None,
"model_sources": self._model_sources(
str(resolved_name or best_repo),
best_preset or {"hf_repo": best_repo},
shard_start,
shard_end,
),
})
@@ -4287,10 +4366,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
]
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
if error:
self._send_json(503, {"error": error})
return
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
if error:
_tracker_log(
server,
"warn",
"route unavailable",
model=model,
route_model=resolved_name or model,
error=error,
candidate_count=len(alive),
candidates=_node_route_summary(alive),
)
self._send_json(503, {"error": error})
return
covered_up_to = required_start - 1
route_with_start: list[tuple] = []