feat(us-024): enhanced availability map with per-node health details

Coverage endpoint now skips node purge so dead nodes remain visible.
Each band in /v1/coverage/<model> includes a `nodes` array with per-node
health: alive status, last_seen_seconds_ago, heartbeat_success_rate,
inference_success_rate, queue_depth, uptime_seconds, and endpoint.
Updated tests to assert new band structure.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
2026-06-30 13:28:12 +03:00
parent 34fb1ec65d
commit fff11c4d8b
2 changed files with 105 additions and 14 deletions

View File

@@ -244,6 +244,81 @@ def _coverage_map(
return coverage
def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
"""Per-node health detail for the availability map."""
age = time.monotonic() - node.last_heartbeat
alive = age <= heartbeat_timeout
hb_expected = max(1, round(node.uptime_seconds / 20.0)) # assume ~20s interval
hb_rate = round(min(1.0, node.heartbeats_received / hb_expected), 4) if node.heartbeats_received else 0.0
total = node.total_requests
inf_rate = round((total - node.failed_requests) / total, 4) if total > 0 else 1.0
return {
"node_id": node.node_id,
"endpoint": node.endpoint,
"alive": alive,
"last_seen_seconds_ago": round(age, 1),
"status": node.status,
"queue_depth": node.queue_depth,
"total_requests": node.total_requests,
"heartbeat_success_rate": hb_rate,
"inference_success_rate": inf_rate,
}
def _coverage_map_detailed(
nodes: list["_NodeEntry"],
required_start: int,
required_end: int,
heartbeat_timeout: float,
) -> list[dict]:
"""Like _coverage_map but with per-node identity and health in each band.
Includes all nodes (alive and stale) so operators can see both coverage
holes and which dead nodes used to fill them.
"""
now = time.monotonic()
def covers(node: "_NodeEntry", layer: int) -> bool:
return (
node.shard_start is not None
and node.shard_end is not None
and node.shard_start <= layer <= node.shard_end
)
# Build per-layer list of nodes (with alive flag)
layer_nodes: list[list[tuple["_NodeEntry", bool]]] = []
for layer in range(required_start, required_end + 1):
ns = [
(n, (now - n.last_heartbeat) <= heartbeat_timeout)
for n in nodes
if covers(n, layer)
]
layer_nodes.append(ns)
# Merge consecutive layers with identical node-set (same node_ids, same alive)
coverage: list[dict] = []
for i, layer_idx in enumerate(range(required_start, required_end + 1)):
ns = layer_nodes[i]
node_ids = [n.node_id for n, _ in ns]
alive_flags = [a for _, a in ns]
# Compare with last band
if (
coverage
and coverage[-1]["end_layer"] == layer_idx - 1
and [nd["node_id"] for nd in coverage[-1]["nodes"]] == node_ids
and [nd["alive"] for nd in coverage[-1]["nodes"]] == alive_flags
):
coverage[-1]["end_layer"] = layer_idx
else:
coverage.append({
"start_layer": layer_idx,
"end_layer": layer_idx,
"node_count": len(ns),
"nodes": [_node_health(n, heartbeat_timeout) for n, _ in ns],
})
return coverage
def _coverage_gaps(coverage: list[dict]) -> list[tuple[int, int]]:
return [
(segment["start_layer"], segment["end_layer"])
@@ -624,19 +699,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def _handle_coverage(self, model: str):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
# Do NOT purge before coverage — dead nodes are included with alive=false
# so operators can see what was covering each layer band before failure.
with server.lock:
self._purge_expired_nodes()
resolved = _nodes_and_bounds_for_model(server, model)
if resolved is None:
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
return
alive, required_start, required_end = resolved
all_nodes, required_start, required_end = resolved
if server.contracts is not None:
alive = [
node for node in alive
all_nodes = [
node for node in all_nodes
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
]
coverage = _coverage_map(alive, required_start, required_end)
coverage = _coverage_map_detailed(all_nodes, required_start, required_end, server.heartbeat_timeout)
self._send_json(200, {"model": model, "coverage": coverage})
def _handle_tracker_nodes(self, model: str):