feat(us-024): enhanced availability map with per-node health details
Coverage endpoint now skips node purge so dead nodes remain visible. Each band in /v1/coverage/<model> includes a `nodes` array with per-node health: alive status, last_seen_seconds_ago, heartbeat_success_rate, inference_success_rate, queue_depth, uptime_seconds, and endpoint. Updated tests to assert new band structure. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -244,6 +244,81 @@ def _coverage_map(
|
||||
return coverage
|
||||
|
||||
|
||||
def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
|
||||
"""Per-node health detail for the availability map."""
|
||||
age = time.monotonic() - node.last_heartbeat
|
||||
alive = age <= heartbeat_timeout
|
||||
hb_expected = max(1, round(node.uptime_seconds / 20.0)) # assume ~20s interval
|
||||
hb_rate = round(min(1.0, node.heartbeats_received / hb_expected), 4) if node.heartbeats_received else 0.0
|
||||
total = node.total_requests
|
||||
inf_rate = round((total - node.failed_requests) / total, 4) if total > 0 else 1.0
|
||||
return {
|
||||
"node_id": node.node_id,
|
||||
"endpoint": node.endpoint,
|
||||
"alive": alive,
|
||||
"last_seen_seconds_ago": round(age, 1),
|
||||
"status": node.status,
|
||||
"queue_depth": node.queue_depth,
|
||||
"total_requests": node.total_requests,
|
||||
"heartbeat_success_rate": hb_rate,
|
||||
"inference_success_rate": inf_rate,
|
||||
}
|
||||
|
||||
|
||||
def _coverage_map_detailed(
|
||||
nodes: list["_NodeEntry"],
|
||||
required_start: int,
|
||||
required_end: int,
|
||||
heartbeat_timeout: float,
|
||||
) -> list[dict]:
|
||||
"""Like _coverage_map but with per-node identity and health in each band.
|
||||
|
||||
Includes all nodes (alive and stale) so operators can see both coverage
|
||||
holes and which dead nodes used to fill them.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
|
||||
def covers(node: "_NodeEntry", layer: int) -> bool:
|
||||
return (
|
||||
node.shard_start is not None
|
||||
and node.shard_end is not None
|
||||
and node.shard_start <= layer <= node.shard_end
|
||||
)
|
||||
|
||||
# Build per-layer list of nodes (with alive flag)
|
||||
layer_nodes: list[list[tuple["_NodeEntry", bool]]] = []
|
||||
for layer in range(required_start, required_end + 1):
|
||||
ns = [
|
||||
(n, (now - n.last_heartbeat) <= heartbeat_timeout)
|
||||
for n in nodes
|
||||
if covers(n, layer)
|
||||
]
|
||||
layer_nodes.append(ns)
|
||||
|
||||
# Merge consecutive layers with identical node-set (same node_ids, same alive)
|
||||
coverage: list[dict] = []
|
||||
for i, layer_idx in enumerate(range(required_start, required_end + 1)):
|
||||
ns = layer_nodes[i]
|
||||
node_ids = [n.node_id for n, _ in ns]
|
||||
alive_flags = [a for _, a in ns]
|
||||
# Compare with last band
|
||||
if (
|
||||
coverage
|
||||
and coverage[-1]["end_layer"] == layer_idx - 1
|
||||
and [nd["node_id"] for nd in coverage[-1]["nodes"]] == node_ids
|
||||
and [nd["alive"] for nd in coverage[-1]["nodes"]] == alive_flags
|
||||
):
|
||||
coverage[-1]["end_layer"] = layer_idx
|
||||
else:
|
||||
coverage.append({
|
||||
"start_layer": layer_idx,
|
||||
"end_layer": layer_idx,
|
||||
"node_count": len(ns),
|
||||
"nodes": [_node_health(n, heartbeat_timeout) for n, _ in ns],
|
||||
})
|
||||
return coverage
|
||||
|
||||
|
||||
def _coverage_gaps(coverage: list[dict]) -> list[tuple[int, int]]:
|
||||
return [
|
||||
(segment["start_layer"], segment["end_layer"])
|
||||
@@ -624,19 +699,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
|
||||
def _handle_coverage(self, model: str):
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
# Do NOT purge before coverage — dead nodes are included with alive=false
|
||||
# so operators can see what was covering each layer band before failure.
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
resolved = _nodes_and_bounds_for_model(server, model)
|
||||
if resolved is None:
|
||||
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
|
||||
return
|
||||
alive, required_start, required_end = resolved
|
||||
all_nodes, required_start, required_end = resolved
|
||||
if server.contracts is not None:
|
||||
alive = [
|
||||
node for node in alive
|
||||
all_nodes = [
|
||||
node for node in all_nodes
|
||||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||||
]
|
||||
coverage = _coverage_map(alive, required_start, required_end)
|
||||
coverage = _coverage_map_detailed(all_nodes, required_start, required_end, server.heartbeat_timeout)
|
||||
self._send_json(200, {"model": model, "coverage": coverage})
|
||||
|
||||
def _handle_tracker_nodes(self, model: str):
|
||||
|
||||
Reference in New Issue
Block a user