feat(us-023): heartbeat stats payload, request counters, dynamic reassignment response
Node now sends cumulative stats in heartbeat body: total_requests, failed_requests, queue_depth, uptime_seconds, status Stats are tracked thread-safely in _TorchHTTPServer; buffered locally during outage streak and flushed on next successful heartbeat. Tracker stores stats on _NodeEntry (total_requests, failed_requests, queue_depth, uptime_seconds, status, heartbeats_expected/received) and returns new_assignment in heartbeat response when pending_new_assignment is set. Node logs received new_assignment (hot-reload wired in US-026). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -61,6 +61,12 @@ class _NodeEntry:
|
||||
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
|
||||
"pending_directives", "last_heartbeat", "tracker_mode",
|
||||
"relay_addr", "cert_fingerprint", "peer_id",
|
||||
# heartbeat stats (reported by node, cumulative)
|
||||
"total_requests", "failed_requests", "queue_depth", "uptime_seconds",
|
||||
"status", # "ready" | "loading"
|
||||
"heartbeats_expected", "heartbeats_received",
|
||||
# dynamic reassignment queued by the tracker
|
||||
"pending_new_assignment",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -110,6 +116,14 @@ class _NodeEntry:
|
||||
self.peer_id = peer_id
|
||||
self.pending_directives: list[dict] = []
|
||||
self.last_heartbeat: float = time.monotonic()
|
||||
self.total_requests: int = 0
|
||||
self.failed_requests: int = 0
|
||||
self.queue_depth: int = 0
|
||||
self.uptime_seconds: float = 0.0
|
||||
self.status: str = "ready"
|
||||
self.heartbeats_expected: int = 0
|
||||
self.heartbeats_received: int = 0
|
||||
self.pending_new_assignment: dict | None = None
|
||||
|
||||
|
||||
def _select_route(
|
||||
@@ -1045,21 +1059,39 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._send_json(404, {"error": "node not found"})
|
||||
return
|
||||
entry.last_heartbeat = time.monotonic()
|
||||
entry.heartbeats_received += 1
|
||||
# P2P metadata
|
||||
if body.get("relay_addr"):
|
||||
entry.relay_addr = body["relay_addr"]
|
||||
if body.get("cert_fingerprint"):
|
||||
entry.cert_fingerprint = body["cert_fingerprint"]
|
||||
if body.get("peer_id"):
|
||||
entry.peer_id = body["peer_id"]
|
||||
# Node stats (cumulative — node always sends totals, tracker replaces)
|
||||
if "total_requests" in body:
|
||||
entry.total_requests = int(body["total_requests"])
|
||||
if "failed_requests" in body:
|
||||
entry.failed_requests = int(body["failed_requests"])
|
||||
if "queue_depth" in body:
|
||||
entry.queue_depth = int(body["queue_depth"])
|
||||
if "uptime_seconds" in body:
|
||||
entry.uptime_seconds = float(body["uptime_seconds"])
|
||||
if "status" in body and body["status"] in ("ready", "loading"):
|
||||
entry.status = body["status"]
|
||||
_rebalance_model_locked(server, entry.model or "stub-model")
|
||||
directives = list(entry.pending_directives)
|
||||
entry.pending_directives.clear()
|
||||
new_assignment = entry.pending_new_assignment
|
||||
if new_assignment is not None:
|
||||
entry.pending_new_assignment = None
|
||||
if server.gossip is not None:
|
||||
server.gossip.record(node_id)
|
||||
resp: dict = {}
|
||||
if directives:
|
||||
self._send_json(200, {"directives": directives})
|
||||
else:
|
||||
self._send_json(200, {})
|
||||
resp["directives"] = directives
|
||||
if new_assignment is not None:
|
||||
resp["new_assignment"] = new_assignment
|
||||
self._send_json(200, resp)
|
||||
|
||||
# ---------------------------------------------------------------- Raft handlers
|
||||
|
||||
|
||||
Reference in New Issue
Block a user