feat(us-023): heartbeat stats payload, request counters, dynamic reassignment response

Node now sends cumulative stats in heartbeat body:
  total_requests, failed_requests, queue_depth, uptime_seconds, status
Stats are tracked thread-safely in _TorchHTTPServer; buffered locally during
outage streak and flushed on next successful heartbeat.

Tracker stores stats on _NodeEntry (total_requests, failed_requests,
queue_depth, uptime_seconds, status, heartbeats_expected/received) and
returns new_assignment in heartbeat response when pending_new_assignment is set.

Node logs received new_assignment (hot-reload wired in US-026).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
2026-06-30 13:20:52 +03:00
parent 96af82892f
commit 34fb1ec65d
3 changed files with 102 additions and 7 deletions

View File

@@ -61,6 +61,12 @@ class _NodeEntry:
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
"pending_directives", "last_heartbeat", "tracker_mode",
"relay_addr", "cert_fingerprint", "peer_id",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "uptime_seconds",
"status", # "ready" | "loading"
"heartbeats_expected", "heartbeats_received",
# dynamic reassignment queued by the tracker
"pending_new_assignment",
)
def __init__(
@@ -110,6 +116,14 @@ class _NodeEntry:
self.peer_id = peer_id
self.pending_directives: list[dict] = []
self.last_heartbeat: float = time.monotonic()
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
self.uptime_seconds: float = 0.0
self.status: str = "ready"
self.heartbeats_expected: int = 0
self.heartbeats_received: int = 0
self.pending_new_assignment: dict | None = None
def _select_route(
@@ -1045,21 +1059,39 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(404, {"error": "node not found"})
return
entry.last_heartbeat = time.monotonic()
entry.heartbeats_received += 1
# P2P metadata
if body.get("relay_addr"):
entry.relay_addr = body["relay_addr"]
if body.get("cert_fingerprint"):
entry.cert_fingerprint = body["cert_fingerprint"]
if body.get("peer_id"):
entry.peer_id = body["peer_id"]
# Node stats (cumulative — node always sends totals, tracker replaces)
if "total_requests" in body:
entry.total_requests = int(body["total_requests"])
if "failed_requests" in body:
entry.failed_requests = int(body["failed_requests"])
if "queue_depth" in body:
entry.queue_depth = int(body["queue_depth"])
if "uptime_seconds" in body:
entry.uptime_seconds = float(body["uptime_seconds"])
if "status" in body and body["status"] in ("ready", "loading"):
entry.status = body["status"]
_rebalance_model_locked(server, entry.model or "stub-model")
directives = list(entry.pending_directives)
entry.pending_directives.clear()
new_assignment = entry.pending_new_assignment
if new_assignment is not None:
entry.pending_new_assignment = None
if server.gossip is not None:
server.gossip.record(node_id)
resp: dict = {}
if directives:
self._send_json(200, {"directives": directives})
else:
self._send_json(200, {})
resp["directives"] = directives
if new_assignment is not None:
resp["new_assignment"] = new_assignment
self._send_json(200, resp)
# ---------------------------------------------------------------- Raft handlers