feat(us-023): heartbeat stats payload, request counters, dynamic reassignment response

Node now sends cumulative stats in heartbeat body:
  total_requests, failed_requests, queue_depth, uptime_seconds, status
Stats are tracked thread-safely in _TorchHTTPServer; buffered locally during
outage streak and flushed on next successful heartbeat.

Tracker stores stats on _NodeEntry (total_requests, failed_requests,
queue_depth, uptime_seconds, status, heartbeats_expected/received) and
returns new_assignment in heartbeat response when pending_new_assignment is set.

Node logs received new_assignment (hot-reload wired in US-026).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
2026-06-30 13:20:52 +03:00
parent 96af82892f
commit 34fb1ec65d
3 changed files with 102 additions and 7 deletions

View File

@@ -46,6 +46,10 @@ class _TorchHTTPServer(http.server.HTTPServer):
self.tracker_mode = tracker_mode
self.tracker_url = tracker_url
self.route_timeout = route_timeout
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
self._stats_lock = threading.Lock()
class _TorchHandler(http.server.BaseHTTPRequestHandler):
@@ -219,6 +223,21 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
def _handle_chat_completions(self) -> None:
server: _TorchHTTPServer = self.server # type: ignore[assignment]
with server._stats_lock:
server.total_requests += 1
server.queue_depth += 1
try:
self._do_chat_completions(server)
finally:
with server._stats_lock:
server.queue_depth -= 1
def _record_failed_request(self) -> None:
server: _TorchHTTPServer = self.server # type: ignore[assignment]
with server._stats_lock:
server.failed_requests += 1
def _do_chat_completions(self, server: "_TorchHTTPServer") -> None:
body = self._read_json_body()
if body is None:
return
@@ -244,6 +263,7 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
text = server.backend.generate_text(messages, max_tokens, temperature, top_p)
self._send_openai_response(text, model_name, False, messages)
except Exception as exc:
self._record_failed_request()
self._send_json(500, {"error": f"generation failed: {exc}"})
return
@@ -620,6 +640,18 @@ class TorchNodeServer:
def forward_chunk_count(self) -> int:
return self._server.forward_chunk_count if self._server is not None else 0
@property
def total_requests(self) -> int:
return self._server.total_requests if self._server is not None else 0
@property
def failed_requests(self) -> int:
return self._server.failed_requests if self._server is not None else 0
@property
def queue_depth(self) -> int:
return self._server.queue_depth if self._server is not None else 0
def start(self) -> int:
if self._server is not None:
raise RuntimeError("TorchNodeServer is already running")