feat(us-023): heartbeat stats payload, request counters, dynamic reassignment response
Node now sends cumulative stats in heartbeat body: total_requests, failed_requests, queue_depth, uptime_seconds, status Stats are tracked thread-safely in _TorchHTTPServer; buffered locally during outage streak and flushed on next successful heartbeat. Tracker stores stats on _NodeEntry (total_requests, failed_requests, queue_depth, uptime_seconds, status, heartbeats_expected/received) and returns new_assignment in heartbeat response when pending_new_assignment is set. Node logs received new_assignment (hot-reload wired in US-026). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -46,6 +46,10 @@ class _TorchHTTPServer(http.server.HTTPServer):
|
||||
self.tracker_mode = tracker_mode
|
||||
self.tracker_url = tracker_url
|
||||
self.route_timeout = route_timeout
|
||||
self.total_requests: int = 0
|
||||
self.failed_requests: int = 0
|
||||
self.queue_depth: int = 0
|
||||
self._stats_lock = threading.Lock()
|
||||
|
||||
|
||||
class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
||||
@@ -219,6 +223,21 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
||||
|
||||
def _handle_chat_completions(self) -> None:
|
||||
server: _TorchHTTPServer = self.server # type: ignore[assignment]
|
||||
with server._stats_lock:
|
||||
server.total_requests += 1
|
||||
server.queue_depth += 1
|
||||
try:
|
||||
self._do_chat_completions(server)
|
||||
finally:
|
||||
with server._stats_lock:
|
||||
server.queue_depth -= 1
|
||||
|
||||
def _record_failed_request(self) -> None:
|
||||
server: _TorchHTTPServer = self.server # type: ignore[assignment]
|
||||
with server._stats_lock:
|
||||
server.failed_requests += 1
|
||||
|
||||
def _do_chat_completions(self, server: "_TorchHTTPServer") -> None:
|
||||
body = self._read_json_body()
|
||||
if body is None:
|
||||
return
|
||||
@@ -244,6 +263,7 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
||||
text = server.backend.generate_text(messages, max_tokens, temperature, top_p)
|
||||
self._send_openai_response(text, model_name, False, messages)
|
||||
except Exception as exc:
|
||||
self._record_failed_request()
|
||||
self._send_json(500, {"error": f"generation failed: {exc}"})
|
||||
return
|
||||
|
||||
@@ -620,6 +640,18 @@ class TorchNodeServer:
|
||||
def forward_chunk_count(self) -> int:
|
||||
return self._server.forward_chunk_count if self._server is not None else 0
|
||||
|
||||
@property
|
||||
def total_requests(self) -> int:
|
||||
return self._server.total_requests if self._server is not None else 0
|
||||
|
||||
@property
|
||||
def failed_requests(self) -> int:
|
||||
return self._server.failed_requests if self._server is not None else 0
|
||||
|
||||
@property
|
||||
def queue_depth(self) -> int:
|
||||
return self._server.queue_depth if self._server is not None else 0
|
||||
|
||||
def start(self) -> int:
|
||||
if self._server is not None:
|
||||
raise RuntimeError("TorchNodeServer is already running")
|
||||
|
||||
Reference in New Issue
Block a user