wip -more responsive UI, better routing
This commit is contained in:
@@ -349,25 +349,38 @@ def _attach_relay_bridge(node: StubNodeServer | TorchNodeServer, bridge: RelayHt
|
||||
_PENDING_NODE_ID = "pending"
|
||||
|
||||
|
||||
_HEARTBEAT_INTERVAL_IDLE = 20.0
|
||||
_HEARTBEAT_INTERVAL_BUSY = 3.0
|
||||
|
||||
|
||||
def _start_heartbeat(
|
||||
tracker_url: str,
|
||||
node_id: str,
|
||||
register_payload: dict,
|
||||
interval: float = 20.0,
|
||||
interval: float = _HEARTBEAT_INTERVAL_IDLE,
|
||||
node_ref: Any | None = None,
|
||||
start_time: float | None = None,
|
||||
) -> threading.Thread:
|
||||
"""Daemon thread: sends heartbeats and re-registers automatically after tracker restarts.
|
||||
|
||||
Heartbeat body carries cumulative stats (total_requests, failed_requests,
|
||||
queue_depth, uptime_seconds, status). Stats are buffered locally during
|
||||
outage and flushed on next successful heartbeat.
|
||||
queue_depth, current_requests, uptime_seconds, status). Stats are buffered
|
||||
locally during outage and flushed on next successful heartbeat.
|
||||
|
||||
Heartbeat response may include new_assignment: {model, shard_start, shard_end}
|
||||
which is logged for now (hot-reload implemented in US-026).
|
||||
"""
|
||||
_start_time = start_time or time.monotonic()
|
||||
|
||||
def _current_requests_snapshot() -> list[dict]:
|
||||
if node_ref is None:
|
||||
return []
|
||||
getter = getattr(node_ref, "current_requests", None)
|
||||
if getter is None:
|
||||
return []
|
||||
current = getter() if callable(getter) else getter
|
||||
return list(current) if isinstance(current, list) else []
|
||||
|
||||
def _get_stats() -> dict:
|
||||
uptime = time.monotonic() - _start_time
|
||||
stats: dict = {"uptime_seconds": round(uptime, 1), "status": "ready"}
|
||||
@@ -379,8 +392,16 @@ def _start_heartbeat(
|
||||
)
|
||||
stats["failed_requests"] = getattr(node_ref, "failed_requests", 0)
|
||||
stats["queue_depth"] = getattr(node_ref, "queue_depth", 0)
|
||||
current_requests = _current_requests_snapshot()
|
||||
if current_requests:
|
||||
stats["current_requests"] = current_requests
|
||||
return stats
|
||||
|
||||
def _sleep_interval() -> float:
|
||||
if _current_requests_snapshot() or (node_ref is not None and getattr(node_ref, "queue_depth", 0) > 0):
|
||||
return _HEARTBEAT_INTERVAL_BUSY
|
||||
return interval
|
||||
|
||||
def _reregister() -> bool:
|
||||
nonlocal node_id
|
||||
try:
|
||||
@@ -442,7 +463,7 @@ def _start_heartbeat(
|
||||
outage_streak = 1 if node_id == _PENDING_NODE_ID else 0
|
||||
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
time.sleep(_sleep_interval())
|
||||
|
||||
if outage_streak > 0:
|
||||
# Tracker was down — attempt re-registration first (it may have restarted
|
||||
|
||||
Reference in New Issue
Block a user