feat(us-016): connection/heartbeat visibility for tracker and node
Tracker now prints a line when a node registers and on every heartbeat received. Node prints its assigned node_id after successful registration and starts a daemon heartbeat thread (30s interval) that logs each send. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,8 @@ from __future__ import annotations
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
@@ -32,6 +34,23 @@ def _get_json(url: str, timeout: float = 10.0) -> dict:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
def _start_heartbeat(tracker_url: str, node_id: str, interval: float = 30.0) -> threading.Thread:
|
||||
"""Daemon thread that sends periodic heartbeats to the tracker."""
|
||||
def _loop() -> None:
|
||||
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
try:
|
||||
_post_json(hb_url, {})
|
||||
print(f" [node] heartbeat sent → tracker (node {node_id[:8]})", flush=True)
|
||||
except Exception as exc:
|
||||
print(f" [node] WARNING: heartbeat failed: {exc}", flush=True)
|
||||
|
||||
t = threading.Thread(target=_loop, daemon=True, name="heartbeat")
|
||||
t.start()
|
||||
return t
|
||||
|
||||
|
||||
def run_startup(
|
||||
tracker_url: str,
|
||||
port: int = 0,
|
||||
@@ -119,7 +138,7 @@ def run_startup(
|
||||
# Register with tracker so other nodes can auto-join this model.
|
||||
total_layers = getattr(node.backend, "total_layers", None)
|
||||
try:
|
||||
_post_json(
|
||||
reg_resp = _post_json(
|
||||
f"{tracker_url}/v1/nodes/register",
|
||||
{
|
||||
"endpoint": endpoint,
|
||||
@@ -135,6 +154,9 @@ def run_startup(
|
||||
"tracker_mode": (shard_start == 0),
|
||||
},
|
||||
)
|
||||
node_id = reg_resp.get("node_id", "?")
|
||||
print(f" Registered with tracker — node ID: {node_id}", flush=True)
|
||||
_start_heartbeat(tracker_url, node_id)
|
||||
except Exception as exc:
|
||||
print(f" Warning: tracker registration failed: {exc}", flush=True)
|
||||
|
||||
@@ -187,7 +209,7 @@ def run_startup(
|
||||
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
||||
endpoint = f"http://{public_host}:{actual_port}"
|
||||
try:
|
||||
_post_json(
|
||||
reg_resp = _post_json(
|
||||
f"{tracker_url}/v1/nodes/register",
|
||||
{
|
||||
"endpoint": endpoint,
|
||||
@@ -203,6 +225,9 @@ def run_startup(
|
||||
"tracker_mode": (assigned_shard_start == 0),
|
||||
},
|
||||
)
|
||||
node_id = reg_resp.get("node_id", "?")
|
||||
print(f" Registered with tracker — node ID: {node_id}", flush=True)
|
||||
_start_heartbeat(tracker_url, node_id)
|
||||
except Exception as exc:
|
||||
print(f" Warning: tracker registration failed: {exc}", flush=True)
|
||||
shard_count = assigned_shard_end - assigned_shard_start + 1
|
||||
|
||||
Reference in New Issue
Block a user