popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Retry tracker registration when initial connect fails.

Browse Source 
Start background re-registration when the tracker is unreachable at startup so nodes do not stay permanently unregistered.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Dobromir Popov
2026-07-07 17:59:27 +02:00
parent 50e8904f1c
commit c38e36f685
2 changed files with 85 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
packages/node/meshnet_node/startup.py
View File

@@ -331,6 +331,9 @@ def _attach_relay_bridge(node: StubNodeServer | TorchNodeServer, bridge: RelayHt
node.stop = _stop_with_bridge # type: ignore[method-assign]
_PENDING_NODE_ID = "pending"
def _start_heartbeat(
tracker_url: str,
node_id: str,
@@ -421,7 +424,7 @@ def _start_heartbeat(
def _loop() -> None:
nonlocal node_id
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
outage_streak = 0 # consecutive intervals where tracker was unreachable
outage_streak = 1 if node_id == _PENDING_NODE_ID else 0
while True:
time.sleep(interval)
@@ -476,9 +479,6 @@ def _start_heartbeat(
return t
_PENDING_NODE_ID = "pending"
def _register_with_tracker(
tracker_url: str,
reg_payload: dict,
@@ -782,16 +782,9 @@ def run_startup(
**registration_capabilities,
**relay_fields,
}
tracker_node_id: str | None = None
try:
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", reg_payload)
tracker_node_id = str(reg_resp.get("node_id") or "?")
setattr(node, "tracker_node_id", tracker_node_id)
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
_start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=_node_start_time)
except Exception as exc:
setattr(node, "tracker_node_id", None)
print(f" Warning: tracker registration failed: {exc}", flush=True)
tracker_node_id = _register_with_tracker(
tracker_url, reg_payload, node, _node_start_time,
)
print(
f"\n{'=' * 32}\n"
@@ -917,16 +910,9 @@ def run_startup(
**registration_capabilities,
**relay_fields,
}
tracker_node_id = None
try:
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", auto_reg_payload)
tracker_node_id = str(reg_resp.get("node_id") or "?")
setattr(node, "tracker_node_id", tracker_node_id)
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
_start_heartbeat(tracker_url, tracker_node_id, auto_reg_payload, node_ref=node, start_time=_node_start_time)
except Exception as exc:
setattr(node, "tracker_node_id", None)
print(f" Warning: tracker registration failed: {exc}", flush=True)
tracker_node_id = _register_with_tracker(
tracker_url, auto_reg_payload, node, _node_start_time,
)
shard_count = assigned_shard_end - assigned_shard_start + 1
print(
f"\n{'=' * 32}\n"