popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: robust tracker reconnect — re-register proactively after outage

Browse Source 
Previous logic caught 404 on heartbeat and re-registered, but the re-
registration failed silently if the tracker wasn't fully ready yet.

New approach: after any connection-refused streak, the heartbeat loop
switches to re-registration mode and keeps retrying until the tracker
accepts the registration (instead of sending heartbeats into a fresh
registry that doesn't know this node). Falls back to the 404 path for
the case where a node is purged without a full tracker restart.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Dobromir Popov
2026-06-30 02:12:06 +03:00
parent be37048145
commit 0be35d257b
1 changed files with 38 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
packages/node/meshnet_node/startup.py
View File

@@ -40,28 +40,58 @@ def _start_heartbeat(
register_payload: dict,
interval: float = 20.0,
) -> threading.Thread:
"""Daemon thread: sends heartbeats; re-registers automatically on 404 (tracker restart)."""
"""Daemon thread: sends heartbeats and re-registers automatically after tracker restarts."""
def _reregister() -> bool:
nonlocal node_id
try:
resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload)
node_id = resp.get("node_id", node_id)
return True
except Exception:
return False
def _loop() -> None:
nonlocal node_id
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
outage_streak = 0 # consecutive intervals where tracker was unreachable
while True:
time.sleep(interval)
if outage_streak > 0:
# Tracker was down — attempt re-registration first (it may have restarted
# with a clean slate and won't know this node).
if _reregister():
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
print(f" [node] re-registered after outage — node ID: {node_id}", flush=True)
outage_streak = 0
else:
outage_streak += 1
if outage_streak <= 3 or outage_streak % 10 == 0:
print(
f" [node] WARNING: tracker still unreachable "
f"({outage_streak * interval:.0f}s)",
flush=True,
)
continue
try:
_post_json(hb_url, {})
except urllib.error.HTTPError as exc:
if exc.code == 404:
# Node was purged (e.g. long gap before restart noticed) — re-register now.
print(" [node] tracker lost registration — re-registering...", flush=True)
try:
resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload)
node_id = resp.get("node_id", node_id)
if _reregister():
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
print(f" [node] re-registered — node ID: {node_id}", flush=True)
except Exception as re_exc:
print(f" [node] WARNING: re-registration failed: {re_exc}", flush=True)
else:
print(f" [node] WARNING: heartbeat failed: {exc}", flush=True)
print(" [node] WARNING: re-registration failed", flush=True)
outage_streak = 1
else:
print(f" [node] WARNING: heartbeat failed ({exc.code}): {exc}", flush=True)
except Exception as exc:
print(f" [node] WARNING: heartbeat failed: {exc}", flush=True)
outage_streak = 1
print(f" [node] WARNING: tracker unreachable: {exc}", flush=True)
t = threading.Thread(target=_loop, daemon=True, name="heartbeat")
t.start()