Merge branch cursor/fix-meshnet-node-param-parsing into master.

Combine shard label formatting with model/shard flag parsing and tracker registration retry.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Dobromir Popov
2026-07-07 18:02:01 +02:00
5 changed files with 309 additions and 44 deletions

View File

@@ -346,6 +346,9 @@ def _attach_relay_bridge(node: StubNodeServer | TorchNodeServer, bridge: RelayHt
node.stop = _stop_with_bridge # type: ignore[method-assign]
_PENDING_NODE_ID = "pending"
def _start_heartbeat(
tracker_url: str,
node_id: str,
@@ -383,6 +386,8 @@ def _start_heartbeat(
try:
resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload)
node_id = resp.get("node_id", node_id)
if node_ref is not None:
setattr(node_ref, "tracker_node_id", node_id)
return True
except Exception:
return False
@@ -434,7 +439,7 @@ def _start_heartbeat(
def _loop() -> None:
nonlocal node_id
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
outage_streak = 0 # consecutive intervals where tracker was unreachable
outage_streak = 1 if node_id == _PENDING_NODE_ID else 0
while True:
time.sleep(interval)
@@ -489,6 +494,34 @@ def _start_heartbeat(
return t
def _register_with_tracker(
tracker_url: str,
reg_payload: dict,
node: Any,
start_time: float,
) -> str | None:
"""Register with the tracker, or start background retries when it is unreachable."""
try:
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", reg_payload)
tracker_node_id = str(reg_resp.get("node_id") or "?")
setattr(node, "tracker_node_id", tracker_node_id)
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
_start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=start_time)
return tracker_node_id
except Exception as exc:
setattr(node, "tracker_node_id", None)
print(f" Warning: tracker registration failed: {exc}", flush=True)
print(" [node] will retry registration in the background", flush=True)
_start_heartbeat(
tracker_url,
_PENDING_NODE_ID,
reg_payload,
node_ref=node,
start_time=start_time,
)
return None
def _warn_virtual_network_ip(ip: str | None) -> None:
"""Print a warning when the auto-detected advertise IP is in a known virtual-network range.
@@ -648,8 +681,11 @@ def run_startup(
if probationary_line is not None:
print(f" {probationary_line}", flush=True)
pinned_shard_start = shard_start
pinned_shard_end = shard_end
user_pinned_shard = pinned_shard_start is not None or pinned_shard_end is not None
if model_id: # treat "" the same as None — no explicit model given
user_pinned_shard = shard_start is not None or shard_end is not None
full_sources: list[dict] = []
# Auto-detect shard range from model config if not explicitly provided
if shard_start is None or shard_end is None:
@@ -757,16 +793,9 @@ def run_startup(
**registration_capabilities,
**relay_fields,
}
tracker_node_id: str | None = None
try:
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", reg_payload)
tracker_node_id = str(reg_resp.get("node_id") or "?")
setattr(node, "tracker_node_id", tracker_node_id)
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
_start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=_node_start_time)
except Exception as exc:
setattr(node, "tracker_node_id", None)
print(f" Warning: tracker registration failed: {exc}", flush=True)
tracker_node_id = _register_with_tracker(
tracker_url, reg_payload, node, _node_start_time,
)
print(
f"\n{'=' * 32}\n"
@@ -784,8 +813,8 @@ def run_startup(
flush=True,
)
return node
if shard_start is not None or shard_end is not None:
raise ValueError("--shard-start / --shard-end require --model-id")
if user_pinned_shard and not model:
raise ValueError("--shard-start / --shard-end require --model")
# 3a. Auto-join: query tracker for network-wide HF model assignment.
# Skipped when the user explicitly requested a model — the shard-assignment
@@ -892,16 +921,9 @@ def run_startup(
**registration_capabilities,
**relay_fields,
}
tracker_node_id = None
try:
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", auto_reg_payload)
tracker_node_id = str(reg_resp.get("node_id") or "?")
setattr(node, "tracker_node_id", tracker_node_id)
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
_start_heartbeat(tracker_url, tracker_node_id, auto_reg_payload, node_ref=node, start_time=_node_start_time)
except Exception as exc:
setattr(node, "tracker_node_id", None)
print(f" Warning: tracker registration failed: {exc}", flush=True)
tracker_node_id = _register_with_tracker(
tracker_url, auto_reg_payload, node, _node_start_time,
)
shard_label = _format_shard_label(
assigned_shard_start,
assigned_shard_end,
@@ -947,8 +969,13 @@ def run_startup(
print(f" ERROR: Cannot reach tracker at {tracker_url}: {exc}", file=sys.stderr, flush=True)
raise
shard_start: int = assignment["shard_start"]
shard_end: int = assignment["shard_end"]
shard_start = assignment["shard_start"]
shard_end = assignment["shard_end"]
if user_pinned_shard:
if pinned_shard_start is not None:
shard_start = pinned_shard_start
if pinned_shard_end is not None:
shard_end = pinned_shard_end
assigned_model: str = assignment.get("model", model)
hf_repo: str | None = assignment.get("hf_repo")
peers: list[dict] = assignment.get("peers", [])
@@ -960,10 +987,15 @@ def run_startup(
if model_layers_end is not None
else None
)
print(
f" Shard: {_format_shard_label(shard_start, shard_end, assigned_total_layers, model_name=assigned_model)}",
flush=True,
shard_label = _format_shard_label(
shard_start,
shard_end,
assigned_total_layers,
model_name=assigned_model,
)
if user_pinned_shard:
shard_label = f"{shard_label} (pinned)"
print(f" Shard: {shard_label}", flush=True)
# 4. Download shard
print("Downloading shard...", flush=True)
@@ -1025,6 +1057,7 @@ def run_startup(
"hardware_profile": hw,
"wallet_address": address,
"score": 1.0,
"managed_assignment": not user_pinned_shard,
**registration_capabilities,
**relay_fields,
}