Merge branch cursor/fix-meshnet-node-param-parsing into master.
Combine shard label formatting with model/shard flag parsing and tracker registration retry. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -346,6 +346,9 @@ def _attach_relay_bridge(node: StubNodeServer | TorchNodeServer, bridge: RelayHt
|
||||
node.stop = _stop_with_bridge # type: ignore[method-assign]
|
||||
|
||||
|
||||
_PENDING_NODE_ID = "pending"
|
||||
|
||||
|
||||
def _start_heartbeat(
|
||||
tracker_url: str,
|
||||
node_id: str,
|
||||
@@ -383,6 +386,8 @@ def _start_heartbeat(
|
||||
try:
|
||||
resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload)
|
||||
node_id = resp.get("node_id", node_id)
|
||||
if node_ref is not None:
|
||||
setattr(node_ref, "tracker_node_id", node_id)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -434,7 +439,7 @@ def _start_heartbeat(
|
||||
def _loop() -> None:
|
||||
nonlocal node_id
|
||||
hb_url = f"{tracker_url}/v1/nodes/{node_id}/heartbeat"
|
||||
outage_streak = 0 # consecutive intervals where tracker was unreachable
|
||||
outage_streak = 1 if node_id == _PENDING_NODE_ID else 0
|
||||
|
||||
while True:
|
||||
time.sleep(interval)
|
||||
@@ -489,6 +494,34 @@ def _start_heartbeat(
|
||||
return t
|
||||
|
||||
|
||||
def _register_with_tracker(
|
||||
tracker_url: str,
|
||||
reg_payload: dict,
|
||||
node: Any,
|
||||
start_time: float,
|
||||
) -> str | None:
|
||||
"""Register with the tracker, or start background retries when it is unreachable."""
|
||||
try:
|
||||
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", reg_payload)
|
||||
tracker_node_id = str(reg_resp.get("node_id") or "?")
|
||||
setattr(node, "tracker_node_id", tracker_node_id)
|
||||
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
|
||||
_start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=start_time)
|
||||
return tracker_node_id
|
||||
except Exception as exc:
|
||||
setattr(node, "tracker_node_id", None)
|
||||
print(f" Warning: tracker registration failed: {exc}", flush=True)
|
||||
print(" [node] will retry registration in the background", flush=True)
|
||||
_start_heartbeat(
|
||||
tracker_url,
|
||||
_PENDING_NODE_ID,
|
||||
reg_payload,
|
||||
node_ref=node,
|
||||
start_time=start_time,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _warn_virtual_network_ip(ip: str | None) -> None:
|
||||
"""Print a warning when the auto-detected advertise IP is in a known virtual-network range.
|
||||
|
||||
@@ -648,8 +681,11 @@ def run_startup(
|
||||
if probationary_line is not None:
|
||||
print(f" {probationary_line}", flush=True)
|
||||
|
||||
pinned_shard_start = shard_start
|
||||
pinned_shard_end = shard_end
|
||||
user_pinned_shard = pinned_shard_start is not None or pinned_shard_end is not None
|
||||
|
||||
if model_id: # treat "" the same as None — no explicit model given
|
||||
user_pinned_shard = shard_start is not None or shard_end is not None
|
||||
full_sources: list[dict] = []
|
||||
# Auto-detect shard range from model config if not explicitly provided
|
||||
if shard_start is None or shard_end is None:
|
||||
@@ -757,16 +793,9 @@ def run_startup(
|
||||
**registration_capabilities,
|
||||
**relay_fields,
|
||||
}
|
||||
tracker_node_id: str | None = None
|
||||
try:
|
||||
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", reg_payload)
|
||||
tracker_node_id = str(reg_resp.get("node_id") or "?")
|
||||
setattr(node, "tracker_node_id", tracker_node_id)
|
||||
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
|
||||
_start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=_node_start_time)
|
||||
except Exception as exc:
|
||||
setattr(node, "tracker_node_id", None)
|
||||
print(f" Warning: tracker registration failed: {exc}", flush=True)
|
||||
tracker_node_id = _register_with_tracker(
|
||||
tracker_url, reg_payload, node, _node_start_time,
|
||||
)
|
||||
|
||||
print(
|
||||
f"\n{'=' * 32}\n"
|
||||
@@ -784,8 +813,8 @@ def run_startup(
|
||||
flush=True,
|
||||
)
|
||||
return node
|
||||
if shard_start is not None or shard_end is not None:
|
||||
raise ValueError("--shard-start / --shard-end require --model-id")
|
||||
if user_pinned_shard and not model:
|
||||
raise ValueError("--shard-start / --shard-end require --model")
|
||||
|
||||
# 3a. Auto-join: query tracker for network-wide HF model assignment.
|
||||
# Skipped when the user explicitly requested a model — the shard-assignment
|
||||
@@ -892,16 +921,9 @@ def run_startup(
|
||||
**registration_capabilities,
|
||||
**relay_fields,
|
||||
}
|
||||
tracker_node_id = None
|
||||
try:
|
||||
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", auto_reg_payload)
|
||||
tracker_node_id = str(reg_resp.get("node_id") or "?")
|
||||
setattr(node, "tracker_node_id", tracker_node_id)
|
||||
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
|
||||
_start_heartbeat(tracker_url, tracker_node_id, auto_reg_payload, node_ref=node, start_time=_node_start_time)
|
||||
except Exception as exc:
|
||||
setattr(node, "tracker_node_id", None)
|
||||
print(f" Warning: tracker registration failed: {exc}", flush=True)
|
||||
tracker_node_id = _register_with_tracker(
|
||||
tracker_url, auto_reg_payload, node, _node_start_time,
|
||||
)
|
||||
shard_label = _format_shard_label(
|
||||
assigned_shard_start,
|
||||
assigned_shard_end,
|
||||
@@ -947,8 +969,13 @@ def run_startup(
|
||||
print(f" ERROR: Cannot reach tracker at {tracker_url}: {exc}", file=sys.stderr, flush=True)
|
||||
raise
|
||||
|
||||
shard_start: int = assignment["shard_start"]
|
||||
shard_end: int = assignment["shard_end"]
|
||||
shard_start = assignment["shard_start"]
|
||||
shard_end = assignment["shard_end"]
|
||||
if user_pinned_shard:
|
||||
if pinned_shard_start is not None:
|
||||
shard_start = pinned_shard_start
|
||||
if pinned_shard_end is not None:
|
||||
shard_end = pinned_shard_end
|
||||
assigned_model: str = assignment.get("model", model)
|
||||
hf_repo: str | None = assignment.get("hf_repo")
|
||||
peers: list[dict] = assignment.get("peers", [])
|
||||
@@ -960,10 +987,15 @@ def run_startup(
|
||||
if model_layers_end is not None
|
||||
else None
|
||||
)
|
||||
print(
|
||||
f" Shard: {_format_shard_label(shard_start, shard_end, assigned_total_layers, model_name=assigned_model)}",
|
||||
flush=True,
|
||||
shard_label = _format_shard_label(
|
||||
shard_start,
|
||||
shard_end,
|
||||
assigned_total_layers,
|
||||
model_name=assigned_model,
|
||||
)
|
||||
if user_pinned_shard:
|
||||
shard_label = f"{shard_label} (pinned)"
|
||||
print(f" Shard: {shard_label}", flush=True)
|
||||
|
||||
# 4. Download shard
|
||||
print("Downloading shard...", flush=True)
|
||||
@@ -1025,6 +1057,7 @@ def run_startup(
|
||||
"hardware_profile": hw,
|
||||
"wallet_address": address,
|
||||
"score": 1.0,
|
||||
"managed_assignment": not user_pinned_shard,
|
||||
**registration_capabilities,
|
||||
**relay_fields,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user