Retry tracker registration when initial connect fails.
Start background re-registration when the tracker is unreachable at startup so nodes do not stay permanently unregistered. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -5,8 +5,10 @@ import io
|
||||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
import types
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
@@ -1644,6 +1646,79 @@ def test_preset_model_startup_honors_pinned_shard_range(tmp_path, monkeypatch):
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_torch_startup_retries_registration_when_tracker_unreachable(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""Failed initial registration should start background retry, not stay unregistered."""
|
||||
import meshnet_node.startup as startup_mod
|
||||
|
||||
class FakeBackend:
|
||||
total_layers = 24
|
||||
|
||||
class FakeTorchNodeServer:
|
||||
def __init__(self, **kwargs):
|
||||
self.backend = FakeBackend()
|
||||
self.port = None
|
||||
self.chat_completion_count = 0
|
||||
self.tracker_node_id = None
|
||||
|
||||
def start(self):
|
||||
self.port = 7000
|
||||
return self.port
|
||||
|
||||
def stop(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(
|
||||
startup_mod,
|
||||
"detect_hardware",
|
||||
lambda: {"device": "cuda", "gpu_name": "Test GPU", "vram_mb": 8192, "ram_mb": 16 * 1024},
|
||||
)
|
||||
monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer)
|
||||
monkeypatch.setattr(
|
||||
startup_mod,
|
||||
"_detect_num_layers",
|
||||
lambda *_args, **_kwargs: 24,
|
||||
)
|
||||
|
||||
heartbeat_calls = []
|
||||
monkeypatch.setattr(
|
||||
startup_mod,
|
||||
"_start_heartbeat",
|
||||
lambda *args, **kwargs: heartbeat_calls.append((args, kwargs)) or threading.Thread(),
|
||||
)
|
||||
|
||||
register_calls = {"count": 0}
|
||||
|
||||
def flaky_register(url, payload):
|
||||
register_calls["count"] += 1
|
||||
raise urllib.error.URLError("connection refused")
|
||||
|
||||
monkeypatch.setattr(startup_mod, "_post_json", flaky_register)
|
||||
|
||||
tracker = TrackerServer()
|
||||
tracker_port = tracker.start()
|
||||
tracker_url = f"http://127.0.0.1:{tracker_port}"
|
||||
try:
|
||||
node = run_startup(
|
||||
tracker_url=tracker_url,
|
||||
model_id="Qwen/Qwen2.5-0.5B-Instruct",
|
||||
wallet_path=tmp_path / "wallet.json",
|
||||
)
|
||||
try:
|
||||
assert register_calls["count"] == 1
|
||||
assert node.tracker_node_id is None
|
||||
assert len(heartbeat_calls) == 1
|
||||
args, kwargs = heartbeat_calls[0]
|
||||
assert args[1] == startup_mod._PENDING_NODE_ID
|
||||
assert kwargs["node_ref"] is node
|
||||
finally:
|
||||
node.stop()
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_real_model_startup_registers_downloaded_inventory_without_checksum(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
|
||||
Reference in New Issue
Block a user