diff --git a/QUICKSTART.md b/QUICKSTART.md index 97d2eb3..91411e2 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -129,8 +129,7 @@ $env:HF_HOME = "D:\DEV\models" One-line variants: ```powershell -// only this works - when not behind NAT(wsl), and via IP. Revisit when relaying/RPC is implemented -.\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 +.\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 ``` @@ -186,6 +185,22 @@ Then a node only needs the public tracker address: --model Qwen/Qwen2.5-0.5B-Instruct ``` +No authentication is required in the prototype. The first public node for a model +must still choose that model with `--model` or a saved wizard config. After at +least one HF model node is registered, later nodes can join the public network +with only the tracker URL; the tracker assigns an uncovered shard if one exists: + +```bash +.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com +``` + +Use the public tracker to verify registration and routing: + +```bash +curl -s "https://ai.neuron.d-popov.com/v1/network/map" | python3 -m json.tool +curl -s "https://ai.neuron.d-popov.com/v1/route?model=qwen2.5-0.5b" | python3 -m json.tool +``` + --- ## Step 1 — Start the tracker (Terminal 1) @@ -258,6 +273,9 @@ For gated models (Llama), run `huggingface-cli login` first. ## Step 3 — Send an inference request (Terminal 3) +If you started the node with `--port 8001`, send the request directly to that +head node: + ```bash Qwen2.5-0.5B-Instruct curl -s http://localhost:8001/v1/chat/completions \ -H "Content-Type: application/json" \ @@ -268,6 +286,22 @@ curl -s http://localhost:8001/v1/chat/completions \ }' | python3 -m json.tool ``` +If you did not pass `--port`, `meshnet-node start` uses the first free port at +or above `7000`. Use the `Endpoint:` printed by the node instead of `8001`. + +To test tracker routing/proxying, send the same OpenAI-compatible request to the +tracker, using either the full HuggingFace repo or the quick alias: + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen2.5-0.5b", + "messages": [{"role": "user", "content": "What is 7 times 8? Answer in one word."}], + "stream": false + }' | python3 -m json.tool +``` + Or use the test script: ```bash @@ -313,6 +347,12 @@ activations to Node B, and streams the final response back. See `docs/TWO_MACHINE_TEST.md` (created by US-018). +For WSL2 nodes, registration only proves the node can reach the tracker +outbound. Tracker-routed inference also requires the tracker to reach the node's +advertised endpoint inbound. Either run the node in native Windows PowerShell, +configure Windows port forwarding into WSL for the node port, or start the +tracker with a relay URL so the node registers a `relay_addr`. + --- ## Browse available models diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 7b92fd2..97eaf0d 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -56,6 +56,27 @@ DEFAULT_QUANTIZATIONS = ["bfloat16"] DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0 +def _model_aliases(model: str | None) -> set[str]: + """Return stable lookup aliases for a model repo or display name.""" + if not model: + return set() + aliases = {model} + short = model.rsplit("/", 1)[-1] + aliases.add(short) + lowered = short.lower() + aliases.add(lowered) + if lowered.endswith("-instruct"): + aliases.add(lowered.removesuffix("-instruct")) + return aliases + + +def _node_matches_model(node: "_NodeEntry", model: str) -> bool: + requested = _model_aliases(model) + if not requested: + return False + return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo))) + + class _RollingCounter: """Circular-bucket request counter. @@ -549,7 +570,7 @@ def _nodes_and_bounds_for_model( nodes = [ node for node in server.registry.values() - if (node.hf_repo == model or node.model == model) + if _node_matches_model(node, model) and node.shard_start is not None and node.shard_end is not None and node.num_layers is not None @@ -1028,7 +1049,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): self._purge_expired_nodes() candidates = [ n for n in server.registry.values() - if n.tracker_mode and (n.model == model or n.hf_repo == model) + if n.tracker_mode and _node_matches_model(n, model) ] if not candidates: @@ -1036,7 +1057,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): with server.lock: candidates = [ n for n in server.registry.values() - if n.shard_start == 0 and (n.model == model or n.hf_repo == model) + if n.shard_start == 0 and _node_matches_model(n, model) ] if not candidates: @@ -1064,7 +1085,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): else: all_nodes = [ n for n in server.registry.values() - if (n.hf_repo == route_model or n.model == route_model) + if _node_matches_model(n, route_model) and n.shard_start is not None and n.num_layers is not None ] rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1) @@ -1745,7 +1766,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): # HF model routing: match by hf_repo (full) or model short name. alive = [ node for node in server.registry.values() - if (node.hf_repo == model or node.model == model) + if _node_matches_model(node, model) and node.shard_start is not None and node.shard_end is not None and node.num_layers is not None diff --git a/tests/test_node_startup.py b/tests/test_node_startup.py index e170dbe..a9f52e3 100644 --- a/tests/test_node_startup.py +++ b/tests/test_node_startup.py @@ -391,6 +391,171 @@ def test_real_model_startup_summary_shows_total_layers(tmp_path, monkeypatch, ca assert "Node ID: node-test-123" in output +def test_public_tracker_model_node_registers_relay_metadata_from_tracker_url_only( + tmp_path, + monkeypatch, +): + """A node only needs the public tracker URL to discover relay metadata and register.""" + import meshnet_node.startup as startup_mod + + class FakeBackend: + total_layers = 24 + + class FakeTorchNodeServer: + def __init__(self, **kwargs): + self.kwargs = kwargs + self.backend = FakeBackend() + self.port = None + self.chat_completion_count = 0 + self.total_requests = 0 + self.failed_requests = 0 + self.queue_depth = 0 + + def start(self): + self.port = 8001 + return self.port + + def stop(self): + pass + + class FakeRelayHttpBridge: + def __init__(self, relay_url, peer_id, local_base_url, advertised_addr): + self.relay_url = relay_url + self.peer_id = peer_id + self.local_base_url = local_base_url + self.advertised_addr = advertised_addr + + @property + def relay_addr(self): + return f"ws://public-relay.example/rpc/{self.peer_id}" + + def start(self): + return types.SimpleNamespace( + peer_id=self.peer_id, + relay_addr=self.relay_addr, + ) + + def wait_connected(self, timeout=5.0): + return True + + def stop(self): + pass + + monkeypatch.setattr( + startup_mod, + "detect_hardware", + lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0}, + ) + monkeypatch.setattr(startup_mod, "_detect_num_layers", lambda _model_id: 24) + monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer) + monkeypatch.setattr(startup_mod, "RelayHttpBridge", FakeRelayHttpBridge) + + tracker = TrackerServer(relay_url="ws://public-relay.example/ws") + tracker_port = tracker.start() + tracker_url = f"http://127.0.0.1:{tracker_port}" + try: + node = run_startup( + tracker_url=tracker_url, + model_id="Qwen/Qwen2.5-0.5B-Instruct", + advertise_host="203.0.113.10", + wallet_path=tmp_path / "wallet.json", + ) + try: + network_map = _get_json(f"{tracker_url}/v1/network/map") + finally: + node.stop() + finally: + tracker.stop() + + assert network_map["relay_url"] == "ws://public-relay.example/ws" + assert len(network_map["nodes"]) == 1 + registered = network_map["nodes"][0] + assert registered["hf_repo"] == "Qwen/Qwen2.5-0.5B-Instruct" + assert registered["endpoint"] == "http://203.0.113.10:8001" + assert registered["relay_addr"].startswith("ws://public-relay.example/rpc/") + assert registered["peer_id"] + + +def test_later_node_auto_joins_existing_public_hf_model_with_only_tracker_url( + tmp_path, + monkeypatch, +): + """After a model exists, a node can join by knowing only the public tracker URL.""" + import meshnet_node.startup as startup_mod + + captured = {} + + class FakeBackend: + total_layers = 24 + + class FakeTorchNodeServer: + def __init__(self, **kwargs): + captured.update(kwargs) + self.backend = FakeBackend() + self.port = None + self.chat_completion_count = 0 + self.total_requests = 0 + self.failed_requests = 0 + self.queue_depth = 0 + + def start(self): + self.port = 8002 + return self.port + + def stop(self): + pass + + monkeypatch.setattr( + startup_mod, + "detect_hardware", + lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0}, + ) + monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer) + + tracker = TrackerServer() + tracker_port = tracker.start() + tracker_url = f"http://127.0.0.1:{tracker_port}" + try: + data = json.dumps({ + "endpoint": "http://203.0.113.20:8001", + "model": "Qwen2.5-0.5B-Instruct", + "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", + "num_layers": 24, + "shard_start": 0, + "shard_end": 11, + "tracker_mode": True, + "hardware_profile": {}, + "score": 1.0, + }).encode() + req = urllib.request.Request( + f"{tracker_url}/v1/nodes/register", + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + resp.read() + + node = run_startup( + tracker_url=tracker_url, + advertise_host="203.0.113.21", + wallet_path=tmp_path / "wallet.json", + ) + try: + route_resp = _get_json( + f"{tracker_url}/v1/route?model=Qwen/Qwen2.5-0.5B-Instruct" + ) + finally: + node.stop() + finally: + tracker.stop() + + assert captured["model_id"] == "Qwen/Qwen2.5-0.5B-Instruct" + assert captured["shard_start"] == 12 + assert captured["shard_end"] == 23 + assert route_resp["route"] == ["http://203.0.113.20:8001", "http://203.0.113.21:8002"] + + # --------------------------------------------------------------------------- # Full startup integration test # --------------------------------------------------------------------------- diff --git a/tests/test_tracker_routing.py b/tests/test_tracker_routing.py index 9c02e61..6f3c4c4 100644 --- a/tests/test_tracker_routing.py +++ b/tests/test_tracker_routing.py @@ -113,6 +113,90 @@ def test_tracker_serves_health_while_proxy_request_is_in_flight(): slow_thread.join(timeout=1.0) +def test_tracker_routes_hf_model_alias_from_quickstart(): + """The documented qwen2.5-0.5b alias resolves a full HF repo registration.""" + tracker = TrackerServer() + tracker_port = tracker.start() + try: + _post_json( + f"http://127.0.0.1:{tracker_port}/v1/nodes/register", + {"endpoint": "http://127.0.0.1:9100", + "model": "Qwen2.5-0.5B-Instruct", + "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", + "num_layers": 24, + "shard_start": 0, + "shard_end": 23, + "tracker_mode": True, + "hardware_profile": {}, + "score": 1.0}, + ) + + route_resp = _get_json( + f"http://127.0.0.1:{tracker_port}/v1/route?model=qwen2.5-0.5b" + ) + finally: + tracker.stop() + + assert route_resp["route"] == ["http://127.0.0.1:9100"] + + +def test_tracker_proxy_accepts_hf_model_alias_from_quickstart(): + """The tracker proxy accepts the same model alias used by the quickstart curl.""" + + class ChatHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + pass + + def do_POST(self): + if self.path != "/v1/chat/completions": + self.send_response(404) + self.end_headers() + return + length = int(self.headers.get("Content-Length", 0)) + request_body = json.loads(self.rfile.read(length) or b"{}") + body = json.dumps({ + "model": request_body["model"], + "choices": [{"message": {"content": "56"}}], + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + node = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler) + node_thread = threading.Thread(target=node.serve_forever, daemon=True) + node_thread.start() + tracker = TrackerServer() + tracker_port = tracker.start() + try: + _post_json( + f"http://127.0.0.1:{tracker_port}/v1/nodes/register", + {"endpoint": f"http://127.0.0.1:{node.server_address[1]}", + "model": "Qwen2.5-0.5B-Instruct", + "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", + "num_layers": 24, + "shard_start": 0, + "shard_end": 23, + "tracker_mode": True, + "hardware_profile": {}, + "score": 1.0}, + ) + + response = _post_json( + f"http://127.0.0.1:{tracker_port}/v1/chat/completions", + {"model": "qwen2.5-0.5b", + "messages": [{"role": "user", "content": "What is 7 times 8?"}]}, + ) + finally: + tracker.stop() + node.shutdown() + node.server_close() + node_thread.join(timeout=1.0) + + assert response["choices"][0]["message"]["content"] == "56" + + def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix(): tracker = TrackerServer() tracker_port = tracker.start()