diff --git a/QUICKSTART.md b/QUICKSTART.md index a393d86..7b1fe69 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -135,6 +135,13 @@ Via public hostname with relay (works from behind NAT, WSL2, 5G — no `--advert .\.venv\Scripts\meshnet-node.exe start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct ``` +WSL (same relay path — no `--advertise-host`): + +```bash +.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +.venv/bin/meshnet-node start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct +``` + `--host 0.0.0.0` binds the node to all Windows interfaces. `--advertise-host` is what the tracker gives to other nodes for direct connections; omit it when using the relay path since all traffic flows through the relay tunnel instead. @@ -176,26 +183,114 @@ Client → HTTPS → ai.neuron.d-popov.com (nginx) └─ /rpc/* → meshnet-relay :8765 (caller opens WS per hop) ``` -### Start the relay and tracker (server side) +### Nginx Proxy Manager (Docker) + +Use **one** proxy host for the domain. Do not create a second host for the same +domain to reach another port — path routing is done with **Custom locations** on +that same host. + +**1. Details tab** (default `/` → tracker) + +| Field | Value | +|-------|--------| +| Domain Names | `ai.neuron.d-popov.com` | +| Scheme | `http` | +| Forward Hostname / IP | LAN IP of the tracker machine (e.g. `192.168.0.179`) | +| Forward Port | `8081` | +| Websockets Support | ON | + +This serves `/v1/network/map`, `/v1/chat/completions`, and the rest of the tracker API. + +**2. Custom locations tab** (sub-paths → relay) + +The Custom locations form has **no separate Websockets toggle** — only location, +scheme, forward host, optional sub-folder path, and port. Add **two** locations +(both pointing at the relay process on port `8765`). Leave **“Add a path for +sub-folder forwarding”** empty so the full URI reaches the relay +(`/ws`, `/rpc/`). + +Location A — persistent node connections: + +| Field | Value | +|-------|--------| +| Define location | `/ws` | +| Scheme | `http` | +| Forward Hostname / IP | `192.168.0.179` | +| Forward Port | `8765` | +| Sub-folder path | *(leave empty)* | + +Location B — per-hop RPC: + +| Field | Value | +|-------|--------| +| Define location | `/rpc` | +| Scheme | `http` | +| Forward Hostname / IP | `192.168.0.179` | +| Forward Port | `8765` | +| Sub-folder path | *(leave empty)* | + +Nginx matches the longer prefixes first: `/ws` and `/rpc/…` go to relay; everything +else stays on `8081`. + +**3. SSL tab** + +Use your existing Let’s Encrypt certificate (unchanged). + +**4. Advanced tab** (only if WebSocket upgrade fails on `/ws` or `/rpc`) + +Custom locations do not expose a Websockets checkbox. If nodes show +`Relay configured but not connected yet` while `/v1/network/map` works, add this +snippet on the **proxy host** Advanced tab: + +```nginx +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $http_connection; +proxy_read_timeout 3600s; +proxy_send_timeout 3600s; +``` + +**5. Verify routing** ```bash -# Terminal 1 — relay (WebSocket hub) +# Tracker (8081 via default location) +curl -s https://ai.neuron.d-popov.com/v1/network/map | python3 -m json.tool + +# Relay paths should not 502/404 from the tracker — check response headers/status +curl -sI https://ai.neuron.d-popov.com/ws +curl -sI https://ai.neuron.d-popov.com/rpc/test-peer +``` + +After NPM is correct, start relay and tracker on the LAN machine: + +```bash +# Terminal 1 — relay .venv/bin/meshnet-relay --host 0.0.0.0 --port 8765 -# Terminal 2 — tracker (advertises relay URL to nodes) +# Terminal 2 — tracker (advertises relay to nodes) .venv/bin/meshnet-tracker start \ --host 0.0.0.0 \ --port 8081 \ --relay-url wss://ai.neuron.d-popov.com/ws ``` +Nodes using `https://ai.neuron.d-popov.com` should then log: + +```text +Relay advertised by tracker — using outbound tunnel wss://ai.neuron.d-popov.com/ws + Relay connected — wss://ai.neuron.d-popov.com/rpc/ +``` + The `--relay-url` flag embeds the relay address in `/v1/network/map`. Every node queries that endpoint on startup and auto-connects if a relay URL is present. ### Start a node (any machine, any network) -No `--advertise-host` needed. The node discovers the relay URL from the tracker -and opens a persistent outbound WebSocket: +No `--advertise-host`, firewall rule, port forwarding, relay URL, or peer URL is +needed on the node. The public tracker is the only bootstrap URL the user types. +The node queries the tracker for `/v1/network/map`, discovers the relay URL, and +opens a persistent outbound WebSocket. If the relay connection drops, the node +keeps retrying it. ```bash .venv/bin/meshnet-node start \ diff --git a/packages/node/meshnet_node/startup.py b/packages/node/meshnet_node/startup.py index e2c5c0c..b791fd1 100644 --- a/packages/node/meshnet_node/startup.py +++ b/packages/node/meshnet_node/startup.py @@ -35,13 +35,32 @@ def _get_json(url: str, timeout: float = 10.0) -> dict: return json.loads(r.read()) +def _infer_relay_url_from_tracker(tracker_url: str) -> str | None: + """Infer relay WebSocket URL from a public HTTPS tracker origin. + + Public deployments colocate relay at /ws on the same host as the tracker API + (see QUICKSTART nginx layout). Local LAN trackers use a separate relay port + and must advertise relay_url explicitly via /v1/network/map. + """ + parsed = urllib.parse.urlparse(tracker_url) + if parsed.scheme != "https": + return None + host = parsed.hostname + if not host or host in ("127.0.0.1", "localhost"): + return None + return f"wss://{parsed.netloc}/ws" + + def _discover_relay_url(tracker_url: str) -> str | None: + relay_url: str | None = None try: network_map = _get_json(f"{tracker_url}/v1/network/map", timeout=5.0) + raw = network_map.get("relay_url") + if isinstance(raw, str) and raw: + relay_url = raw except Exception: - return None - relay_url = network_map.get("relay_url") - return relay_url if isinstance(relay_url, str) and relay_url else None + pass + return relay_url or _infer_relay_url_from_tracker(tracker_url) def _start_relay_bridge_if_available( @@ -49,8 +68,9 @@ def _start_relay_bridge_if_available( wallet_address: str, local_base_url: str, advertised_endpoint: str, + relay_url: str | None = None, ) -> tuple[RelayHttpBridge | None, dict]: - relay_url = _discover_relay_url(tracker_url) + relay_url = relay_url or _discover_relay_url(tracker_url) if not relay_url: return None, {} peer_id = peer_id_from_wallet(wallet_address) @@ -222,7 +242,8 @@ def _warn_virtual_network_ip(ip: str | None) -> None: f" This range is used by Docker, WSL2, and virtual machines and is\n" f" NOT reachable from other physical machines on your LAN.\n" f" Cross-host pipeline hops WILL time out.\n" - f" Fix: pass --advertise-host (e.g. 192.168.x.x).\n", + f" Fix: use a public tracker with relay (wss://…/ws), or pass\n" + f" --advertise-host (e.g. 192.168.x.x).\n", flush=True, ) except Exception: @@ -260,6 +281,7 @@ def run_startup( """ tracker_url = tracker_url.rstrip("/") + relay_url = _discover_relay_url(tracker_url) # 1. Hardware detection if advertise_host is None and host == "0.0.0.0": @@ -275,7 +297,10 @@ def run_startup( except Exception: advertise_host = socket.getfqdn() - _warn_virtual_network_ip(advertise_host) + if relay_url: + print(f"Relay advertised by tracker — using outbound tunnel {relay_url}", flush=True) + else: + _warn_virtual_network_ip(advertise_host) print("Detecting hardware...", flush=True) hw = detect_hardware() @@ -361,6 +386,7 @@ def run_startup( address, local_base_url, endpoint, + relay_url=relay_url, ) _attach_relay_bridge(node, relay_bridge) # Register with tracker so other nodes can auto-join this model. @@ -451,6 +477,7 @@ def run_startup( address, local_base_url, endpoint, + relay_url=relay_url, ) _attach_relay_bridge(node, relay_bridge) auto_reg_payload = { @@ -548,6 +575,7 @@ def run_startup( address, local_base_url, endpoint, + relay_url=relay_url, ) _attach_relay_bridge(node, relay_bridge) diff --git a/packages/tracker/meshnet_tracker/cli.py b/packages/tracker/meshnet_tracker/cli.py index 8e6a8f7..c7da977 100644 --- a/packages/tracker/meshnet_tracker/cli.py +++ b/packages/tracker/meshnet_tracker/cli.py @@ -4,7 +4,7 @@ import argparse import sys import time -from .server import TrackerServer +from .server import TrackerServer, derive_relay_url_from_public_tracker_url def main() -> None: @@ -52,6 +52,7 @@ def main() -> None: if args.command in {None, "start"}: cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()] + relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url) server = TrackerServer( host=args.host, port=args.port, @@ -59,7 +60,7 @@ def main() -> None: cluster_peers=cluster_peers or None, cluster_self_url=args.self_url, stats_db=getattr(args, "stats_db", None), - relay_url=args.relay_url, + relay_url=relay_url, ) port = server.start() print(f"meshnet-tracker listening on http://{args.host}:{port}", flush=True) diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 927b8a6..3344638 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -37,6 +37,19 @@ from .gossip import NodeGossip from .raft import RaftNode +def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None: + """Return wss://host/ws when url is a public HTTPS tracker origin.""" + if not url: + return None + parsed = urllib.parse.urlparse(url) + if parsed.scheme != "https": + return None + host = parsed.hostname + if not host or host in ("127.0.0.1", "localhost"): + return None + return f"wss://{parsed.netloc}/ws" + + DEFAULT_MODEL_PRESETS: dict[str, dict] = { "stub-model": { "layers_start": 0, @@ -1228,6 +1241,33 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): if auth: req.add_header("Authorization", auth) + relay_headers = { + "Content-Type": "application/json", + "X-Meshnet-Route": downstream_urls, + **({"Authorization": auth} if auth else {}), + } + + if node.relay_addr: + print( + f"[tracker] proxy via relay {request_id}: {node.relay_addr} " + f"(direct endpoint {target_url})", + flush=True, + ) + relayed = _relay_http_request( + node.relay_addr, + path="/v1/chat/completions", + body=raw_body, + headers=relay_headers, + ) + if relayed is not None: + self._send_relayed_response(relayed) + return + print( + f"[tracker] relay proxy failed {request_id}: {node.relay_addr}; " + f"trying direct {target_url}", + flush=True, + ) + try: upstream = urllib.request.urlopen(req, timeout=300.0) print(f"[tracker] proxy connected {request_id}: {target_url}", flush=True) @@ -1247,23 +1287,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): if node.relay_addr: print( f"[tracker] direct proxy failed {request_id}: {target_url}: {exc}; " - f"trying relay {node.relay_addr}", + f"relay already attempted for {node.relay_addr}", flush=True, ) - relayed = _relay_http_request( - node.relay_addr, - path="/v1/chat/completions", - body=raw_body, - headers={ - "Content-Type": "application/json", - "X-Meshnet-Route": downstream_urls, - **({"Authorization": auth} if auth else {}), - }, - ) - if relayed is not None: - self._send_relayed_response(relayed) - return - print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True) + else: + print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True) self._send_json(503, {"error": { "message": f"upstream node unreachable: {exc}", "type": "service_unavailable", @@ -2040,6 +2068,11 @@ class TrackerServer: if self._server is not None: raise RuntimeError("TrackerServer is already running") + effective_relay_url = ( + self._relay_url + or derive_relay_url_from_public_tracker_url(self._cluster_self_url) + ) + # Start HTTP server first so we know our port self._server = _TrackerHTTPServer( (self._host, self._requested_port), @@ -2050,7 +2083,7 @@ class TrackerServer: self._model_presets, self._contracts, self._minimum_stake, - relay_url=self._relay_url, + relay_url=effective_relay_url, stats=self._stats, ) self.port = self._server.server_address[1] diff --git a/tests/test_gossip_and_relay.py b/tests/test_gossip_and_relay.py index 7344444..5530a04 100644 --- a/tests/test_gossip_and_relay.py +++ b/tests/test_gossip_and_relay.py @@ -350,6 +350,51 @@ def test_relay_rpc_round_trips_http_request_to_peer(): assert json.loads(response["body"]) == {"ok": True, "path": "/v1/chat/completions"} +def test_node_relay_bridge_reconnects_after_failed_connection(monkeypatch): + """Node-side relay bridge keeps retrying its outbound WebSocket connection.""" + import websockets.sync.client as wsc # type: ignore[import] + from meshnet_node.relay_bridge import RelayHttpBridge + + attempts = [] + + class FakeWebSocket: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def send(self, _message): + pass + + def recv(self, timeout=1): + raise TimeoutError + + def fake_connect(url, open_timeout=5): + attempts.append((url, open_timeout)) + if len(attempts) == 1: + raise OSError("temporary relay outage") + return FakeWebSocket() + + monkeypatch.setattr(wsc, "connect", fake_connect) + + bridge = RelayHttpBridge( + relay_url="ws://relay.example/ws", + peer_id="peer-reconnect", + local_base_url="http://127.0.0.1:8001", + advertised_addr="http://127.0.0.1:8001", + reconnect_interval=0.01, + ) + try: + info = bridge.start() + assert info.relay_addr == "ws://relay.example/rpc/peer-reconnect" + assert bridge.wait_connected(timeout=1.0) + finally: + bridge.stop() + + assert len(attempts) >= 2 + + # --------------------------------------------------------------------------- # Tracker gossip fields tests # --------------------------------------------------------------------------- @@ -389,6 +434,27 @@ def _start_tracker_and_register(extra_fields: dict) -> dict: return resp +def test_tracker_derives_relay_url_from_public_self_url(): + from meshnet_tracker.server import TrackerServer, derive_relay_url_from_public_tracker_url + + assert derive_relay_url_from_public_tracker_url("https://ai.neuron.d-popov.com") == ( + "wss://ai.neuron.d-popov.com/ws" + ) + assert derive_relay_url_from_public_tracker_url("http://127.0.0.1:8081") is None + + tracker = TrackerServer(cluster_self_url="https://ai.neuron.d-popov.com") + port = tracker.start() + try: + import json as _json + import urllib.request + + with urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map", timeout=5) as resp: + body = _json.loads(resp.read()) + assert body["relay_url"] == "wss://ai.neuron.d-popov.com/ws" + finally: + tracker.stop() + + def test_tracker_accepts_relay_addr_in_registration(): resp = _start_tracker_and_register({ "relay_addr": "ws://relay.meshnet.ai:8765/relay/abc123", diff --git a/tests/test_node_startup.py b/tests/test_node_startup.py index a9f52e3..d560e84 100644 --- a/tests/test_node_startup.py +++ b/tests/test_node_startup.py @@ -11,7 +11,11 @@ import pytest from meshnet_node.downloader import download_shard, write_shard_archive from meshnet_node.hardware import detect_hardware -from meshnet_node.startup import _probationary_status_line, run_startup +from meshnet_node.startup import ( + _infer_relay_url_from_tracker, + _probationary_status_line, + run_startup, +) from meshnet_node.wallet import _b58encode, load_or_create_wallet from meshnet_contracts import LocalSolanaContracts from meshnet_tracker.server import TrackerServer @@ -347,6 +351,94 @@ def test_tracker_assign_lists_peers_for_same_model_shard(): tracker.stop() +def test_infer_relay_url_from_public_https_tracker(): + assert _infer_relay_url_from_tracker("https://ai.neuron.d-popov.com") == ( + "wss://ai.neuron.d-popov.com/ws" + ) + assert _infer_relay_url_from_tracker("https://ai.neuron.d-popov.com/v1/network/map") == ( + "wss://ai.neuron.d-popov.com/ws" + ) + assert _infer_relay_url_from_tracker("http://192.168.0.179:8081") is None + assert _infer_relay_url_from_tracker("http://127.0.0.1:8081") is None + + +def test_public_https_tracker_infers_relay_when_network_map_omits_relay_url( + tmp_path, + monkeypatch, + capsys, +): + """Nodes bootstrap relay from the tracker origin when map relay_url is null.""" + import meshnet_node.startup as startup_mod + + class FakeBackend: + total_layers = 24 + + class FakeTorchNodeServer: + def __init__(self, **kwargs): + self.backend = FakeBackend() + self.port = None + self.chat_completion_count = 0 + self.total_requests = 0 + self.failed_requests = 0 + self.queue_depth = 0 + + def start(self): + self.port = 8001 + return self.port + + def stop(self): + pass + + class FakeRelayHttpBridge: + def __init__(self, relay_url, peer_id, local_base_url, advertised_addr): + self.relay_url = relay_url + self.peer_id = peer_id + + @property + def relay_addr(self): + return f"{self.relay_url.replace('/ws', '')}/rpc/{self.peer_id}" + + def start(self): + return types.SimpleNamespace(peer_id=self.peer_id, relay_addr=self.relay_addr) + + def wait_connected(self, timeout=5.0): + return True + + def stop(self): + pass + + monkeypatch.setattr( + startup_mod, + "detect_hardware", + lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0}, + ) + monkeypatch.setattr(startup_mod, "_detect_num_layers", lambda _model_id: 24) + monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer) + monkeypatch.setattr(startup_mod, "RelayHttpBridge", FakeRelayHttpBridge) + monkeypatch.setattr( + startup_mod, + "_get_json", + lambda url, timeout=10.0: {"relay_url": None, "nodes": []}, + ) + + tracker_url = "https://ai.neuron.d-popov.com" + node = run_startup( + tracker_url=tracker_url, + model_id="Qwen/Qwen2.5-0.5B-Instruct", + advertise_host="172.29.104.23", + wallet_path=tmp_path / "wallet.json", + ) + try: + pass + finally: + node.stop() + + output = capsys.readouterr().out + assert "Relay advertised by tracker" in output + assert "wss://ai.neuron.d-popov.com/ws" in output + assert "Cross-host pipeline hops WILL time out" not in output + + def test_real_model_startup_summary_shows_total_layers(tmp_path, monkeypatch, capsys): """Real-model startup summary prints the shard range plus total model layers.""" import meshnet_node.startup as startup_mod @@ -394,6 +486,7 @@ def test_real_model_startup_summary_shows_total_layers(tmp_path, monkeypatch, ca def test_public_tracker_model_node_registers_relay_metadata_from_tracker_url_only( tmp_path, monkeypatch, + capsys, ): """A node only needs the public tracker URL to discover relay metadata and register.""" import meshnet_node.startup as startup_mod @@ -457,7 +550,6 @@ def test_public_tracker_model_node_registers_relay_metadata_from_tracker_url_onl node = run_startup( tracker_url=tracker_url, model_id="Qwen/Qwen2.5-0.5B-Instruct", - advertise_host="203.0.113.10", wallet_path=tmp_path / "wallet.json", ) try: @@ -471,9 +563,91 @@ def test_public_tracker_model_node_registers_relay_metadata_from_tracker_url_onl assert len(network_map["nodes"]) == 1 registered = network_map["nodes"][0] assert registered["hf_repo"] == "Qwen/Qwen2.5-0.5B-Instruct" - assert registered["endpoint"] == "http://203.0.113.10:8001" + assert registered["endpoint"].startswith("http://") + assert registered["endpoint"].endswith(":8001") assert registered["relay_addr"].startswith("ws://public-relay.example/rpc/") assert registered["peer_id"] + output = capsys.readouterr().out + assert "Relay advertised by tracker" in output + assert "Cross-host pipeline hops WILL time out" not in output + + +def test_public_tracker_relay_suppresses_virtual_ip_warning( + tmp_path, + monkeypatch, + capsys, +): + """A WSL/Docker endpoint is acceptable when the tracker advertises relay RPC.""" + import meshnet_node.startup as startup_mod + + class FakeBackend: + total_layers = 24 + + class FakeTorchNodeServer: + def __init__(self, **kwargs): + self.backend = FakeBackend() + self.port = None + self.chat_completion_count = 0 + self.total_requests = 0 + self.failed_requests = 0 + self.queue_depth = 0 + + def start(self): + self.port = 8001 + return self.port + + def stop(self): + pass + + class FakeRelayHttpBridge: + def __init__(self, relay_url, peer_id, local_base_url, advertised_addr): + self.relay_url = relay_url + self.peer_id = peer_id + + @property + def relay_addr(self): + return f"ws://public-relay.example/rpc/{self.peer_id}" + + def start(self): + return types.SimpleNamespace(peer_id=self.peer_id, relay_addr=self.relay_addr) + + def wait_connected(self, timeout=5.0): + return True + + def stop(self): + pass + + monkeypatch.setattr( + startup_mod, + "detect_hardware", + lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0}, + ) + monkeypatch.setattr(startup_mod, "_detect_num_layers", lambda _model_id: 24) + monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer) + monkeypatch.setattr(startup_mod, "RelayHttpBridge", FakeRelayHttpBridge) + + tracker = TrackerServer(relay_url="ws://public-relay.example/ws") + tracker_port = tracker.start() + tracker_url = f"http://127.0.0.1:{tracker_port}" + try: + node = run_startup( + tracker_url=tracker_url, + model_id="Qwen/Qwen2.5-0.5B-Instruct", + advertise_host="172.29.104.23", + wallet_path=tmp_path / "wallet.json", + ) + try: + network_map = _get_json(f"{tracker_url}/v1/network/map") + finally: + node.stop() + finally: + tracker.stop() + + assert network_map["nodes"][0]["endpoint"] == "http://172.29.104.23:8001" + assert network_map["nodes"][0]["relay_addr"].startswith("ws://public-relay.example/rpc/") + output = capsys.readouterr().out + assert "Relay advertised by tracker" in output + assert "Cross-host pipeline hops WILL time out" not in output def test_later_node_auto_joins_existing_public_hf_model_with_only_tracker_url(