2 Commits

Author SHA1 Message Date
Dobromir Popov
5fe471d8ca less verbose node hop if not in debug mode 2026-06-30 17:44:21 +03:00
Dobromir Popov
6e4f755e71 fix(lan): warn when auto-detected endpoint IP is in Docker/WSL2 172.16/12 range
Nodes running inside WSL2 or Docker have a virtual network interface whose IP
(172.16-31.x.x) is not reachable from physical machines on the LAN.  The UDP
socket probe for outbound IP returns this virtual IP, which gets registered with
the tracker — causing downstream pipeline hops to time out with "urlopen error
timed out".

_warn_virtual_network_ip() now prints a clear multi-line warning at startup
when the auto-detected advertise IP falls in 172.16.0.0/12, including the
fix: --advertise-host <LAN-ip>.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-30 17:40:13 +03:00
5 changed files with 1564 additions and 2 deletions

View File

@@ -30,6 +30,7 @@ def _run_node(cfg: dict) -> None:
advertise_host=cfg.get("advertise_host"),
route_timeout=float(cfg.get("route_timeout", 30.0)),
vram_mb_override=cfg.get("vram_mb_override"),
debug=bool(cfg.get("debug", False)),
)
except Exception as exc:
print(f"\nERROR: {exc}", file=sys.stderr, flush=True)
@@ -109,6 +110,8 @@ def _cmd_default(args) -> int:
overrides["route_timeout"] = args.route_timeout
if getattr(args, "memory", None) is not None:
overrides["vram_mb_override"] = args.memory
if args.debug:
overrides["debug"] = True
if overrides:
cfg = merge_cli_overrides(cfg, **overrides)
@@ -202,6 +205,7 @@ def _cmd_start(args) -> int:
advertise_host=getattr(args, "advertise_host", None),
route_timeout=getattr(args, "route_timeout", 30.0),
vram_mb_override=getattr(args, "memory", None),
debug=getattr(args, "debug", False),
)
except Exception as exc:
print(f"ERROR: {exc}", file=sys.stderr, flush=True)
@@ -246,6 +250,7 @@ def main() -> None:
help="Seconds to wait for tracker route lookup (default 30)")
parser.add_argument("--memory", type=int, metavar="MB", default=None,
help="Override autodetected VRAM/RAM budget in MB used for shard assignment")
parser.add_argument("--debug", action="store_true", help="Enable verbose node debug logging")
parser.add_argument("--no-tui", action="store_true", help="Plain-text output (no rich dashboard)")
parser.add_argument("--compact", action="store_true", help="Single-line status output")
parser.add_argument("--reset-config", action="store_true", help="Re-run wizard even if config exists")
@@ -278,6 +283,7 @@ def main() -> None:
help="Seconds to wait for tracker route lookup (default 30)")
start_cmd.add_argument("--memory", type=int, default=None, metavar="MB",
help="Override autodetected VRAM/RAM budget in MB used for shard assignment")
start_cmd.add_argument("--debug", action="store_true", help="Enable verbose node debug logging")
args = parser.parse_args()

View File

@@ -25,6 +25,7 @@ DEFAULTS = {
"shard_end": None,
"port": 7000,
"host": "0.0.0.0",
"debug": False,
}

View File

@@ -179,6 +179,35 @@ def _start_heartbeat(
return t
def _warn_virtual_network_ip(ip: str | None) -> None:
"""Print a warning when the auto-detected advertise IP is in a known virtual-network range.
172.16.0.0/12 is used by Docker, WSL2, and most hypervisors. Nodes behind these
adapters are NOT directly reachable from other physical machines on the LAN, so
cross-host pipeline hops will time out. The user must pass --advertise-host with
their actual LAN IP (e.g. 192.168.x.x) to fix this.
"""
if ip is None:
return
try:
parts = [int(p) for p in ip.split(".")]
if len(parts) != 4:
return
a, b = parts[0], parts[1]
# 172.16.0.0/12 → 172.1631.x.x
if a == 172 and 16 <= b <= 31:
print(
f"\n WARNING: auto-detected endpoint IP {ip} is in 172.16.0.0/12.\n"
f" This range is used by Docker, WSL2, and virtual machines and is\n"
f" NOT reachable from other physical machines on your LAN.\n"
f" Cross-host pipeline hops WILL time out.\n"
f" Fix: pass --advertise-host <your-LAN-ip> (e.g. 192.168.x.x).\n",
flush=True,
)
except Exception:
pass
def run_startup(
tracker_url: str,
port: int = 0,
@@ -194,6 +223,7 @@ def run_startup(
contracts: Any | None = None,
route_timeout: float = 30.0,
vram_mb_override: int | None = None,
debug: bool = False,
) -> StubNodeServer | TorchNodeServer:
"""Execute the full startup sequence and return a running node server.
@@ -224,6 +254,8 @@ def run_startup(
except Exception:
advertise_host = socket.getfqdn()
_warn_virtual_network_ip(advertise_host)
print("Detecting hardware...", flush=True)
hw = detect_hardware()
device: str = hw["device"]
@@ -289,6 +321,7 @@ def run_startup(
quantization=quantization,
tracker_url=tracker_url,
route_timeout=route_timeout,
debug=debug,
)
_node_start_time = time.monotonic()
actual_port = node.start()
@@ -383,6 +416,7 @@ def run_startup(
quantization=quantization,
tracker_url=tracker_url,
route_timeout=route_timeout,
debug=debug,
)
_node_start_time = time.monotonic()
actual_port = node.start()

View File

@@ -38,6 +38,7 @@ class _TorchHTTPServer(http.server.HTTPServer):
tracker_mode: bool = False,
tracker_url: str | None = None,
route_timeout: float = 30.0,
debug: bool = False,
):
super().__init__(addr, handler)
self.backend = backend
@@ -46,6 +47,7 @@ class _TorchHTTPServer(http.server.HTTPServer):
self.tracker_mode = tracker_mode
self.tracker_url = tracker_url
self.route_timeout = route_timeout
self.debug = debug
self.total_requests: int = 0
self.failed_requests: int = 0
self.queue_depth: int = 0
@@ -406,7 +408,8 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
current_pos = pos_ids
for hop_index, (node_url, start_layer) in enumerate(route):
print(f" [node] pipeline hop {hop_index}: {node_url} start_layer={start_layer}", flush=True)
if server.debug:
print(f" [node] pipeline hop {hop_index}: {node_url} start_layer={start_layer}", flush=True)
headers: dict[str, str] = {
"Content-Type": "application/octet-stream",
"X-Meshnet-Wire": _WIRE_VERSION,
@@ -440,7 +443,8 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
try:
data = json.loads(resp_body)
text = str(data.get("text", ""))
print(f" [node] pipeline hop {hop_index} returned text={text!r}", flush=True)
if server.debug:
print(f" [node] pipeline hop {hop_index} returned text={text!r}", flush=True)
return text
except json.JSONDecodeError:
return resp_body.decode("utf-8", errors="replace")
@@ -611,6 +615,7 @@ class TorchNodeServer:
tracker_mode: bool | None = None,
tracker_url: str | None = None,
route_timeout: float = 30.0,
debug: bool = False,
) -> None:
self._host = host
self._requested_port = port
@@ -624,6 +629,7 @@ class TorchNodeServer:
self._tracker_mode = tracker_mode if tracker_mode is not None else (shard_start == 0)
self._tracker_url = tracker_url
self._route_timeout = route_timeout
self._debug = debug
self._server: _TorchHTTPServer | None = None
self._thread: threading.Thread | None = None
self.port: int | None = None
@@ -666,6 +672,7 @@ class TorchNodeServer:
self._tracker_mode,
self._tracker_url,
self._route_timeout,
self._debug,
)
self.port = self._server.server_address[1]
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)

1514
uv.lock generated Normal file

File diff suppressed because it is too large Load Diff