From be370481458ef59e1e0ea7a4a71328e5767b6745 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 30 Jun 2026 01:37:33 +0300 Subject: [PATCH] feat(us-018): WSL2 install guide, two-machine LAN test docs, and test script - docs/INSTALL_WINDOWS.md: step-by-step WSL2 + CUDA + meshnet-node install on Windows 11, including port-proxy setup and known issues - docs/TWO_MACHINE_TEST.md: two-machine LAN test procedure, start order, verification steps, latency reading, and Known Issues section - scripts/test_lan_inference.py: stdlib-only test script; sends 3 chat completions, validates OpenAI response format, prints tokens + latency, exits 0 on success; auto-discovers gateway from tracker if --gateway omitted Co-Authored-By: Claude Sonnet 4.6 --- .../distributed-inference-network/prd.json | 16 +- docs/INSTALL_WINDOWS.md | 212 ++++++++++++++++++ docs/TWO_MACHINE_TEST.md | 200 +++++++++++++++++ scripts/test_lan_inference.py | 163 ++++++++++++++ 4 files changed, 584 insertions(+), 7 deletions(-) create mode 100644 docs/INSTALL_WINDOWS.md create mode 100644 docs/TWO_MACHINE_TEST.md create mode 100644 scripts/test_lan_inference.py diff --git a/.scratch/distributed-inference-network/prd.json b/.scratch/distributed-inference-network/prd.json index ed95c0d..518830e 100644 --- a/.scratch/distributed-inference-network/prd.json +++ b/.scratch/distributed-inference-network/prd.json @@ -452,24 +452,24 @@ "Commit only this story changes" ], "priority": 18, - "status": "open", + "status": "done", "notes": "Source issue: .scratch/distributed-inference-network/issues/18-two-machine-lan-test.md", "dependsOn": [ "US-016", "US-017" ], - "completionNotes": null + "completionNotes": "docs/INSTALL_WINDOWS.md: WSL2+CUDA+meshnet-node install guide. docs/TWO_MACHINE_TEST.md: two-machine LAN test procedure with known issues. scripts/test_lan_inference.py: stdlib-only test script, 3 requests, exit 0 on success, auto-discovers gateway from tracker." }, { "id": "US-019", - "title": "19 — Distributed tracker consensus (Raft assignments + CRDT heartbeats)", - "description": "Replace the single-point-of-failure tracker with a fault-tolerant cluster. Tracker nodes elect a leader via Raft and commit shard assignments as log entries — all tracker nodes agree on who owns what. Node liveness (heartbeats) uses CRDT gossip (eventual consistency, high frequency OK). A node registers with any tracker node; the write is forwarded to the leader and replicated to followers. A 3-node tracker cluster survives one tracker failure without losing assignment state. The relay/gossip layer already built in US-017 handles peer heartbeats; this story wires Raft on top for authoritative assignments.", + "title": "19 \u2014 Distributed tracker consensus (Raft assignments + CRDT heartbeats)", + "description": "Replace the single-point-of-failure tracker with a fault-tolerant cluster. Tracker nodes elect a leader via Raft and commit shard assignments as log entries \u2014 all tracker nodes agree on who owns what. Node liveness (heartbeats) uses CRDT gossip (eventual consistency, high frequency OK). A node registers with any tracker node; the write is forwarded to the leader and replicated to followers. A 3-node tracker cluster survives one tracker failure without losing assignment state. The relay/gossip layer already built in US-017 handles peer heartbeats; this story wires Raft on top for authoritative assignments.", "acceptanceCriteria": [ "3 tracker nodes can be started and form a Raft cluster (leader election, log replication)", - "A node registers with any follower — the registration is forwarded to the leader and replicated", + "A node registers with any follower \u2014 the registration is forwarded to the leader and replicated", "Killing the leader causes a new election within 5 seconds; registrations continue working", "Shard assignments returned by any tracker node are identical (strong consistency)", - "Node heartbeats use CRDT gossip (not Raft) — high-frequency, eventual consistency", + "Node heartbeats use CRDT gossip (not Raft) \u2014 high-frequency, eventual consistency", "meshnet-tracker CLI gains --cluster-peers flag to specify peer tracker URLs", "Integration test: 3 tracker nodes, kill leader mid-test, verify assignment still works", "QUICKSTART.md updated with multi-tracker setup section" @@ -477,7 +477,9 @@ "priority": 19, "status": "open", "notes": "Architecture decision: Raft for assignments (strong consistency) + CRDT gossip for liveness (eventual consistency). User approved 2026-06-29.", - "dependsOn": ["US-017"], + "dependsOn": [ + "US-017" + ], "completionNotes": null } ], diff --git a/docs/INSTALL_WINDOWS.md b/docs/INSTALL_WINDOWS.md new file mode 100644 index 0000000..3507363 --- /dev/null +++ b/docs/INSTALL_WINDOWS.md @@ -0,0 +1,212 @@ +# Installing meshnet-node on Windows 11 with WSL2 + +This guide covers setting up a meshnet-node on a Windows 11 machine using WSL2 with CUDA passthrough so it can join an existing inference network over LAN. + +## Prerequisites + +- Windows 11 with WSL2 support (most systems with Windows 10 version 2004+ qualify) +- NVIDIA GPU with CUDA support (driver ≥ 525.x recommended for WSL2 CUDA) +- At least 8 GB RAM + enough VRAM for the model shard you intend to serve +- The Linux machine (other node) is reachable on your LAN + +--- + +## Step 1 — Enable WSL2 and install Ubuntu + +Open **PowerShell as Administrator** and run: + +```powershell +wsl --install -d Ubuntu-24.04 +``` + +This installs WSL2 with Ubuntu 24.04. Reboot when prompted. + +After reboot, Ubuntu starts and asks you to create a UNIX username/password. Choose anything convenient. + +Verify WSL version: + +```powershell +wsl -l -v +``` + +Output should show `VERSION 2`. + +--- + +## Step 2 — Install NVIDIA GPU driver on Windows (NOT inside WSL) + +WSL2 CUDA passthrough works through the Windows host driver. **Do not install CUDA inside WSL2.** + +1. Download the latest Game Ready or Studio driver for your GPU from https://www.nvidia.com/drivers +2. Install on Windows normally (standard installer). +3. Inside WSL2 (Ubuntu terminal), verify: + +```bash +nvidia-smi +``` + +Expected output: your GPU name, driver version, CUDA version. If this command fails, the Windows driver is too old — update it. + +> **Note:** The `cuda-toolkit` package inside WSL2 is optional and only needed if you compile CUDA kernels. For inference with `torch`, the Windows host driver is sufficient. + +--- + +## Step 3 — Install Python 3.11+ inside WSL2 + +Ubuntu 24.04 ships Python 3.12. Confirm: + +```bash +python3 --version +``` + +If it shows 3.10 or older: + +```bash +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt update +sudo apt install python3.12 python3.12-venv python3.12-dev +``` + +Install pip: + +```bash +curl -sS https://bootstrap.pypa.io/get-pip.py | python3 +``` + +--- + +## Step 4 — Clone the repository + +Inside WSL2: + +```bash +# Store the repo in the Linux filesystem (faster I/O than /mnt/c) +cd ~ +git clone https://github.com/YOUR_ORG/d-popov.com.git +cd d-popov.com/AI +``` + +--- + +## Step 5 — Create a virtualenv and install meshnet-node + +```bash +python3 -m venv .venv +source .venv/bin/activate + +# Install node + PyTorch (CUDA build) +pip install torch --index-url https://download.pytorch.org/whl/cu124 +pip install -e "packages/node[torch]" +``` + +Verify the install: + +```bash +meshnet-node --help +``` + +--- + +## Step 6 — Pre-download the model shard + +Download the model before starting the node so the startup process doesn't time out on the tracker side: + +```bash +python3 - <<'EOF' +from transformers import AutoConfig +AutoConfig.from_pretrained("microsoft/Phi-3-medium-128k-instruct") +EOF +``` + +For the full model weights (needed at runtime), `transformers` downloads them automatically on first `meshnet-node` start. If you want to pre-fetch: + +```bash +python3 -c " +from transformers import AutoModelForCausalLM +AutoModelForCausalLM.from_pretrained('microsoft/Phi-3-medium-128k-instruct', device_map='cpu') +" +``` + +This can take 10–30 minutes on first run. + +--- + +## Step 7 — Expose the node port to your LAN + +WSL2 runs behind a NAT with a virtual IP (typically `172.x.x.x`). Your LAN sees the Windows host IP. You need to forward the node port. + +**Option A — Windows port proxy (recommended for simple setups):** + +In **PowerShell as Administrator**: + +```powershell +# Get the current WSL2 IP (changes on each WSL restart) +$wslIp = (wsl hostname -I).Trim() + +# Forward Windows host port 8001 → WSL2 port 8001 +netsh interface portproxy add v4tov4 ` + listenport=8001 listenaddress=0.0.0.0 ` + connectport=8001 connectaddress=$wslIp + +# Allow inbound on Windows Firewall +New-NetFirewallRule -DisplayName "meshnet-node" ` + -Direction Inbound -Protocol TCP -LocalPort 8001 -Action Allow +``` + +Verify: from the Linux machine, `curl http://WINDOWS_LAN_IP:8001/v1/health` should return a response once the node is running. + +**Redo this after every WSL2 restart** — the WSL2 IP changes. + +**Option B — P2P relay (US-017, no port forwarding needed):** + +Start a relay node on the Linux machine. The WSL2 node connects outbound through the relay. No firewall rules needed. See `docs/TWO_MACHINE_TEST.md` for details. + +--- + +## Step 8 — Start the node + +Replace `192.168.1.10` with the actual LAN IP of the Linux machine running the tracker. +Replace shard range with the complementary range to what the Linux node is serving. + +```bash +source .venv/bin/activate + +meshnet-node \ + --model microsoft/Phi-3-medium-128k-instruct \ + --quantization bf16 \ + --shard-start 20 --shard-end 39 \ + --tracker http://192.168.1.10:8080 \ + --port 8001 \ + --host 0.0.0.0 \ + --advertise-host WINDOWS_LAN_IP +``` + +The `--advertise-host` flag tells the tracker what IP the Linux machine should use to reach this node. Use your Windows machine's LAN IP (e.g. `192.168.1.20`), **not** the WSL2 internal IP. + +Expected startup output: + +``` +Detecting hardware... + GPU: NVIDIA GeForce RTX 3080 (10240 MB VRAM) +Loading wallet... + Wallet: 5K7r... +Loading real PyTorch model shard... + Auto-detected 40 layers → shard 20–39 +================================ +meshnet-node ready + Model ID: microsoft/Phi-3-medium-128k-instruct + Shard: layers 20–39; 20 of 40 + Endpoint: http://192.168.1.20:8001 + Hardware: CUDA +================================ +``` + +--- + +## Known issues + +- **WSL2 IP changes on restart.** Always re-run the `netsh` port-proxy command after restarting WSL2 or Windows. +- **CUDA not visible in WSL2.** If `nvidia-smi` fails inside WSL2, update the Windows host GPU driver to ≥ 525.x. Installing CUDA inside WSL2 will not fix it. +- **Model download is slow.** HuggingFace downloads happen over HTTPS. Pre-fetch the model before a timed test (see Step 6). +- **Port 8001 already in use.** Change `--port` to another value and update the firewall/portproxy rules accordingly. +- **`bf16` not supported on older GPUs.** Use `--quantization int8` on Turing (RTX 20xx) cards or earlier if bfloat16 ops fail. diff --git a/docs/TWO_MACHINE_TEST.md b/docs/TWO_MACHINE_TEST.md new file mode 100644 index 0000000..b082414 --- /dev/null +++ b/docs/TWO_MACHINE_TEST.md @@ -0,0 +1,200 @@ +# Two-machine LAN inference test + +This guide proves that distributed inference works across two physical machines: a Linux rig (tracker + first shard) and a Windows 11 / WSL2 rig (second shard). A test script sends real inference requests and validates the output. + +## Network topology + +``` +[Linux machine — 192.168.1.10] + meshnet-tracker :8080 + meshnet-node A :8001 shard 0–19 (tracker-mode, entry point) + +[Windows 11 / WSL2 — 192.168.1.20] + meshnet-node B :8001 shard 20–39 + +[Client — either machine] + scripts/test_lan_inference.py --tracker http://192.168.1.10:8080 +``` + +Adjust the IPs and shard ranges to match your hardware. Use a model that fits (sharded) in both GPUs combined. The example uses `microsoft/Phi-3-medium-128k-instruct` (40 layers, BF16 ~15 GB each shard ~7.5 GB). + +--- + +## Prerequisites + +**Both machines:** +- Python 3.11+ with `meshnet-node` installed (see `docs/INSTALL_WINDOWS.md` for Windows) +- Model weights already downloaded (pre-fetch prevents timeout on first startup) +- LAN connectivity verified: `ping 192.168.1.10` from Windows, `ping 192.168.1.20` from Linux + +**Linux machine ports open:** + +```bash +# ufw (skip if firewall is off) +sudo ufw allow 8080/tcp # tracker +sudo ufw allow 8001/tcp # node A +``` + +**Windows machine port forwarded (WSL2 only):** + +```powershell +# Run in PowerShell as Administrator — redo after every WSL restart +$wsl = (wsl hostname -I).Trim() +netsh interface portproxy add v4tov4 listenport=8001 listenaddress=0.0.0.0 connectport=8001 connectaddress=$wsl +New-NetFirewallRule -DisplayName "meshnet-node" -Direction Inbound -Protocol TCP -LocalPort 8001 -Action Allow +``` + +--- + +## Start sequence + +**Always start in this order: tracker → node A → node B → test.** + +### Terminal 1 — Linux: tracker + +```bash +meshnet-tracker --port 8080 +``` + +Expected: + +``` +[tracker] listening on 0.0.0.0:8080 +``` + +### Terminal 2 — Linux: node A (shard 0–19, tracker-mode) + +```bash +meshnet-node \ + --model microsoft/Phi-3-medium-128k-instruct \ + --quantization bf16 \ + --shard-start 0 --shard-end 19 \ + --tracker http://localhost:8080 \ + --port 8001 \ + --host 0.0.0.0 +``` + +`shard_start=0` auto-sets `tracker_mode=True` — this node accepts inference requests. + +Wait until you see `meshnet-node ready` before continuing. + +### Terminal 3 — Windows WSL2: node B (shard 20–39) + +```bash +meshnet-node \ + --model microsoft/Phi-3-medium-128k-instruct \ + --quantization bf16 \ + --shard-start 20 --shard-end 39 \ + --tracker http://192.168.1.10:8080 \ + --port 8001 \ + --host 0.0.0.0 \ + --advertise-host 192.168.1.20 +``` + +`--advertise-host` must be the Windows **LAN IP** (not the WSL2 internal 172.x.x.x IP) so the Linux node can reach it. + +--- + +## Verify nodes are registered + +From any machine with `curl`: + +```bash +# List all registered nodes +curl http://192.168.1.10:8080/v1/nodes + +# Check route for the model — should list both node endpoints in order +curl "http://192.168.1.10:8080/v1/route?model=microsoft/Phi-3-medium-128k-instruct" +``` + +Expected route response: + +```json +{ + "route": [ + "http://192.168.1.10:8001", + "http://192.168.1.20:8001" + ] +} +``` + +If only one endpoint appears, node B hasn't registered yet — wait a few seconds and retry. + +--- + +## Run the test script + +```bash +# From any machine that can reach the tracker +python3 scripts/test_lan_inference.py \ + --tracker http://192.168.1.10:8080 \ + --gateway http://192.168.1.10:8001 +``` + +Expected output: + +``` +Inference endpoint: http://192.168.1.10:8001 +Tracker: http://192.168.1.10:8080 + +Route: ['http://192.168.1.10:8001', 'http://192.168.1.20:8001'] + +[1] Q: What is 7 × 8? Answer in one word. + A: 56 + 3 tokens 2.41s 1.2 t/s + +[2] Q: Name the capital of France in one word. + A: Paris + 2 tokens 1.87s 1.1 t/s + +[3] Q: Complete the sequence: 1, 1, 2, 3, 5, ___ + A: 8 + 2 tokens 1.93s 1.0 t/s + +All 3 requests completed successfully. +Exit code: 0 +``` + +The script exits 0 if all 3 requests complete with valid OpenAI-format responses. + +--- + +## Reading latency from node logs + +The node logs show per-hop timing. On node A terminal look for: + +``` +[node] forwarding to downstream: http://192.168.1.20:8001 (took 1.23s) +``` + +Approximate breakdown: +- **client → node A (encode + first shard):** full request latency minus the downstream time +- **node A → node B (pipeline):** the `forwarding to downstream` duration +- **node B → node A (tail decode + token):** included in downstream duration + +Full end-to-end latency = prompt encode + shard A forward + network transfer + shard B forward + decode. + +With LAN latency < 1 ms, the network transfer is negligible. Bottleneck is GPU compute. + +--- + +## Known Issues + +**WSL2 IP changes after restart.** +The `netsh portproxy` forwarding rule uses a fixed WSL2 IP. If Windows or WSL2 restarts, the IP changes and the rule breaks. Redo the `netsh` and `New-NetFirewallRule` commands. To automate this, add a Task Scheduler job on WSL start. + +**Node B registers with internal WSL2 IP (172.x.x.x) instead of LAN IP.** +Symptom: route response lists `172.x.x.x` and node A cannot reach it. +Fix: always pass `--advertise-host 192.168.1.20` (your Windows LAN IP) when starting node B. + +**Model download times out node registration.** +If the model hasn't been pre-fetched, `transformers` downloads it during node startup, which can take 20+ minutes. The tracker heartbeat timeout (90s) will expire, and node A will deregister node B. Pre-download the model weights before starting the node (see `docs/INSTALL_WINDOWS.md` Step 6). Node B re-registers automatically via the heartbeat re-registration loop once it's up. + +**`bf16` unsupported on older NVIDIA GPUs.** +GPUs before Ampere (RTX 30xx) have limited bfloat16 support. Use `--quantization int8` on RTX 20xx and earlier. + +**Windows Defender blocks inbound connection on WSL2.** +Even with the firewall rule added, Windows Defender SmartScreen or a corporate security policy can block the connection. Verify by checking Windows Event Viewer → Security → Filtering Platform Connection for blocked connections on port 8001. + +**Route returns only one node.** +If node B registers but the route only returns one endpoint, check that both nodes use the same `--model` string (full HuggingFace repo path). Route lookup matches on `hf_repo` — a short name vs. full path mismatch causes the node to be excluded. diff --git a/scripts/test_lan_inference.py b/scripts/test_lan_inference.py new file mode 100644 index 0000000..9b5ba87 --- /dev/null +++ b/scripts/test_lan_inference.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +End-to-end LAN inference test for meshnet distributed inference. + +Sends 3 chat-completion requests to a meshnet node, validates OpenAI-format +responses, and prints token counts + latency per request. + +Usage: + python scripts/test_lan_inference.py \\ + --tracker http://192.168.1.10:8080 \\ + --gateway http://192.168.1.10:8001 + +Exit 0 on success, 1 on any failure. +""" +from __future__ import annotations + +import argparse +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + + +PROMPTS = [ + {"role": "user", "content": "What is 7 × 8? Answer in one word."}, + {"role": "user", "content": "Name the capital of France in one word."}, + {"role": "user", "content": "Complete the sequence: 1, 1, 2, 3, 5, ___. Answer in one word."}, +] + +MODEL = "microsoft/Phi-3-medium-128k-instruct" + + +def _get(url: str, timeout: float = 10.0) -> dict: + with urllib.request.urlopen(url, timeout=timeout) as r: + return json.loads(r.read()) + + +def _post(url: str, payload: dict, timeout: float = 60.0) -> dict: + data = json.dumps(payload).encode() + req = urllib.request.Request( + url, data=data, headers={"Content-Type": "application/json"}, method="POST" + ) + with urllib.request.urlopen(req, timeout=timeout) as r: + return json.loads(r.read()) + + +def discover_gateway(tracker_url: str) -> str: + """Return the first tracker-mode node endpoint for MODEL.""" + nodes = _get(f"{tracker_url}/v1/nodes", timeout=5.0) + if isinstance(nodes, dict): + nodes = list(nodes.values()) + tracker_nodes = [ + n for n in nodes + if n.get("tracker_mode") and ( + n.get("hf_repo") == MODEL or n.get("model") == MODEL.split("/")[-1] + ) + ] + if not tracker_nodes: + raise RuntimeError( + f"No tracker-mode nodes found for {MODEL!r}. " + "Is the first-shard node running and registered?" + ) + endpoint: str = tracker_nodes[0]["endpoint"] + return endpoint.rstrip("/") + + +def check_route(tracker_url: str, gateway_url: str) -> list[str]: + """Return the full inference route for MODEL.""" + url = f"{tracker_url}/v1/route?model={urllib.parse.quote(MODEL)}" + try: + resp = _get(url, timeout=5.0) + return resp.get("route", []) + except Exception as exc: + print(f" Warning: could not fetch route: {exc}", file=sys.stderr) + return [gateway_url] + + +def run_inference(gateway_url: str, messages: list[dict]) -> tuple[str, int, float]: + """Send one chat-completion request. Returns (content, tokens, elapsed_s).""" + t0 = time.monotonic() + resp = _post( + f"{gateway_url}/v1/chat/completions", + {"model": MODEL, "messages": messages, "stream": False}, + timeout=120.0, + ) + elapsed = time.monotonic() - t0 + + choices = resp.get("choices") + if not choices: + raise ValueError(f"No choices in response: {resp}") + content: str = choices[0].get("message", {}).get("content", "") + if not isinstance(content, str): + raise TypeError(f"Expected string content, got {type(content)}: {content}") + + usage = resp.get("usage", {}) + tokens: int = usage.get("completion_tokens", len(content.split())) + + return content, tokens, elapsed + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tracker", required=True, help="Tracker URL, e.g. http://192.168.1.10:8080") + p.add_argument( + "--gateway", + default=None, + help="Inference entry point URL. Auto-discovered from tracker if omitted.", + ) + args = p.parse_args(argv) + + tracker_url = args.tracker.rstrip("/") + + print(f"Tracker: {tracker_url}") + + # Resolve gateway + gateway_url = args.gateway.rstrip("/") if args.gateway else None + if gateway_url is None: + try: + gateway_url = discover_gateway(tracker_url) + print(f"Gateway (auto-discovered): {gateway_url}") + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + else: + print(f"Gateway: {gateway_url}") + + # Show route + route = check_route(tracker_url, gateway_url) + print(f"Route: {route}") + if len(route) < 2: + print(" Warning: only one node in route — is the second-shard node registered?") + print() + + failures = 0 + for i, msg in enumerate(PROMPTS, start=1): + print(f"[{i}] Q: {msg['content']}") + try: + content, tokens, elapsed = run_inference(gateway_url, [msg]) + tps = tokens / elapsed if elapsed > 0 else 0.0 + print(f" A: {content.strip()}") + print(f" {tokens} tokens {elapsed:.2f}s {tps:.1f} t/s") + except urllib.error.HTTPError as exc: + body = exc.read().decode(errors="replace") + print(f" ERROR {exc.code}: {body}", file=sys.stderr) + failures += 1 + except Exception as exc: + print(f" ERROR: {exc}", file=sys.stderr) + failures += 1 + print() + + if failures == 0: + print(f"All {len(PROMPTS)} requests completed successfully.") + print("Exit code: 0") + return 0 + else: + print(f"{failures}/{len(PROMPTS)} requests failed.", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main())