diff --git a/QUICKSTART.md b/QUICKSTART.md index ec3821b..6f4b452 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -1,761 +1,783 @@ -# Quickstart — Running a node and testing inference - -This guide gets you from zero to a live inference request in three terminals. -Tested on: AMD Ryzen AI Max (Strix Halo APU), 124 GB RAM, Linux, CPU inference. - ---- - -## Prerequisites - -```bash -# Clone and enter repo -cd /run/media/popov/d/DEV/repos/d-popov.com/AI - -# Create the virtualenv if it does not exist yet -python3 -m venv .venv - -# Keep packaging tools current enough for editable installs -.venv/bin/python -m pip install --upgrade pip setuptools wheel - -# Install Python packages (editable — picks up code changes immediately) -.venv/bin/pip install -e packages/tracker -e packages/node -e packages/p2p -e packages/gateway -e packages/relay - -# CPU-only PyTorch (skip if you have CUDA/ROCm already) -.venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu - -# HuggingFace model libraries -.venv/bin/pip install transformers accelerate -``` - -> **NVIDIA GPU (CUDA):** replace the torch line with `pip install torch` (default index). -> **AMD GPU (ROCm):** `pip install torch --index-url https://download.pytorch.org/whl/rocm6.2` - -## Bootstrap a tracker on a new machine - -Use this when provisioning a fresh LAN/public tracker host. The tracker itself is -lightweight; install the relay too if nodes will connect from NAT, WSL2, mobile, -or other networks where inbound node ports are not reachable. - -```bash -# 1. Get the repo onto the tracker host -git clone https://git.d-popov.com/popov/neuron-tai.git AI -cd AI - -# 2. Create an isolated Python environment -python3 -m venv .venv -.venv/bin/python -m pip install --upgrade pip setuptools wheel - -# 3. Install only the services needed by the tracker host -.venv/bin/pip install -e packages/tracker -e packages/relay -e packages/gateway -``` - -For a private LAN tracker, start only the tracker and open the selected TCP port -on the host firewall if other machines will join: - -```bash -.venv/bin/meshnet-tracker start --host 0.0.0.0 --port 8080 -# --starting-credit 1 --devnet-topup 10 -``` - -Verify from the tracker host: - -```bash -curl -s http://localhost:8080/v1/network/map | python3 -m json.tool -``` - -Verify from another LAN machine, replacing the IP with the tracker host's LAN IP: - -```bash -curl -s http://192.168.0.179:8080/v1/network/map | python3 -m json.tool -``` - -For a public tracker with relay support, run both services. The relay listens on -`8765`; the tracker below listens on `8081` and advertises the public WebSocket -URL that nodes should use for outbound relay connections: - -```bash -# Terminal 1 — relay -.venv/bin/meshnet-relay --host 0.0.0.0 --port 8765 - -# Terminal 2 — tracker -.venv/bin/meshnet-tracker start \ - --host 0.0.0.0 \ - --port 8081 \ - --relay-url wss://ai.neuron.d-popov.com/ws -``` - -If this host sits behind Nginx Proxy Manager, point `/` and `/v1/*` at tracker -port `8081`, and point `/ws` plus `/rpc` at relay port `8765` as shown in the -public tracker section below. After the proxy is configured, verify the public -bootstrap endpoint: - -```bash -curl -s https://ai.neuron.d-popov.com/v1/network/map | python3 -m json.tool -``` - -Nodes can then join with either the LAN tracker URL or the public URL: - -```bash -.venv/bin/meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen/Qwen2.5-0.5B-Instruct -.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct -.venv/bin/meshnet-node start --tracker https://ai.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct -``` - -### Windows / WSL2 - -Run the Linux commands from WSL, not Git Bash. From the repo opened in Git Bash: - -```bash -wsl -cd /mnt/d/DEV/workspace/REPOS/git.d-popov.com/neuron-tai -python3 -m venv .venv -.venv/bin/python -m pip install --upgrade pip setuptools wheel -.venv/bin/pip install -e packages/tracker -e packages/node -e packages/p2p -e packages/gateway -e packages/relay -.venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu -.venv/bin/pip install transformers accelerate -.venv/bin/meshnet-node --help -``` - -If `.venv/bin/meshnet-node` is missing, the editable install step did not finish -successfully. Re-run the `.venv/bin/pip install -e ...` command above inside WSL. - -WSL2 sits behind Windows NAT and is **not directly reachable** from other LAN machines. -Direct cross-host hops time out. The relay path (see below) solves this: the WSL2 node -opens an outbound WebSocket to the relay server and all traffic flows through that tunnel. -No firewall rules, no `--advertise-host` needed — just point at the public tracker URL. - -### Native Windows PowerShell node (not WSL) - -Use this when the tracker is on another machine and you want Windows to host a -reachable node on the LAN. - -#### Option A — existing conda/miniforge environment with CUDA torch (recommended if you already have it) - -First, make sure the conda base environment is active so that `python` and `pip` both -resolve to the same miniforge installation: - -```powershell -conda activate base -deactivate # drop any .venv that may be layered on top; safe no-op if none active -``` - -Install project packages into the active conda/miniforge env: - -```powershell -cd D:\DEV\workspace\REPOS\git.d-popov.com\neuron-tai - -pip install -e packages\tracker -e packages\node -e packages\p2p -e packages\gateway -e packages\relay -pip install transformers accelerate safetensors # torch is already present -``` - -Verify torch is importable and CUDA is live **before** starting the node: - -```powershell -python -c "import torch; print(torch.__version__, torch.cuda.is_available())" -# Expected: 2.x.x+cuXXX True -``` - -If you get `ModuleNotFoundError: No module named 'torch'` even though `pip install torch` -says "already satisfied", the `torch/` package directory is missing while the metadata -stub remains (can happen after a conda-managed install). Force-reinstall all three -PyTorch packages together so their versions stay in sync: - -```powershell -pip install --force-reinstall torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 -``` - -> **Important:** always reinstall `torch`, `torchvision`, and `torchaudio` as a group. -> Upgrading only `torch` leaves `torchvision` on an incompatible version, which causes -> `RuntimeError: operator torchvision::nms does not exist` and makes transformers fail -> to import any model class (the error surfaces as `Could not import module 'Qwen2ForCausalLM'`). - -Then re-run the verify step above. - -If that prints `True` but `meshnet-node` still can't find torch, the venv entry point -is shadowing the conda one. Check which binary wins: - -```powershell -(Get-Command meshnet-node).Source -# Should show: C:\Users\\miniforge3\Scripts\meshnet-node.exe -# If it shows .venv\Scripts\meshnet-node.exe, use the full path below instead -``` - -To start a node: - -```powershell -$env:HF_HOME = "D:\DEV\models" -meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct -``` - -If the wrong entry point is shadowing, invoke via the full conda path: - -```powershell -C:\Users\popov\miniforge3\Scripts\meshnet-node.exe start ` - --tracker https://ai.neuron.d-popov.com ` - --model Qwen/Qwen2.5-0.5B-Instruct -``` - -#### Option B — isolated virtualenv (fresh machine, no existing torch) - -1. Install prerequisites on Windows: - - Python 3.11 or 3.12 from - - Git for Windows from - -2. Open **PowerShell** in the cloned repo and install the node packages: - -```powershell -# Example repo path; adjust to wherever you cloned it -cd D:\DEV\workspace\REPOS\git.d-popov.com\neuron-tai - -python -m venv .venv -.\.venv\Scripts\python.exe -m pip install --upgrade pip setuptools wheel -.\.venv\Scripts\pip.exe install -e packages\tracker -e packages\node -e packages\p2p -e packages\gateway -e packages\relay - -# CPU-only PyTorch. For NVIDIA CUDA, use `pip install torch` instead. -.\.venv\Scripts\pip.exe install torch --index-url https://download.pytorch.org/whl/cpu -.\.venv\Scripts\pip.exe install transformers accelerate - -.\.venv\Scripts\meshnet-node.exe --help -``` - -For `start`-specific flags, run: - -```powershell -.\.venv\Scripts\meshnet-node.exe start --help -``` - -3. Find the Windows LAN IP address: - -```powershell -ipconfig -``` - -Use the IPv4 address on the active Ethernet/Wi-Fi adapter, for example -`192.168.0.42`. Avoid WSL/Docker/Hyper-V adapter addresses like `172.16.x.x`, -`172.17.x.x`, or other virtual adapter IPs. - -4. Allow inbound traffic for the node port in Windows Firewall. Run PowerShell as -Administrator once: - -```powershell -New-NetFirewallRule ` - -DisplayName "Meshnet node 8005" ` - -Direction Inbound ` - -Action Allow ` - -Protocol TCP ` - -LocalPort 8005 -``` - -5. Start the Windows node from normal PowerShell. Replace the tracker and -advertised host values with your actual LAN addresses: - -```powershell -$env:HF_HOME = "D:\DEV\models" - -.\.venv\Scripts\meshnet-node.exe start ` - --tracker http://192.168.0.179:8081 ` - --model Qwen/Qwen2.5-0.5B-Instruct ` - --shard-start 12 --shard-end 23 ` - --quantization bfloat16 ` - --host 0.0.0.0 ` - --advertise-host 192.168.0.42 ` - --port 8005 -``` - -One-line variants (direct LAN — node must be reachable by IP from other machines): - -```powershell -.\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 -``` - -Via public hostname with relay (works from behind NAT, WSL2, 5G — no `--advertise-host` needed): - -```powershell -.\.venv\Scripts\meshnet-node.exe start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct -``` - -WSL (same relay path — no `--advertise-host`): - -```bash -.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct -.venv/bin/meshnet-node start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct -``` - -`--host 0.0.0.0` binds the node to all Windows interfaces. `--advertise-host` -is what the tracker gives to other nodes for direct connections; omit it when using -the relay path since all traffic flows through the relay tunnel instead. - -If you want verbose per-hop pipeline logs while debugging a split model, add -`--debug`. Leave it off for normal runs; otherwise every generated token logs -lines like: - -```text - [node] pipeline hop 0: http://127.0.0.1:8005 start_layer=22 - [node] pipeline hop 0 returned text=' token' - [node] pipeline hop 1: wss://ai.neuron.d-popov.com/rpc/abc123 relay start_layer=12 -``` - -6. From the tracker machine, verify Windows is reachable: - -```bash -curl http://192.168.0.42:8005/v1/health -``` - -If that endpoint returns 404 or 501, that is okay: it still proves the TCP -connection reached the node process. If it times out or connection-refuses, check -the Windows Firewall rule, `--host 0.0.0.0`, the selected LAN IP, and that the -node is still running. - ---- - -## Public tracker + relay (internet / NAT nodes) - -This setup lets nodes connect from anywhere — behind home NAT, 5G, WSL2, or -on a different continent — without opening firewall ports. - -### Architecture - -``` -Client → HTTPS → ai.neuron.d-popov.com (nginx) - ├─ /v1/* → meshnet-tracker :8081 - ├─ /ws → meshnet-relay :8765 (node persistent outbound WS) - └─ /rpc/* → meshnet-relay :8765 (caller opens WS per hop) -``` - -### Nginx Proxy Manager (Docker) - -Use **one** proxy host for the domain. Do not create a second host for the same -domain to reach another port — path routing is done with **Custom locations** on -that same host. - -**1. Details tab** (default `/` → tracker) - -| Field | Value | -|-------|--------| -| Domain Names | `ai.neuron.d-popov.com` | -| Scheme | `http` | -| Forward Hostname / IP | LAN IP of the tracker machine (e.g. `192.168.0.179`) | -| Forward Port | `8081` | -| Websockets Support | ON | - -This serves `/v1/network/map`, `/v1/chat/completions`, and the rest of the tracker API. - -**2. Custom locations tab** (sub-paths → relay) - -The Custom locations form has **no separate Websockets toggle** — only location, -scheme, forward host, optional sub-folder path, and port. Add **two** locations -(both pointing at the relay process on port `8765`). Leave **“Add a path for -sub-folder forwarding”** empty so the full URI reaches the relay -(`/ws`, `/rpc/`). - -Location A — persistent node connections: - -| Field | Value | -|-------|--------| -| Define location | `/ws` | -| Scheme | `http` | -| Forward Hostname / IP | `192.168.0.179` | -| Forward Port | `8765` | -| Sub-folder path | *(leave empty)* | - -Location B — per-hop RPC: - -| Field | Value | -|-------|--------| -| Define location | `/rpc` | -| Scheme | `http` | -| Forward Hostname / IP | `192.168.0.179` | -| Forward Port | `8765` | -| Sub-folder path | *(leave empty)* | - -Nginx matches the longer prefixes first: `/ws` and `/rpc/…` go to relay; everything -else stays on `8081`. - -**3. SSL tab** - -Use your existing Let’s Encrypt certificate (unchanged). - -**4. Advanced tab** (only if WebSocket upgrade fails on `/ws` or `/rpc`) - -Custom locations do not expose a Websockets checkbox. If nodes show -`Relay configured but not connected yet` while `/v1/network/map` works, add this -snippet on the **proxy host** Advanced tab: - -```nginx -proxy_http_version 1.1; -proxy_set_header Upgrade $http_upgrade; -proxy_set_header Connection $http_connection; -proxy_read_timeout 3600s; -proxy_send_timeout 3600s; -``` - -**5. Verify routing** - -```bash -# Tracker (8081 via default location) -curl -s https://ai.neuron.d-popov.com/v1/network/map | python3 -m json.tool - -# Relay paths should not 502/404 from the tracker — check response headers/status -curl -sI https://ai.neuron.d-popov.com/ws -curl -sI https://ai.neuron.d-popov.com/rpc/test-peer -``` - -After NPM is correct, start relay and tracker on the LAN machine: - -```bash -# Terminal 1 — relay -.venv/bin/meshnet-relay --host 0.0.0.0 --port 8765 - -# Terminal 2 — tracker (advertises relay to nodes) -.venv/bin/meshnet-tracker start \ - --host 0.0.0.0 \ - --port 8081 \ - --relay-url wss://ai.neuron.d-popov.com/ws -``` - -Nodes using `https://ai.neuron.d-popov.com` should then log: - -```text -Relay advertised by tracker — using outbound tunnel wss://ai.neuron.d-popov.com/ws - Relay connected — wss://ai.neuron.d-popov.com/rpc/ -``` - -The `--relay-url` flag embeds the relay address in `/v1/network/map`. Every node -queries that endpoint on startup and auto-connects if a relay URL is present. - -### Start a node (any machine, any network) - -No `--advertise-host`, firewall rule, port forwarding, relay URL, or peer URL is -needed on the node. The public tracker is the only bootstrap URL the user types. -The node queries the tracker for `/v1/network/map`, discovers the relay URL, and -opens a persistent outbound WebSocket. If the relay connection drops, the node -keeps retrying it. - -```bash -.venv/bin/meshnet-node start \ - --tracker https://ai.neuron.d-popov.com \ - --model Qwen/Qwen2.5-0.5B-Instruct -``` - -No authentication is required in the prototype. The first public node for a model -must still choose that model with `--model` or a saved wizard config. After at -least one HF model node is registered, later nodes can join the public network -with only the tracker URL; the tracker assigns an uncovered shard if one exists: - -```bash -.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com -``` - -Use the public tracker to verify registration and routing: - -```bash -curl -s "https://ai.neuron.d-popov.com/v1/network/map" | python3 -m json.tool -curl -s "https://ai.neuron.d-popov.com/v1/route?model=qwen2.5-0.5b" | python3 -m json.tool -``` - -Expected startup output (relay path): - -``` - Auto-detected 24 layers → shard 0–23 - Relay connected — wss://ai.neuron.d-popov.com/rpc/abc1def2ef3f4567 -================================ -meshnet-node ready - Wallet:
- Model ID: Qwen/Qwen2.5-0.5B-Instruct - Shard: layers 0–23; 24 of 24 - Quantization: bfloat16 - Endpoint: http://172.29.104.23:7001 - Node ID: - Hardware: CPU -================================ -``` - -The `Endpoint` shown is the local IP (unreachable from outside). Other nodes reach -this one via `wss://ai.neuron.d-popov.com/rpc/` instead. - -### How relay hops work - -When node A needs to forward activations to node B (behind NAT): - -1. Tracker injects `X-Meshnet-Route` with `relay_addr` for each behind-NAT hop. -2. Node A opens a WebSocket to `wss://relay/rpc/{peer_id_B}`. -3. Relay forwards the `relay-http-request` envelope to Node B's persistent connection. -4. Node B processes `/forward` locally, returns `relay-http-response`. -5. Relay sends the response back to Node A over the same WebSocket. -6. Node A closes the WebSocket and continues the pipeline. - -Binary activation tensors (bfloat16) are Base64-encoded through the relay JSON -protocol and decoded on both sides — no precision loss. - -If the relay hop fails (relay down, peer disconnected), the node logs a warning and -falls back to a direct HTTP attempt before returning an error. - -### Test from WSL2 using the public tracker - -In WSL2 (which gets a `172.x.x.x` virtual IP — unreachable from other machines): - -```bash -# WSL2 Terminal 1 — head node (layers 0–11, handles chat requests) -.venv/bin/meshnet-node start \ - --tracker https://ai.neuron.d-popov.com \ - --model Qwen/Qwen2.5-0.5B-Instruct \ - --shard-start 0 --shard-end 11 - -# WSL2 Terminal 2 — tail node (layers 12–23) -.venv/bin/meshnet-node start \ - --tracker https://ai.neuron.d-popov.com \ - --model Qwen/Qwen2.5-0.5B-Instruct \ - --shard-start 12 --shard-end 23 -``` - -Both nodes connect to the relay automatically. When a chat request arrives at Node A, -it forwards activations to Node B via `wss://ai.neuron.d-popov.com/rpc/{peer_id_B}`. - -Send inference through the tracker (which picks the head node and injects the route): - -```bash -curl -s https://ai.neuron.d-popov.com/v1/chat/completions \ - -H "Content-Type: application/json" \ - -H "Authorization: Bearer sk-mesh-" \ - -d '{ - "model": "Qwen/Qwen2.5-0.5B-Instruct", - "messages": [{"role": "user", "content": "What is 7 times 8?"}], - "stream": false - }' | python3 -m json.tool -``` - -Or send directly to Node A's local port (within WSL): - -```bash -curl -s http://localhost:7001/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{"model": "Qwen/Qwen2.5-0.5B-Instruct", "messages": [{"role": "user", "content": "Hi"}]}' -``` - -## Accounts, API keys, and credit (billing-enabled trackers) - -Public trackers run with billing on: `/v1/chat/completions` requires a real -API key from a registered account. Unknown bearer strings get `401`; a key -with no balance gets `402 insufficient balance`. - -**Dashboard flow (easiest):** open `https:///dashboard`, register with -an email + password, then click **+ new key**. The key (`sk-mesh-…`) shows its -balance next to it. If the tracker was started with `--starting-credit`, your -first key arrives pre-funded (Caller Credit, once per account). If it was -started with `--devnet-topup`, every key row has a **+$N (devnet)** button to -refill during testing. - -**Curl flow:** - -```bash -# 1. Register (once) -curl -s https:///v1/auth/register \ - -H "Content-Type: application/json" \ - -d '{"email": "you@example.com", "password": "hunter22-or-better"}' -# → {"session_token": "...", ...} - -# 2. Create an API key (session token from step 1) -curl -s https:///v1/account/keys -X POST \ - -H "Authorization: Bearer " -# → {"api_key": "sk-mesh-...", "caller_credit_granted": true} - -# 3. Check balance / usage -curl -s https:///v1/account -H "Authorization: Bearer " - -# 4. (devnet trackers only) top up a key -curl -s https:///v1/account/topup -X POST \ - -H "Authorization: Bearer " \ - -H "Content-Type: application/json" \ - -d '{"api_key": "sk-mesh-..."}' -``` - -Operator side: both features default to 1 USDT (`--starting-credit` / -`--devnet-topup`). Set both to 0 on mainnet deployments — real deposits flow -through the on-chain USDT treasury watcher instead. - ---- - -## Step 1 — Start the tracker (Terminal 1) - -```bash -cd /run/media/popov/d/DEV/repos/d-popov.com/AI -.venv/bin/meshnet-tracker start --port 8080 -``` - -Expected output: -``` -Tracker listening on 0.0.0.0:8080 -``` - -Keep this terminal open. - ---- - -## Step 2 — Start a node (Terminal 2) - -### Recommended model: Qwen2.5-0.5B-Instruct - -- 0.5B parameters, ~1 GB in BF16 -- No HuggingFace account or license required -- Downloads once to `~/.meshnet/models/`, cached for future runs -- 24 transformer layers (auto-detected — no need to specify) - -```bash -cd /run/media/popov/d/DEV/repos/d-popov.com/AI -HF_HOME=/run/media/popov/d/DEV/models \ -.venv/bin/meshnet-node start \ - --model Qwen/Qwen2.5-0.5B-Instruct \ - --quantization bfloat16 \ - --tracker http://localhost:8080 \ - --port 8001 -``` - -Shard range is **auto-detected** from the curated catalog (no network call for known -models). For unknown repos, the node fetches only `config.json` (~1 KB) to read -`num_hidden_layers`. You can still pass `--shard-start` / `--shard-end` explicitly -to run a partial shard on one machine. - -Expected output (after model loads): -``` - Auto-detected 24 layers → shard 0–23 -================================ -meshnet-node ready - Wallet:
- Model ID: Qwen/Qwen2.5-0.5B-Instruct - Shard: layers 0–23 - Quantization: bfloat16 - Endpoint: http://:8001 - Hardware: CPU -================================ -``` - -### Other model options (all CPU-friendly) - -| Model | HF repo | Layers | BF16 size | Notes | -|-------|---------|--------|-----------|-------| -| Qwen2.5-0.5B | `Qwen/Qwen2.5-0.5B-Instruct` | 24 | ~1 GB | Fastest, no gating | -| Qwen2.5-1.5B | `Qwen/Qwen2.5-1.5B-Instruct` | 28 | ~3 GB | Better quality | -| Phi-3-mini | `microsoft/Phi-3-mini-4k-instruct` | 32 | ~7.5 GB | Best CPU quality | -| Llama-3.2-1B | `meta-llama/Llama-3.2-1B-Instruct` | 16 | ~2 GB | Requires HF login | -| Llama-3.2-3B | `meta-llama/Llama-3.2-3B-Instruct` | 28 | ~6 GB | Requires HF login | - -For gated models (Llama), run `huggingface-cli login` first. - ---- - -## Step 3 — Send an inference request (Terminal 3) - -If you started the node with `--port 8001`, send the request directly to that -head node: - -```bash Qwen2.5-0.5B-Instruct -curl -s http://localhost:8001/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "qwen2.5-0.5b", - "messages": [{"role": "user", "content": "What is 7 times 8? Answer in one word."}], - "stream": false - }' | python3 -m json.tool -``` - -If you did not pass `--port`, `meshnet-node start` uses the first free port at -or above `7000`. Use the `Endpoint:` printed by the node instead of `8001`. - -To test tracker routing/proxying, send the same OpenAI-compatible request to the -tracker, using either the full HuggingFace repo or the quick alias: - -```bash -curl -s http://localhost:8080/v1/chat/completions \ - -H "Content-Type: application/json" \ - -d '{ - "model": "qwen2.5-0.5b", - "messages": [{"role": "user", "content": "What is 7 times 8? Answer in one word."}], - "stream": false - }' | python3 -m json.tool -``` - -Or use the test script: - -```bash -.venv/bin/python scripts/test_lan_inference.py \ - --tracker http://localhost:8080 \ - --gateway http://localhost:8001 -``` - ---- - -## Two-node split (same machine, two terminals) - -Split Qwen2.5-0.5B's 24 layers across two node processes to test the sharded pipeline: - -**Node A — layers 0–11 (tracker mode, serves chat completions):** -```bash -HF_HOME=/run/media/popov/d/DEV/models \ -.venv/bin/meshnet-node start \ - --model Qwen/Qwen2.5-0.5B-Instruct \ - --shard-start 0 --shard-end 11 \ - --quantization bfloat16 \ - --tracker http://localhost:8080 \ - --port 8001 -``` - -**Node B — layers 12–23:** -```bash -HF_HOME=/run/media/popov/d/DEV/models \ -.venv/bin/meshnet-node start \ - --model Qwen/Qwen2.5-0.5B-Instruct \ - --shard-start 12 --shard-end 23 \ - --quantization bfloat16 \ - --tracker http://localhost:8080 \ - --port 8002 -``` - -Send the request to Node A — it tokenizes, runs layers 0–11, passes binary -activations to Node B, and streams the final response back. - ---- - -## Two-machine LAN test (Linux + Windows/WSL2) - -See `docs/TWO_MACHINE_TEST.md` (created by US-018). - -For WSL2 nodes, registration only proves the node can reach the tracker -outbound. Tracker-routed inference also requires the tracker to reach the node's -advertised endpoint inbound. Either run the node in native Windows PowerShell, -configure Windows port forwarding into WSL for the node port, or start the -tracker with a relay URL so the node registers a `relay_addr`. - ---- - -## Browse available models - -```bash -# Show curated list with VRAM requirements -.venv/bin/meshnet-node models - -# Browse HuggingFace Hub top-20 text-generation models -.venv/bin/meshnet-node models --browse -``` - ---- - -## Start with the interactive wizard - -```bash -# First run: wizard detects GPU, shows model list, saves config -.venv/bin/meshnet-node - -# Subsequent runs: starts directly from saved config -.venv/bin/meshnet-node - -# Re-run wizard even with saved config -.venv/bin/meshnet-node --reset-config -``` - ---- - -## Run all tests - -```bash -.venv/bin/python -m pytest -q -``` +# Quickstart — Running a node and testing inference + +This guide gets you from zero to a live inference request in three terminals. +Tested on: AMD Ryzen AI Max (Strix Halo APU), 124 GB RAM, Linux, CPU inference. + +--- + +## Prerequisites + +```bash +# Clone and enter repo +cd /run/media/popov/d/DEV/repos/d-popov.com/AI + +# Create the virtualenv if it does not exist yet +python3 -m venv .venv + +# Keep packaging tools current enough for editable installs +.venv/bin/python -m pip install --upgrade pip setuptools wheel + +# Install Python packages (editable — picks up code changes immediately) +.venv/bin/pip install -e packages/tracker -e packages/node -e packages/p2p -e packages/gateway -e packages/relay + +# CPU-only PyTorch (skip if you have CUDA/ROCm already) +.venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu + +# HuggingFace model libraries +.venv/bin/pip install "transformers>=5.12" accelerate +``` + +> **NVIDIA GPU (CUDA):** replace the torch line with `pip install torch` (default index). +> **AMD GPU (ROCm):** `pip install torch --index-url https://download.pytorch.org/whl/rocm6.2` + +### Version and library notes for Qwen3.5/3.6-MoE models + +- **transformers ≥ 5.12 is required** for Qwen3.5/3.6-MoE (e.g. `Qwen3.6-35B-A3B`). + Older versions fail at load time with + `'Qwen3_5MoeConfig' object has no attribute 'vocab_size'`. Check with + `python -c "import transformers; print(transformers.__version__)"` and upgrade + with `pip install -U transformers` in the environment that runs `meshnet-node` + (conda/miniforge users: upgrade inside that env, not a layered `.venv`). +- The startup warning + `The fast path is not available because one of the required library is not installed` + is **harmless** — transformers falls back to a pure-torch implementation of the + linear-attention layers. The fast-path packages (`flash-linear-attention`, + `causal-conv1d`) are CUDA-only kernels: install them for GPU speed if you want, + skip them entirely on CPU nodes. +- `pip install nvidia-ml-py` silences the pynvml deprecation warning on NVIDIA hosts. + +## Bootstrap a tracker on a new machine + +Use this when provisioning a fresh LAN/public tracker host. The tracker itself is +lightweight; install the relay too if nodes will connect from NAT, WSL2, mobile, +or other networks where inbound node ports are not reachable. + +```bash +# 1. Get the repo onto the tracker host +git clone https://git.d-popov.com/popov/neuron-tai.git AI +cd AI + +# 2. Create an isolated Python environment +python3 -m venv .venv +.venv/bin/python -m pip install --upgrade pip setuptools wheel + +# 3. Install only the services needed by the tracker host +.venv/bin/pip install -e packages/tracker -e packages/relay -e packages/gateway +``` + +For a private LAN tracker, start only the tracker and open the selected TCP port +on the host firewall if other machines will join: + +```bash +.venv/bin/meshnet-tracker start --host 0.0.0.0 --port 8080 +# --starting-credit 1 --devnet-topup 10 +``` + +Verify from the tracker host: + +```bash +curl -s http://localhost:8080/v1/network/map | python3 -m json.tool +``` + +Verify from another LAN machine, replacing the IP with the tracker host's LAN IP: + +```bash +curl -s http://192.168.0.179:8080/v1/network/map | python3 -m json.tool +``` + +For a public tracker with relay support, run both services. The relay listens on +`8765`; the tracker below listens on `8081` and advertises the public WebSocket +URL that nodes should use for outbound relay connections: + +```bash +# Terminal 1 — relay +.venv/bin/meshnet-relay --host 0.0.0.0 --port 8765 + +# Terminal 2 — tracker +.venv/bin/meshnet-tracker start \ + --host 0.0.0.0 \ + --port 8081 \ + --relay-url wss://ai.neuron.d-popov.com/ws +``` + +If this host sits behind Nginx Proxy Manager, point `/` and `/v1/*` at tracker +port `8081`, and point `/ws` plus `/rpc` at relay port `8765` as shown in the +public tracker section below. After the proxy is configured, verify the public +bootstrap endpoint: + +```bash +curl -s https://ai.neuron.d-popov.com/v1/network/map | python3 -m json.tool +``` + +Nodes can then join with either the LAN tracker URL or the public URL: + +```bash +.venv/bin/meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen/Qwen2.5-0.5B-Instruct +.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +.venv/bin/meshnet-node start --tracker https://ai.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +``` + +### Windows / WSL2 + +Run the Linux commands from WSL, not Git Bash. From the repo opened in Git Bash: + +```bash +wsl +cd /mnt/d/DEV/workspace/REPOS/git.d-popov.com/neuron-tai +python3 -m venv .venv +.venv/bin/python -m pip install --upgrade pip setuptools wheel +.venv/bin/pip install -e packages/tracker -e packages/node -e packages/p2p -e packages/gateway -e packages/relay +.venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cpu +.venv/bin/pip install "transformers>=5.12" accelerate +.venv/bin/meshnet-node --help +``` + +If `.venv/bin/meshnet-node` is missing, the editable install step did not finish +successfully. Re-run the `.venv/bin/pip install -e ...` command above inside WSL. + +WSL2 sits behind Windows NAT and is **not directly reachable** from other LAN machines. +Direct cross-host hops time out. The relay path (see below) solves this: the WSL2 node +opens an outbound WebSocket to the relay server and all traffic flows through that tunnel. +No firewall rules, no `--advertise-host` needed — just point at the public tracker URL. + +### Native Windows PowerShell node (not WSL) + +Use this when the tracker is on another machine and you want Windows to host a +reachable node on the LAN. + +#### Option A — existing conda/miniforge environment with CUDA torch (recommended if you already have it) + +First, make sure the conda base environment is active so that `python` and `pip` both +resolve to the same miniforge installation: + +```powershell +conda activate base +deactivate # drop any .venv that may be layered on top; safe no-op if none active +``` + +Install project packages into the active conda/miniforge env: + +```powershell +cd D:\DEV\workspace\REPOS\git.d-popov.com\neuron-tai + +pip install -e packages\tracker -e packages\node -e packages\p2p -e packages\gateway -e packages\relay +pip install "transformers>=5.12" accelerate safetensors # torch is already present +``` + +> Conda/miniforge envs often carry an older `transformers` pinned by other tools +> (aider, etc.). Qwen3.5/3.6-MoE models need **transformers ≥ 5.12** — verify with +> `python -c "import transformers; print(transformers.__version__)"`. The pip +> resolver may print dependency-conflict warnings for those other tools; they don't +> affect `meshnet-node`. + +Verify torch is importable and CUDA is live **before** starting the node: + +```powershell +python -c "import torch; print(torch.__version__, torch.cuda.is_available())" +# Expected: 2.x.x+cuXXX True +``` + +If you get `ModuleNotFoundError: No module named 'torch'` even though `pip install torch` +says "already satisfied", the `torch/` package directory is missing while the metadata +stub remains (can happen after a conda-managed install). Force-reinstall all three +PyTorch packages together so their versions stay in sync: + +```powershell +pip install --force-reinstall torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 +``` + +> **Important:** always reinstall `torch`, `torchvision`, and `torchaudio` as a group. +> Upgrading only `torch` leaves `torchvision` on an incompatible version, which causes +> `RuntimeError: operator torchvision::nms does not exist` and makes transformers fail +> to import any model class (the error surfaces as `Could not import module 'Qwen2ForCausalLM'`). + +Then re-run the verify step above. + +If that prints `True` but `meshnet-node` still can't find torch, the venv entry point +is shadowing the conda one. Check which binary wins: + +```powershell +(Get-Command meshnet-node).Source +# Should show: C:\Users\\miniforge3\Scripts\meshnet-node.exe +# If it shows .venv\Scripts\meshnet-node.exe, use the full path below instead +``` + +To start a node: + +```powershell +$env:HF_HOME = "D:\DEV\models" +meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +``` + +If the wrong entry point is shadowing, invoke via the full conda path: + +```powershell +C:\Users\popov\miniforge3\Scripts\meshnet-node.exe start ` + --tracker https://ai.neuron.d-popov.com ` + --model Qwen/Qwen2.5-0.5B-Instruct +``` + +#### Option B — isolated virtualenv (fresh machine, no existing torch) + +1. Install prerequisites on Windows: + - Python 3.11 or 3.12 from + - Git for Windows from + +2. Open **PowerShell** in the cloned repo and install the node packages: + +```powershell +# Example repo path; adjust to wherever you cloned it +cd D:\DEV\workspace\REPOS\git.d-popov.com\neuron-tai + +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install --upgrade pip setuptools wheel +.\.venv\Scripts\pip.exe install -e packages\tracker -e packages\node -e packages\p2p -e packages\gateway -e packages\relay + +# CPU-only PyTorch. For NVIDIA CUDA, use `pip install torch` instead. +.\.venv\Scripts\pip.exe install torch --index-url https://download.pytorch.org/whl/cpu +.\.venv\Scripts\pip.exe install "transformers>=5.12" accelerate + +.\.venv\Scripts\meshnet-node.exe --help +``` + +For `start`-specific flags, run: + +```powershell +.\.venv\Scripts\meshnet-node.exe start --help +``` + +3. Find the Windows LAN IP address: + +```powershell +ipconfig +``` + +Use the IPv4 address on the active Ethernet/Wi-Fi adapter, for example +`192.168.0.42`. Avoid WSL/Docker/Hyper-V adapter addresses like `172.16.x.x`, +`172.17.x.x`, or other virtual adapter IPs. + +4. Allow inbound traffic for the node port in Windows Firewall. Run PowerShell as +Administrator once: + +```powershell +New-NetFirewallRule ` + -DisplayName "Meshnet node 8005" ` + -Direction Inbound ` + -Action Allow ` + -Protocol TCP ` + -LocalPort 8005 +``` + +5. Start the Windows node from normal PowerShell. Replace the tracker and +advertised host values with your actual LAN addresses: + +```powershell +$env:HF_HOME = "D:\DEV\models" + +.\.venv\Scripts\meshnet-node.exe start ` + --tracker http://192.168.0.179:8081 ` + --model Qwen/Qwen2.5-0.5B-Instruct ` + --shard-start 12 --shard-end 23 ` + --quantization bfloat16 ` + --host 0.0.0.0 ` + --advertise-host 192.168.0.42 ` + --port 8005 +``` + +One-line variants (direct LAN — node must be reachable by IP from other machines): + +```powershell +.\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 +``` + +Via public hostname with relay (works from behind NAT, WSL2, 5G — no `--advertise-host` needed): + +```powershell +.\.venv\Scripts\meshnet-node.exe start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +``` + +WSL (same relay path — no `--advertise-host`): + +```bash +.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +.venv/bin/meshnet-node start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct +``` + +`--host 0.0.0.0` binds the node to all Windows interfaces. `--advertise-host` +is what the tracker gives to other nodes for direct connections; omit it when using +the relay path since all traffic flows through the relay tunnel instead. + +If you want verbose per-hop pipeline logs while debugging a split model, add +`--debug`. Leave it off for normal runs; otherwise every generated token logs +lines like: + +```text + [node] pipeline hop 0: http://127.0.0.1:8005 start_layer=22 + [node] pipeline hop 0 returned text=' token' + [node] pipeline hop 1: wss://ai.neuron.d-popov.com/rpc/abc123 relay start_layer=12 +``` + +6. From the tracker machine, verify Windows is reachable: + +```bash +curl http://192.168.0.42:8005/v1/health +``` + +If that endpoint returns 404 or 501, that is okay: it still proves the TCP +connection reached the node process. If it times out or connection-refuses, check +the Windows Firewall rule, `--host 0.0.0.0`, the selected LAN IP, and that the +node is still running. + +--- + +## Public tracker + relay (internet / NAT nodes) + +This setup lets nodes connect from anywhere — behind home NAT, 5G, WSL2, or +on a different continent — without opening firewall ports. + +### Architecture + +``` +Client → HTTPS → ai.neuron.d-popov.com (nginx) + ├─ /v1/* → meshnet-tracker :8081 + ├─ /ws → meshnet-relay :8765 (node persistent outbound WS) + └─ /rpc/* → meshnet-relay :8765 (caller opens WS per hop) +``` + +### Nginx Proxy Manager (Docker) + +Use **one** proxy host for the domain. Do not create a second host for the same +domain to reach another port — path routing is done with **Custom locations** on +that same host. + +**1. Details tab** (default `/` → tracker) + +| Field | Value | +|-------|--------| +| Domain Names | `ai.neuron.d-popov.com` | +| Scheme | `http` | +| Forward Hostname / IP | LAN IP of the tracker machine (e.g. `192.168.0.179`) | +| Forward Port | `8081` | +| Websockets Support | ON | + +This serves `/v1/network/map`, `/v1/chat/completions`, and the rest of the tracker API. + +**2. Custom locations tab** (sub-paths → relay) + +The Custom locations form has **no separate Websockets toggle** — only location, +scheme, forward host, optional sub-folder path, and port. Add **two** locations +(both pointing at the relay process on port `8765`). Leave **“Add a path for +sub-folder forwarding”** empty so the full URI reaches the relay +(`/ws`, `/rpc/`). + +Location A — persistent node connections: + +| Field | Value | +|-------|--------| +| Define location | `/ws` | +| Scheme | `http` | +| Forward Hostname / IP | `192.168.0.179` | +| Forward Port | `8765` | +| Sub-folder path | *(leave empty)* | + +Location B — per-hop RPC: + +| Field | Value | +|-------|--------| +| Define location | `/rpc` | +| Scheme | `http` | +| Forward Hostname / IP | `192.168.0.179` | +| Forward Port | `8765` | +| Sub-folder path | *(leave empty)* | + +Nginx matches the longer prefixes first: `/ws` and `/rpc/…` go to relay; everything +else stays on `8081`. + +**3. SSL tab** + +Use your existing Let’s Encrypt certificate (unchanged). + +**4. Advanced tab** (only if WebSocket upgrade fails on `/ws` or `/rpc`) + +Custom locations do not expose a Websockets checkbox. If nodes show +`Relay configured but not connected yet` while `/v1/network/map` works, add this +snippet on the **proxy host** Advanced tab: + +```nginx +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $http_connection; +proxy_read_timeout 3600s; +proxy_send_timeout 3600s; +``` + +**5. Verify routing** + +```bash +# Tracker (8081 via default location) +curl -s https://ai.neuron.d-popov.com/v1/network/map | python3 -m json.tool + +# Relay paths should not 502/404 from the tracker — check response headers/status +curl -sI https://ai.neuron.d-popov.com/ws +curl -sI https://ai.neuron.d-popov.com/rpc/test-peer +``` + +After NPM is correct, start relay and tracker on the LAN machine: + +```bash +# Terminal 1 — relay +.venv/bin/meshnet-relay --host 0.0.0.0 --port 8765 + +# Terminal 2 — tracker (advertises relay to nodes) +.venv/bin/meshnet-tracker start \ + --host 0.0.0.0 \ + --port 8081 \ + --relay-url wss://ai.neuron.d-popov.com/ws +``` + +Nodes using `https://ai.neuron.d-popov.com` should then log: + +```text +Relay advertised by tracker — using outbound tunnel wss://ai.neuron.d-popov.com/ws + Relay connected — wss://ai.neuron.d-popov.com/rpc/ +``` + +The `--relay-url` flag embeds the relay address in `/v1/network/map`. Every node +queries that endpoint on startup and auto-connects if a relay URL is present. + +### Start a node (any machine, any network) + +No `--advertise-host`, firewall rule, port forwarding, relay URL, or peer URL is +needed on the node. The public tracker is the only bootstrap URL the user types. +The node queries the tracker for `/v1/network/map`, discovers the relay URL, and +opens a persistent outbound WebSocket. If the relay connection drops, the node +keeps retrying it. + +```bash +.venv/bin/meshnet-node start \ + --tracker https://ai.neuron.d-popov.com \ + --model Qwen/Qwen2.5-0.5B-Instruct +``` + +No authentication is required in the prototype. The first public node for a model +must still choose that model with `--model` or a saved wizard config. After at +least one HF model node is registered, later nodes can join the public network +with only the tracker URL; the tracker assigns an uncovered shard if one exists: + +```bash +.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com +``` + +Use the public tracker to verify registration and routing: + +```bash +curl -s "https://ai.neuron.d-popov.com/v1/network/map" | python3 -m json.tool +curl -s "https://ai.neuron.d-popov.com/v1/route?model=qwen2.5-0.5b" | python3 -m json.tool +``` + +Expected startup output (relay path): + +``` + Auto-detected 24 layers → shard 0–23 + Relay connected — wss://ai.neuron.d-popov.com/rpc/abc1def2ef3f4567 +================================ +meshnet-node ready + Wallet:
+ Model ID: Qwen/Qwen2.5-0.5B-Instruct + Shard: layers 0–23; 24 of 24 + Quantization: bfloat16 + Endpoint: http://172.29.104.23:7001 + Node ID: + Hardware: CPU +================================ +``` + +The `Endpoint` shown is the local IP (unreachable from outside). Other nodes reach +this one via `wss://ai.neuron.d-popov.com/rpc/` instead. + +### How relay hops work + +When node A needs to forward activations to node B (behind NAT): + +1. Tracker injects `X-Meshnet-Route` with `relay_addr` for each behind-NAT hop. +2. Node A opens a WebSocket to `wss://relay/rpc/{peer_id_B}`. +3. Relay forwards the `relay-http-request` envelope to Node B's persistent connection. +4. Node B processes `/forward` locally, returns `relay-http-response`. +5. Relay sends the response back to Node A over the same WebSocket. +6. Node A closes the WebSocket and continues the pipeline. + +Binary activation tensors (bfloat16) are Base64-encoded through the relay JSON +protocol and decoded on both sides — no precision loss. + +If the relay hop fails (relay down, peer disconnected), the node logs a warning and +falls back to a direct HTTP attempt before returning an error. + +### Test from WSL2 using the public tracker + +In WSL2 (which gets a `172.x.x.x` virtual IP — unreachable from other machines): + +```bash +# WSL2 Terminal 1 — head node (layers 0–11, handles chat requests) +.venv/bin/meshnet-node start \ + --tracker https://ai.neuron.d-popov.com \ + --model Qwen/Qwen2.5-0.5B-Instruct \ + --shard-start 0 --shard-end 11 + +# WSL2 Terminal 2 — tail node (layers 12–23) +.venv/bin/meshnet-node start \ + --tracker https://ai.neuron.d-popov.com \ + --model Qwen/Qwen2.5-0.5B-Instruct \ + --shard-start 12 --shard-end 23 +``` + +Both nodes connect to the relay automatically. When a chat request arrives at Node A, +it forwards activations to Node B via `wss://ai.neuron.d-popov.com/rpc/{peer_id_B}`. + +Send inference through the tracker (which picks the head node and injects the route): + +```bash +curl -s https://ai.neuron.d-popov.com/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-mesh-" \ + -d '{ + "model": "Qwen/Qwen2.5-0.5B-Instruct", + "messages": [{"role": "user", "content": "What is 7 times 8?"}], + "stream": false + }' | python3 -m json.tool +``` + +Or send directly to Node A's local port (within WSL): + +```bash +curl -s http://localhost:7001/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{"model": "Qwen/Qwen2.5-0.5B-Instruct", "messages": [{"role": "user", "content": "Hi"}]}' +``` + +## Accounts, API keys, and credit (billing-enabled trackers) + +Public trackers run with billing on: `/v1/chat/completions` requires a real +API key from a registered account. Unknown bearer strings get `401`; a key +with no balance gets `402 insufficient balance`. + +**Dashboard flow (easiest):** open `https:///dashboard`, register with +an email + password, then click **+ new key**. The key (`sk-mesh-…`) shows its +balance next to it. If the tracker was started with `--starting-credit`, your +first key arrives pre-funded (Caller Credit, once per account). If it was +started with `--devnet-topup`, every key row has a **+$N (devnet)** button to +refill during testing. + +**Curl flow:** + +```bash +# 1. Register (once) +curl -s https:///v1/auth/register \ + -H "Content-Type: application/json" \ + -d '{"email": "you@example.com", "password": "hunter22-or-better"}' +# → {"session_token": "...", ...} + +# 2. Create an API key (session token from step 1) +curl -s https:///v1/account/keys -X POST \ + -H "Authorization: Bearer " +# → {"api_key": "sk-mesh-...", "caller_credit_granted": true} + +# 3. Check balance / usage +curl -s https:///v1/account -H "Authorization: Bearer " + +# 4. (devnet trackers only) top up a key +curl -s https:///v1/account/topup -X POST \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{"api_key": "sk-mesh-..."}' +``` + +Operator side: both features default to 1 USDT (`--starting-credit` / +`--devnet-topup`). Set both to 0 on mainnet deployments — real deposits flow +through the on-chain USDT treasury watcher instead. + +--- + +## Step 1 — Start the tracker (Terminal 1) + +```bash +cd /run/media/popov/d/DEV/repos/d-popov.com/AI +.venv/bin/meshnet-tracker start --port 8080 +``` + +Expected output: +``` +Tracker listening on 0.0.0.0:8080 +``` + +Keep this terminal open. + +--- + +## Step 2 — Start a node (Terminal 2) + +### Recommended model: Qwen2.5-0.5B-Instruct + +- 0.5B parameters, ~1 GB in BF16 +- No HuggingFace account or license required +- Downloads once to `~/.meshnet/models/`, cached for future runs +- 24 transformer layers (auto-detected — no need to specify) + +```bash +cd /run/media/popov/d/DEV/repos/d-popov.com/AI +HF_HOME=/run/media/popov/d/DEV/models \ +.venv/bin/meshnet-node start \ + --model Qwen/Qwen2.5-0.5B-Instruct \ + --quantization bfloat16 \ + --tracker http://localhost:8080 \ + --port 8001 +``` + +Shard range is **auto-detected** from the curated catalog (no network call for known +models). For unknown repos, the node fetches only `config.json` (~1 KB) to read +`num_hidden_layers`. You can still pass `--shard-start` / `--shard-end` explicitly +to run a partial shard on one machine. + +Expected output (after model loads): +``` + Auto-detected 24 layers → shard 0–23 +================================ +meshnet-node ready + Wallet:
+ Model ID: Qwen/Qwen2.5-0.5B-Instruct + Shard: layers 0–23 + Quantization: bfloat16 + Endpoint: http://:8001 + Hardware: CPU +================================ +``` + +### Other model options (all CPU-friendly) + +| Model | HF repo | Layers | BF16 size | Notes | +|-------|---------|--------|-----------|-------| +| Qwen2.5-0.5B | `Qwen/Qwen2.5-0.5B-Instruct` | 24 | ~1 GB | Fastest, no gating | +| Qwen2.5-1.5B | `Qwen/Qwen2.5-1.5B-Instruct` | 28 | ~3 GB | Better quality | +| Phi-3-mini | `microsoft/Phi-3-mini-4k-instruct` | 32 | ~7.5 GB | Best CPU quality | +| Llama-3.2-1B | `meta-llama/Llama-3.2-1B-Instruct` | 16 | ~2 GB | Requires HF login | +| Llama-3.2-3B | `meta-llama/Llama-3.2-3B-Instruct` | 28 | ~6 GB | Requires HF login | + +For gated models (Llama), run `huggingface-cli login` first. + +--- + +## Step 3 — Send an inference request (Terminal 3) + +If you started the node with `--port 8001`, send the request directly to that +head node: + +```bash Qwen2.5-0.5B-Instruct +curl -s http://localhost:8001/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen2.5-0.5b", + "messages": [{"role": "user", "content": "What is 7 times 8? Answer in one word."}], + "stream": false + }' | python3 -m json.tool +``` + +If you did not pass `--port`, `meshnet-node start` uses the first free port at +or above `7000`. Use the `Endpoint:` printed by the node instead of `8001`. + +To test tracker routing/proxying, send the same OpenAI-compatible request to the +tracker, using either the full HuggingFace repo or the quick alias: + +```bash +curl -s http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "qwen2.5-0.5b", + "messages": [{"role": "user", "content": "What is 7 times 8? Answer in one word."}], + "stream": false + }' | python3 -m json.tool +``` + +Or use the test script: + +```bash +.venv/bin/python scripts/test_lan_inference.py \ + --tracker http://localhost:8080 \ + --gateway http://localhost:8001 +``` + +--- + +## Two-node split (same machine, two terminals) + +Split Qwen2.5-0.5B's 24 layers across two node processes to test the sharded pipeline: + +**Node A — layers 0–11 (tracker mode, serves chat completions):** +```bash +HF_HOME=/run/media/popov/d/DEV/models \ +.venv/bin/meshnet-node start \ + --model Qwen/Qwen2.5-0.5B-Instruct \ + --shard-start 0 --shard-end 11 \ + --quantization bfloat16 \ + --tracker http://localhost:8080 \ + --port 8001 +``` + +**Node B — layers 12–23:** +```bash +HF_HOME=/run/media/popov/d/DEV/models \ +.venv/bin/meshnet-node start \ + --model Qwen/Qwen2.5-0.5B-Instruct \ + --shard-start 12 --shard-end 23 \ + --quantization bfloat16 \ + --tracker http://localhost:8080 \ + --port 8002 +``` + +Send the request to Node A — it tokenizes, runs layers 0–11, passes binary +activations to Node B, and streams the final response back. + +--- + +## Two-machine LAN test (Linux + Windows/WSL2) + +See `docs/TWO_MACHINE_TEST.md` (created by US-018). + +For WSL2 nodes, registration only proves the node can reach the tracker +outbound. Tracker-routed inference also requires the tracker to reach the node's +advertised endpoint inbound. Either run the node in native Windows PowerShell, +configure Windows port forwarding into WSL for the node port, or start the +tracker with a relay URL so the node registers a `relay_addr`. + +--- + +## Browse available models + +```bash +# Show curated list with VRAM requirements +.venv/bin/meshnet-node models + +# Browse HuggingFace Hub top-20 text-generation models +.venv/bin/meshnet-node models --browse +``` + +--- + +## Start with the interactive wizard + +```bash +# First run: wizard detects GPU, shows model list, saves config +.venv/bin/meshnet-node + +# Subsequent runs: starts directly from saved config +.venv/bin/meshnet-node + +# Re-run wizard even with saved config +.venv/bin/meshnet-node --reset-config +``` + +--- + +## Run all tests + +```bash +.venv/bin/python -m pytest -q +``` diff --git a/docs/INSTALL_WINDOWS.md b/docs/INSTALL_WINDOWS.md index 3507363..6775845 100644 --- a/docs/INSTALL_WINDOWS.md +++ b/docs/INSTALL_WINDOWS.md @@ -1,212 +1,220 @@ -# Installing meshnet-node on Windows 11 with WSL2 - -This guide covers setting up a meshnet-node on a Windows 11 machine using WSL2 with CUDA passthrough so it can join an existing inference network over LAN. - -## Prerequisites - -- Windows 11 with WSL2 support (most systems with Windows 10 version 2004+ qualify) -- NVIDIA GPU with CUDA support (driver ≥ 525.x recommended for WSL2 CUDA) -- At least 8 GB RAM + enough VRAM for the model shard you intend to serve -- The Linux machine (other node) is reachable on your LAN - ---- - -## Step 1 — Enable WSL2 and install Ubuntu - -Open **PowerShell as Administrator** and run: - -```powershell -wsl --install -d Ubuntu-24.04 -``` - -This installs WSL2 with Ubuntu 24.04. Reboot when prompted. - -After reboot, Ubuntu starts and asks you to create a UNIX username/password. Choose anything convenient. - -Verify WSL version: - -```powershell -wsl -l -v -``` - -Output should show `VERSION 2`. - ---- - -## Step 2 — Install NVIDIA GPU driver on Windows (NOT inside WSL) - -WSL2 CUDA passthrough works through the Windows host driver. **Do not install CUDA inside WSL2.** - -1. Download the latest Game Ready or Studio driver for your GPU from https://www.nvidia.com/drivers -2. Install on Windows normally (standard installer). -3. Inside WSL2 (Ubuntu terminal), verify: - -```bash -nvidia-smi -``` - -Expected output: your GPU name, driver version, CUDA version. If this command fails, the Windows driver is too old — update it. - -> **Note:** The `cuda-toolkit` package inside WSL2 is optional and only needed if you compile CUDA kernels. For inference with `torch`, the Windows host driver is sufficient. - ---- - -## Step 3 — Install Python 3.11+ inside WSL2 - -Ubuntu 24.04 ships Python 3.12. Confirm: - -```bash -python3 --version -``` - -If it shows 3.10 or older: - -```bash -sudo add-apt-repository ppa:deadsnakes/ppa -sudo apt update -sudo apt install python3.12 python3.12-venv python3.12-dev -``` - -Install pip: - -```bash -curl -sS https://bootstrap.pypa.io/get-pip.py | python3 -``` - ---- - -## Step 4 — Clone the repository - -Inside WSL2: - -```bash -# Store the repo in the Linux filesystem (faster I/O than /mnt/c) -cd ~ -git clone https://github.com/YOUR_ORG/d-popov.com.git -cd d-popov.com/AI -``` - ---- - -## Step 5 — Create a virtualenv and install meshnet-node - -```bash -python3 -m venv .venv -source .venv/bin/activate - -# Install node + PyTorch (CUDA build) -pip install torch --index-url https://download.pytorch.org/whl/cu124 -pip install -e "packages/node[torch]" -``` - -Verify the install: - -```bash -meshnet-node --help -``` - ---- - -## Step 6 — Pre-download the model shard - -Download the model before starting the node so the startup process doesn't time out on the tracker side: - -```bash -python3 - <<'EOF' -from transformers import AutoConfig -AutoConfig.from_pretrained("microsoft/Phi-3-medium-128k-instruct") -EOF -``` - -For the full model weights (needed at runtime), `transformers` downloads them automatically on first `meshnet-node` start. If you want to pre-fetch: - -```bash -python3 -c " -from transformers import AutoModelForCausalLM -AutoModelForCausalLM.from_pretrained('microsoft/Phi-3-medium-128k-instruct', device_map='cpu') -" -``` - -This can take 10–30 minutes on first run. - ---- - -## Step 7 — Expose the node port to your LAN - -WSL2 runs behind a NAT with a virtual IP (typically `172.x.x.x`). Your LAN sees the Windows host IP. You need to forward the node port. - -**Option A — Windows port proxy (recommended for simple setups):** - -In **PowerShell as Administrator**: - -```powershell -# Get the current WSL2 IP (changes on each WSL restart) -$wslIp = (wsl hostname -I).Trim() - -# Forward Windows host port 8001 → WSL2 port 8001 -netsh interface portproxy add v4tov4 ` - listenport=8001 listenaddress=0.0.0.0 ` - connectport=8001 connectaddress=$wslIp - -# Allow inbound on Windows Firewall -New-NetFirewallRule -DisplayName "meshnet-node" ` - -Direction Inbound -Protocol TCP -LocalPort 8001 -Action Allow -``` - -Verify: from the Linux machine, `curl http://WINDOWS_LAN_IP:8001/v1/health` should return a response once the node is running. - -**Redo this after every WSL2 restart** — the WSL2 IP changes. - -**Option B — P2P relay (US-017, no port forwarding needed):** - -Start a relay node on the Linux machine. The WSL2 node connects outbound through the relay. No firewall rules needed. See `docs/TWO_MACHINE_TEST.md` for details. - ---- - -## Step 8 — Start the node - -Replace `192.168.1.10` with the actual LAN IP of the Linux machine running the tracker. -Replace shard range with the complementary range to what the Linux node is serving. - -```bash -source .venv/bin/activate - -meshnet-node \ - --model microsoft/Phi-3-medium-128k-instruct \ - --quantization bf16 \ - --shard-start 20 --shard-end 39 \ - --tracker http://192.168.1.10:8080 \ - --port 8001 \ - --host 0.0.0.0 \ - --advertise-host WINDOWS_LAN_IP -``` - -The `--advertise-host` flag tells the tracker what IP the Linux machine should use to reach this node. Use your Windows machine's LAN IP (e.g. `192.168.1.20`), **not** the WSL2 internal IP. - -Expected startup output: - -``` -Detecting hardware... - GPU: NVIDIA GeForce RTX 3080 (10240 MB VRAM) -Loading wallet... - Wallet: 5K7r... -Loading real PyTorch model shard... - Auto-detected 40 layers → shard 20–39 -================================ -meshnet-node ready - Model ID: microsoft/Phi-3-medium-128k-instruct - Shard: layers 20–39; 20 of 40 - Endpoint: http://192.168.1.20:8001 - Hardware: CUDA -================================ -``` - ---- - -## Known issues - -- **WSL2 IP changes on restart.** Always re-run the `netsh` port-proxy command after restarting WSL2 or Windows. -- **CUDA not visible in WSL2.** If `nvidia-smi` fails inside WSL2, update the Windows host GPU driver to ≥ 525.x. Installing CUDA inside WSL2 will not fix it. -- **Model download is slow.** HuggingFace downloads happen over HTTPS. Pre-fetch the model before a timed test (see Step 6). -- **Port 8001 already in use.** Change `--port` to another value and update the firewall/portproxy rules accordingly. -- **`bf16` not supported on older GPUs.** Use `--quantization int8` on Turing (RTX 20xx) cards or earlier if bfloat16 ops fail. +# Installing meshnet-node on Windows 11 with WSL2 + +This guide covers setting up a meshnet-node on a Windows 11 machine using WSL2 with CUDA passthrough so it can join an existing inference network over LAN. + +## Prerequisites + +- Windows 11 with WSL2 support (most systems with Windows 10 version 2004+ qualify) +- NVIDIA GPU with CUDA support (driver ≥ 525.x recommended for WSL2 CUDA) +- At least 8 GB RAM + enough VRAM for the model shard you intend to serve +- The Linux machine (other node) is reachable on your LAN + +--- + +## Step 1 — Enable WSL2 and install Ubuntu + +Open **PowerShell as Administrator** and run: + +```powershell +wsl --install -d Ubuntu-24.04 +``` + +This installs WSL2 with Ubuntu 24.04. Reboot when prompted. + +After reboot, Ubuntu starts and asks you to create a UNIX username/password. Choose anything convenient. + +Verify WSL version: + +```powershell +wsl -l -v +``` + +Output should show `VERSION 2`. + +--- + +## Step 2 — Install NVIDIA GPU driver on Windows (NOT inside WSL) + +WSL2 CUDA passthrough works through the Windows host driver. **Do not install CUDA inside WSL2.** + +1. Download the latest Game Ready or Studio driver for your GPU from https://www.nvidia.com/drivers +2. Install on Windows normally (standard installer). +3. Inside WSL2 (Ubuntu terminal), verify: + +```bash +nvidia-smi +``` + +Expected output: your GPU name, driver version, CUDA version. If this command fails, the Windows driver is too old — update it. + +> **Note:** The `cuda-toolkit` package inside WSL2 is optional and only needed if you compile CUDA kernels. For inference with `torch`, the Windows host driver is sufficient. + +--- + +## Step 3 — Install Python 3.11+ inside WSL2 + +Ubuntu 24.04 ships Python 3.12. Confirm: + +```bash +python3 --version +``` + +If it shows 3.10 or older: + +```bash +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt update +sudo apt install python3.12 python3.12-venv python3.12-dev +``` + +Install pip: + +```bash +curl -sS https://bootstrap.pypa.io/get-pip.py | python3 +``` + +--- + +## Step 4 — Clone the repository + +Inside WSL2: + +```bash +# Store the repo in the Linux filesystem (faster I/O than /mnt/c) +cd ~ +git clone https://github.com/YOUR_ORG/d-popov.com.git +cd d-popov.com/AI +``` + +--- + +## Step 5 — Create a virtualenv and install meshnet-node + +```bash +python3 -m venv .venv +source .venv/bin/activate + +# Install node + PyTorch (CUDA build) +pip install torch --index-url https://download.pytorch.org/whl/cu124 +pip install -e "packages/node[torch]" +``` + +Verify the install: + +```bash +meshnet-node --help +python -c "import transformers; print(transformers.__version__)" +``` + +`transformers` must be **≥ 5.12** for Qwen3.5/3.6-MoE models (older versions fail +with `'Qwen3_5MoeConfig' object has no attribute 'vocab_size'`). If you install +into an existing conda/miniforge env instead of a fresh venv, run +`pip install -U transformers` there. The startup warning about +`flash-linear-attention` / `causal-conv1d` ("fast path is not available") is +harmless on CPU — those are optional CUDA-only kernels. + +--- + +## Step 6 — Pre-download the model shard + +Download the model before starting the node so the startup process doesn't time out on the tracker side: + +```bash +python3 - <<'EOF' +from transformers import AutoConfig +AutoConfig.from_pretrained("microsoft/Phi-3-medium-128k-instruct") +EOF +``` + +For the full model weights (needed at runtime), `transformers` downloads them automatically on first `meshnet-node` start. If you want to pre-fetch: + +```bash +python3 -c " +from transformers import AutoModelForCausalLM +AutoModelForCausalLM.from_pretrained('microsoft/Phi-3-medium-128k-instruct', device_map='cpu') +" +``` + +This can take 10–30 minutes on first run. + +--- + +## Step 7 — Expose the node port to your LAN + +WSL2 runs behind a NAT with a virtual IP (typically `172.x.x.x`). Your LAN sees the Windows host IP. You need to forward the node port. + +**Option A — Windows port proxy (recommended for simple setups):** + +In **PowerShell as Administrator**: + +```powershell +# Get the current WSL2 IP (changes on each WSL restart) +$wslIp = (wsl hostname -I).Trim() + +# Forward Windows host port 8001 → WSL2 port 8001 +netsh interface portproxy add v4tov4 ` + listenport=8001 listenaddress=0.0.0.0 ` + connectport=8001 connectaddress=$wslIp + +# Allow inbound on Windows Firewall +New-NetFirewallRule -DisplayName "meshnet-node" ` + -Direction Inbound -Protocol TCP -LocalPort 8001 -Action Allow +``` + +Verify: from the Linux machine, `curl http://WINDOWS_LAN_IP:8001/v1/health` should return a response once the node is running. + +**Redo this after every WSL2 restart** — the WSL2 IP changes. + +**Option B — P2P relay (US-017, no port forwarding needed):** + +Start a relay node on the Linux machine. The WSL2 node connects outbound through the relay. No firewall rules needed. See `docs/TWO_MACHINE_TEST.md` for details. + +--- + +## Step 8 — Start the node + +Replace `192.168.1.10` with the actual LAN IP of the Linux machine running the tracker. +Replace shard range with the complementary range to what the Linux node is serving. + +```bash +source .venv/bin/activate + +meshnet-node \ + --model microsoft/Phi-3-medium-128k-instruct \ + --quantization bf16 \ + --shard-start 20 --shard-end 39 \ + --tracker http://192.168.1.10:8080 \ + --port 8001 \ + --host 0.0.0.0 \ + --advertise-host WINDOWS_LAN_IP +``` + +The `--advertise-host` flag tells the tracker what IP the Linux machine should use to reach this node. Use your Windows machine's LAN IP (e.g. `192.168.1.20`), **not** the WSL2 internal IP. + +Expected startup output: + +``` +Detecting hardware... + GPU: NVIDIA GeForce RTX 3080 (10240 MB VRAM) +Loading wallet... + Wallet: 5K7r... +Loading real PyTorch model shard... + Auto-detected 40 layers → shard 20–39 +================================ +meshnet-node ready + Model ID: microsoft/Phi-3-medium-128k-instruct + Shard: layers 20–39; 20 of 40 + Endpoint: http://192.168.1.20:8001 + Hardware: CUDA +================================ +``` + +--- + +## Known issues + +- **WSL2 IP changes on restart.** Always re-run the `netsh` port-proxy command after restarting WSL2 or Windows. +- **CUDA not visible in WSL2.** If `nvidia-smi` fails inside WSL2, update the Windows host GPU driver to ≥ 525.x. Installing CUDA inside WSL2 will not fix it. +- **Model download is slow.** HuggingFace downloads happen over HTTPS. Pre-fetch the model before a timed test (see Step 6). +- **Port 8001 already in use.** Change `--port` to another value and update the firewall/portproxy rules accordingly. +- **`bf16` not supported on older GPUs.** Use `--quantization int8` on Turing (RTX 20xx) cards or earlier if bfloat16 ops fail. diff --git a/packages/node/pyproject.toml b/packages/node/pyproject.toml index 08cea8f..454aaf2 100644 --- a/packages/node/pyproject.toml +++ b/packages/node/pyproject.toml @@ -1,33 +1,33 @@ -[build-system] -requires = ["setuptools>=64"] -build-backend = "setuptools.build_meta" - -[project] -name = "meshnet-node" -version = "0.1.0" -description = "Distributed Inference Network node client" -requires-python = ">=3.10" - -dependencies = [ - "cryptography>=41", - "huggingface-hub>=0.20", - "accelerate>=0.28", - "bitsandbytes>=0.43", - "rich>=13", - "safetensors>=0.4", - "torch>=2.1", - "transformers>=4.39", - "websockets>=13", - "zstandard>=0.22", - "kernels>=0.11.1,<0.16", -] - -[project.scripts] -meshnet-node = "meshnet_node.cli:main" - -[tool.setuptools.packages.find] -where = ["."] -include = ["meshnet_node*"] - -[tool.setuptools.package-data] -meshnet_node = ["*.json"] +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "meshnet-node" +version = "0.1.0" +description = "Distributed Inference Network node client" +requires-python = ">=3.10" + +dependencies = [ + "cryptography>=41", + "huggingface-hub>=0.20", + "accelerate>=0.28", + "bitsandbytes>=0.43", + "rich>=13", + "safetensors>=0.4", + "torch>=2.1", + "transformers>=5.12", + "websockets>=13", + "zstandard>=0.22", + "kernels>=0.11.1,<0.16", +] + +[project.scripts] +meshnet-node = "meshnet_node.cli:main" + +[tool.setuptools.packages.find] +where = ["."] +include = ["meshnet_node*"] + +[tool.setuptools.package-data] +meshnet_node = ["*.json"]