diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..b17e8eb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.venv +__pycache__ +*.py[cod] +.pytest_cache +*.egg-info +build +dist +.ralph-tui +.scratch +.claude +.env* diff --git a/_DEV_NOTES.md b/_DEV_NOTES.md index 45da67a..35dc33c 100644 --- a/_DEV_NOTES.md +++ b/_DEV_NOTES.md @@ -1,8 +1,16 @@ +# tracker +.venv/bin/meshnet-tracker start --host 0.0.0.0 --port 8080 + .\.venv\Scripts\python.exe -m meshnet_tracker.cli start --host 127.0.0.1 --port 8080 --billing-db .\billing.sqlite + + +# node + .\.venv\Scripts\python.exe -m meshnet_node.cli start --tracker http://localhost:8080 --model Qwen/Qwen2.5-0.5B-Instruct --port 7000 --debug + @win .venv/bin/meshnet-node start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 20 --advertise-host 192.168.0.20 - -.\.venv\Scripts\meshnet-node.exe start http://192.168.0.179:8081 --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 - -.\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 +.\.venv\Scripts\meshnet-node.exe start http://192.168.0.179:8081 --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 + .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 we .\.venv\Scripts\meshnet-node.exe start ` --tracker http://192.168.0.179:8081 ` @@ -13,3 +21,18 @@ HF_HOME=/run/media/popov/d/DEV/models .venv/bin/meshnet-node start --model-id Qwen/Qwen2.5-0.5B-Instruct --shard-start 0 --shard-end 21 --quantization bfloat16 --tracker http://localhost:8081 # win meshnet-node start --tracker http://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct + + + + + Then test tracker API: + + curl http://localhost:8080/v1/health + curl http://localhost:8080/v1/models + + Because billing is enabled, chat calls need a Bearer key: + + curl http://localhost:8080/v1/chat/completions ` + -H "Content-Type: application/json" ` + -H "Authorization: Bearer test-key" ` + -d '{"model":"stub-model","messages":[{"role":"user","content":"hi"}]}' \ No newline at end of file diff --git a/billing.sqlite b/billing.sqlite new file mode 100644 index 0000000..63ec514 Binary files /dev/null and b/billing.sqlite differ diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile new file mode 100644 index 0000000..cbe1bd5 --- /dev/null +++ b/deploy/docker/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +RUN useradd --create-home --uid 10001 meshnet \ + && mkdir -p /var/lib/meshnet \ + && chown meshnet:meshnet /var/lib/meshnet + +COPY packages/contracts /app/packages/contracts +COPY packages/tracker /app/packages/tracker +COPY packages/relay /app/packages/relay + +RUN python -m pip install --upgrade pip setuptools wheel \ + && python -m pip install \ + -e /app/packages/contracts \ + -e /app/packages/tracker \ + -e /app/packages/relay + +USER meshnet + +EXPOSE 8081 8765 + +CMD ["meshnet-tracker", "start", "--host", "0.0.0.0", "--port", "8081"] diff --git a/deploy/portainer/meshnet-tracker-nobuild-stack.yml b/deploy/portainer/meshnet-tracker-nobuild-stack.yml new file mode 100644 index 0000000..c50c7da --- /dev/null +++ b/deploy/portainer/meshnet-tracker-nobuild-stack.yml @@ -0,0 +1,153 @@ +# Meshnet public tracker + relay stack for Portainer without a custom image. +# +# This stack does NOT use deploy/docker/Dockerfile and does NOT require pushing an +# image to a registry. Each service starts from the public python:3.12-slim image, +# downloads a source tarball, installs the tracker/relay packages into a named +# venv volume, then starts the service. +# +# Required Portainer variables: +# SOURCE_TARBALL_URL URL to a .tar.gz archive of this repo +# PUBLIC_TRACKER_URL e.g. https://cloud.neuron.d-popov.com +# PUBLIC_PROXY_NETWORK Docker network shared with nginx/NPM, e.g. npm_proxy +# +# Optional: +# CLUSTER_PEERS e.g. https://ai.neuron.d-popov.com +# PUBLIC_RELAY_URL defaults to PUBLIC_TRACKER_URL with wss/ws + /ws +# SOURCE_STRIP_COMPONENTS defaults to 1 for GitHub/Gitea archive tarballs + +services: + meshnet-tracker: + image: python:3.12-slim + container_name: meshnet-tracker + restart: unless-stopped + environment: + SOURCE_TARBALL_URL: ${SOURCE_TARBALL_URL:?set SOURCE_TARBALL_URL} + SOURCE_STRIP_COMPONENTS: ${SOURCE_STRIP_COMPONENTS:-1} + PUBLIC_TRACKER_URL: ${PUBLIC_TRACKER_URL:?set PUBLIC_TRACKER_URL, e.g. https://cloud.neuron.d-popov.com} + CLUSTER_PEERS: ${CLUSTER_PEERS:-} + PUBLIC_RELAY_URL: ${PUBLIC_RELAY_URL:-} + HEARTBEAT_TIMEOUT: ${HEARTBEAT_TIMEOUT:-30} + ENABLE_BILLING_DB: ${ENABLE_BILLING_DB:-1} + SOLANA_RPC_URL: ${SOLANA_RPC_URL:-} + USDT_MINT: ${USDT_MINT:-} + MESHNET_TREASURY_KEYPAIR_B64: ${MESHNET_TREASURY_KEYPAIR_B64:-} + SETTLE_PERIOD: ${SETTLE_PERIOD:-86400} + PAYOUT_THRESHOLD: ${PAYOUT_THRESHOLD:-5} + PAYOUT_DUST_FLOOR: ${PAYOUT_DUST_FLOOR:-0.01} + command: + - /bin/sh + - -lc + - | + set -eu + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl tar + rm -rf /var/lib/apt/lists/* + + rm -rf /opt/meshnet-src + mkdir -p /opt/meshnet-src + curl -fsSL "$${SOURCE_TARBALL_URL}" -o /tmp/meshnet-src.tar.gz + tar -xzf /tmp/meshnet-src.tar.gz -C /opt/meshnet-src --strip-components "$${SOURCE_STRIP_COMPONENTS:-1}" + + python -m venv /opt/meshnet-venv + /opt/meshnet-venv/bin/python -m pip install --upgrade pip setuptools wheel + /opt/meshnet-venv/bin/pip install \ + -e /opt/meshnet-src/packages/contracts \ + -e /opt/meshnet-src/packages/tracker \ + -e /opt/meshnet-src/packages/relay + + PEER_ARGS="" + if [ -n "$${CLUSTER_PEERS:-}" ]; then + PEER_ARGS="--cluster-peers $${CLUSTER_PEERS}" + fi + RELAY_URL="$${PUBLIC_RELAY_URL:-}" + if [ -z "$${RELAY_URL}" ]; then + RELAY_URL="$$(printf '%s' "$${PUBLIC_TRACKER_URL}" | sed 's#^https://#wss://#; s#^http://#ws://#')/ws" + fi + BILLING_ARGS="" + if [ "$${ENABLE_BILLING_DB:-1}" = "1" ]; then + BILLING_ARGS="--billing-db /var/lib/meshnet/billing.sqlite" + fi + TREASURY_ARGS="" + if [ -n "$${SOLANA_RPC_URL:-}" ] || [ -n "$${USDT_MINT:-}" ] || [ -n "$${MESHNET_TREASURY_KEYPAIR_B64:-}" ]; then + if [ -z "$${SOLANA_RPC_URL:-}" ] || [ -z "$${USDT_MINT:-}" ] || [ -z "$${MESHNET_TREASURY_KEYPAIR_B64:-}" ]; then + echo "SOLANA_RPC_URL, USDT_MINT, and MESHNET_TREASURY_KEYPAIR_B64 must all be set together" >&2 + exit 1 + fi + printf '%s' "$${MESHNET_TREASURY_KEYPAIR_B64}" | base64 -d > /var/lib/meshnet/treasury-keypair.json + chmod 600 /var/lib/meshnet/treasury-keypair.json + TREASURY_ARGS="--solana-rpc-url $${SOLANA_RPC_URL} --usdt-mint $${USDT_MINT} --treasury-keypair /var/lib/meshnet/treasury-keypair.json --settle-period $${SETTLE_PERIOD} --payout-threshold $${PAYOUT_THRESHOLD} --payout-dust-floor $${PAYOUT_DUST_FLOOR}" + fi + exec /opt/meshnet-venv/bin/meshnet-tracker start \ + --host 0.0.0.0 \ + --port 8081 \ + --heartbeat-timeout "$${HEARTBEAT_TIMEOUT}" \ + --self-url "$${PUBLIC_TRACKER_URL}" \ + --relay-url "$${RELAY_URL}" \ + --stats-db /var/lib/meshnet/tracker-stats.sqlite \ + $${BILLING_ARGS} \ + $${TREASURY_ARGS} \ + $${PEER_ARGS} + volumes: + - meshnet-tracker-data:/var/lib/meshnet + - meshnet-tracker-venv:/opt/meshnet-venv + expose: + - "8081" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/v1/health', timeout=3).read()"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s + networks: + - public-proxy + + meshnet-relay: + image: python:3.12-slim + container_name: meshnet-relay + restart: unless-stopped + environment: + SOURCE_TARBALL_URL: ${SOURCE_TARBALL_URL:?set SOURCE_TARBALL_URL} + SOURCE_STRIP_COMPONENTS: ${SOURCE_STRIP_COMPONENTS:-1} + command: + - /bin/sh + - -lc + - | + set -eu + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl tar + rm -rf /var/lib/apt/lists/* + + rm -rf /opt/meshnet-src + mkdir -p /opt/meshnet-src + curl -fsSL "$${SOURCE_TARBALL_URL}" -o /tmp/meshnet-src.tar.gz + tar -xzf /tmp/meshnet-src.tar.gz -C /opt/meshnet-src --strip-components "$${SOURCE_STRIP_COMPONENTS:-1}" + + python -m venv /opt/meshnet-venv + /opt/meshnet-venv/bin/python -m pip install --upgrade pip setuptools wheel + /opt/meshnet-venv/bin/pip install \ + -e /opt/meshnet-src/packages/tracker \ + -e /opt/meshnet-src/packages/relay + + exec /opt/meshnet-venv/bin/meshnet-relay --host 0.0.0.0 --port 8765 --log-level INFO + volumes: + - meshnet-relay-venv:/opt/meshnet-venv + expose: + - "8765" + healthcheck: + test: ["CMD", "python", "-c", "import socket; s=socket.create_connection(('127.0.0.1', 8765), 3); s.close()"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 60s + networks: + - public-proxy + +volumes: + meshnet-tracker-data: + meshnet-tracker-venv: + meshnet-relay-venv: + +networks: + public-proxy: + external: true + name: ${PUBLIC_PROXY_NETWORK:-npm_proxy} diff --git a/deploy/portainer/meshnet-tracker-stack.yml b/deploy/portainer/meshnet-tracker-stack.yml new file mode 100644 index 0000000..0cd42b5 --- /dev/null +++ b/deploy/portainer/meshnet-tracker-stack.yml @@ -0,0 +1,107 @@ +# Meshnet public tracker + relay stack for Portainer. +# +# Intended topology when Nginx Proxy Manager (or another nginx reverse proxy) +# runs on the same Docker host: +# https://YOUR_DOMAIN/v1/* -> meshnet-tracker:8081 +# https://YOUR_DOMAIN/ws -> meshnet-relay:8765 (WebSocket) +# https://YOUR_DOMAIN/rpc/* -> meshnet-relay:8765 (WebSocket) +# +# Before deploying, create or identify the Docker network shared with nginx/NPM, +# then set PUBLIC_PROXY_NETWORK to its name in Portainer environment variables. + +services: + meshnet-tracker: + build: + context: ../.. + dockerfile: deploy/docker/Dockerfile + image: meshnet-tracker-relay:local + container_name: meshnet-tracker + restart: unless-stopped + environment: + PUBLIC_TRACKER_URL: ${PUBLIC_TRACKER_URL:?set PUBLIC_TRACKER_URL, e.g. https://cloud.neuron.d-popov.com} + CLUSTER_PEERS: ${CLUSTER_PEERS:-} + PUBLIC_RELAY_URL: ${PUBLIC_RELAY_URL:-} + HEARTBEAT_TIMEOUT: ${HEARTBEAT_TIMEOUT:-30} + ENABLE_BILLING_DB: ${ENABLE_BILLING_DB:-1} + SOLANA_RPC_URL: ${SOLANA_RPC_URL:-} + USDT_MINT: ${USDT_MINT:-} + MESHNET_TREASURY_KEYPAIR_B64: ${MESHNET_TREASURY_KEYPAIR_B64:-} + SETTLE_PERIOD: ${SETTLE_PERIOD:-86400} + PAYOUT_THRESHOLD: ${PAYOUT_THRESHOLD:-5} + PAYOUT_DUST_FLOOR: ${PAYOUT_DUST_FLOOR:-0.01} + command: + - /bin/sh + - -lc + - | + set -eu + PEER_ARGS="" + if [ -n "$${CLUSTER_PEERS:-}" ]; then + PEER_ARGS="--cluster-peers $${CLUSTER_PEERS}" + fi + RELAY_URL="$${PUBLIC_RELAY_URL:-}" + if [ -z "$${RELAY_URL}" ]; then + RELAY_URL="$$(printf '%s' "$${PUBLIC_TRACKER_URL}" | sed 's#^https://#wss://#; s#^http://#ws://#')/ws" + fi + BILLING_ARGS="" + if [ "$${ENABLE_BILLING_DB:-1}" = "1" ]; then + BILLING_ARGS="--billing-db /var/lib/meshnet/billing.sqlite" + fi + TREASURY_ARGS="" + if [ -n "$${SOLANA_RPC_URL:-}" ] || [ -n "$${USDT_MINT:-}" ] || [ -n "$${MESHNET_TREASURY_KEYPAIR_B64:-}" ]; then + if [ -z "$${SOLANA_RPC_URL:-}" ] || [ -z "$${USDT_MINT:-}" ] || [ -z "$${MESHNET_TREASURY_KEYPAIR_B64:-}" ]; then + echo "SOLANA_RPC_URL, USDT_MINT, and MESHNET_TREASURY_KEYPAIR_B64 must all be set together" >&2 + exit 1 + fi + printf '%s' "$${MESHNET_TREASURY_KEYPAIR_B64}" | base64 -d > /var/lib/meshnet/treasury-keypair.json + chmod 600 /var/lib/meshnet/treasury-keypair.json + TREASURY_ARGS="--solana-rpc-url $${SOLANA_RPC_URL} --usdt-mint $${USDT_MINT} --treasury-keypair /var/lib/meshnet/treasury-keypair.json --settle-period $${SETTLE_PERIOD} --payout-threshold $${PAYOUT_THRESHOLD} --payout-dust-floor $${PAYOUT_DUST_FLOOR}" + fi + exec meshnet-tracker start \ + --host 0.0.0.0 \ + --port 8081 \ + --heartbeat-timeout "$${HEARTBEAT_TIMEOUT}" \ + --self-url "$${PUBLIC_TRACKER_URL}" \ + --relay-url "$${RELAY_URL}" \ + --stats-db /var/lib/meshnet/tracker-stats.sqlite \ + $${BILLING_ARGS} \ + $${TREASURY_ARGS} \ + $${PEER_ARGS} + volumes: + - meshnet-tracker-data:/var/lib/meshnet + expose: + - "8081" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8081/v1/health', timeout=3).read()"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: + - public-proxy + + meshnet-relay: + image: meshnet-tracker-relay:local + container_name: meshnet-relay + restart: unless-stopped + depends_on: + meshnet-tracker: + condition: service_started + command: ["meshnet-relay", "--host", "0.0.0.0", "--port", "8765", "--log-level", "INFO"] + expose: + - "8765" + healthcheck: + test: ["CMD", "python", "-c", "import socket; s=socket.create_connection(('127.0.0.1', 8765), 3); s.close()"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: + - public-proxy + +volumes: + meshnet-tracker-data: + +networks: + public-proxy: + external: true + name: ${PUBLIC_PROXY_NETWORK:-npm_proxy} diff --git a/docs/DEPLOY_PORTAINER.md b/docs/DEPLOY_PORTAINER.md new file mode 100644 index 0000000..ad6a978 --- /dev/null +++ b/docs/DEPLOY_PORTAINER.md @@ -0,0 +1,171 @@ +# One-shot Portainer deploy: public tracker + relay + +No Dockerfile, no custom image, no registry. Portainer runs `python:3.12-slim`, downloads this repo tarball, installs tracker/relay, and starts both services. + +## 1. One-time host setup + +DNS points to the Docker host, e.g. `cloud.neuron.d-popov.com`. +Firewall exposes only `80`, `443`, and admin SSH. +Nginx Proxy Manager and this stack share a Docker network, e.g. `npm_proxy`. + +If needed: + +```bash +docker network create npm_proxy +docker network connect npm_proxy +``` + +## 2. Portainer stack + +Portainer → Stacks → Add stack → Repository. + +Stack file: + +```text +deploy/portainer/meshnet-tracker-nobuild-stack.yml +``` + +Deploy only after the current branch has been pushed. The stack passes the +current tracker billing flags (`--billing-db`, and optional treasury flags), so +an older remote `master.tar.gz` will crash on startup with unrecognized +arguments. + +Variables: + +```text +SOURCE_TARBALL_URL=https://git.d-popov.com///archive/master.tar.gz +PUBLIC_TRACKER_URL=https://cloud.neuron.d-popov.com +PUBLIC_PROXY_NETWORK=npm_proxy +CLUSTER_PEERS=https://ai.neuron.d-popov.com +PUBLIC_RELAY_URL=wss://cloud.neuron.d-popov.com/ws +HEARTBEAT_TIMEOUT=30 +ENABLE_BILLING_DB=1 +``` + +For first cloud-only test, use `CLUSTER_PEERS=`. Click **Deploy the stack**. + +`ENABLE_BILLING_DB=1` makes billing public behavior active: `/v1/chat/completions` +requires `Authorization: Bearer `. Any key is accepted and starts with +the configured starter credit; calls return `402` after that balance is +exhausted. Set `ENABLE_BILLING_DB=0` if existing unauthenticated clients must +keep working during the first redeploy. + +Optional Solana treasury settlement variables: + +```text +SOLANA_RPC_URL=https://api.devnet.solana.com +USDT_MINT= +MESHNET_TREASURY_KEYPAIR_B64= +SETTLE_PERIOD=86400 +PAYOUT_THRESHOLD=5 +PAYOUT_DUST_FLOOR=0.01 +``` + +Set all three treasury identity values together (`SOLANA_RPC_URL`, `USDT_MINT`, +`MESHNET_TREASURY_KEYPAIR_B64`) or leave all three empty. The stack writes the +decoded keypair into the tracker data volume at startup and passes it to +`meshnet-tracker`; do not put this key on relay-only hosts or non-settlement +trackers. + +Expected containers: + +```text +meshnet-tracker internal 8081 +meshnet-relay internal 8765 +``` + +## 3. Nginx Proxy Manager + +One Proxy Host for `cloud.neuron.d-popov.com`: + +```text +Forward Hostname/IP: meshnet-tracker +Forward Port: 8081 +Websockets Support: ON +SSL: Let's Encrypt + Force SSL +``` + +Custom locations on the same proxy host: + +```text +/ws -> http://meshnet-relay:8765 +/rpc -> http://meshnet-relay:8765 +``` + +Leave sub-folder forwarding empty. + +If WebSockets fail, Advanced: + +```nginx +proxy_http_version 1.1; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $http_connection; +proxy_read_timeout 3600s; +proxy_send_timeout 3600s; +``` + +## 4. Smoke test + +```bash +curl -s https://cloud.neuron.d-popov.com/v1/health +curl -s https://cloud.neuron.d-popov.com/v1/network/map | python3 -m json.tool +curl -s https://cloud.neuron.d-popov.com/v1/raft/status | python3 -m json.tool +``` + +Plain curl to `/ws` or `/rpc/test-peer` may show `426 Upgrade Required`; OK. It must not show nginx `502`. + +Dashboard: + +```bash +curl -s https://cloud.neuron.d-popov.com/dashboard | head +``` + +Then open: + +```text +https://cloud.neuron.d-popov.com/dashboard +``` + +The dashboard is served by the tracker through the same proxy host; no extra NPM +location is required. + +If you previously deployed the build-image variant before `/var/lib/meshnet` +was created as the `meshnet` user, the named volume may already be root-owned. +Recreate that volume or chown it once before retrying the fixed image. + +## 5. Start a node + +`meshnet-node` is the worker/miner process. It will ask for a model and load a +model shard. Do not use it to test the tracker dashboard. + +```bash +meshnet-node start --tracker https://cloud.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct +``` + +Expected: + +```text +Relay advertised by tracker — using outbound tunnel wss://cloud.neuron.d-popov.com/ws +Relay connected — wss://cloud.neuron.d-popov.com/rpc/ +``` + +## 6. Two tracker sync + +Cloud stack: + +```text +PUBLIC_TRACKER_URL=https://cloud.neuron.d-popov.com +CLUSTER_PEERS=https://ai.neuron.d-popov.com +PUBLIC_RELAY_URL=wss://cloud.neuron.d-popov.com/ws +``` + +Local tracker: + +```bash +meshnet-tracker start --host 0.0.0.0 --port 8081 \ + --self-url https://ai.neuron.d-popov.com \ + --cluster-peers https://cloud.neuron.d-popov.com \ + --relay-url wss://ai.neuron.d-popov.com/ws +``` + +Two Raft peers are enough for sync testing; real HA needs three trackers. diff --git a/docs/adr/0009-coverage-first-shard-assignment-and-tracker-as-node.md b/docs/adr/0009-coverage-first-shard-assignment-and-tracker-as-node.md index 49a4873..18aa31b 100644 --- a/docs/adr/0009-coverage-first-shard-assignment-and-tracker-as-node.md +++ b/docs/adr/0009-coverage-first-shard-assignment-and-tracker-as-node.md @@ -1,6 +1,6 @@ -# Coverage-first shard assignment and tracker-as-first-layer-node +# Coverage-first shard assignment and tracker-routed inference -The tracker assigns shard ranges to nodes using a coverage-first, speed-weighted bin-packing algorithm. Tracker nodes must host at least the first layer shard of every model they coordinate, making them the natural inference entry point. Any node serving layers[0..k] can become a tracker node for that model. +The tracker assigns shard ranges to worker nodes using a coverage-first, speed-weighted bin-packing algorithm. The tracker is a control-plane service and public inference API endpoint: it stores registry state, selects routes, enforces billing, and proxies OpenAI-compatible requests to the selected head worker. It does not download or load model weights. ## Problem @@ -23,24 +23,29 @@ Example: 700B NF4 model (~350GB weights). Node A has 128GB, Node B and C each ha - Node C gets layers[k_b..N] (the remainder) - If Node B benchmarks 2× faster than Node C, the tracker shifts the B/C boundary so B carries more layers -### Tracker-as-first-layer-node +### Tracker-routed head worker -Any node that advertises a new model to the network becomes a **tracker node** for that model. Tracker nodes have one hard requirement: they must hold and serve `layers[0..k]` (the first-layer shard) for every model they coordinate. +A worker that serves `layers[0..k]` is the **head worker** for that model. The tracker forwards `/v1/chat/completions` to a live head worker and injects the remaining downstream route. The worker, not the tracker process: -The reason is functional: a tracker node is also the inference entry point. When a client request arrives, the tracker node: +When a client request arrives, the tracker: +1. Authenticates/bills the request +2. Selects a live head worker and full downstream route from the coverage map +3. Proxies the request to that head worker +4. Records usage and credits node shares after completion + +The head worker: 1. Tokenizes the input (owns the tokenizer) 2. Runs `model.embed_tokens` + `model.layers[0..k]` -3. Selects the optimal onward route from the coverage map -4. Forwards activations to the next node in the route -5. Receives the final hidden state back and streams tokens to the client +3. Forwards activations to the next node in the route +4. Receives the final hidden state back and streams tokens to the client -This collapses the separate "gateway" role: the tracker node that starts an inference request IS the gateway for that request. A standalone HTTP proxy/load-balancer may sit in front to pick which tracker node handles the request, but it carries no model weights. +This keeps the public tracker lightweight: a standalone HTTP proxy/load-balancer may sit in front to pick which tracker handles the request, but neither proxy nor tracker carries model weights. -Multiple tracker nodes for the same model = multiple entry points = horizontal scale for both routing decisions and the first-layer compute. +Multiple head workers for the same model = multiple inference entry points = horizontal scale for first-layer compute. Multiple trackers scale routing and billing, not model execution. ### Last-layer node (tail) -The node assigned `layers[N-k..N]` also runs `model.norm` and `model.lm_head`. It returns decoded token IDs (not hidden states) to the tracker node, which assembles the response. The tail shard assignment is marked `is_tail: true` in the shard registry. +The node assigned `layers[N-k..N]` also runs `model.norm` and `model.lm_head`. It returns decoded token IDs (not hidden states) to the head worker, which assembles the response. The tail shard assignment is marked `is_tail: true` in the shard registry. ### Adaptive quantization @@ -78,8 +83,8 @@ Nodes obey directives asynchronously; the tracker waits up to a configurable tim ## Consequences -- The standalone `meshnet-gateway` service from US-005 becomes a thin load-balancer that routes to tracker nodes; tracker nodes do the actual inference orchestration -- Tracker nodes must download more model data (tokenizer + first-layer shard) — this is the price of being an entry point +- The standalone `meshnet-gateway` service from US-005 becomes a thin compatibility proxy/load-balancer; the public tracker can also serve the OpenAI-compatible endpoint directly +- Tracker processes do not download or load model data. Only worker nodes load model shards. - Benchmark data is self-reported by nodes at registration; the validator can detect fraudulent benchmarks (a node claiming 100 tokens/sec but delivering 2 gets slashed for under-performance) - VRAM reservation for KV cache means nodes can host fewer layers than their raw VRAM suggests — this is intentional; running out of KV cache during inference causes OOM crashes -- New CONTEXT.md terms: **Tracker Node** (node serving first-layer shard + inference routing for a model), **Coverage Map** (tracker's per-model layer-range → node-count mapping), **Rebalance Directive** (tracker instruction to a node to load or drop a shard) +- New CONTEXT.md terms: **Head Worker** (worker node serving first-layer shard for a model), **Coverage Map** (tracker's per-model layer-range → node-count mapping), **Rebalance Directive** (tracker instruction to a node to load or drop a shard) diff --git a/packages/contracts/pyproject.toml b/packages/contracts/pyproject.toml index ca0061f..7923cda 100644 --- a/packages/contracts/pyproject.toml +++ b/packages/contracts/pyproject.toml @@ -7,6 +7,10 @@ name = "meshnet-contracts" version = "0.1.0" description = "Distributed Inference Network Solana smart contract wrappers" requires-python = ">=3.10" +dependencies = [ + "solana>=0.36", + "solders>=0.23", +] [tool.setuptools.packages.find] where = ["."] diff --git a/packages/gateway/meshnet_gateway/server.py b/packages/gateway/meshnet_gateway/server.py index 4223e54..7a31ef8 100644 --- a/packages/gateway/meshnet_gateway/server.py +++ b/packages/gateway/meshnet_gateway/server.py @@ -159,7 +159,9 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler): "shard_coverage_percentage": float(model.get("shard_coverage_percentage", 0.0)), } for model in models_resp.get("data", []) - if isinstance(model, dict) and isinstance(model.get("id"), str) + if isinstance(model, dict) + and isinstance(model.get("id"), str) + and float(model.get("shard_coverage_percentage", 0.0)) > 0.0 ] self._send_json(200, {"data": data}) return @@ -244,7 +246,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler): def _handle_chat_completions(self): server: _GatewayHTTPServer = self.server # type: ignore[assignment] - # Read raw bytes first so we can proxy them if tracker-nodes are available + # Read raw bytes first so we can proxy them if head workers are available. length = int(self.headers.get("Content-Length", 0)) raw_body = self.rfile.read(length) try: @@ -257,12 +259,11 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler): return model = str(body.get("model", "stub-model")) - tracker_nodes = _get_tracker_nodes(server, model) - if tracker_nodes: - # Proxy to a tracker-node (round-robin by request count) - target = tracker_nodes[server.request_count % len(tracker_nodes)] + head_workers = _get_head_workers(server, model) + if head_workers: + target = head_workers[server.request_count % len(head_workers)] server.request_count += 1 - return self._proxy_to_tracker_node(target, raw_body) + return self._proxy_to_head_worker(target, raw_body) # Fallback: use existing direct pipeline (backward compat) streaming = bool(body.get("stream", False)) @@ -284,8 +285,8 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler): else: self._send_json(200, completion) - def _proxy_to_tracker_node(self, url: str, body_bytes: bytes) -> None: - """Forward a raw request body to a tracker-node and stream the response back.""" + def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None: + """Forward a raw request body to a head worker and stream the response back.""" target_url = f"{url}/v1/chat/completions" req = urllib.request.Request( target_url, @@ -307,7 +308,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler): self.wfile.write(resp_body) return except Exception as exc: - self._send_json_error(503, f"tracker-node unreachable: {exc}") + self._send_json_error(503, f"head worker unreachable: {exc}") return self.send_response(status) self.send_header("Content-Type", content_type) @@ -816,13 +817,13 @@ def _get_json(url: str, timeout: float = 5.0) -> dict: return json.loads(r.read()) -def _get_tracker_nodes(server: _GatewayHTTPServer, model: str) -> list[str]: - """Return tracker-node endpoint URLs for this model, or empty list on any failure.""" +def _get_head_workers(server: _GatewayHTTPServer, model: str) -> list[str]: + """Return head-worker endpoint URLs for this model, or empty list on failure.""" if server.tracker_url is None: return [] try: - resp = _get_json(f"{server.tracker_url}/v1/tracker-nodes/{urllib.parse.quote(model)}") - return [n["endpoint"] for n in resp.get("tracker_nodes", [])] + resp = _get_json(f"{server.tracker_url}/v1/head-workers/{urllib.parse.quote(model)}") + return [n["endpoint"] for n in resp.get("head_workers", [])] except Exception: return [] diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 77ddb4c..c553aac 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -1182,6 +1182,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): elif parsed.path.startswith("/v1/tracker-nodes/"): model = urllib.parse.unquote(parsed.path.removeprefix("/v1/tracker-nodes/")) self._handle_tracker_nodes(model) + elif parsed.path.startswith("/v1/head-workers/"): + model = urllib.parse.unquote(parsed.path.removeprefix("/v1/head-workers/")) + self._handle_tracker_nodes(model) elif parsed.path == "/v1/raft/status": self._handle_raft_status() elif parsed.path == "/v1/stats": @@ -1311,7 +1314,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): self._send_json(200, {"model": model, "coverage": coverage}) def _handle_tracker_nodes(self, model: str): - """Return nodes registered with tracker_mode=True whose shard starts at layer 0.""" + """Return head workers: worker nodes that can start inference for a model. + + The historical endpoint name is /v1/tracker-nodes, but these are not + tracker processes and they are the only machines that load model shards. + """ server: _TrackerHTTPServer = self.server # type: ignore[assignment] resolved_name, preset = _resolve_model_preset(server.model_presets, model) if preset is None: @@ -1337,6 +1344,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): ] self._send_json(200, { "model": resolved_name, + "head_workers": [ + { + "node_id": node.node_id, + "endpoint": node.endpoint, + "relay_addr": node.relay_addr, + "peer_id": node.peer_id, + } + for node in tracker_nodes + ], "tracker_nodes": [ { "node_id": node.node_id, @@ -2862,7 +2878,7 @@ class TrackerServer: treasury = self._treasury assert billing is not None and treasury is not None while not self._settlement_stop.wait(self._settlement_check_interval): - if self._raft is not None and not self._raft.is_leader(): + if self._raft is not None and not self._raft.is_leader: continue # followers replicate the ledger but never sign # resend batches whose transaction never confirmed for settlement_id, payouts in billing.unconfirmed_settlements().items(): diff --git a/pyproject.toml b/pyproject.toml index 2fbd9d4..c6fa7b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,9 @@ requires-python = ">=3.10" [project.optional-dependencies] dev = ["pytest>=8", "openai>=1", "langchain-openai>=0.1"] +[tool.setuptools] +packages = [] + [tool.pytest.ini_options] testpaths = ["tests"] markers = [ diff --git a/tests/test_tracker_as_node.py b/tests/test_tracker_as_node.py index e28be81..4b3528b 100644 --- a/tests/test_tracker_as_node.py +++ b/tests/test_tracker_as_node.py @@ -1,8 +1,9 @@ -"""US-014 integration test: tracker-as-first-layer-node inference entry point. +"""US-014 integration test: head-worker inference entry point. -Two stub tracker-nodes (shard 0-5, tracker_mode=True) + two mid-shard stub nodes -(shard 6-11) for openai-community/gpt2. Ten requests via gateway assert round-robin -load distribution across tracker-nodes and valid OpenAI-format responses. +Two stub head workers (shard 0-5, tracker_mode=True for legacy wire +compatibility) + two mid-shard stub nodes (shard 6-11) for openai-community/gpt2. +Ten requests via gateway assert round-robin load distribution across head +workers and valid OpenAI-format responses. """ import json @@ -51,7 +52,7 @@ def _register_node( @pytest.fixture def tracker_node_setup(): - """Start tracker, two tracker-nodes (shard 0-5), two mid-shard nodes (shard 6-11), + """Start tracker, two head workers (shard 0-5), two mid-shard nodes (shard 6-11), and a gateway. Yields (gateway_url, tracker_node_a, tracker_node_b).""" tracker = TrackerServer( @@ -66,12 +67,12 @@ def tracker_node_setup(): tracker_port = tracker.start() tracker_url = f"http://127.0.0.1:{tracker_port}" - # Two tracker-nodes: serve shard 0-5, expose /v1/chat/completions + # Two head workers: serve shard 0-5, expose /v1/chat/completions. tracker_node_a = StubNodeServer( shard_start=0, shard_end=5, is_last_shard=False, - response_prefix="tracker-node-A:", + response_prefix="head-worker-A:", model=GPT2_MODEL, tracker_mode=True, ) @@ -81,7 +82,7 @@ def tracker_node_setup(): shard_start=0, shard_end=5, is_last_shard=False, - response_prefix="tracker-node-B:", + response_prefix="head-worker-B:", model=GPT2_MODEL, tracker_mode=True, ) @@ -106,7 +107,7 @@ def tracker_node_setup(): ) mid_port_b = mid_node_b.start() - # Register all nodes with tracker (tracker-nodes get tracker_mode=True) + # Register all nodes with tracker (head workers keep legacy tracker_mode=True). _register_node(tracker_url, f"http://127.0.0.1:{port_a}", 0, 5, tracker_mode=True) _register_node(tracker_url, f"http://127.0.0.1:{port_b}", 0, 5, tracker_mode=True) _register_node(tracker_url, f"http://127.0.0.1:{mid_port_a}", 6, 11) @@ -155,22 +156,22 @@ def test_all_responses_valid_openai_format(tracker_node_setup): def test_both_tracker_nodes_receive_load(tracker_node_setup): - """Both tracker-nodes handle at least one request each out of ten.""" + """Both head workers handle at least one request each out of ten.""" gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup for i in range(10): _send_chat_request(gateway_url, f"message {i}") assert tracker_node_a.chat_completion_count >= 1, ( - f"tracker-node-A received no requests (count={tracker_node_a.chat_completion_count})" + f"head-worker-A received no requests (count={tracker_node_a.chat_completion_count})" ) assert tracker_node_b.chat_completion_count >= 1, ( - f"tracker-node-B received no requests (count={tracker_node_b.chat_completion_count})" + f"head-worker-B received no requests (count={tracker_node_b.chat_completion_count})" ) total = tracker_node_a.chat_completion_count + tracker_node_b.chat_completion_count - assert total == 10, f"total requests handled by tracker-nodes: {total}, expected 10" + assert total == 10, f"total requests handled by head workers: {total}, expected 10" def test_tracker_nodes_endpoint_returns_registered_nodes(tracker_node_setup): - """GET /v1/tracker-nodes/ on the tracker returns both registered tracker-nodes.""" + """GET /v1/tracker-nodes/ remains as a legacy alias for head workers.""" _, tracker_node_a, tracker_node_b = tracker_node_setup # Find the tracker URL by inspecting the fixture indirectly # We need the tracker URL — use the gateway's tracker_url @@ -181,13 +182,13 @@ def test_tracker_nodes_endpoint_returns_registered_nodes(tracker_node_setup): def test_load_is_distributed_evenly(tracker_node_setup): - """With 10 requests and round-robin, each tracker-node gets exactly 5.""" + """With 10 requests and round-robin, each head worker gets exactly 5.""" gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup for i in range(10): _send_chat_request(gateway_url, f"round-robin test {i}") assert tracker_node_a.chat_completion_count == 5, ( - f"expected 5 requests on tracker-node-A, got {tracker_node_a.chat_completion_count}" + f"expected 5 requests on head-worker-A, got {tracker_node_a.chat_completion_count}" ) assert tracker_node_b.chat_completion_count == 5, ( - f"expected 5 requests on tracker-node-B, got {tracker_node_b.chat_completion_count}" + f"expected 5 requests on head-worker-B, got {tracker_node_b.chat_completion_count}" ) diff --git a/tests/test_tracker_control_plane.py b/tests/test_tracker_control_plane.py new file mode 100644 index 0000000..e983f6d --- /dev/null +++ b/tests/test_tracker_control_plane.py @@ -0,0 +1,45 @@ +"""Tracker control-plane boundaries.""" + +import subprocess +import sys +import textwrap + + +def test_tracker_startup_does_not_import_or_load_model_backends(): + """The public tracker is a router/API endpoint, not an inference worker.""" + code = textwrap.dedent( + """ + import builtins + import sys + + blocked_roots = {"meshnet_node", "torch", "transformers"} + real_import = builtins.__import__ + + def guarded_import(name, globals=None, locals=None, fromlist=(), level=0): + root = name.split(".", 1)[0] + if root in blocked_roots: + raise AssertionError(f"tracker imported model backend dependency: {name}") + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = guarded_import + + from meshnet_tracker.server import TrackerServer + + tracker = TrackerServer() + port = tracker.start() + assert port > 0 + tracker.stop() + + imported = blocked_roots & set(sys.modules) + assert not imported, f"unexpected model modules imported: {sorted(imported)}" + """ + ) + + result = subprocess.run( + [sys.executable, "-c", code], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr