From 6b9caecd90de92396c1ae27fb675a1d9d3596782 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 29 Jun 2026 18:37:33 +0300 Subject: [PATCH] feat: add US-019 distributed tracker consensus (Raft + CRDT) Captures the architecture decision: Raft for shard assignments (strong consistency, leader-elected) + CRDT gossip for node heartbeats (high-frequency, eventual consistency). Approved 2026-06-29. Co-Authored-By: Claude Sonnet 4.6 --- .../distributed-inference-network/prd.json | 99 ++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/.scratch/distributed-inference-network/prd.json b/.scratch/distributed-inference-network/prd.json index 6b5a153..ed95c0d 100644 --- a/.scratch/distributed-inference-network/prd.json +++ b/.scratch/distributed-inference-network/prd.json @@ -382,10 +382,107 @@ "notes": "Source issue: .scratch/distributed-inference-network/issues/15-ralph-agent-agnostic-status-aware.md", "dependsOn": [], "completionNotes": "Implemented by agent: status-aware helpers (_is_done, _needs_attention, _is_active, _is_in_design), 6-bucket _story_sets, attention dashboard section, _review_report Attention Required block, auto --include-revise, set-agent subcommand with persistent agent-config.json, _run_openrouter stub, custom agent support, list-parallel subcommand, and auto --parallel N worktree orchestration. All 65 tests pass." + }, + { + "id": "US-016", + "title": "16 \u2014 Mining-style node startup CLI + live dashboard", + "description": "Replace the bare flag-driven node CLI with a wizard-guided first-run experience (like a GPU mining client) followed by a live terminal dashboard once the node is running. On first run, the wizard auto-detects GPU VRAM, presents a curated list of compatible models with VRAM requirements at each quantization level, lets the user pick a download location, and writes a persistent config file so subsequent starts are one command. Once the node is running, the wizard gives way to a rich live status panel showing: GPU temp + VRAM used, tokens/sec, requests served, peers connected, TAI earned (stub until US-006 is live). A Browse HuggingFace option calls the HF Hub API so users can load any HF model beyond the curated list.", + "acceptanceCriteria": [ + " with no args and no config file enters the interactive setup wizard", + "Wizard step 1: auto-detect GPU(s) via torch.cuda / torch.version.hip; print GPU name + total VRAM", + "Wizard step 2: show curated model list (name, HF repo, layers, VRAM@NF4/INT8/BF16); mark models that do NOT fit available VRAM as [too large]", + "Wizard step 3: offer [B] Browse HuggingFace \u2014 calls HF Hub API (huggingface_hub.list_models filtered by pipeline_tag=text-generation, sorted by downloads, top 20) and lets user enter a custom HF repo ID", + "Wizard step 4: prompt for download directory (default ~/.meshnet/models/); validate writable; show estimated disk usage for chosen model+quantization", + "Wizard step 5: prompt for tracker URL (default http://localhost:8080); validate connection", + "Wizard writes ~/.config/meshnet/config.json; second run skips wizard and starts directly", + "All wizard values overridable via CLI flags: --model, --download-dir, --quantization [bf16|int8|nf4], --tracker, --wallet, --reset-config", + "Once node is running, wizard clears and a live dashboard renders every 2s (rich.live): GPU util%, VRAM used/total, tokens/sec (EMA), requests served, TAI earned (stub 0.0), peers connected, uptime, current model/shard range", + "Dashboard exits cleanly on Ctrl-C with a summary line", + "Works inside WSL2 (no termios/ioctl calls that fail on Windows terminal; fall back to plain-text status if rich is not available)", + " passes from repo root", + "Commit only this story changes" + ], + "priority": 16, + "status": "done", + "notes": "Source issue: .scratch/distributed-inference-network/issues/16-mining-cli-ux.md", + "dependsOn": [ + "US-004", + "US-012" + ], + "completionNotes": "Implemented: mining-style wizard with GPU detection, curated model list (7 models with NF4/INT8/BF16 VRAM requirements), HF Hub browse, persistent config, rich live dashboard with plain-text WSL2 fallback. 19 tests, 97 passed total." + }, + { + "id": "US-017", + "title": "17 \u2014 P2P gossip, NAT-traversal relay node, and SSL/TLS", + "description": "Add a gossip layer so nodes discover each other and propagate coverage-map changes without polling the tracker continuously. Introduce a publicly-hosted relay node (run by the team) that solves NAT traversal using circuit relay (Petals-style) and serves as the bootstrap peer list for new nodes. Encrypt all node-to-node and node-to-tracker communications with TLS. Gossip protocol: WebSocket-based PubSub over wss://, topics: node-join / node-leave / coverage-update / heartbeat. Peer discovery: mDNS (zeroconf) for LAN, public relay bootstrap list for internet. NAT traversal: relay node acts as TCP-level circuit relay when direct connection fails (hole-punching first, relay second). Architecture is designed to migrate to libp2p GossipSub + Kademlia DHT in a future story without breaking the message schema.", + "acceptanceCriteria": [ + "All HTTP between nodes and tracker uses HTTPS (TLS 1.3); self-signed cert generated on first run and fingerprint pinned in config; relay node uses Let's Encrypt", + "Nodes broadcast node-join / node-leave events over wss:// to known peers within 1s of registration", + "mDNS peer discovery (Python zeroconf) finds other meshnet nodes on the same LAN segment without manual tracker URL entry", + "Public relay bootstrap list (hardcoded relay URL + ) is consulted when no LAN peers found", + "Relay node is a standalone meshnet package () with CLI: starts a WebSocket relay server + circuit relay + optional tracker proxy", + "When a node behind NAT cannot accept inbound connections, the relay forwards its traffic; node advertises relay address (relay_url/node_id) to tracker as its effective endpoint", + "Tracker accepts both direct node URLs and relay-proxied URLs in heartbeat payloads", + "Integration test: two nodes in separate processes on localhost (simulating NAT) communicate via a local relay process; inference request routes correctly", + "ADR-0010 documents the gossip protocol, relay architecture, and migration path to libp2p", + " passes from repo root", + "Commit only this story changes" + ], + "priority": 17, + "status": "done", + "notes": "Source issue: .scratch/distributed-inference-network/issues/17-p2p-gossip-relay-ssl.md", + "dependsOn": [ + "US-013", + "US-014" + ], + "completionNotes": "Implemented: packages/p2p (identity, TLS cert+fingerprint, GossipClient WSS PubSub, MdnsDiscovery with zeroconf optional), packages/relay (RelayServer gossip hub + circuit relay proxy, meshnet-relay CLI), tracker extended with relay_addr/cert_fingerprint/peer_id, relay_bootstrap.json, ADR-0010. 18 new tests; 115 total passed." + }, + { + "id": "US-018", + "title": "18 \u2014 End-to-end two-machine LAN inference test", + "description": "Prove the network works across two real machines: the Linux rig (this machine) and a Windows 11 rig running WSL2. One machine runs the tracker + first-shard node (inference entry point). The other machine runs a second-shard node. A client sends a real inference request and receives a streamed response. This story is primarily a test plan + setup guide + test execution script; it produces documented evidence (logs, timing, token output) that real distributed inference works. It also surfaces any real-world issues (port forwarding, CUDA driver version mismatches, WSL2 CUDA passthrough, model download paths) that need fixing.", + "acceptanceCriteria": [ + "docs/INSTALL_WINDOWS.md exists: step-by-step WSL2 + CUDA + meshnet-node install on Windows 11", + "docs/TWO_MACHINE_TEST.md exists: how to start tracker on machine A, node on machine B, run inference, interpret output", + "A test script scripts/test_lan_inference.py: given --tracker-url, --gateway-url, sends 3 chat completion requests, asserts valid OpenAI format, prints token count + latency + which nodes served each request", + "Both machines can reach each other on LAN (documented: firewall rules, port list)", + "At least one successful inference recorded: the test script exits 0 with output showing tokens generated and node IDs", + "Latency breakdown logged: gateway\u2192node-A, node-A\u2192node-B, node-B\u2192gateway (approximate, from server logs)", + "Known issues during test documented in docs/TWO_MACHINE_TEST.md under a Known Issues section", + "Commit only this story changes" + ], + "priority": 18, + "status": "open", + "notes": "Source issue: .scratch/distributed-inference-network/issues/18-two-machine-lan-test.md", + "dependsOn": [ + "US-016", + "US-017" + ], + "completionNotes": null + }, + { + "id": "US-019", + "title": "19 — Distributed tracker consensus (Raft assignments + CRDT heartbeats)", + "description": "Replace the single-point-of-failure tracker with a fault-tolerant cluster. Tracker nodes elect a leader via Raft and commit shard assignments as log entries — all tracker nodes agree on who owns what. Node liveness (heartbeats) uses CRDT gossip (eventual consistency, high frequency OK). A node registers with any tracker node; the write is forwarded to the leader and replicated to followers. A 3-node tracker cluster survives one tracker failure without losing assignment state. The relay/gossip layer already built in US-017 handles peer heartbeats; this story wires Raft on top for authoritative assignments.", + "acceptanceCriteria": [ + "3 tracker nodes can be started and form a Raft cluster (leader election, log replication)", + "A node registers with any follower — the registration is forwarded to the leader and replicated", + "Killing the leader causes a new election within 5 seconds; registrations continue working", + "Shard assignments returned by any tracker node are identical (strong consistency)", + "Node heartbeats use CRDT gossip (not Raft) — high-frequency, eventual consistency", + "meshnet-tracker CLI gains --cluster-peers flag to specify peer tracker URLs", + "Integration test: 3 tracker nodes, kill leader mid-test, verify assignment still works", + "QUICKSTART.md updated with multi-tracker setup section" + ], + "priority": 19, + "status": "open", + "notes": "Architecture decision: Raft for assignments (strong consistency) + CRDT gossip for liveness (eventual consistency). User approved 2026-06-29.", + "dependsOn": ["US-017"], + "completionNotes": null } ], "metadata": { - "updatedAt": "2026-06-29T13:30:00.000Z", + "updatedAt": "2026-06-29T15:35:00.000Z", "statusVocabulary": { "open": "Not started", "in-design": "Decisions pending before implementation can begin",