From 5d87e81bc98dc6687c73e2a17778ecfd15e7151a Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 13 Jul 2026 21:58:08 +0200 Subject: [PATCH] feat: harden node placement and partial model loading --- .claude/memory/MEMORY.md | 4 +- .claude/memory/project-status.md | 2 +- .scratch/alpha-hardening/README.md | 5 +- .../21-honest-noise-calibration-corpus.md | 6 +- .scratch/alpha-hardening/prd.json | 6 +- .../runbooks/04-toploc-calibration-run.md | 70 +++++++ .scratch/node-capability-admission/PRD.md | 2 + .scratch/node-capability-admission/README.md | 1 + .scratch/qwen3.6-27b-demand-placement/PRD.md | 13 +- docs/PRD.md | 36 ++-- docs/adr/0024-distributed-gguf-runtime.md | 8 + ...ignment-ownership-and-managed-placement.md | 51 +++++ ...dget-shard-slots-and-dropout-relocation.md | 10 + docs/issues/42-gguf-llamacpp-node-backend.md | 12 +- ...4-tracker-shard-source-partial-download.md | 15 +- ...9-mainnet-usdt-cutover-two-wallet-pilot.md | 2 +- .../issues/50-qwen3.6-27b-demand-placement.md | 15 ++ docs/prd.json | 61 ++++-- packages/node/meshnet_node/cli.py | 14 ++ packages/node/meshnet_node/model_backend.py | 35 +++- tests/test_real_model_backend.py | 184 ++++++++++++++++++ 21 files changed, 497 insertions(+), 55 deletions(-) create mode 100644 .scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md create mode 100644 docs/adr/0026-node-assignment-ownership-and-managed-placement.md create mode 100644 docs/issues/20-memory-budget-shard-slots-and-dropout-relocation.md create mode 100644 docs/issues/50-qwen3.6-27b-demand-placement.md diff --git a/.claude/memory/MEMORY.md b/.claude/memory/MEMORY.md index bdeecd9..598c8e8 100644 --- a/.claude/memory/MEMORY.md +++ b/.claude/memory/MEMORY.md @@ -2,9 +2,9 @@ - [Product selling points](product-selling-points.md) — key differentiators and landing page angles for neuron-tai - [User profile](user-profile.md) — who Dobromir is and how to work with him -- [Project status](project-status.md) — 35/35 stories done; alpha hardening next +- [Project status](project-status.md) — US-001…US-035 done; US-036…US-050 in docs/prd.json; alpha hardening + scratch features next - **Alpha hardening** — `.scratch/alpha-hardening/` (22 issues, ADRs 0016–0019, [README](../../.scratch/alpha-hardening/README.md), [handoff](../../.scratch/alpha-hardening/handoff.md)) - [Alpha hardening navigation](alpha-hardening-navigation.md) — locked fraud/auth decisions, Bucket-1 order, handoff pointers -- **Node capability admission** — `.scratch/node-capability-admission/` (P0 plan: generic doctor/real-forward validation, fail-closed readiness, tracker admission gate; [PRD](../../.scratch/node-capability-admission/PRD.md), [README](../../.scratch/node-capability-admission/README.md), ADR-0023) +- **Node capability admission** — `.scratch/node-capability-admission/` (P0 plan; [ADR-0023](../../docs/adr/0023-model-agnostic-node-capability-admission.md), [ADR-0026](../../docs/adr/0026-node-assignment-ownership-and-managed-placement.md)) - **Distributed relay performance** — relay `/rpc` requester sockets are persistent per Route Session and Activation Seam as of 2026-07-10; `request_id` remains unique per activation while `X-Meshnet-Session` remains stable for KV state. Next low-risk priorities: persistent direct/loopback HTTP, seam byte/latency telemetry, then trace-driven zstd tuning. - **Distributed GGUF direction** — benchmark-gated native runtime: compare controlled Transformers/safetensors and whole-model llama.cpp lanes before expensive work; ship only for measured speed or model-fit advantage. Public parallelism is contiguous Shards in an Inference Route; concurrency comes from per-node continuous batching across isolated Route Sessions, while tensor/expert collectives stay inside optional trusted composite providers. Native data plane uses versioned Protobuf over long-lived gRPC/HTTP2 seam streams, with existing relay carrying the same opaque frames when needed. llama.cpp/GGML remains the substrate behind a project-owned standalone worker and small pinned fork; vLLM is an optional complete managed provider and concept donor, not a fork. Nakshatra, `prima.cpp`, `llama-gguf`, LiGGUF and historical GPUStack are source/test donors only. Active plan: [README](../../.scratch/distributed-gguf-runtime/README.md), [architecture](../../.scratch/distributed-gguf-runtime/architecture.md), [PRD](../../.scratch/distributed-gguf-runtime/PRD.md), [Ralph backlog](../../.scratch/distributed-gguf-runtime/prd.json). ADR: [0024](../../docs/adr/0024-distributed-gguf-runtime.md). Research: [landscape](../../docs/research/distributed-gguf-landscape.md), [GitHub follow-up](../../docs/research/distributed-gguf-github-followup.md), [vLLM](../../docs/research/vllm-distributed-gguf-assessment.md). diff --git a/.claude/memory/project-status.md b/.claude/memory/project-status.md index 0c4dd4c..8c26a8b 100644 --- a/.claude/memory/project-status.md +++ b/.claude/memory/project-status.md @@ -8,7 +8,7 @@ metadata: # Project Status (2026-07-13) -> Doc reconciliation 2026-07-13: `docs/PRD.md`, ADR numbering (0024 = distributed GGUF), and `docs/issues/` statuses updated to match `docs/prd.json` (US-001…US-035 done). **US-036…US-047** added to `docs/prd.json` for the friends-test / LAN-serving arc. Post-035 scratch work remains in `.scratch/`. +> Doc reconciliation 2026-07-13: `docs/prd.json` tracks US-001…US-050 (048 memory budget, 049 mainnet pilot, 050 Qwen demand placement). ADRs 0025–0026 added (TAI phase B/C, assignment ownership). All 35 user stories in docs/prd.json are done (35/35), including the reward-system arc US-030…US-035 completed 2026-07-02: diff --git a/.scratch/alpha-hardening/README.md b/.scratch/alpha-hardening/README.md index d6aaecd..ca29d6c 100644 --- a/.scratch/alpha-hardening/README.md +++ b/.scratch/alpha-hardening/README.md @@ -2,9 +2,9 @@ Pre-release alpha audit + grilling (2026-07-04). Bucket 1 trust-boundary blockers + fraud arc: **done** (16/22 original issues). Bucket 2 (12-15, multi-tracker) and 17 (doc dedup) remain deferred/human-gated — not launch blockers. -**Launch-readiness grilling (2026-07-06):** locked plan is devnet dev/test run now, then real mainnet SOL/USDT for the first cohort — friends (API clients) + hired VPS/VPC hosts (own test infra, not third-party volunteers, stake-free). No new public token; TAI stays dormant per ADR-0002's existing volume/legal gates. Two new issues came out of this session: +**Launch-readiness grilling (2026-07-06):** locked plan is devnet dev/test run now, then real mainnet USDT for the first cohort — friends (API clients) + hired VPS/VPC hosts (own test infra, not third-party volunteers; no upfront stake, probation only). No new public token; TAI stays dormant per ADR-0002's existing volume/legal gates. Two new issues came out of this session: -- **[21 — Honest-noise calibration corpus](./issues/21-honest-noise-calibration-corpus.md)** — rescoped from "prod gate" to a hard **alpha-release blocker**. `Status: ready-for-human` — engineering (audit.py raw divergence, tracker dispatch endpoint, SQLite corpus, p99 envelope) done 2026-07-06; blocked on a human running the calibration job against the real hired-VPS fleet before launch. +- **[21 — Honest-noise calibration corpus](./issues/21-honest-noise-calibration-corpus.md)** — rescoped from "prod gate" to a hard **alpha-release blocker**. `Status: ready-for-human` — engineering (audit.py raw divergence, tracker dispatch endpoint, SQLite corpus, p99 envelope) done 2026-07-06; blocked on a human running the calibration job against the real hired-VPS fleet before launch. Runbook: [04-toploc-calibration-run](./runbooks/04-toploc-calibration-run.md). - **[23 — Dynamic HF-benchmarked pricing](./issues/23-dynamic-hf-pricing_completed.md)** — new, high priority but not a release blocker. `Status: done` — engineering complete 2026-07-06 (hf_pricing.py, opt-in daily refresh loop, GET /v1/pricing/hf/history); real `hf_aliases` curation per model is a follow-up human sign-off, not a completion blocker. Locked scope: one settlement tracker, open node join, devnet mock-USDT, reputation carries forward → fraud must be bounded. See [ADR-0016](../../docs/adr/0016-alpha-scope-and-known-limitations.md). @@ -77,6 +77,7 @@ Locked scope: one settlement tracker, open node join, devnet mock-USDT, reputati | [17 Duplicate US-020 dedup](./issues/17-doc-duplicate-us020-dedup.md) | | [18 Operational runbooks](./issues/18-doc-operational-runbooks_completed.md) | | [19 Cryptography + test env](./issues/19-doc-cryptography-test-env_completed.md) | +| [04 TOPLOC calibration run](./runbooks/04-toploc-calibration-run.md) (issue 21 ops) | | [22 MEMORY + project-status index](./issues/22-doc-memory-project-status_completed.md) (done) | | [21 Honest-noise calibration corpus](./issues/21-honest-noise-calibration-corpus.md) (ops; prod gate for audits) | diff --git a/.scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md b/.scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md index 3e64004..d7a114a 100644 --- a/.scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md +++ b/.scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md @@ -1,6 +1,6 @@ Status: ready-for-human -**BLOCKS ALPHA RELEASE.** Scoped 2026-07-06 during alpha-launch-readiness grilling session — must complete before real-money (mainnet SOL/USDT) traffic goes live for the friends + hired-VPS-host launch. Loose/uncalibrated thresholds + manual admin slash-reversal are the stopgap only until this closes. +**BLOCKS ALPHA RELEASE.** Scoped 2026-07-06 during alpha-launch-readiness grilling session — must complete before real-money mainnet USDT traffic goes live for the friends + hired-VPS-host launch. Loose/uncalibrated thresholds + manual admin slash-reversal are the stopgap only until this closes. **Engineering complete 2026-07-06; blocked on a human running it against the real hired-VPS fleet before launch.** The three code gaps below are closed and unit-tested (see Deliverables), but nothing in a dev session can stand in for actually dispatching the job at real hardware — that step, plus the threshold/FPR write-up that depends on its output, needs an operator with the live fleet. See the validator README's "Honest-noise calibration corpus" section for the operational how-to. @@ -14,7 +14,7 @@ Per [ADR-0018 consequences](../../docs/adr/0018-fraud-detection-verification-and Research anchor: `.scratch/alpha-hardening/research-verifiable-inference.md` §8 layer 3 — "collect this first — run identical jobs across the current node fleet to measure the honest divergence envelope before setting thresholds." -**Launch context (why this is buildable now, not a research project):** first-launch nodes are hired VPS/VPC hosts under our own direct control (test infrastructure we pay for, not third-party volunteers) — not a long-term topology, but risk-free for calibration purposes since there's no external party to dispute a bad reading. Friends are client-side users of the API in this phase, not node operators. Run the calibration pass against this small, fully-controlled fleet first; hired hosts stay stake-free until it's done, then move to real staking once thresholds derive from their own hardware. +**Launch context (why this is buildable now, not a research project):** first-launch nodes are hired VPS/VPC hosts under our own direct control (test infrastructure we pay for, not third-party volunteers) — not a long-term topology, but risk-free for calibration purposes since there's no external party to dispute a bad reading. Friends are client-side users of the API in this phase, not node operators. Run the calibration pass against this small, fully-controlled fleet first; hired hosts stay on probation (no upfront stake) until it's done, then move to paid USDT serving once thresholds derive from their own hardware. **Current gap (historical — closed 2026-07-06):** the three engineering pieces below were missing when this issue was filed; all are now implemented and unit-tested. Remaining work is the human calibration run on the live hired-VPS fleet. @@ -36,7 +36,7 @@ Research anchor: `.scratch/alpha-hardening/research-verifiable-inference.md` §8 - [ ] Threshold constants in validator config derived from corpus, not guessed — mechanically ready (`envelope()` returns them) but depends on the real corpus above; not yet wired into `ToplocAuditConfig` as enforced thresholds (deliberately — enforcing unvalidated thresholds would be worse than today's guessed bool). - [ ] False-positive rate estimate documented at chosen thresholds — `envelope()` returns `estimated_false_positive_rate` (in-sample: fraction of the recorded corpus the recommended thresholds would themselves flag); needs the real corpus to be a meaningful number, and should be written up in the runbook once collected. - [x] README / runbook cross-link: **do not enable production audits** until this issue closes — `packages/validator/README.md` "TOPLOC audit contract" section, updated with the full operational how-to. -- [x] Note in the runbook that this alpha corpus must be re-run once the fleet grows beyond the hired-VPS set (different hardware mix invalidates the envelope) — same README section. +- [x] Note in the runbook that this alpha corpus must be re-run once the fleet grows beyond the hired-VPS set (different hardware mix invalidates the envelope) — same README section; [runbook 04](../runbooks/04-toploc-calibration-run.md). ## ADR links diff --git a/.scratch/alpha-hardening/prd.json b/.scratch/alpha-hardening/prd.json index 7ae4d40..babf5a9 100644 --- a/.scratch/alpha-hardening/prd.json +++ b/.scratch/alpha-hardening/prd.json @@ -440,12 +440,12 @@ "Run relevant pytest tests; run the full suite when practical or document why not" ], "priority": 21, - "passes": true, - "notes": "Source issue: .scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md. BLOCKS ALPHA RELEASE (real-money friends+hired-VPS launch) — rescoped 2026-07-06, no longer a Ralph-skip.", + "passes": false, + "notes": "Source issue: .scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md. BLOCKS ALPHA RELEASE (real-money mainnet USDT). Operator runbook: .scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md", "dependsOn": [ "AH-006" ], - "completionNotes": "Engineering complete and unit-tested (validator audit.py detailed-verify aggregation, tracker calibration.py corpus store, calibration dispatch endpoints). Marked ready-for-human, not done: real corpus collection against the live hired-VPS fleet, and the threshold/FPR write-up that depends on its output, need a human operator — see .ralph-tui/progress.md and packages/validator/README.md." + "completionNotes": "Engineering complete and unit-tested. Remaining: human runs POST /v1/calibration/toploc/run on live hired-VPS fleet, records envelope/FPR, wires thresholds — see runbook 04 and packages/validator/README.md." }, { "id": "AH-022", diff --git a/.scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md b/.scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md new file mode 100644 index 0000000..ed7991e --- /dev/null +++ b/.scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md @@ -0,0 +1,70 @@ +# Runbook 04 — Honest-noise TOPLOC calibration (issue 21) + +**Status:** engineering complete; **operator action required** before production audit thresholds. + +**Blocks:** enabling calibrated TOPLOC thresholds on a mainnet / friends-test fleet (issue 21, ADR-0018). + +## When to run + +- Before first real-money traffic with audit enforcement enabled. +- Again whenever the fleet’s **hardware mix** changes materially (new GPU generation, CPU-only nodes added, precision/recipe change per model). + +Alpha exception: with a **small hired-VPS-only** fleet, `gate_status.ready` may mean “covers every node we operate today” (`--toploc-calibration-gate-min-hardware-profiles 1`). + +## Prerequisites + +- Tracker running with billing + registry + `--toploc-calibration-db PATH` (or default under tracker cwd). +- At least one **solo-capable** node per hardware profile you want in the corpus (full model coverage — partial shards are skipped). +- Admin or validator credentials (`Authorization` header or validator service token per ADR-0017). +- Reference validator can replay the fixed calibration prompt (same model/seed as dispatch uses). + +## Steps + +1. **Register the fleet** — all nodes you intend to pay on mainnet should be up, admitted (NCA when enabled), and solo-serving the calibration model. + +2. **Dispatch the job** (admin/validator only): + + ```bash + curl -X POST "https:///v1/calibration/toploc/run" \ + -H "Authorization: Bearer " \ + -H "Content-Type: application/json" \ + -d '{}' + ``` + + Partial-shard nodes appear under `skipped_partial_shard_node_ids`. Per-node failures appear under `skipped` with reasons. + +3. **Wait for completion** — watch tracker logs and node consoles until every solo-capable node has a row in the corpus. + +4. **Fetch results**: + + ```bash + curl "https:///v1/calibration/toploc/results" \ + -H "Authorization: Bearer " + ``` + + Record: + - `envelope` — p99 metrics + 20% safety margin (recommended tolerances). + - `gate_status.ready` and `gate_status.hardware_profiles`. + - `estimated_false_positive_rate` (in-sample sanity check only). + +5. **Write up thresholds** — paste envelope values into operator notes / issue 21 comment. Do **not** wire into production `ToplocAuditConfig` until you have reviewed FPR on this fleet. + +6. **Mark issue 21 done** — when corpus covers the launch fleet and thresholds are documented. + +## Two-wallet / minimal pilot variant + +If your “fleet” is one node machine + one client: + +- Run calibration against the **node** profile only (one hardware row is enough for `gate_status` with min profiles = 1). +- Client wallet is irrelevant to calibration — it never serves inference. + +## Do not + +- Enable stricter production audit thresholds before this completes. +- Reuse a corpus collected on devnet/mock hardware for a different mainnet GPU mix without re-running. + +## References + +- Issue: `.scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md` +- Code: `packages/tracker/meshnet_tracker/calibration.py`, `POST /v1/calibration/toploc/run` +- Validator: `packages/validator/README.md` — TOPLOC audit contract diff --git a/.scratch/node-capability-admission/PRD.md b/.scratch/node-capability-admission/PRD.md index 3aa76d1..f857bf7 100644 --- a/.scratch/node-capability-admission/PRD.md +++ b/.scratch/node-capability-admission/PRD.md @@ -71,6 +71,8 @@ As an operator and release engineer, I need clear doctor output and opt-in hardw Add a small generic capability domain object in the node package. `doctor` loads the requested generic model path through the same backend startup uses, executes a bounded real forward at the assigned Shard, and emits the report. Startup gates routable registration on the successful report. Registration carries validated capabilities; the tracker persists/exposes them and filters route candidates at the model/shard/recipe seam. +**Assignment ownership:** NCA validates whatever the node loads; it does not assign models. Pinned vs tracker-managed assignment rules are in [ADR-0026](../../docs/adr/0026-node-assignment-ownership-and-managed-placement.md). Demand-driven managed placement (Qwen scratch PRD) may only consume spare capacity; admission applies equally to pinned and managed loads. + The future signed-update contract is represented only by a local manifest version and generic schema in P0. A future Tracker Model Artifact Manifest may be signed data, but Node executable behavior remains supplied by signed Node releases. ## Success measures diff --git a/.scratch/node-capability-admission/README.md b/.scratch/node-capability-admission/README.md index 0f10c7b..4786ccc 100644 --- a/.scratch/node-capability-admission/README.md +++ b/.scratch/node-capability-admission/README.md @@ -7,6 +7,7 @@ This P0 makes a Node prove it can serve its selected Model Artifact and Shard be ## Locked decisions - A Node explicitly asked to serve a Model Preset fails closed when no validated recipe can execute it; it must not register as ready or accept paid inference. +- **Assignment ownership:** startup/`--model` loads are **pinned**; tracker-managed demand placement (Qwen US-050) may use **spare capacity only** — [ADR-0026](../../docs/adr/0026-node-assignment-ownership-and-managed-placement.md). - Default validation covers the selected model/shard only. `meshnet-node doctor --all-recipes` is reserved for support and CI. - A Model Preset may have multiple named recipes. Each independently proves a real forward; the Tracker schedules only validated recipes while considering measured performance. - Compatibility schemas are generic. A future Tracker may publish signed, data-only Model Artifact Manifests, but executable recipes arrive only through signed Node releases. diff --git a/.scratch/qwen3.6-27b-demand-placement/PRD.md b/.scratch/qwen3.6-27b-demand-placement/PRD.md index e6f5e44..fc65b0d 100644 --- a/.scratch/qwen3.6-27b-demand-placement/PRD.md +++ b/.scratch/qwen3.6-27b-demand-placement/PRD.md @@ -46,13 +46,12 @@ model rather than waiting for an operator to request a load. ## Node ownership -- A startup-assigned `(model, shard range, quantization)` is pinned and never - changed by the tracker. -- Spare capacity on a pinned node, and all capacity on a model-less node, is - available for tracker-managed assignments. -- Tracker-added assignments are explicitly marked managed and may be moved or - removed by the tracker under the safety policy. Runtime UI controls are a - later feature. +Reconciled with [ADR-0026](../../docs/adr/0026-node-assignment-ownership-and-managed-placement.md) and NCA (ADR-0023): + +- A **startup-assigned** `(model, shard range, quantization)` from explicit `--model` or accepted bootstrap assign is **pinned** until the operator restarts. +- **Tracker-managed** assignments (this feature) use only **spare capacity** — model-less nodes or (future, US-048) unused shard slots — and are marked `managed: true`. +- The tracker may move or remove managed assignments under the safety policy below; it must not retarget a pinned serving assignment to satisfy demand. +- Every assignment, pinned or managed, must pass NCA `doctor` before becoming routable when admission is enabled. ## Pricing diff --git a/docs/PRD.md b/docs/PRD.md index c497673..85d8827 100644 --- a/docs/PRD.md +++ b/docs/PRD.md @@ -1,4 +1,4 @@ -Status: done (base program US-001…US-035 complete; friends-test arc US-036…US-047 in `docs/prd.json`. Payment/settlement superseded by ADR-0015; fraud by ADR-0018.) +Status: done (US-001…US-035 complete; friends-test arc US-036…US-049 in `docs/prd.json`; US-048/050 tracked. See ADRs 0015–0018, 0023, 0025–0026.) # Distributed Inference Network — PRD @@ -8,10 +8,12 @@ Running large language models requires expensive dedicated hardware that most pe ## Solution -A volunteer GPU network where anyone can share their GPU by running a single command and immediately start earning tokens. Nodes each load a shard of a large model; a tracker routes inference requests through the optimal chain of nodes whose shards collectively cover all layers. Developers access the network through an OpenAI-compatible API — a one-line change from any existing LLM integration. Clients pay in **USDT** (alpha: devnet mock-USDT; production: mainnet USDT). Node operators earn USDT payouts from the custodial treasury (ADR-0015); the TAI reward token (ADR-0002) remains deferred. Everything is auto-configured: GPU detection, shard download, wallet creation, and network registration happen automatically on first start. +A volunteer GPU network where anyone can share their GPU by running a single command and immediately start earning **USDT**. Nodes each load a shard of a large model; a tracker routes inference requests through the optimal chain of nodes whose shards collectively cover all layers. Developers access the network through an OpenAI-compatible API — a one-line change from any existing LLM integration. Clients pay in **USDT** (alpha: devnet mock-USDT; production: mainnet USDT). Node operators earn USDT payouts from the custodial treasury (ADR-0015); the TAI reward token (ADR-0002) remains deferred. Everything is auto-configured: GPU detection, shard download, wallet creation, and network registration happen automatically on first start. ## User Stories +> **Status (2026-07-13):** Stories below are the original product intent. **Shipped behavior** is in [Implementation Decisions](#implementation-decisions) and ADRs 0015–0018, 0023, 0025–0026. Superseded lines are marked inline. + ### Node Operator 1. As a node operator, I want to install the node client with a single command (`pip install meshnet-node`), so that I can start contributing without reading documentation. @@ -21,10 +23,10 @@ A volunteer GPU network where anyone can share their GPU by running a single com 5. As a node operator, I want my assigned shard to download automatically from HuggingFace on first start, so that I don't have to manually find or download model weights. 6. As a node operator, I want to seed my shard to other nodes via P2P once I have it, so that new nodes with the same shard assignment don't need to download from HuggingFace. 7. As a node operator, I want the node client to register with the tracker automatically and begin serving inference requests, so that I start earning as soon as setup is complete. -8. As a node operator, I want to see my current node score, shard assignment, and token earnings in the terminal, so that I can verify my node is contributing correctly. -9. As a node operator, I want to stake tokens before serving paid inference, so that I have skin in the game and the network can trust my outputs. +8. As a node operator, I want to see my current node score, shard assignment, and USDT earnings in the terminal, so that I can verify my node is contributing correctly. +9. As a node operator, I want to serve paid inference without upfront stake deposits, with my accrued USDT pending balance as fraud collateral and probation as the anti-sybil cost, so that onboarding stays frictionless. *(Supersedes stake-before-serving; ADR-0015/0018.)* 10. As a node operator, I want my first N jobs to run without earning (probationary period), so that the network can establish trust before paying me. -11. As a node operator, I want to be notified immediately if my stake is slashed due to a fraud detection event, so that I can investigate and fix the issue. +11. As a node operator, I want to be notified when my pending balance is forfeited due to a failed audit, so that I can investigate and fix the issue. *(Supersedes stake slash; ADR-0018 forfeiture.)* 12. As a node operator, I want to receive a strike and a warning before being banned, so that accidental failures don't immediately end my participation. 13. As a node operator, I want to be automatically reassigned to a different shard when the tracker determines another shard is more in demand, so that my hardware is always optimally used. 14. As a node operator, I want the node client to reconnect automatically if the tracker is temporarily unavailable, so that transient network issues don't stop me from earning. @@ -34,8 +36,8 @@ A volunteer GPU network where anyone can share their GPU by running a single com ### Client Developer 17. As a client developer, I want to send `POST /v1/chat/completions` requests to the gateway in the same format as the OpenAI API, so that I can switch to the network with a one-line code change. -18. As a client developer, I want to authenticate with an API key funded by SOL or USDC, so that I never need to acquire or hold our native token. -19. As a client developer, I want to top up my API key balance by sending SOL or USDC to a Solana address, so that payment is simple and familiar. +18. As a client developer, I want to authenticate with an API key funded by USDT, so that I never need to acquire or hold our native token. *(ADR-0015.)* +19. As a client developer, I want to top up my API key balance by sending USDT to the treasury Solana address, so that payment is simple and familiar. *(ADR-0015; wallet binding US-039/041.)* 20. As a client developer, I want to see a per-request cost estimate before sending a request, so that I can budget inference costs accurately. 21. As a client developer, I want to receive streaming responses (`text/event-stream`) in OpenAI-compatible format, so that I can build low-latency user experiences. 22. As a client developer, I want `GET /v1/models` to return the list of available model presets on the network, so that I know what I can request. @@ -46,15 +48,15 @@ A volunteer GPU network where anyone can share their GPU by running a single com ### End User (via a client app) -27. As an end user, I want to buy SOL on any exchange and use it to pay for inference, so that I don't need to understand blockchain technology to use the service. +27. As an end user, I want to buy USDT on an exchange and use it to pay for inference via Solana, so that I don't need deep crypto knowledge to use the service. *(Clients pay USDT; SOL is only for network fees if they self-custody.)* 28. As an end user, I want responses of equivalent quality to centralised providers, so that I don't have to trade quality for cost savings. 29. As an end user, I want low latency on first token, so that conversational applications feel responsive. ### Validator -30. As a validator, I want to automatically re-run a random sample (~5%) of completed inference requests on a reference node, so that I can detect nodes returning fraudulent outputs. -31. As a validator, I want to submit a fraud proof on-chain when a node's output diverges beyond tolerance, so that the slash event is recorded trustlessly. -32. As a validator, I want to earn a reward for each successful fraud detection, so that there is an economic incentive to run validation. +30. As a validator, I want to automatically re-run a random sample (~5%) of completed inference requests on a reference node with TOPLOC activation verification, so that I can detect nodes returning fraudulent outputs. *(ADR-0018.)* +31. As a validator, I want the tracker to record forfeiture and strikes when an audit fails, so that penalties are applied consistently. *(Supersedes on-chain fraud proof in alpha; ADR-0018.)* +32. As a validator, I want economic incentive to run validation, so that fraud detection is not purely altruistic. *(Validator reward share deferred; forfeiture to protocol cut today.)* ### Network (tracker / system) @@ -62,7 +64,7 @@ A volunteer GPU network where anyone can share their GPU by running a single com 34. As the tracker, I want to rebalance shard assignments across nodes when demand for a model preset changes, so that the network always covers the most-requested models. 35. As the tracker, I want to instruct a node to download a new shard when no other node covers it, so that model preset coverage is maintained automatically. 36. As the tracker, I want to exclude banned wallets from route selection, so that fraudulent nodes cannot serve paid inference. -37. As the tracker, I want to read stake, slash, strike, and ban state exclusively from Solana smart contracts, so that I cannot manipulate payouts even with full control of the routing layer. +37. As the tracker, I want strike and ban state persisted in the registry and enforced on route selection, so that fraudulent wallets cannot serve paid inference. *(Supersedes on-chain-only stake/slash registry; ADR-0018; on-chain deferred per ADR-0007/0015.)* 38. As the network, I want new model presets to be addable by submitting a HuggingFace model ID and shard count, so that the set of available models can grow without code changes. ## Implementation Decisions @@ -73,7 +75,7 @@ The codebase is organized as a Python monorepo with the following top-level pack - `packages/gateway` — OpenAI-compatible HTTP gateway and route orchestration - `packages/tracker` — centralized tracker service (node registry, scoring, route selection) - `packages/sdk` — `meshnet` Python SDK wrapping gateway + wallet controls -- `packages/contracts` — Solana L2 smart contracts (stake, slash, strike, ban, settlement) +- `packages/contracts` — Solana adapter boundary (custodial USDT treasury, local registry prototype) - `packages/p2p` — P2P gossip layer and shard swarm seeding ### Inference engine (ADR-0001; native GGUF path ADR-0024) @@ -98,7 +100,7 @@ Clients pre-fund an API key with USDT. The tracker meters each request against t Validators re-run ~5% of completed requests with TOPLOC activation verification. Caught cheaters forfeit pending balance and receive strikes; three strikes bans the wallet. Probation (first N unpaid jobs) remains the anti-sybil re-entry cost. ### Tracker architecture (ADR-0004) -Centralized tracker service (HTTP + WebSocket) for fast routing. Nodes gossip state via a lightweight P2P layer so the node client can discover routes during tracker outages. Solana is the authoritative source of truth for all incentive-relevant state. +Centralized tracker service (HTTP + WebSocket) for fast routing. Nodes gossip state via a lightweight P2P layer so the node client can discover routes during tracker outages. **Alpha:** strike/ban/forfeiture state lives in the tracker registry (ADR-0018); USDT settlement via custodial treasury (ADR-0015). On-chain programs deferred (ADR-0007). ### Shard distribution (ADR-0005) Shards are identified by `(model_preset, shard_index)`. On assignment, the node downloads the shard layers from HuggingFace using `huggingface_hub`. Once downloaded, the node joins the P2P shard swarm and seeds to other nodes requesting the same shard. Popular shards propagate entirely via P2P; cold shards fall back to HuggingFace. @@ -115,9 +117,9 @@ The gateway exposes OpenAI-compatible endpoints (`/v1/chat/completions`, `/v1/mo **Per-component seams:** - **Tracker**: given a set of registered nodes with known shard coverage and node scores, assert `select_route(model_preset)` returns an optimal ordered list of node endpoints. - **Node shard serving**: given an activation tensor for the node's layer range, assert the output tensor shape and dtype are correct. -- **Fraud detection**: given a validator that re-runs a known-bad node response, assert a slash transaction is submitted on-chain with correct attribution. +- **Fraud detection**: given a validator that re-runs a known-bad node response, assert strike/forfeiture state updates with correct attribution (ADR-0018; on-chain slash deferred). - **Shard swarm**: given a node that has a shard, assert a second node with the same assignment downloads it via P2P rather than HuggingFace. -- **Payment settlement**: given a completed inference session with known compute attribution, assert token balances change by the expected amounts after epoch settlement. +- **Payment settlement**: given a completed inference session with known compute attribution, assert USDT ledger balances change by the expected amounts after epoch settlement (ADR-0015). ## Out of Scope @@ -135,4 +137,4 @@ The gateway exposes OpenAI-compatible endpoints (`/v1/chat/completions`, `/v1/mo - The `meshnet-node` CLI is the primary viral growth vector. Every friction point in the install/start sequence costs node operators. The startup sequence must complete without any manual configuration on a machine with a CUDA-capable GPU. - The name "meshnet" is a working name. The actual package and token names are TBD. - The Solana L2 chain selection (vs Base/Arbitrum) is not yet finalised — both are cheap, EVM-compatible fallbacks. The contracts package should abstract chain-specific details. -- The probationary period length (N free jobs) and slash amounts are economic parameters that will need tuning once the network has real usage data. Hardcode sensible defaults; make them on-chain governable. +- The probationary period length (N free jobs) and forfeiture amounts are economic parameters that will need tuning once the network has real usage data. Hardcode sensible defaults; governance TBD (ADR-0018). diff --git a/docs/adr/0024-distributed-gguf-runtime.md b/docs/adr/0024-distributed-gguf-runtime.md index 14ed027..4d948ea 100644 --- a/docs/adr/0024-distributed-gguf-runtime.md +++ b/docs/adr/0024-distributed-gguf-runtime.md @@ -123,3 +123,11 @@ Rejected. gRPC/HTTP2 already provides mature streaming, flow control, deadlines, 4. Real two-machine execution using both Shards. 5. End-to-end performance/fit advantage over the current distributed route. 6. Separate Qwen3-family architecture certification. + +## Relationship to US-042 (whole-model GGUF shortcut) + +[US-042](../issues/42-gguf-llamacpp-node-backend.md) **phase C** ships first: a node with enough RAM serves a **full** GGUF via llama.cpp on a single-hop Inference Route using the existing HTTP activation seam and PyTorch-era tracker integration. That is intentionally small and does not require this ADR's gRPC worker or llama.cpp patch stack. + +This ADR's track starts only after **DGR-001** (controlled safetensors-vs-GGUF benchmark) shows a meaningful speed or fit benefit. Then implement the native worker (DGR-002+) — which subsumes US-042 direction A (layer-range GGUF + boundary tensors) if the benchmark warrants it. + +Do not run US-042 phase C and DGR-008+ in parallel on the same node backend without an explicit integration plan; phase C uses llama-cpp-python (or equivalent) whole-model path; ADR-0024 uses the standalone C++ worker. diff --git a/docs/adr/0026-node-assignment-ownership-and-managed-placement.md b/docs/adr/0026-node-assignment-ownership-and-managed-placement.md new file mode 100644 index 0000000..9051ca3 --- /dev/null +++ b/docs/adr/0026-node-assignment-ownership-and-managed-placement.md @@ -0,0 +1,51 @@ +# ADR-0026: Node assignment ownership — pinned startup vs managed demand placement + +## Status: Accepted + +## Context + +Three features define how a node gets its `(model, shard range, recipe/quantization)`: + +1. **ADR-0011 / US-013** — tracker suggests a gap from coverage map on startup or auto-join. +2. **Node capability admission (ADR-0023 / NCA)** — a node must pass `doctor` + real forward before becoming routable; startup-assigned work is validated, not blindly trusted. +3. **Qwen demand placement** (`.scratch/qwen3.6-27b-demand-placement/`) — tracker deploys a model when chat demand appears and spare capacity exists. + +These looked contradictory: NCA and the Qwen PRD both say startup assignments are "pinned," while demand placement wants the tracker to assign models dynamically. + +## Decision + +### Three assignment tiers + +| Tier | How it is created | Mutable by tracker? | Admission | +|---|---|---|---| +| **Operator-initiated** | Node starts with explicit `--model` / shard flags | **No** — pinned until operator restarts or explicitly reloads | Must pass NCA `doctor` before routable | +| **Network bootstrap** | `/v1/network/assign` or `/v1/nodes/assign` on first join (ADR-0011) | **No** for the active loaded shard — treated as operator-equivalent once accepted at startup | Must pass NCA before routable | +| **Tracker-managed** | Demand-driven placement (Qwen PRD) on spare capacity | **Yes** — marked `managed: true`; subject to cooldown / safety policy | Must pass NCA for the new assignment before routable | + +### Spare capacity rule (unifies NCA + Qwen) + +- A node’s **active** `(model, shard, recipe)` from startup is **pinned** — the tracker does not silently retarget a serving node to a different model. +- **Spare capacity** — memory/slots not holding the pinned assignment, or a node registered without a model — may receive **tracker-managed** assignments to satisfy demand. +- Until multi-shard runtime exists (US-048), “spare capacity” effectively means **model-less nodes** or nodes explicitly registered for managed placement; do not overload a single-shard node with a second assignment. + +### Demand placement interaction + +- First chat request for an unrouted model queues **demand**; leader tracker may assign **managed** nodes only when eligible spare capacity exists (Qwen PRD). +- Until complete coverage + validated recipes exist, return retryable `503 model_loading` with coverage metadata. +- Managed assignments must not evict pinned assignments on other nodes without the Qwen safety policy (≥3 copies, 1.5× demand multiplier, cooldown). + +### NCA is not optional for any tier + +Regardless of assignment source, registration carries **validated capability** only after `doctor` succeeds. The tracker excludes nodes with absent, stale, or failed capability reports (ADR-0023). + +## Consequences + +- NCA and Qwen demand placement are complementary: NCA gates *quality*; demand placement gates *where new coverage comes from*. +- US-048 (multi-shard slots) extends spare capacity — until then, demand placement primarily targets nodes that join without `--model`. +- Rebalance / dropout relocation (US-013, US-048) applies to **coverage gaps**, not retroactive retargeting of pinned nodes for demand convenience. + +## Verification + +- NCA tests: unvalidated nodes never routed. +- Demand-placement tests (when implemented): managed flag set; pinned nodes unchanged. +- Documented in Qwen scratch PRD and NCA README cross-links. diff --git a/docs/issues/20-memory-budget-shard-slots-and-dropout-relocation.md b/docs/issues/20-memory-budget-shard-slots-and-dropout-relocation.md new file mode 100644 index 0000000..b4328fb --- /dev/null +++ b/docs/issues/20-memory-budget-shard-slots-and-dropout-relocation.md @@ -0,0 +1,10 @@ +Status: superseded + +# Superseded — renumbered to US-048 + +This issue slot was a duplicate of tracker-node-hardening (US-020). Memory budget / shard slots / dropout relocation work lives at: + +- **Issue:** [48-memory-budget-shard-slots-and-dropout-relocation.md](./48-memory-budget-shard-slots-and-dropout-relocation.md) +- **PRD:** `docs/prd.json` → `US-048` + +Do not implement from this file. diff --git a/docs/issues/42-gguf-llamacpp-node-backend.md b/docs/issues/42-gguf-llamacpp-node-backend.md index 9f8645b..dc26ac4 100644 --- a/docs/issues/42-gguf-llamacpp-node-backend.md +++ b/docs/issues/42-gguf-llamacpp-node-backend.md @@ -35,7 +35,17 @@ to it (single-hop route). Smallest step, no cross-node activation work, and already useful: Strix Halo 128 GB serves DeepSeek-V4-Flash IQ3_XXS (114 GB) via llama.cpp Vulkan today. -Recommended sequencing: C first (small, real value), then A/B investigation. +Recommended sequencing: **C first** (US-042), then **ADR-0024 benchmark gate** (DGR-001), then distributed native worker (DGR-002+). Direction B (llama.cpp RPC) is rejected per ADR-0024. + +## Runtime sequencing + +| Stage | Track | Delivers | +|---|---|---| +| **C — Whole-model GGUF** | US-042 (this issue) | Single-hop llama.cpp, billing, relay streaming | +| **0 — Benchmark gate** | ADR-0024 DGR-001 | Safetensors vs GGUF measured contract | +| **1 — Distributed GGUF** | ADR-0024 `.scratch/distributed-gguf-runtime/` | gRPC C++ worker, layer-range GGUF | + +Phase C uses the existing tracker hop path (whole model, one node). ADR-0024 direction A (layer-range GGUF + activations) merges into the native worker track after the benchmark gate — not in parallel with phase C on the same backend without an integration plan. ## Also in scope diff --git a/docs/issues/44-tracker-shard-source-partial-download.md b/docs/issues/44-tracker-shard-source-partial-download.md index 2f19111..5d5dccc 100644 --- a/docs/issues/44-tracker-shard-source-partial-download.md +++ b/docs/issues/44-tracker-shard-source-partial-download.md @@ -86,10 +86,10 @@ What exists already (build on it, don't duplicate): - [ ] Two-machine test: machine A (tracker + node, holds full snapshot) serves layers 0–k; machine B joins with no model and receives **only** the files for its assigned range from A — nothing fetched from HF -- [ ] Machine B's resident memory scales with its shard size, not model size -- [ ] Checksums verified end-to-end; corrupted transfer falls back cleanly +- [x] Machine B's resident memory scales with its shard size, not model size +- [x] Checksums verified end-to-end; corrupted transfer falls back cleanly - [x] Single-node/full-model flows unchanged -- [ ] `python -m pytest` passes from repo root +- [x] `python -m pytest` passes from repo root ## Implementation notes @@ -98,6 +98,13 @@ What exists already (build on it, don't duplicate): `full_url`; HuggingFace remains fallback-only, and when it is used the node computes `allow_patterns` from the repo's remote SafeTensors index so it stays layer-filtered even without tracker-cached files. Remaining hard half - is true partial model materialization: the backend can prefer a downloaded + is partial model materialization: the backend can prefer a downloaded local model directory, but Transformers still needs a `meta`-device load path that materializes only assigned layers. +- 2026-07-13: Partial LOAD implemented. `_load_partial_model_from_snapshot` builds + on `meta` via `init_empty_weights`, materializes only layer-scoped checkpoint + tensors, and finalizes device placement without copying unmaterialized meta + weights (`_finalize_active_shard_modules_on_device`). Tests cover memory + scaling (`test_partial_snapshot_resident_weight_numel_scales_with_shard`) + and real-torch meta-vs-materialized counts. Remaining: live two-machine LAN + verification. diff --git a/docs/issues/49-mainnet-usdt-cutover-two-wallet-pilot.md b/docs/issues/49-mainnet-usdt-cutover-two-wallet-pilot.md index 116cb7e..fc9e01f 100644 --- a/docs/issues/49-mainnet-usdt-cutover-two-wallet-pilot.md +++ b/docs/issues/49-mainnet-usdt-cutover-two-wallet-pilot.md @@ -75,7 +75,7 @@ meshnet-tracker start \ - [ ] `--starting-credit 0` — no free inference credit - [ ] Treasury keypair not committed to git; file mode 600 - [ ] Plan multisig migration before large float ([alpha runbook](../../.scratch/alpha-hardening/runbooks/02-treasury-key-rotation.md) intent) -- [ ] Issue **21** (TOPLOC calibration) before production audit thresholds on untrusted nodes +- [ ] Issue **21** (TOPLOC calibration) before production audit thresholds on untrusted nodes — runbook: [04-toploc-calibration-run](../../.scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md) ## Cost estimate (this pilot) diff --git a/docs/issues/50-qwen3.6-27b-demand-placement.md b/docs/issues/50-qwen3.6-27b-demand-placement.md new file mode 100644 index 0000000..4f6eab2 --- /dev/null +++ b/docs/issues/50-qwen3.6-27b-demand-placement.md @@ -0,0 +1,15 @@ +Status: in-design + +# US-050 — Qwen3.6-27B demand-driven managed placement + +> Full spec: [.scratch/qwen3.6-27b-demand-placement/PRD.md](../../.scratch/qwen3.6-27b-demand-placement/PRD.md) +> Assignment rules: [ADR-0026](../adr/0026-node-assignment-ownership-and-managed-placement.md) +> Admission: [ADR-0023](../adr/0023-model-agnostic-node-capability-admission.md) + +## Summary + +Deploy `Qwen/Qwen3.6-27B` when chat demand appears and **spare** fleet capacity exists. Startup `--model` assignments stay **pinned**; tracker-managed loads fill gaps on model-less or (future US-048) unused slot capacity only. + +## Acceptance criteria + +See scratch PRD and `docs/prd.json` US-050. diff --git a/docs/prd.json b/docs/prd.json index 4ed7cf2..b73b830 100644 --- a/docs/prd.json +++ b/docs/prd.json @@ -1,6 +1,6 @@ { "name": "Distributed Inference Network", - "description": "Distributed inference network: base program US-001…US-035 complete; post-alpha friends-test and LAN-serving arc US-036…US-047 tracked here. Active scratch features (alpha hardening, NCA, distributed GGUF) have separate prd.json files under .scratch/.", + "description": "Distributed inference network: base program US-001…US-035 complete; friends-test arc US-036…US-049; capacity/placement US-048/050. Scratch features (alpha hardening AH-001…AH-025, NCA, distributed GGUF, Qwen demand) have separate prd.json files under .scratch/.", "branchName": "ralph/distributed-inference-network", "userStories": [ { @@ -851,11 +851,12 @@ "python -m pytest passes from repo root" ], "priority": 37, - "status": "open", + "status": "done", "notes": "Source issue: docs/issues/37-relay-bridge-concurrency.md. Critical for public friends-test; blocks concurrent head + hop on same node.", "dependsOn": [ "US-036" - ] + ], + "completionNotes": "ThreadPoolExecutor dispatch in relay_bridge.py; per-frame WS send lock; test_relay_bridge_serves_concurrent_requests; --relay-concurrency CLI flag sets MESHNET_RELAY_CONCURRENCY." }, { "id": "US-038", @@ -890,12 +891,13 @@ "python -m pytest passes from repo root" ], "priority": 39, - "status": "open", + "status": "done", "notes": "Source issue: docs/issues/39-caller-credit-account-keys.md. Critical for friends-test inference.", "dependsOn": [ "US-031", "US-035" - ] + ], + "completionNotes": "Caller credit granted once per account on first API key via deterministic event id; tests/test_accounts.py covers grant, revoke, and invented-bearer rejection." }, { "id": "US-040", @@ -908,11 +910,12 @@ "python -m pytest passes from repo root" ], "priority": 40, - "status": "open", + "status": "done", "notes": "Source issue: docs/issues/40-devnet-dashboard-topup.md. Mainnet deployments set --devnet-topup 0.", "dependsOn": [ "US-039" - ] + ], + "completionNotes": "POST /v1/account/topup with session auth and flag gating; tests/test_accounts.py covers flag off/on, own-account credit, and cross-account 403." }, { "id": "US-041", @@ -946,7 +949,7 @@ ], "priority": 42, "status": "in-design", - "notes": "Source issue: docs/issues/42-gguf-llamacpp-node-backend.md. Distributed native path: ADR-0024. Sequencing: phase C before A/B investigation.", + "notes": "Source issue: docs/issues/42-gguf-llamacpp-node-backend.md. Phase C before ADR-0024 distributed worker; see runtime sequencing in issue file.", "dependsOn": [ "US-036" ] @@ -984,12 +987,12 @@ ], "priority": 44, "status": "in-progress", - "notes": "Source issue: docs/issues/44-tracker-shard-source-partial-download.md. Download path largely implemented 2026-07-06; partial LOAD (meta-device materialization) and two-machine acceptance remain.", + "notes": "Source issue: docs/issues/44-tracker-shard-source-partial-download.md. Download path and partial LOAD implemented; live two-machine LAN verification remains.", "dependsOn": [ "US-004", "US-012" ], - "completionNotes": "Tracker models-dir indexing, layer-scoped tar stream, HF allow_patterns client-side from remote index, per-file download API with retries, symlink dereference in tar writers. Remaining: true partial model load and live two-machine verification." + "completionNotes": "Tracker models-dir indexing, layer-scoped tar stream, HF allow_patterns client-side from remote index, per-file download API with retries, symlink dereference in tar writers. Partial LOAD via init_empty_weights + layer-scoped safetensors materialization; memory-scaling and checksum fallback tests pass. Remaining: live two-machine test (machine B receives only assigned files from A, no HF)." }, { "id": "US-045", @@ -1072,10 +1075,46 @@ "US-033", "US-039" ] + }, + { + "id": "US-048", + "title": "48 — Memory budget, shard slots, and dropout relocation hardening", + "description": "Harden the capacity contract around US-013 coverage-first assignment: enforce memory budget and max_loaded_shards in rebalance/dropout relocation, and decide whether one node may host multiple concurrent shard backends or max_loaded_shards remains metadata until runtime support lands.", + "acceptanceCriteria": [ + "Assignment/rebalance never exceeds memory budget or max_loaded_shards", + "Dropout test restores full coverage without violating capacity limits", + "CLI --memory and --max-shards reflected in registration payload", + "python -m pytest tests/test_tracker_routing.py tests/test_node_startup.py passes" + ], + "priority": 48, + "status": "open", + "notes": "Source issue: docs/issues/48-memory-budget-shard-slots-and-dropout-relocation.md. Renumbered from duplicate slot 20. Enables spare shard slots for ADR-0026 managed placement.", + "dependsOn": [ + "US-013" + ] + }, + { + "id": "US-050", + "title": "50 — Qwen3.6-27B demand-driven managed placement", + "description": "Offer pinned Qwen/Qwen3.6-27B as a recommended text-only chat model. Valid chat requests prove demand; when spare fleet capacity exists, the tracker assigns managed nodes to reach complete coverage. Pinned startup assignments remain immutable per ADR-0026; NCA admission required before routable.", + "acceptanceCriteria": [ + "First valid request for an uncovered variant queues demand and returns 503 model_loading until complete validated coverage exists", + "Managed assignments use only spare capacity and carry managed: true", + "Pinned startup assignments are never silently retargeted", + "Optional quantization field (bfloat16/int8/nf4) with coverage-vote UI semantics per scratch PRD", + "python -m pytest passes from repo root" + ], + "priority": 50, + "status": "in-design", + "notes": "Source: .scratch/qwen3.6-27b-demand-placement/PRD.md and docs/issues/50-qwen3.6-27b-demand-placement.md. Reconciled with ADR-0026 and ADR-0023.", + "dependsOn": [ + "US-035", + "US-048" + ] } ], "metadata": { - "updatedAt": "2026-07-13T16:40:00.000Z", + "updatedAt": "2026-07-13T17:00:00.000Z", "statusVocabulary": { "open": "Not started", "in-design": "Decisions pending before implementation can begin", diff --git a/packages/node/meshnet_node/cli.py b/packages/node/meshnet_node/cli.py index a986276..f3d588a 100644 --- a/packages/node/meshnet_node/cli.py +++ b/packages/node/meshnet_node/cli.py @@ -36,6 +36,12 @@ def _load_env_file(path: Path) -> None: os.environ[key] = value +def _apply_relay_concurrency_flag(value: int | None) -> None: + """Expose relay bridge worker cap via CLI (env MESHNET_RELAY_CONCURRENCY).""" + if value is not None: + os.environ["MESHNET_RELAY_CONCURRENCY"] = str(max(1, value)) + + def _load_env_defaults() -> None: """Load machine-specific, local, and user-level node env defaults.""" machine = socket.gethostname().strip() @@ -189,6 +195,8 @@ def _cmd_default(args) -> int: if getattr(args, "cpu", False): overrides["force_cpu"] = True + _apply_relay_concurrency_flag(getattr(args, "relay_concurrency", None)) + if overrides: cfg = merge_cli_overrides(cfg, **overrides) @@ -349,6 +357,8 @@ def _cmd_start(args) -> int: if getattr(args, "node_name", None): cfg["node_name"] = args.node_name + _apply_relay_concurrency_flag(getattr(args, "relay_concurrency", None)) + # Legacy start: just run without the dashboard (keep original blocking loop) from .startup import run_startup @@ -433,6 +443,8 @@ def main() -> None: help="Set PyTorch inter-op CPU worker threads") parser.add_argument("--cpu", action="store_true", help="Force CPU inference even when a GPU is available") + parser.add_argument("--relay-concurrency", type=int, metavar="N", + help="Max concurrent relay-http-request workers (env MESHNET_RELAY_CONCURRENCY)") parser.add_argument("--debug", action="store_true", help="Enable verbose node debug logging") parser.add_argument("--no-tui", action="store_true", help="Plain-text output (no rich dashboard)") parser.add_argument("--compact", action="store_true", help="Single-line status output") @@ -510,6 +522,8 @@ def main() -> None: help="Set PyTorch inter-op CPU worker threads") start_cmd.add_argument("--cpu", action="store_true", help="Force CPU inference even when a GPU is available") + start_cmd.add_argument("--relay-concurrency", type=int, metavar="N", + help="Max concurrent relay-http-request workers (env MESHNET_RELAY_CONCURRENCY)") start_cmd.add_argument("--debug", action="store_true", help="Enable verbose node debug logging") start_cmd.add_argument("--tracker-source-disabled", action="store_true", help="Skip tracker/peer model-file sources and download from HuggingFace directly") diff --git a/packages/node/meshnet_node/model_backend.py b/packages/node/meshnet_node/model_backend.py index ec40cfe..7f2ae1d 100644 --- a/packages/node/meshnet_node/model_backend.py +++ b/packages/node/meshnet_node/model_backend.py @@ -899,12 +899,41 @@ def _load_partial_model_from_snapshot( dtype=dtype, ) - for module in _active_modules_for_shard(model, shard_start, shard_end): - if hasattr(module, "to"): - module.to(device) + _finalize_active_shard_modules_on_device(model, shard_start, shard_end, device) return model +def _finalize_active_shard_modules_on_device( + model: Any, shard_start: int, shard_end: int, device: Any +) -> None: + """Place active shard modules on device without copying unmaterialized meta weights.""" + for module in _active_modules_for_shard(model, shard_start, shard_end): + parameters = getattr(module, "parameters", None) + if not callable(parameters): + if hasattr(module, "to"): + module.to(device) + continue + params = list(parameters(recurse=True)) + buffers_fn = getattr(module, "buffers", None) + buffers = list(buffers_fn(recurse=True)) if callable(buffers_fn) else [] + tensors = params + buffers + if not tensors: + if hasattr(module, "to"): + module.to(device) + continue + if all(tensor.device.type == "meta" for tensor in tensors): + to_empty = getattr(module, "to_empty", None) + if callable(to_empty): + to_empty(device) + continue + if all(tensor.device.type != "meta" for tensor in tensors): + if hasattr(module, "to"): + module.to(device) + continue + # Partially materialized: set_module_tensor_to_device already placed loaded + # weights on the target device; leave remaining meta parameters untouched. + + def _model_load_plan( auto_config: Any, model_id: str, diff --git a/tests/test_real_model_backend.py b/tests/test_real_model_backend.py index 6f533b3..a27b0cd 100644 --- a/tests/test_real_model_backend.py +++ b/tests/test_real_model_backend.py @@ -1075,6 +1075,190 @@ def test_partial_snapshot_loader_materializes_only_assigned_tensors(tmp_path): assert model.model.rotary_emb.to_calls == ["cpu:0"] +def _build_partial_snapshot_fixture(snapshot_dir: Path, *, num_layers: int = 4) -> dict[str, dict[str, int]]: + """Minimal HF snapshot with per-tensor numel metadata for memory-scaling tests.""" + weight_map: dict[str, str] = { + "model.embed_tokens.weight": "shard-head.safetensors", + "model.norm.weight": "shard-tail.safetensors", + "lm_head.weight": "shard-tail.safetensors", + } + tensor_numel: dict[str, dict[str, int]] = { + "shard-head.safetensors": {"model.embed_tokens.weight": 10_000}, + "shard-tail.safetensors": { + "model.norm.weight": 512, + "lm_head.weight": 10_000, + }, + } + for layer in range(num_layers): + rel = f"shard-layer-{layer}.safetensors" + weight_map[f"model.layers.{layer}.self_attn.q_proj.weight"] = rel + weight_map[f"model.layers.{layer}.mlp.down_proj.weight"] = rel + per_layer = 1_000 * (layer + 1) + tensor_numel[rel] = { + f"model.layers.{layer}.self_attn.q_proj.weight": per_layer, + f"model.layers.{layer}.mlp.down_proj.weight": per_layer, + } + (snapshot_dir / rel).write_bytes(b"stub") + + (snapshot_dir / "config.json").write_text(json.dumps({"num_hidden_layers": num_layers})) + (snapshot_dir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map})) + (snapshot_dir / "shard-head.safetensors").write_bytes(b"stub") + (snapshot_dir / "shard-tail.safetensors").write_bytes(b"stub") + return tensor_numel + + +def _partial_load_materialized_numel( + snapshot_dir: Path, + shard_start: int, + shard_end: int, + tensor_numel: dict[str, dict[str, int]], +) -> int: + """Return the summed numel of checkpoint tensors assigned to one shard load.""" + + class FakeModule: + def to(self, device): + return self + + class FakeModel: + def __init__(self, num_layers: int): + self.model = types.SimpleNamespace( + embed_tokens=FakeModule(), + layers=[FakeModule() for _ in range(num_layers)], + rotary_emb=FakeModule(), + norm=FakeModule(), + ) + self.lm_head = FakeModule() + + def tie_weights(self): + pass + + class AutoConfigStub: + @staticmethod + def from_pretrained(model_id): + assert model_id == str(snapshot_dir) + return types.SimpleNamespace(num_hidden_layers=num_layers) + + class AutoModelStub: + @staticmethod + def from_config(cfg, torch_dtype=None): + return FakeModel(cfg.num_hidden_layers) + + class UnusedContext: + def __enter__(self): + return None + + def __exit__(self, exc_type, exc, tb): + return False + + num_layers = json.loads((snapshot_dir / "config.json").read_text())["num_hidden_layers"] + loaded_numel = 0 + + def fake_set_tensor(module, tensor_name, device, value=None, dtype=None): + nonlocal loaded_numel + loaded_numel += int(getattr(value, "numel", lambda: 0)()) + + class FakeTensor: + def __init__(self, numel: int): + self._numel = numel + + def numel(self) -> int: + return self._numel + + class FakeSafeOpen: + def __init__(self, filename, framework, device): + self.filename = Path(filename).name + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def get_tensor(self, tensor_name): + return FakeTensor(tensor_numel[self.filename][tensor_name]) + + _load_partial_model_from_snapshot( + AutoConfigStub, + AutoModelStub, + types.SimpleNamespace(), + str(snapshot_dir), + shard_start, + shard_end, + "bf16", + "cpu:0", + init_empty_weights_fn=UnusedContext, + set_tensor_fn=fake_set_tensor, + safe_open_fn=FakeSafeOpen, + ) + return loaded_numel + + +def test_partial_snapshot_resident_weight_numel_scales_with_shard(tmp_path): + "Partial load materializes only assigned-layer weights, not the full checkpoint.\n\nTags: model, node, real-inference" + snapshot_dir = tmp_path / "snapshot" + snapshot_dir.mkdir() + tensor_numel = _build_partial_snapshot_fixture(snapshot_dir, num_layers=4) + + middle_numel = _partial_load_materialized_numel(snapshot_dir, 1, 1, tensor_numel) + full_numel = _partial_load_materialized_numel(snapshot_dir, 0, 3, tensor_numel) + + # Layer 1 only: two tensors at 2000 numel each. + assert middle_numel == 4_000 + # Head embed + four layers (two tensors each, increasing sizes) + tail norm/lm_head. + assert full_numel == 10_000 + 512 + 10_000 + 2_000 + 4_000 + 6_000 + 8_000 + assert middle_numel < full_numel // 4 + + +@pytest.mark.integration +def test_partial_snapshot_materialized_param_count_with_real_torch(tmp_path): + "When torch is installed, partial load leaves unassigned params on meta device.\n\nTags: model, node, real-inference" + torch = _require_functional_torch() + pytest.importorskip("transformers") + pytest.importorskip("safetensors") + pytest.importorskip("accelerate") + from safetensors.torch import save_file + from transformers import AutoConfig, AutoModelForCausalLM, GPT2Config + + n_layer = 4 + n_embd = 16 + snapshot_dir = tmp_path / "snapshot" + snapshot_dir.mkdir() + config = GPT2Config(n_layer=n_layer, n_embd=n_embd, n_head=2, n_positions=32) + (snapshot_dir / "config.json").write_text(config.to_json_string()) + + weight_map: dict[str, str] = {} + for layer in range(n_layer): + key = f"transformer.h.{layer}.attn.c_attn.weight" + rel = f"layer-{layer}.safetensors" + weight_map[key] = rel + save_file({key: torch.ones(n_embd, 3 * n_embd)}, snapshot_dir / rel) + (snapshot_dir / "model.safetensors.index.json").write_text(json.dumps({"weight_map": weight_map})) + + def count_materialized(model) -> tuple[int, int]: + materialized = 0 + meta = 0 + for param in model.parameters(): + if param.device.type == "meta": + meta += param.numel() + else: + materialized += param.numel() + return materialized, meta + + middle = _load_partial_model_from_snapshot( + AutoConfig, + AutoModelForCausalLM, + torch, + str(snapshot_dir), + 1, + 1, + torch.float32, + torch.device("cpu"), + ) + mid_mat, mid_meta = count_materialized(middle) + assert mid_mat == n_embd * 3 * n_embd + assert mid_meta > mid_mat * 10 + + def test_partial_snapshot_loader_requires_known_layer_count(tmp_path): "Partial snapshot loader requires known layer count\n\nTags: model, node, real-inference" snapshot_dir = tmp_path / "snapshot"