diff --git a/.ralph-tui/config.toml b/.ralph-tui/config.toml deleted file mode 100644 index 956f0d4..0000000 --- a/.ralph-tui/config.toml +++ /dev/null @@ -1,13 +0,0 @@ -# Ralph TUI Configuration -# Generated by setup wizard -# See: ralph-tui config help - -configVersion = "2.1" -tracker = "json" -agent = "codex" -model = "gpt-5.6-terra" -maxIterations = 0 -autoCommit = true - -[trackerOptions] -[agentOptions] diff --git a/.scratch/distributed-gguf-runtime/PRD.md b/.scratch/distributed-gguf-runtime/PRD.md index cd72358..c412386 100644 --- a/.scratch/distributed-gguf-runtime/PRD.md +++ b/.scratch/distributed-gguf-runtime/PRD.md @@ -1,252 +1,714 @@ -# PRD: Performant Concurrent Distributed GGUF Runtime +# PRD: Distributed GGUF Runtime -## Overview +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -Build one lean native GGUF execution path that lets an Inference Route combine consumer machines to serve models larger than any one node can hold. Reuse the existing Meshnet control plane and llama.cpp/GGML execution engine. Adopt gRPC/HTTP2 and Protocol Buffers for the native Shard worker data plane rather than inventing a transport. +## Goal -The program is benchmark-gated. GGUF is not assumed faster merely because it is quantized or uses a different file format. The first story compares the current Transformers/safetensors backend against whole-model llama.cpp on controlled model/hardware/quality lanes and locks a performance contract. Native distributed work proceeds only when GGUF provides a meaningful speed or fit benefit. +Deliver benchmark-gated, concurrent, distributed GGUF inference across consumer machines through existing Meshnet control-plane behavior and a standalone native worker around upstream llama.cpp. The accepted target is DeepSeek V4 Flash, not the superseded legacy target. -## Goals +## Locked scope -- Execute one GGUF model across independently addressable contiguous Shards. -- Retain Hot KV State locally for each Shard and isolate concurrent Route Sessions. -- Batch compatible decode steps across active sessions for aggregate throughput. -- Use consumer CPU, AMD, NVIDIA, Vulkan, Metal, and mixed routes only where a real certified forward passes. -- Beat the current distributed safetensors route under a controlled performance contract or enable a larger otherwise-unroutable model at useful measured speed. -- Keep the critical path to Meshnet plus a small pinned llama.cpp fork and standalone C++ worker. -- Produce narrow upstream collaboration material for llama.cpp without placing Meshnet networking or economics inside upstream. +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. -## Quality Gates +## Target identities -Every story must: +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. -- Run its targeted `pytest` tests. -- Run `python -m compileall packages tests` for Python changes. -- Run `git diff --check`. -- Keep default tests deterministic, model-download-free, API-credit-free, and GPU-free. -- Preserve existing Transformers/safetensors behavior unless the story explicitly changes a versioned compatibility contract. +## Quality gates -Stories touching the native worker must also: +The canonical gate groups live in `prd.json`. Every story explicitly requires its applicable shared gates: deterministic targeted tests, Python compile checks where applicable, native CMake/CTest and exact patch verification where applicable, real artifact/hardware provenance for opt-in lanes, `git diff --check`, and a durable evidence handoff. -- Build the pinned C++ target with CMake. -- Run focused C++/protocol tests through CTest or the documented equivalent. -- Verify the llama.cpp patch stack applies cleanly to the exact pinned commit. +## Milestones -Real-model/hardware stories must: +- **M0 — Truth and contracts (DGR-017..DGR-020):** Reconciled legacy truth, canonical metadata, immutable gates, and a controlled whole-model baseline. +- **M1 — Protocol and native substrate (DGR-021..DGR-033):** Versioned gRPC protocol, exact identities/artifacts, pinned upstream, reproducible builds, ShardEngine, and fake worker. +- **M2 — Dense vertical proof (DGR-034..DGR-043):** Dense ranged execution, parity, local state, worker integration, and GGUF inputs to existing routing. +- **M3 — DeepSeek V4 Flash alpha (DGR-044..DGR-054):** Pinned V4 adapter around upstream llama.cpp, real route certification, and pre-locked alpha decision with MTP off. +- **M4 — Performance and beta hardening (DGR-055..DGR-067):** Batching, backpressure, recovery, scale certification, optimization, MTP, and hardware matrix. +- **M5 — Release and maintenance (DGR-068..DGR-071):** Reproducible packages, upstream collaboration, beta decision, and sustainable recertification. -- Require `MESHNET_ENABLE_REAL_INFERENCE_TESTS=1`. -- Use the machine-specific mounted-drive model path and the certified runtime environment; never place model artifacts under `/home`. -- Record exact model revision, artifact hash, runtime recipe, hardware, driver/backend, commands, raw JSON metrics, and output-quality result. -- Label synthetic tests as unit coverage rather than distributed acceptance. +## User stories -Before a story is marked complete, run the full deterministic `pytest -q` suite or record the exact pre-existing unrelated failure with a clean-tree reproduction. +### DGR-017: Reconcile and clean the superseded DGR backlog -## User Stories +**Milestone:** M0 · **Mode:** AFK · **State:** complete · **Depends on:** none -### DGR-001: Lock the safetensors-versus-GGUF performance contract -**Description:** As a runtime engineer, I need a controlled baseline so that GGUF work proceeds from measured speed, memory, and quality rather than reputation. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md`, and evidence READMEs for dependencies (none) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Audit implementation reality, void inherited completion credit, and clean misleading backlog/stub baggage while preserving attributable evidence and accepted research. -**Acceptance Criteria:** -- [ ] Benchmark the same model architecture/revision, machine, prompts, context lengths, output lengths, sampling policy, and concurrency across the current Transformers/safetensors recipe and whole-model llama.cpp recipes. -- [ ] Separate correctness/quality lanes from quantized performance/fit lanes instead of claiming BF16 and Q4 are numerically equivalent. -- [ ] Report TTFT, prefill tok/s, decode tok/s, p50/p95 latency, aggregate throughput, RSS, VRAM, artifact size, failures, and output drift in machine-readable JSON. -- [ ] Add concurrency levels 1 and 4 where memory permits. -- [ ] Write a versioned performance contract consumed by later release gates, including an explicit stop condition when llama.cpp/GGUF has no meaningful speed or fit benefit. +- [x] Compare the branch, old DGR-001..016 issue/pass states, evidence, and actual runtime sources; classify each output as reusable, reference-only, blocked, obsolete, or absent. +- [x] Record an authoritative old-to-new disposition and provenance; explicitly give no completion credit to any new story and note absent implementation/evidence. +- [x] Remove or archive only artifacts the audit proves obsolete while preserving accepted ADRs, useful research, raw benchmark evidence, and attributable reusable work. +- [x] Protect ignored build workspaces, generated protobuf outputs, Ralph logs, and model artifacts from accidental commits, and document every retained legacy artifact. +- [x] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-002: Adopt the versioned gRPC Shard protocol -**Description:** As a node developer, I need a battle-proven streaming protocol so that Python and C++ Shards communicate without a custom socket protocol. +### DGR-018: Define canonical Ralph and Gitea metadata schema -**Acceptance Criteria:** -- [ ] Add a Protocol Buffers schema for capability, health, session stream, release, and cancellation operations. -- [ ] Define one long-lived bidirectional gRPC stream per Route Session Activation Seam with deadlines, cancellation, flow control, and structured errors. -- [ ] Define bounded chunking for prefill and a small decode fast path. -- [ ] Carry schema version, request/work ID, Route Session ID, route epoch, artifact/recipe fingerprint, Shard range/effective start, phase, position, idempotency step, cache expectation, compression, and checksum. -- [ ] Define a versioned named-tensor bundle with per-tensor name, shape, dtype, byte order, and payload fragments. -- [ ] Add generated-schema round-trip and compatibility tests in Python and C++. +**Milestone:** M0 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-017 -### DGR-003: Define exact Artifact and runtime recipe identity -**Description:** As the Tracker, I need exact compatibility identity so that only numerically and operationally compatible Shards form an Inference Route. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Make `prd.json` the validated source from which Markdown and Gitea issues can later be generated losslessly. -**Acceptance Criteria:** -- [ ] Separate weight quantization, activation dtype, compute dtype, KV dtype/layout, tokenizer revision, architecture adapter, backend, and runtime version. -- [ ] Bind derivative or split artifacts to an exact source Model Artifact hash and Shard range. -- [ ] Produce a stable compatibility fingerprint used by capability admission and the gRPC handshake. -- [ ] Fail closed on mismatched artifact, tokenizer, architecture, range, boundary schema, activation recipe, or cache layout. -- [ ] Keep unsupported recipes registered-but-dark until a real distributed forward certifies them. +- [ ] Define fields for stable ID/title, labels, milestone, type, `dependsOn`, derived `blocks`, triage, evidence class, and hardware/model/upstream flags. +- [ ] Validate that all stories start `passes: false`, use known dependencies, and have unique stable IDs. +- [ ] Reject cycles, missing dependencies, mismatched generated `blocks`, duplicate titles/IDs, and generated artifacts claiming authority over `prd.json`. +- [ ] Add deterministic model-free tests for parse, validation, and generation round trips. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-004: Create the reproducible pinned llama.cpp patch stack -**Description:** As a maintainer, I need a small auditable fork boundary so that upstream updates do not turn the runtime into an unmaintainable stitched codebase. +### DGR-019: Lock alpha and beta performance contracts -**Acceptance Criteria:** -- [ ] Pin one exact llama.cpp commit through a reproducible source dependency mechanism. -- [ ] Store a numbered minimal patch stack separately from Meshnet networking code. -- [ ] Add a build script that applies/checks patches and builds the standalone worker without manual source copying. -- [ ] Record upstream file/ABI assumptions and fail clearly when the pin changes. -- [ ] Preserve upstream license and attribution notices. -- [ ] Add a clean rebuild smoke test that does not download a model. +**Milestone:** M0 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-017 -### DGR-005: Implement dense-Llama range-aware GGUF ownership -**Description:** As a node, I need to map only my assigned dense-Llama Shard so that aggregate consumer memory can hold a model larger than one node. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Freeze useful speed, correctness, memory-fit, and stop/go thresholds before implementation results are visible. -**Acceptance Criteria:** -- [ ] Register and allocate only `blk.N.*` tensors in the assigned range. -- [ ] Load embeddings only for the head and final norm/LM head only for the tail, including tied embeddings. -- [ ] Prefer range-aware mapping from one exact source GGUF; if derivative sub-GGUFs are used temporarily, verify source/slice hashes and avoid claiming final artifact semantics. -- [ ] Report authoritative loaded range and endpoint ownership from the model, not operator CLI claims. -- [ ] Demonstrate mapped/resident memory scales with owned tensors rather than full model size. +- [ ] Define controlled safetensors, whole-model GGUF, dense distributed GGUF, and V4 Flash distributed lanes with fixed prompts, context/output lengths, sampling, concurrency, hardware, and metrics. +- [ ] Alpha requires correctness plus a human-approved useful-speed threshold; beta adds concurrency, long-context, failure, and sustained-throughput thresholds. +- [ ] Separate quantization/model-fit gains from runtime, transport, batching, and kernel gains. +- [ ] Treat quants and 2–4/10+ stage counts only as named certification scenarios; no product logic may hardcode them. +- [ ] Lock thresholds and stop conditions in versioned machine-readable data before benchmark result ingestion. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-006: Implement architecture-defined boundary input/output -**Description:** As a Shard, I need to consume and emit the correct transformer boundary state so that disjoint processes reproduce whole-model execution. +### DGR-020: Run the controlled whole-model GGUF baseline -**Acceptance Criteria:** -- [ ] Head accepts token IDs and owns token embedding. -- [ ] Middle/tail bypass token embedding and accept the named boundary bundle. -- [ ] Non-tail emits the unnormalized architecture-defined residual/boundary before final norm/head and before tail-only row pruning. -- [ ] Tail emits logits or token output through an explicit sampling contract. -- [ ] Dense-Llama whole-model versus two-range prefill and greedy-decode parity passes the documented tolerance. -- [ ] The adapter interface fails closed for uncertified architectures. +**Milestone:** M0 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-019 -### DGR-007: Add isolated concurrent local Hot KV State -**Description:** As a client, I need concurrent Route Sessions to retain independent per-Shard cache so that one request cannot clear or corrupt another. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md`, and evidence READMEs for dependencies (DGR-019) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Execute the locked safetensors and whole-model llama.cpp lanes before distributed implementation results can influence the decision. -**Acceptance Criteria:** -- [ ] Map `(Route Session ID, route epoch)` to an isolated llama sequence or bounded context. -- [ ] Allocate KV only for owned layers. -- [ ] Support prefill append, decode append, truncate, release, TTL/LRU eviction, and explicit cache-miss response. -- [ ] Reject stale epochs and incompatible cache recipes. -- [ ] At least four concurrent sessions on a small model complete without token or KV cross-talk. -- [ ] Cancellation/release of one session leaves other sessions intact and memory returns to the configured budget. +- [ ] Run the exact DGR-019 safetensors and whole-model llama.cpp benchmark lanes with locked prompts, lengths, sampling, concurrency, hardware, and artifact/runtime identities. +- [ ] Record raw machine-readable correctness, TTFT, prefill/decode, throughput, latency, memory, artifact-size, failure, and quality-drift metrics without ingesting distributed implementation results. +- [ ] Separate quantization/model-fit effects from runtime/kernel effects and preserve failed or unavailable lanes honestly. +- [ ] Publish a threshold-based `go`, `optimize baseline`, or `stop` decision without changing the locked contract. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-008: Build the standalone C++ gRPC Shard worker -**Description:** As a node runtime, I need one supervised native process so that llama.cpp internals remain behind a stable project-owned protocol. +### DGR-021: Define the versioned named-tensor stream envelope -**Acceptance Criteria:** -- [ ] Worker exposes capability, health, session stream, release, cancellation, and metrics services from DGR-002. -- [ ] Worker loads one exact Artifact/recipe/Shard identity and refuses mismatched requests. -- [ ] Streaming path enforces bounded messages, flow control, deadlines, idempotency, and independent session cancellation. -- [ ] Worker does not expose raw llama.cpp RPC or arbitrary GGML graph execution. -- [ ] Graceful shutdown releases sessions; crash behavior is bounded and observable. -- [ ] Python integration tests run against a fake model mode without model downloads. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-018 -### DGR-009: Integrate the native worker with Meshnet -**Description:** As the existing node service, I need a GGUF Shard backend adapter so that the Tracker, relay, billing, telemetry, and capability admission remain the sole control plane. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md`, and evidence READMEs for dependencies (DGR-018) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish the backend-neutral protobuf envelope used by direct and relayed Shard activation traffic. -**Acceptance Criteria:** -- [ ] Implement the existing model-backend surface without changing Transformers behavior. -- [ ] Registration carries exact validated GGUF recipe, Shard, backend and concurrency/KV capacity. -- [ ] Tracker forms only complete compatible routes and keeps uncertified recipes dark. -- [ ] Direct routes use gRPC streams; relayed routes carry the same versioned protobuf frames as opaque binary through the existing relay seam. -- [ ] Existing request/work IDs, cancellation, Generation Telemetry, billing, and per-node attribution remain correlated. -- [ ] No vLLM, Nakshatra, prima.cpp, or custom-engine control plane becomes a core dependency. +- [ ] Define schema version, request/work ID, route session/epoch, shard range/effective start, phase, position, and idempotency step. +- [ ] Define named tensors with shape, dtype, byte order, bounded fragments, compression identity, and checksum. +- [ ] Reserve extensible fields for token-ID sidebands, architecture state, recurrent state, and MTP without claiming implementations. +- [ ] Add deterministic serialization, fragmentation, checksum, unknown-field, and size-limit tests. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-010: Pass local real-model two-process acceptance -**Description:** As a release engineer, I need real local distributed parity before involving network variability. +### DGR-022: Define Shard lifecycle and structured status RPCs -**Acceptance Criteria:** -- [ ] Two local worker processes open disjoint dense-Llama ranges from the certified Artifact. -- [ ] Prefill and at least 32 greedy decode tokens match whole-model llama.cpp within the certified tolerance. -- [ ] Each worker retains only its own tensors and Hot KV State. -- [ ] Four concurrent Route Sessions pass isolation and cleanup checks. -- [ ] Report TTFT, prefill/decode throughput, seam bytes/latency, worker RSS/VRAM, KV memory, batch size, and queue time. -- [ ] Killing one worker produces a bounded structured failure rather than a deadlock. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-021 -### DGR-011: Pass a real heterogeneous two-machine route -**Description:** As a consumer-hardware operator, I need two physical machines to execute one GGUF model so that the distributed claim is real. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/022-define-shard-lifecycle-and-structured-status-rpcs.md`, and evidence READMEs for dependencies (DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Complete the gRPC contract for worker capability, health, sessions, cancellation, release, and metrics. -**Acceptance Criteria:** -- [ ] Tracker selects two physical nodes with disjoint Shards and one exact certified recipe/compatibility class. -- [ ] Actual CPU/GPU execution occurs on both nodes; synthetic workers do not satisfy acceptance. -- [ ] Prefill/decode, concurrent-session isolation, telemetry, cancellation, and cleanup pass over the real transport/relay path. -- [ ] Exact hardware, network, backend, model hash, route, commands, and raw metrics are recorded. -- [ ] A model or recipe larger than one participating node's admitted memory is exercised when available. -- [ ] Output drift is measured and incompatible mixed backends fail closed. +- [ ] Define capability, health, bidirectional session stream, cancellation, release, and metrics RPCs. +- [ ] Specify deadlines, cancellation propagation, bounded flow control, cache expectations/results, and structured error taxonomy. +- [ ] Specify TLS/auth hooks without moving Meshnet authentication or billing into the worker. +- [ ] Add compatibility tests for supported versions and fail-closed tests for unsupported versions and malformed lifecycle transitions. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-012: Implement continuous batching and bounded admission -**Description:** As a node operator, I need active sessions batched safely so that concurrency increases aggregate throughput rather than serializing every request. +### DGR-023: Make Python and C++ protobuf generation reproducible -**Acceptance Criteria:** -- [ ] Node scheduler admits sessions against weight, KV, scratch, and queue budgets. -- [ ] Compatible decode steps from multiple sessions form llama.cpp batches while preserving per-session positions and outputs. -- [ ] Prefill does not starve decode; scheduling policy and bounds are explicit. -- [ ] Backpressure prevents unbounded queued activations or KV growth. -- [ ] Capability telemetry reports active sessions, queue depth, batch occupancy, KV pressure, prefill/decode rates, and rejected admissions. -- [ ] Concurrency 1/2/4/8 benchmark identifies saturation and shows no cross-session corruption. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-021 -### DGR-013: Harden failure, cancellation, and restart semantics -**Description:** As a client, I need failures to be bounded and explicit so that distributed speed does not come with hanging or corrupted generations. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md`, and evidence READMEs for dependencies (DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Generate identical Python/C++ protocol bindings without manual copying or checked-in build debris. -**Acceptance Criteria:** -- [ ] Deadlines and heartbeat/health loss terminate blocked stream operations. -- [ ] Cancellation propagates across every Shard and releases local KV and queued buffers. -- [ ] Duplicate steps are idempotent; uncertain mutations are never replayed silently. -- [ ] Alpha failover restarts from token zero on a newly compatible route rather than importing unverified KV. -- [ ] Worker death, stream reset, malformed bundle, stale epoch, and cache miss tests pass. -- [ ] Billing/work records distinguish completed, cancelled, failed, and unverified work. +- [ ] Pin protoc, gRPC, and plugin versions or declare a verified compatible range. +- [ ] Generate Python and C++ bindings into out-of-tree build/package locations through documented commands. +- [ ] Add Python↔C++ round-trip and descriptor compatibility tests. +- [ ] A clean checkout regenerates bindings deterministically or fails with an actionable toolchain error. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-014: Enforce the GGUF-versus-safetensors release gate -**Description:** As the product owner, I need an end-to-end comparison so that the native runtime ships only if it advances model access or performance. +### DGR-024: Implement in-memory fake gRPC seam transport -**Acceptance Criteria:** -- [ ] Run current distributed safetensors and distributed GGUF routes on the same certified model/hardware/network scenario where technically comparable. -- [ ] Report quality, TTFT, prefill/decode throughput, aggregate concurrency throughput, p95 latency, seam cost, memory, KV pressure, failures, and cleanup. -- [ ] Evaluate against the DGR-001 performance contract without changing thresholds after seeing results. -- [ ] Ship recommendation is one of: promote GGUF, optimize a measured bottleneck with a new bounded task, or stop the native track. -- [ ] Results clearly separate quantization gains from transport/runtime gains. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-022, DGR-023 -### DGR-015: Add and certify a Qwen3/Qwen3-MoE adapter -**Description:** As a client seeking top models, I need a separately certified MoE-capable architecture after the dense runtime proves stable. +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md`, and evidence READMEs for dependencies (DGR-022, DGR-023) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Exercise the complete streaming protocol deterministically before a real model or worker exists. -**Acceptance Criteria:** -- [ ] Implement explicit tensor ownership, router/top-k, expert/shared-expert, Q/K normalization, boundary bundle, and cache semantics for the selected Qwen3 family recipe. -- [ ] Do not reuse the dense-Llama adapter through unchecked name substitutions. -- [ ] Whole-model versus distributed prefill/decode parity passes the architecture-specific tolerance. -- [ ] Expert memory ownership and communication are measured. -- [ ] Real consumer-hardware acceptance and capability admission pass before the recipe becomes routable. +- [ ] Provide a fake bidirectional stream supporting prefill fragments, decode fast-path frames, release, cancel, and structured errors. +- [ ] Test flow-control blocking, deadlines, malformed fragments, checksum failure, duplicates, and stale epochs. +- [ ] Verify direct and opaque-relay framing preserve identical protobuf bytes. +- [ ] Tests require no sockets outside localhost, model downloads, or native accelerator. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -### DGR-016: Produce the upstream llama.cpp collaboration package -**Description:** As a maintainer, I need narrow upstreamable proposals so that our patch burden can shrink without asking llama.cpp to own Meshnet networking. +### DGR-025: Define exact artifact and runtime recipe identity -**Acceptance Criteria:** -- [ ] Separate generic llama.cpp hooks from Meshnet protocol/control-plane code. -- [ ] Prepare minimal reproducible examples and tests for range-aware loading, boundary input/output, and layer-filtered KV. -- [ ] Compare the proposal with Nakshatra and prima.cpp evidence and explain why the API is generally useful. -- [ ] Preserve one scoped commit/patch per concern against the exact upstream pin. -- [ ] Produce an outreach document suitable for Georgi/llama.cpp maintainers; actual sending remains a human action. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-018, DGR-021 -## Functional Requirements +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md`, and evidence READMEs for dependencies (DGR-018, DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure the tracker and worker only combine numerically and operationally compatible shards. -1. The public distributed primitive is an ordered Inference Route of contiguous Shards. -2. The native runtime uses llama.cpp/GGML; vLLM remains optional as a complete managed provider. -3. Native worker communication uses gRPC/HTTP2 and Protocol Buffers with one stable stream per Route Session Activation Seam. -4. Artifact identity, runtime recipe, boundary schema, activation dtype and cache layout must match exactly before routing. -5. Hot KV State remains local to the node serving the Shard. -6. Multiple Route Sessions must execute concurrently without shared-cache corruption. -7. Nodes batch compatible active decode steps and enforce bounded admission/backpressure. -8. Unsupported architectures and hardware recipes remain non-routable until real certification passes. -9. Default tests never download models or require GPUs; real tests are explicit and preserve artifacts off `/home`. -10. The release decision is based on measured performance, fit, quality, concurrency, and reliability relative to the safetensors baseline. +- [ ] Fingerprint source artifact SHA, tokenizer revision, architecture adapter/version, boundary schema, runtime pin/patch stack, backend, quant, activation/compute dtype, and KV/state layout. +- [ ] Bind each shard to an exact half-open range without hardcoding a topology or quant. +- [ ] Fail closed on any artifact, adapter, boundary, cache, backend, or runtime mismatch. +- [ ] Unsupported recipes remain registered-but-dark until real-hardware evidence certifies them. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -## Non-Goals +### DGR-026: Provision exact split-GGUF artifacts outside /home -- Forking vLLM or importing its PagedAttention/Torch distributed runtime. -- Adopting Nakshatra, prima.cpp, llama-gguf, LiGGUF, or GPUStack as the control plane. -- Public WAN tensor/expert parallel collectives. -- QUIC, WebRTC, or a custom socket protocol. -- Automatic KV migration or mid-generation route repair in the first release. -- Speculative decoding or disaggregated prefill before the core release gate. -- Supporting every GGUF architecture before dense Llama and Qwen3-family certification. -- A marketing-scale model demo that bypasses parity, concurrency, admission, or performance gates. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-025 -## Success Metrics +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md`, and evidence READMEs for dependencies (DGR-025) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Make exact split-GGUF inputs reproducibly available from mounted-drive storage without embedding a quantization or topology assumption in product code. -- A real model larger than one admitted node can execute across consumer machines when suitable hardware/artifacts are available. -- Four or more concurrent sessions complete without cross-talk; hardware-specific saturation is measured. -- Distributed GGUF passes the locked performance/fit contract against the existing safetensors route. -- Worker and Tracker recover all resources after completion, cancellation, malformed input, and node failure. -- The critical runtime remains Meshnet plus one standalone worker and a small auditable llama.cpp patch stack. +- [ ] Create an exact manifest that binds the source artifact, tokenizer/revision, every split file name, size, range/role, and cryptographic hash. +- [ ] Provide resumable, hash-verifying download/provision tooling targeting configured mounted-drive storage; refuse paths under `/home` and incomplete or mismatched splits. +- [ ] Keep quantization and split topology as manifest/recipe inputs with no hardcoded quant, node count, or range layout. +- [ ] Add deterministic model-download-free tests using tiny local split fixtures, including interrupted resume, missing split, hash mismatch, and `/home` rejection. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. -## Open Questions +### DGR-027: Add exact llama.cpp provenance manifest and fetch workspace -- Exact benchmark model and quantization lanes are selected by DGR-001 from currently supported, legally redistributable artifacts. -- Final hardware-specific concurrency and useful-speed thresholds are locked by measured baselines rather than guessed globally. -- Upstream llama.cpp acceptance is desirable but not a prerequisite for the first narrow pinned fork. +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-017 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Pin llama.cpp exactly through an in-repo manifest while fetching source only into an ignored build workspace. + +- [ ] Manifest records upstream URL, exact commit, expected source archive/tree hash, license, and retrieval method. +- [ ] Fetch tooling verifies identity before use and refuses an unpinned branch/tag. +- [ ] Source is fetched into an ignored build workspace; no submodule, vendored source tree, or permanent fork is introduced. +- [ ] Offline reuse is supported only after the cached tree’s exact identity is verified. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-028: Implement numbered patch-stack apply and verification + +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-027 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md`, and evidence READMEs for dependencies (DGR-027) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Maintain a minimal auditable llama.cpp delta with one numbered patch per concern. + +- [ ] Add deterministic apply/check/reverse verification against the exact manifest pin. +- [ ] Separate range loading, boundary I/O, filtered state, and worker hooks into scoped patches. +- [ ] Record upstream file/API assumptions and fail with the first incompatible patch when the pin changes. +- [ ] Verify license/attribution and prove no Meshnet routing, billing, relay, or authentication code enters the patch stack. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-029: Create the native CMake skeleton and deterministic CPU lane + +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-027, DGR-028 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md`, and evidence READMEs for dependencies (DGR-027, DGR-028) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish an out-of-tree standalone native build with a deterministic CPU lane before accelerator matrix work. + +- [ ] Create the standalone native CMake target/skeleton and isolated out-of-tree configure/build preset for CPU. +- [ ] Build and run a deterministic model-free CPU smoke/CTest lane from a clean checkout with actionable toolchain failures. +- [ ] Keep fetched upstream sources, generated bindings, and all build outputs ignored and out of tree. +- [ ] Ensure build success alone does not advertise any backend/model/recipe capability. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-030: Add accelerator build presets and native CI matrix + +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-029 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md`, and evidence READMEs for dependencies (DGR-029) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Compile all required accelerator lanes reproducibly while keeping untested real-hardware capabilities registered-dark. + +- [ ] Add isolated out-of-tree presets for CUDA, ROCm, Vulkan, and Metal without changing the deterministic CPU default. +- [ ] Add a native CI/build matrix that reports unavailable SDKs as explicit unavailable/skipped lanes rather than false success. +- [ ] Compile each available lane and preserve exact compiler, SDK, upstream pin, patch-stack, and build-option evidence. +- [ ] Keep every backend/model/recipe lane registered-dark until a separate real-hardware certification record exists. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-031: Introduce the project-owned `ShardEngine` interface + +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-021, DGR-025 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md`, and evidence READMEs for dependencies (DGR-021, DGR-025) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Isolate worker/protocol code from llama.cpp internals behind a stable project-owned engine contract. + +- [ ] Define load, capabilities, prefill/decode, boundary/logits result, cancel, release, health, and metrics operations. +- [ ] Use project-owned request/result/state types; expose no `ggml_tensor`, llama context, scheduler, or ABI-owned structure. +- [ ] Reserve typed MTP and architecture auxiliary-state hooks without enabling them. +- [ ] Add contract tests proving fake and future llama implementations obey identical lifecycle semantics. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-032: Implement deterministic fake `ShardEngine` + +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-031 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md`, and evidence READMEs for dependencies (DGR-031) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Provide an engine fixture that deterministically transforms typed boundary bundles and session state. + +- [ ] Support head, middle, tail, prefill, decode, cancellation, and release with deterministic outputs. +- [ ] Model isolated session/epoch state and deterministic cache-miss/stale-epoch failures. +- [ ] Support configurable delay, memory pressure, malformed output, and crash injection. +- [ ] Contract tests distinguish fixture evidence from real-model certification. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-033: Build a standalone fake C++ gRPC Shard worker + +**Milestone:** M1 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-022, DGR-024, DGR-032 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md`, and evidence READMEs for dependencies (DGR-022, DGR-024, DGR-032) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove the standalone worker process, stream, lifecycle, and supervision shape before llama.cpp integration. + +- [ ] A standalone C++ executable serves the complete lifecycle and stream RPC contract using the fake engine. +- [ ] Python integration tests cover startup, health, capability, fragmented prefill, decode, release, cancellation, and graceful shutdown. +- [ ] Bounded messages, deadlines, flow control, and independent session cancellation are enforced. +- [ ] The worker exposes neither llama.cpp RPC nor arbitrary graph execution. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-034: Implement dense-Llama range-aware GGUF ownership + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-028, DGR-029, DGR-031 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md`, and evidence READMEs for dependencies (DGR-028, DGR-029, DGR-031) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Load only the dense-Llama tensors owned by a shard range. + +- [ ] Load only `blk.N.*` tensors in the assigned range, embeddings only at the head, and norm/output or tied output only at the tail. +- [ ] Derive authoritative range and endpoint ownership from the loaded engine state. +- [ ] Reject invalid/gapped/out-of-model ranges and unexpected required tensors. +- [ ] Real-model evidence shows mapped/resident memory scales with owned tensors rather than full artifact size. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-035: Implement dense architecture boundary input/output + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-021, DGR-031, DGR-034 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md`, and evidence READMEs for dependencies (DGR-021, DGR-031, DGR-034) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Execute dense head/middle/tail ranges through an architecture-defined residual boundary. + +- [ ] Head accepts token IDs and owns embedding; middle/tail bypass embedding and accept a named boundary bundle. +- [ ] Non-tail returns the unnormalized residual before final norm/head and before tail-only row pruning. +- [ ] Tail returns logits or sampled-token output under an explicit contract. +- [ ] Uncertified architectures and incompatible boundary schemas fail closed. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-036: Prove dense fixture and real-model range parity + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-033, DGR-035 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md`, and evidence READMEs for dependencies (DGR-033, DGR-035) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove the engine/protocol seam before adding session concurrency. + +- [ ] Model-free two-stage tests pass through two fake worker processes with disjoint ranges. +- [ ] A small real dense GGUF passes whole-model versus two-range prefill parity. +- [ ] At least 32 greedy decode tokens match the locked tolerance. +- [ ] Evidence distinguishes deterministic fixture proof from opt-in real-model proof. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-037: Bind llama.cpp to the standalone worker + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-022, DGR-023, DGR-031, DGR-034, DGR-035 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md`, and evidence READMEs for dependencies (DGR-022, DGR-023, DGR-031, DGR-034, DGR-035) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Replace the fake engine with the pinned llama.cpp `ShardEngine` implementation without changing the public worker contract. + +- [ ] Worker loads exactly one artifact/recipe/range identity and rejects mismatched stream requests. +- [ ] All execution passes through `ShardEngine`; llama.cpp implementation types remain private. +- [ ] Health and metrics expose loaded identity, authoritative ownership, memory, and execution state. +- [ ] Graceful shutdown releases model/session resources; injected process death is observable and bounded. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-038: Implement isolated shard-local Hot KV State + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-037 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md`, and evidence READMEs for dependencies (DGR-037) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Map each route session and epoch to isolated state containing only locally owned layers. + +- [ ] Map `(route_session_id, route_epoch)` to an isolated llama sequence or bounded context. +- [ ] Support prefill/decode append, truncate, release, TTL/LRU eviction, cache miss, and stale-epoch rejection. +- [ ] Four concurrent sessions complete without token, KV, position, or cancellation cross-talk. +- [ ] Release/eviction returns memory to the configured budget without affecting other sessions. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-039: Pass local two-process dense acceptance + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-036, DGR-037, DGR-038 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md`, and evidence READMEs for dependencies (DGR-036, DGR-037, DGR-038) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish the first real, independently executing distributed GGUF route on one host. + +- [ ] Two worker processes open disjoint dense ranges and both execute real prefill/decode work. +- [ ] Whole-model parity, 32-token greedy decode, four-session isolation, cancellation, and cleanup pass. +- [ ] Record TTFT, prefill/decode rates, seam bytes/latency, RSS/VRAM, KV, queue, and failure metrics. +- [ ] Killing one worker returns a bounded structured failure rather than hanging. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-040: Add node-side native worker supervision + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-033, DGR-037 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md`, and evidence READMEs for dependencies (DGR-033, DGR-037) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Let the existing node service safely start, probe, stop, and restart the standalone worker. + +- [ ] Supervision owns process startup, readiness, log capture, graceful shutdown, and bounded forced termination. +- [ ] Startup verifies worker binary, artifact identity, recipe, and range before registration. +- [ ] Crashes or health loss make the capability unavailable without corrupting the Transformers backend. +- [ ] Tests use the fake worker and deterministic crash injection. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-041: Register native Shard capabilities without redesigning Meshnet + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-025, DGR-040 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md`, and evidence READMEs for dependencies (DGR-025, DGR-040) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Adapt native worker identity and capacity to existing backend-agnostic capability admission. + +- [ ] Registration carries exact recipe fingerprint, authoritative range, backend, memory/KV capacity, concurrency, and certification status. +- [ ] Existing tracker, billing, routing, telemetry, and provider semantics remain backend-agnostic. +- [ ] Uncertified backend/model/recipe combinations are visible but unroutable. +- [ ] Existing Transformers registration and route tests remain unchanged in behavior. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-042: Carry native frames through direct and existing relay seams + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-024, DGR-040 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md`, and evidence READMEs for dependencies (DGR-024, DGR-040) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Use direct gRPC where available and the existing relay as opaque protobuf transport without redesigning relay behavior. + +- [ ] Direct paths use the long-lived gRPC activation stream. +- [ ] Relayed paths carry byte-identical versioned protobuf frames through the existing relay contract. +- [ ] Request/work identity, cancellation, deadlines, telemetry, billing correlation, and per-node attribution survive both paths. +- [ ] Fake-worker tests cover direct, relay, disconnect, cancellation, and bounded buffering. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-043: Expose GGUF compatibility and measured cost inputs to existing routing + +**Milestone:** M2 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-041 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md`, and evidence READMEs for dependencies (DGR-041) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Adapt exact GGUF recipe/range compatibility and measured capacity, queue, seam-cost, health, and reliability data into the existing backend-agnostic routing inputs without redesigning routing or load balancing. + +- [ ] Expose exact recipe, range coverage, capacity, queue/load, seam-cost, health, reliability, backend, and certification measurements through existing tracker input contracts. +- [ ] Prove existing routing forms complete compatible coverage and excludes dark or mismatched candidates using its current backend-agnostic mechanisms. +- [ ] Regression-test unchanged Transformers behavior and unchanged tracker routing, load-balancing, billing, relay, and provider semantics. +- [ ] Regression-test that no quant, stage count, fixed split, architecture, backend sequence, or DeepSeek-specific policy is hardcoded. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-044: Pin the DeepSeek V4 Flash target contract + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-019, DGR-025, DGR-026, DGR-027 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md`, and evidence READMEs for dependencies (DGR-019, DGR-025, DGR-026, DGR-027) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Fix the alpha/beta model identity and upstream maturity assumptions before implementing the adapter. + +- [ ] Pin Hugging Face target SHA `60d8d70770c6776ff598c94bb586a859a38244f1`. +- [ ] Record the selected GGUF provenance and exact source binding without assuming one quant. +- [ ] Record that llama.cpp support began with PR 24162/commit `8c146a8` and remains maturing; the build manifest still pins one exact validated upstream commit. +- [ ] Record 43 main layers plus MTP and all known architecture-specific state obligations. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-045: Inventory V4 GGUF tensors and layer ownership + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-026, DGR-044 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md`, and evidence READMEs for dependencies (DGR-026, DGR-044) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Derive exact head, repeated-layer, tail, expert, auxiliary-state, and MTP ownership from the pinned artifact. + +- [ ] Produce a machine-readable inventory tied to the target SHA and exact GGUF artifact. +- [ ] Classify embeddings, 43 main layers, final head, MTP, mHC, attention variants, indexer/compressor, and MoE tensors. +- [ ] Validate half-open range ownership and fail on unclassified required tensors. +- [ ] Inventory tooling works across quants without embedding quant names in architecture logic. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-046: Define the V4 typed architecture boundary schema + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-021, DGR-045 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md`, and evidence READMEs for dependencies (DGR-021, DGR-045) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Define the exact cross-stage V4 architecture boundary while keeping per-layer attention and auxiliary caches shard-local. + +- [ ] Define a versioned named bundle for the mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations. +- [ ] Explicitly exclude per-layer CSA, HCA, SWA, indexer, compressor, KV, and MTP caches/state from the WAN boundary; those remain local to the owning shard and session/epoch. +- [ ] Reserve typed MTP boundary fields but mark MTP execution unsupported and unroutable for alpha. +- [ ] Fingerprint independently of quant/topology and fail closed on missing, incompatible, incorrectly shaped, or stale boundary/cache expectations. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-047: Adapt the upstream V4 mHC boundary for ranged ownership + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-045, DGR-046 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md`, and evidence READMEs for dependencies (DGR-045, DGR-046) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Add range-boundary adapters around upstream llama.cpp V4 mHC execution without reimplementing the V4 graph or kernels. + +- [ ] Represent and validate the upstream V4 4×4096 mHC boundary without flattening semantic axes. +- [ ] Add only head/intermediate/tail range ownership and boundary conversion hooks around the pinned upstream llama.cpp graph. +- [ ] Compare deterministic fixture vectors and single-process ranged outputs with upstream whole-model execution. +- [ ] Document that llama.cpp owns V4 mHC graph/kernels and that quantized storage does not alter the logical boundary schema. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-048: Carry token-ID sideband through the first three hash-routed layers + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-046, DGR-047 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md`, and evidence READMEs for dependencies (DGR-046, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure any stage containing the first three hash-routed layers receives exact token identities. + +- [ ] Head emits token IDs in the typed sideband for boundaries that require them. +- [ ] Intermediate stages preserve or consume the sideband according to their layer range. +- [ ] Routes splitting before, within, and after the first three layers pass deterministic partition tests. +- [ ] Missing or mismatched token IDs fail closed rather than silently selecting incorrect experts. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-049: Keep V4 attention and auxiliary state shard-local + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-046, DGR-047 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md`, and evidence READMEs for dependencies (DGR-046, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Bind V4 CSA/HCA/SWA/indexer/compressor and KV state to the owning shard session/epoch rather than transferring caches over the WAN. + +- [ ] Map CSA, HCA, SWA, indexer, compressor, and KV ownership from the pinned upstream graph to each owned layer range. +- [ ] Key all such caches/state by route session and epoch with bounded lifecycle, memory accounting, stale-epoch rejection, release, and eviction. +- [ ] Prove cross-stage messages carry only the typed architecture boundary and cache expectations, never per-layer cache contents. +- [ ] On route change or state loss, use explicit cache miss and re-prefill/restart; do not migrate V4 caches across workers. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-050: Validate upstream V4 MoE and hash-routing execution under ranged ownership + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-045, DGR-047 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md`, and evidence READMEs for dependencies (DGR-045, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Validate the pinned upstream llama.cpp V4 MoE and hash-routing graph when layers are range-owned, without reimplementing routing or expert kernels in Meshnet. + +- [ ] Load range-owned tensors for 256 routed plus one shared expert and validate upstream execution activates six routed experts where specified. +- [ ] Validate the first three upstream hash-routed layers receive exact token-ID sideband across splits before, within, and after them. +- [ ] Compare upstream whole-model and ranged router/expert outputs, shared-expert contribution, and malformed sideband failures. +- [ ] Document that llama.cpp owns V4 MoE, attention, hash routing, graph, and kernels; no expert-parallel WAN collective or Meshnet reimplementation is introduced. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-051: Assemble the DeepSeek V4 Flash `ShardEngine` adapter + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-034, DGR-037, DGR-047, DGR-048, DGR-049, DGR-050 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md`, and evidence READMEs for dependencies (DGR-034, DGR-037, DGR-047, DGR-048, DGR-049, DGR-050) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Assemble range ownership hooks and boundary/local-state adapters around upstream llama.cpp V4 execution behind the stable ShardEngine interface. + +- [ ] Support valid head, middle, tail, and whole-model ranges over all 43 main layers using the pinned upstream llama.cpp V4 graph/kernels. +- [ ] Integrate exact range ownership, mHC boundary, positions, required token-ID sideband, and shard-local session/epoch state adapters. +- [ ] Report MTP reserved-but-not-executable for alpha and reject unsupported split points or cache expectations. +- [ ] Include exact adapter/schema/runtime/artifact identity and valid split restrictions without reimplementing V4 MoE, attention, mHC, or hash routing. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-052: Pass local V4 whole-model versus ranged parity + +**Milestone:** M3 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-038, DGR-051 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md`, and evidence READMEs for dependencies (DGR-038, DGR-051) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove V4 prefill and decode correctness locally before network certification. + +- [ ] Compare pinned whole-model llama.cpp with at least two valid ranged partitions. +- [ ] Prefill intermediates and locked-length greedy decode pass architecture-specific tolerances. +- [ ] Test splits around the first three hash-routed layers and at least one CSA/HCA/SWA/indexer/compressor transition. +- [ ] Four concurrent sessions pass KV/state isolation and cleanup. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-053: Certify a real 2–4-stage V4 route + +**Milestone:** M3 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-030, DGR-043, DGR-052 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md`, and evidence READMEs for dependencies (DGR-030, DGR-043, DGR-052) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove real Tracker-selected V4 execution across physical machines before alpha. + +- [ ] Run one documented 2–4-stage certification scenario using exact compatible artifacts/recipes; the count and chosen quant are evidence inputs, not product constants. +- [ ] Actual CPU/GPU work executes on every stage; fake workers do not satisfy acceptance. +- [ ] Record parity, TTFT, prefill/decode speed, seam cost, memory, cache/state isolation, cancellation, and cleanup. +- [ ] Tracker selection remains dynamic and rejects an injected incompatible backend/recipe. +- [ ] Only the exact real-hardware lanes exercised become eligible for advertisement. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-054: Enforce the DeepSeek V4 Flash alpha gate + +**Milestone:** M3 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-017, DGR-019, DGR-020, DGR-039, DGR-042, DGR-043, DGR-052, DGR-053 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md`, and evidence READMEs for dependencies (DGR-017, DGR-019, DGR-020, DGR-039, DGR-042, DGR-043, DGR-052, DGR-053) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Release alpha only when V4 is correct and clears the pre-locked useful-speed requirement. + +- [ ] Evaluate correctness, model fit, TTFT, prefill/decode speed, seam cost, memory, and failure rate against DGF-004 without changing thresholds. +- [ ] Separate quant/model-fit gains from runtime/transport gains. +- [ ] Decision is `alpha`, `optimize measured bottleneck`, or `stop`; failures cannot be relabeled as certification. +- [ ] Alpha documentation states MTP is reserved but not implemented and identifies every dark hardware lane. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-055: Implement compatible continuous decode batching + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-038, DGR-040, DGR-054 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md`, and evidence READMEs for dependencies (DGR-038, DGR-040, DGR-054) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Batch compatible active sessions without violating route, recipe, position, or architecture state. + +- [ ] Batch only sessions with compatible engine, recipe, phase, boundary schema, and state layout. +- [ ] Preserve per-session positions, sequence IDs, outputs, cancellation, and accounting. +- [ ] Decode receives bounded service while prefill remains schedulable. +- [ ] Deterministic 1/2/4/8 concurrency tests show no cross-session corruption. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-056: Enforce bounded admission, queues, and backpressure + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-055 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md`, and evidence READMEs for dependencies (DGR-055) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prevent weight, KV, state, scratch, and stream buffers from exceeding admitted capacity. + +- [ ] Admission accounts separately for weights, KV/state, scratch, in-flight fragments, and queue slots. +- [ ] Queue and stream limits produce structured retryable or terminal responses. +- [ ] Prefill cannot starve decode and one route session cannot monopolize queue capacity. +- [ ] Telemetry reports active sessions, queue depth/time, batch occupancy, memory pressure, and rejects. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-057: Benchmark batching saturation and fairness + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-055, DGR-056 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md`, and evidence READMEs for dependencies (DGR-055, DGR-056) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Identify useful concurrency and saturation points without assuming one global batch size. + +- [ ] Run controlled concurrency 1/2/4/8 and record aggregate throughput, per-request p50/p95, TTFT, queue time, occupancy, and memory. +- [ ] Measure dense and V4 lanes separately where hardware permits. +- [ ] Verify bounded fairness and no prefill/decode starvation. +- [ ] Produce machine-readable saturation recommendations consumed by capability reporting, not hardcoded constants. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-058: Harden worker and stream failure semantics + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-040, DGR-042, DGR-056 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md`, and evidence READMEs for dependencies (DGR-040, DGR-042, DGR-056) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure malformed input, process death, transport reset, and cancellation terminate cleanly. + +- [ ] Test worker death, stream reset, malformed bundle, checksum failure, deadline, cache miss, stale epoch, and partial fragment loss. +- [ ] Cancellation propagates across all stages and releases KV/state, queue entries, and fragment buffers. +- [ ] Duplicate steps are idempotent; uncertain mutations are never silently replayed. +- [ ] Billing/work outcomes distinguish completed, cancelled, failed, and unverified work. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-059: Implement alpha-safe route recovery + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-043, DGR-058 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md`, and evidence READMEs for dependencies (DGR-043, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Recover from route loss without importing or continuing unverified state. + +- [ ] On route loss or local-state cache miss, restart from token zero or re-prefill on a newly compatible route according to the locked alpha contract. +- [ ] Never import KV, CSA, HCA, SWA, indexer, compressor, recurrent, or MTP cache/state across workers. +- [ ] Keep exact same-route duplicate delivery idempotent while cross-route continuation fails closed. +- [ ] Test direct/relay loss, cache miss, replacement selection, client cancellation, and billing reconciliation. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-060: Certify V4 long-context state correctness + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-051, DGR-056, DGR-058 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md`, and evidence READMEs for dependencies (DGR-051, DGR-056, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove V4’s KV and auxiliary state remain correct and bounded at long contexts. + +- [ ] Exercise pre-locked context lengths covering multiple prefill chunks and sustained decode. +- [ ] Validate KV plus CSA/HCA/SWA/indexer/compressor state positions across every stage. +- [ ] Measure memory growth, fragment sizes, TTFT, decode degradation, cancellation latency, and cleanup. +- [ ] Cache mismatch, truncation, and context-limit behavior fail deterministically. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-061: Certify existing routing with 10+ GGUF stage candidates + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-043, DGR-057, DGR-058 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md`, and evidence READMEs for dependencies (DGR-043, DGR-057, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Certify that existing backend-agnostic routing consumes GGUF compatibility and measured-cost inputs for a 10+ candidate scenario without algorithm or policy redesign. + +- [ ] Generate deterministic pools with 10+ compatible GGUF stage candidates plus slower, overloaded, unhealthy, and incompatible alternatives. +- [ ] Run the existing routing implementation unchanged and verify complete coverage, compatibility, acyclicity, and use of measured inputs. +- [ ] Regression-test that no stage count, quant, fixed split, architecture, or backend sequence is encoded in production routing policy. +- [ ] Verify existing route recomputation reacts to measured load/failure while tracker, load-balancing, billing, relay, and provider semantics remain unchanged. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-062: Certify a dynamic real 10+ stage V4 scenario + +**Milestone:** M4 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-053, DGR-060, DGR-061 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md`, and evidence READMEs for dependencies (DGR-053, DGR-060, DGR-061) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Demonstrate that a large real route can be dynamically selected and usefully execute V4. + +- [ ] Run one real 10+-stage V4 certification scenario; stage count, quant, and split remain scenario metadata only. +- [ ] Every selected stage proves real work, exact compatible identity, local state ownership, and bounded resources. +- [ ] Record end-to-end and per-stage TTFT, prefill/decode, seam cost, queueing, memory, failures, and cleanup. +- [ ] Inject at least one incompatible or degraded candidate and verify dynamic exclusion/reselection. +- [ ] The scenario must meet its pre-locked usefulness threshold or remain failed evidence. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-063: Profile and optimize the critical seam + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-057, DGR-061 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md`, and evidence READMEs for dependencies (DGR-057, DGR-061) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Optimize only the measured dominant compute, serialization, copy, or transport bottleneck. + +- [ ] Profile worker compute, tensor conversion, copies, protobuf framing, compression, relay/direct transport, queueing, and synchronization. +- [ ] Select one dominant bottleneck from evidence and lock its before/after benchmark. +- [ ] Implement one bounded optimization without weakening correctness or protocol identity. +- [ ] Report whether the optimization changes TTFT, prefill, decode, throughput, or memory and preserve raw data. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-064: Add negotiated activation compression + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-060, DGR-063 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md`, and evidence READMEs for dependencies (DGR-060, DGR-063) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Reduce V4 seam cost only when a certified compression recipe improves end-to-end performance acceptably. + +- [ ] Negotiate compression through recipe identity; unnegotiated or mismatched compression fails closed. +- [ ] Preserve names, shapes, semantic axes, token-ID sideband, and checksums. +- [ ] Measure quality drift, encode/decode cost, bytes, TTFT, and throughput against uncompressed boundaries. +- [ ] Compression remains dark unless it meets pre-locked quality and speed thresholds. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-065: Define V4 MTP ownership and distributed state contract + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-051, DGR-054, DGR-060 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md`, and evidence READMEs for dependencies (DGR-051, DGR-054, DGR-060) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Lock MTP tensor, execution, boundary, acceptance/rejection, and session-state ownership before implementation while keeping MTP disabled for alpha. + +- [ ] Derive exact MTP tensor and execution ownership from the pinned V4 target and upstream llama.cpp graph, distinguishing main-layer, tail, and MTP responsibilities. +- [ ] Define typed MTP inputs/outputs plus accepted-token, rejection, rollback/truncation, position, cache, and session/epoch lifecycle semantics. +- [ ] Specify which MTP state remains shard-local and which typed boundary values cross a seam; prohibit WAN migration of KV or V4 auxiliary caches. +- [ ] Keep every MTP recipe disabled and unroutable for alpha, with fail-closed capability negotiation and deterministic contract fixtures. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-066: Implement and benchmark V4 MTP + +**Milestone:** M4 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-054, DGR-060, DGR-064, DGR-065 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md`, and evidence READMEs for dependencies (DGR-054, DGR-060, DGR-064, DGR-065) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Implement the locked MTP contract around upstream llama.cpp before beta, without making MTP an alpha prerequisite. + +- [ ] Implement the exact DGR-065 MTP ownership and typed state contract against the pinned upstream V4 graph while preserving shard-local caches. +- [ ] Pass whole-model versus ranged MTP correctness, accepted-token, rejection, rollback/truncation, and session-isolation tests. +- [ ] Benchmark MTP-off versus MTP-on quality, accepted-token rate, TTFT, decode speed, seam bytes, and memory on the locked lane. +- [ ] Distinguish implemented, certified, disabled, and unsupported MTP recipes in fail-closed capability advertisement. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-067: Certify the backend capability matrix + +**Milestone:** M4 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-030, DGR-053, DGR-062, DGR-066 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md`, and evidence READMEs for dependencies (DGR-030, DGR-053, DGR-062, DGR-066) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Publish only backend/model/recipe lanes proven on actual CPU, CUDA, ROCm, Vulkan, or Metal hardware. + +- [ ] Maintain independent build and certification status for CPU, CUDA, ROCm, Vulkan, and Metal. +- [ ] Each advertised lane includes real hardware/driver/runtime/model evidence for parity, concurrency, cancellation, cleanup, and useful speed. +- [ ] Build success alone never makes a lane routable; unavailable hardware remains explicitly dark. +- [ ] Tracker admission consumes signed/versioned certification records rather than backend-name allowlists. +- [ ] Mixed-backend routes require exact compatibility evidence and fail closed otherwise. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-068: Package reproducible native worker releases + +**Milestone:** M5 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-030, DGR-054, DGR-067 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md`, and evidence READMEs for dependencies (DGR-030, DGR-054, DGR-067) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Produce installable worker packages that preserve exact source, patch, protocol, and certification identity. + +- [ ] Package worker binaries/configuration for supported host/backend combinations without bundling an unverified llama.cpp tree. +- [ ] Embed protocol, engine, upstream pin, patch-stack, build-toolchain, and certification fingerprints. +- [ ] Installation/startup refuses mismatched artifacts, runtime recipes, or unsupported hardware lanes. +- [ ] Produce checksums, SBOM/license attribution, reproducible build instructions, and smoke tests. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-069: Prepare narrow upstream llama.cpp collaboration patches + +**Milestone:** M5 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-028, DGR-035, DGR-038, DGR-051 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md`, and evidence READMEs for dependencies (DGR-028, DGR-035, DGR-038, DGR-051) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Reduce long-term patch burden through generic upstreamable hooks without asking llama.cpp to own Meshnet. + +- [ ] Separate generic range loading, boundary I/O, and filtered state hooks from project worker/protocol code. +- [ ] Provide one scoped patch, focused test, and minimal reproducer per concern against the exact upstream pin. +- [ ] Compare the proposal with prior-art evidence and current upstream V4 support/maturity. +- [ ] Prepare human-ready design/outreach text; actual issue/PR submission remains a human action. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-070: Enforce the DeepSeek V4 Flash beta gate + +**Milestone:** M5 · **Mode:** HITL · **Triage:** `ready-for-human` · **Depends on:** DGR-057, DGR-059, DGR-060, DGR-062, DGR-064, DGR-066, DGR-067, DGR-068 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md`, and evidence READMEs for dependencies (DGR-057, DGR-059, DGR-060, DGR-062, DGR-064, DGR-066, DGR-067, DGR-068) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Promote beta only after concurrency, resilience, long-context, MTP, scale, packaging, and speed gates pass. + +- [ ] Evaluate all beta metrics against DGF-004 without changing thresholds after observing results. +- [ ] Require V4 parity, batching/fairness, bounded backpressure, failure recovery, long-context correctness, MTP evidence, and dynamic 10+ stage evidence. +- [ ] Publish the exact certified hardware/backend/recipe matrix; all other lanes remain dark. +- [ ] Decision is `beta`, `targeted optimization`, or `stop/rollback`, with unresolved failures listed explicitly. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +### DGR-071: Establish upstream pin, patch, and certification maintenance + +**Milestone:** M5 · **Mode:** AFK · **Triage:** `ready-for-agent` · **Depends on:** DGR-069, DGR-070 + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md`, and evidence READMEs for dependencies (DGR-069, DGR-070) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Keep beta reproducible and detect upstream/model/hardware drift before it reaches routable nodes. + +- [ ] Add a documented update procedure for llama.cpp pin, numbered patches, protocol/engine versions, V4 target revision, and certification records. +- [ ] A candidate pin update must apply/build/test in isolation and cannot replace the production pin automatically. +- [ ] Any changed graph, tensor inventory, boundary, cache/state layout, backend behavior, or performance invalidates affected certification. +- [ ] Add scheduled model-free checks plus opt-in real-hardware recertification instructions and rollback procedure. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Non-goals + +- Replacing or redesigning Tracker routing/load-balancing, billing, relay, or provider policy. +- Reimplementing DeepSeek V4 architecture graphs or kernels in Meshnet. +- Hardcoding a quant, stage count, node count, split, architecture, or backend sequence. +- WAN migration of KV or V4 auxiliary caches. +- Advertising a compile-only or fixture-only hardware lane. +- Making MTP an alpha dependency. diff --git a/.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md b/.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md index d5c5e6b..3ccbd55 100644 --- a/.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md +++ b/.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md @@ -1,309 +1,57 @@ -# Ralph execution context: Performant Concurrent Distributed GGUF Runtime +# Ralph context: Distributed GGUF Runtime -Status: authoritative context for every fresh Ralph iteration -Last updated: 2026-07-13 +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -## Mandatory startup sequence +## Mandatory startup for every fresh story -Before changing code, every Ralph agent must: +1. Read this file and authoritative `prd.json` completely. +2. Read the generated source issue named in the selected story description. +3. Read every dependency evidence README; legacy DGR-001..016 evidence is provenance only. +4. Read `docs/adr/0024-distributed-gguf-runtime.md`, root `CONTEXT.md`, `.claude/memory/MEMORY.md`, and relevant live source/tests. +5. Inspect `git status`; preserve unrelated work. Never infer implementation from planning text or old pass states. +6. If blocked or oversized, keep `passes: false` and write an honest `BLOCKED.md`/`DECOMPOSITION.md`; never weaken criteria or fabricate evidence. -1. Read this file completely. -2. Read the selected issue under `.scratch/distributed-gguf-runtime/issues/`. -3. Read `docs/adr/0024-distributed-gguf-runtime.md` and the relevant part of `architecture.md`. -4. Read `.claude/memory/MEMORY.md` and root `CONTEXT.md` for current project vocabulary and constraints. -5. Inspect the current implementation and tests; do not assume historical scratch text describes live code. -6. Read the evidence/handoff directories for every declared dependency. -7. Inspect `git status` and preserve all pre-existing working-tree changes. +## Locked scope -A fresh Ralph iteration has no conversational memory. These files are the context contract. +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. -## Story sizing and interruption rule +## Target identities -Each story is intended to fit one focused Ralph context. Before implementation, estimate whether every acceptance criterion can be completed and verified in the current iteration. +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. -If the story is too large, an external dependency is unavailable, or the context/provider limit prevents completion: +## Control/data-plane contract -- Do not weaken criteria. -- Do not mark the issue done or set `passes: true`. -- Avoid leaving an unverified cross-cutting partial implementation when a smaller safe spike is possible. -- Write `evidence//DECOMPOSITION.md` or `BLOCKED.md` with the exact blocker, current verified state, proposed child stories, dependency graph and rollback/continuation instructions. -- Stop for supervised review. +Meshnet continues to own registration, coverage, existing route selection/load balancing, route epochs/sessions, direct/relay behavior, capability admission, cancellation, telemetry, billing, validation, and attribution. The GGUF adapter exposes measured inputs to those existing mechanisms. Direct seams use long-lived gRPC streams; relay seams carry byte-identical protobuf frames opaquely. -If interrupted after code changes, record every changed file, command result and unresolved invariant so the next fresh loop can verify rather than guess. +The project-owned `ShardEngine` hides llama.cpp internals. A worker loads one exact artifact/recipe/range identity. Default tests use fake/tiny fixtures. Real runs are opt-in, preserve raw metrics, and never download models under `/home`. -## Product objective +## Gitea issue synchronization -Build performant, concurrent distributed inference that combines consumer machines to serve top open models that exceed one node's RAM/VRAM. +Gitea is a projection of `prd.json`, never a competing source of truth. Before and after every supervised Ralph run, invoke: -A distributed demo is not success. The product must provide: - -- Useful measured prefill and decode speed. -- Multiple concurrent Route Sessions. -- No KV/token cross-talk. -- Bounded memory, queues, cancellation and failures. -- Real execution on every participating node. -- A model-fit or performance advantage over the current Transformers/safetensors route. - -## Critical-path architecture - -```text -Existing Meshnet control plane - | -Versioned Protobuf over gRPC/HTTP2 - | -Project-owned standalone C++ Shard worker - | -Small exact-commit llama.cpp patch stack +```bash +python3 scripts/ralph_gitea_sync.py sync ``` -Meshnet remains the only control plane and owns: +For a complete Ralph invocation with automatic state reconciliation, use: -- Tracker registration, Coverage Map, route selection and route epochs. -- Route Sessions and Activation Seams. -- Direct/relay routing. -- Capability admission. -- Cancellation, Generation Telemetry and backpressure. -- Billing, validation and per-node work attribution. - -Do not introduce another scheduler/control plane from vLLM, Nakshatra, prima.cpp, llama-gguf, GPUStack or another project. - -## Runtime decisions that are not open - -1. Public-network Shards are contiguous transformer layer ranges. -2. llama.cpp/GGML is the native GGUF execution substrate. -3. The project owns a small standalone worker and a narrow pinned llama.cpp patch stack. -4. The native Shard protocol is Protocol Buffers over gRPC/HTTP2. -5. One long-lived bidirectional stream serves one Route Session Activation Seam. -6. The public activation boundary is a versioned named-tensor bundle. -7. Hot KV State remains local to the node serving the Shard. -8. `(Route Session ID, route epoch)` maps to an isolated llama sequence or bounded context. -9. Concurrency uses continuous batching of compatible active sessions inside each node. -10. Transformers/safetensors remains the correctness and performance baseline. -11. vLLM may be an optional complete managed provider and concept donor; it is not forked into public Shards. -12. Tensor/expert collectives are deferred to a trusted composite provider, not public WAN routes. -13. Unsupported architectures/backends remain registered-but-dark until real certification passes. -14. Alpha failure retries from token zero; unverified KV is never migrated silently. -15. Model artifacts must remain on mounted-drive storage and never under `/home`. - -Changing one of these requires an explicit ADR update and human review, not an incidental story implementation. - -## Performance discipline - -GGUF performance is a hypothesis. Never write “GGUF is faster” without measurements. - -DGR-001 locks controlled benchmark lanes and thresholds. DGR-014 enforces the final distributed comparison. - -Always distinguish: - -- Weight quantization from activation/compute/KV dtype. -- Runtime/kernel gains from quantization/model-fit gains. -- Single-request latency from aggregate concurrency throughput. -- Synthetic unit coverage from real distributed acceptance. - -Required metrics where applicable: - -```text -TTFT -prefill tokens/sec -decode tokens/sec -aggregate throughput -p50/p95 latency -seam bytes and latency -queue and batch occupancy -RSS and VRAM -KV pressure -output-quality drift -failures and cleanup +```bash +scripts/ralph-gitea-run.sh ralph-tui run --prd .scratch/distributed-gguf-runtime/prd.json --agent claude --model sonnet --iterations 1 --no-tui --no-setup --direct-merge --no-sandbox ``` -Do not weaken or move performance thresholds after seeing implementation results. +The sync creates/reconciles one Gitea issue per `DGR-*` story, creates missing labels/milestones, closes issues whose `passes` is true, marks the selected next eligible story `status:in-progress`, and marks blocked stories `status:blocked`. `gitea-issues.json` is a derived mapping only. -## Transport discipline +## Evidence and completion -Do not invent a raw TCP protocol, new WebSocket protocol, QUIC layer or bespoke binary control format. - -The `.proto` schema is the semantic contract. Direct transport uses gRPC. Existing relay infrastructure may carry the same serialized protobuf frames as opaque binary. - -Protocol requirements: - -- Schema/version negotiation. -- Request/work ID. -- Route Session ID and route epoch. -- Exact Model Artifact/runtime recipe fingerprint. -- Shard range and effective overlap-safe start. -- Prefill/decode/release/cancel phases. -- Position/token range and idempotency step. -- Named tensors with shape, dtype, byte order and bounded fragments. -- Compression/checksum. -- Cache expectation/result. -- Deadlines, cancellation, flow control and structured status. - -Avoid per-token channel creation and unbounded unary payloads. Generated code and build tooling must be reproducible; do not require manual copying. - -## Native runtime discipline - -Reuse llama.cpp for GGUF, mmap, kernels, architecture graphs, tokenizer, KV, sequences and heterogeneous backends. - -The project patch stack is limited to: - -- Range-aware tensor registration/loading. -- Endpoint-specific embedding/final head ownership. -- Architecture-defined intermediate input/output. -- Intermediate output before final norm/head. -- Layer-filtered KV and session mapping. - -Do not place Meshnet routing, transport, billing or authentication inside llama.cpp. Keep patches numbered, scoped, pinned and upstreamable. - -Dense Llama-family is first. Qwen3/Qwen3-MoE is a separate adapter after the dense release gate. Do not generalize through unchecked tensor-name substitutions. - -## Existing code seams to inspect first - -- `packages/node/meshnet_node/model_backend.py` — backend abstraction. -- `packages/node/meshnet_node/torch_server.py` — reference ranged execution and session behavior. -- `packages/node/meshnet_node/activation_compression.py` — current activation framing/compression. -- `packages/node/meshnet_node/route_session_benchmark.py` — existing benchmark infrastructure. -- `packages/tracker/meshnet_tracker/server.py` — registration, route and proxy behavior. -- `packages/tracker/meshnet_tracker/capability.py` — fail-closed capability admission. -- `tests/test_real_model_backend.py` — real backend coverage. -- `tests/test_tracker_routing.py` — route/session behavior. -- `tests/test_tracker_capability_admission.py` — recipe admission. -- `tests/test_route_session_benchmark.py` and `tests/test_manual_route_benchmark.py` — benchmark patterns. -- `docs/adr/0008-binary-activation-wire-format.md` — existing wire compatibility. -- `docs/adr/0012-start-layer-overlapping-shards.md` — effective start semantics. -- `docs/adr/0022-sharded-per-node-kv-cache.md` — Hot KV State contract. -- `docs/adr/0023-model-agnostic-node-capability-admission.md` — certification/admission. - -Do not edit generated `build/`, `__pycache__`, egg-info, Ralph logs or unrelated scratch features. - -## Planned source layout - -Use these paths unless current code inspection proves a better project-consistent location. If changed, document the reason in task evidence. - -```text -packages/node/native/ - proto/shard_runtime.proto - cmake/ - llama/ - UPSTREAM_COMMIT - patches/ - gguf_worker/ - tests/ - -packages/node/meshnet_node/ - native_protocol/ - gguf_backend.py - runtime_recipe.py - -.scratch/distributed-gguf-runtime/evidence// - README.md - commands.txt - results.json or other machine-readable evidence -``` - -Generated protobuf/C++ build outputs belong in build directories unless packaging explicitly requires checked-in generated Python modules. The story must document the generation command and version. - -## Story output map - -| Story | Required durable outputs | -|---|---| -| DGR-001 | benchmark harness/tests; `evidence/DGR-001/performance-contract.json`; raw/summary benchmark evidence | -| DGR-002 | `packages/node/native/proto/shard_runtime.proto`; reproducible Python/C++ generation/build wiring; protocol round-trip/compatibility tests; `evidence/DGR-002/` | -| DGR-003 | exact runtime-recipe/fingerprint implementation and admission tests; `evidence/DGR-003/` | -| DGR-004 | exact upstream pin, numbered patch series, reproducible fetch/apply/build smoke; `evidence/DGR-004/` | -| DGR-005 | dense-Llama range ownership loader and memory evidence; `evidence/DGR-005/` | -| DGR-006 | architecture boundary adapter/parity tests and results; `evidence/DGR-006/` | -| DGR-007 | concurrent session/KV manager, isolation/cleanup tests; `evidence/DGR-007/` | -| DGR-008 | standalone C++ gRPC worker, fake-model integration tests, lifecycle evidence; `evidence/DGR-008/` | -| DGR-009 | Meshnet backend/registration/relay integration and tests; `evidence/DGR-009/` | -| DGR-010 | real local two-process commands, raw metrics and parity report; `evidence/DGR-010/` | -| DGR-011 | two-machine configuration, commands, hardware/network manifest and raw results; `evidence/DGR-011/` | -| DGR-012 | continuous scheduler/admission implementation and 1/2/4/8 concurrency report; `evidence/DGR-012/` | -| DGR-013 | failure/cancel/restart test matrix and resource-cleanup evidence; `evidence/DGR-013/` | -| DGR-014 | immutable final comparison against DGR-001 thresholds and ship/stop recommendation; `evidence/DGR-014/` | -| DGR-015 | Qwen3-family adapter, architecture-specific parity/admission/performance evidence; `evidence/DGR-015/` | -| DGR-016 | narrow upstream patches/tests, design note and human-ready outreach package; `evidence/DGR-016/` | - -## Dependency handoff rule - -For every dependency listed by Ralph: - -1. Confirm its `passes` state in `prd.json`. -2. Read `.scratch/distributed-gguf-runtime/evidence//README.md`. -3. Verify referenced source paths and commands still exist. -4. Do not repeat completed work unless verification exposes a concrete defect. -5. If dependency evidence is missing or contradictory, stop and repair the dependency instead of guessing. - -## Testing and hardware rules - -Default tests must be deterministic, GPU-free, model-download-free and API-credit-free. - -Real model tests require: - -```text -MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 -``` - -On this machine: - -- Use `.venv-rocm` for real Radeon 8060S ROCm execution. -- The default Python 3.14 `.venv` is unsuitable for real ROCm inference. -- Resolve model storage through the machine-specific `.env.` configuration. -- Never download model artifacts under `/home`. -- Real acceptance must exercise actual Tracker-routed CPU/GPU computation; synthetic workers are only unit tests. - -Record exact: - -- Model/revision and Artifact hash. -- Quantization and runtime recipe. -- Host/hardware/backend/driver. -- Commands and environment names without secrets. -- Raw output and metrics. -- Whether the evidence is synthetic, local-real, or multi-machine-real. - -## Worktree and commit discipline - -This repository may contain pre-existing changes from research or another feature. - -- Inspect `git status` before editing. -- Never reset, checkout over, stash, delete or reformat unrelated changes. -- Stage only files belonging to the selected story. -- Exclude `.ralph-tui`, iteration logs, caches, generated builds, FUSE artifacts and unrelated scratch work. -- Keep one scoped commit per completed story when the supervising loop requests commits. -- Do not modify `passes` for another story. - -## Mandatory finish/handoff sequence - -Before emitting `COMPLETE`: - -1. Verify every acceptance criterion with real command output or file evidence. -2. Run story-specific gates and repository quality gates. -3. Write `.scratch/distributed-gguf-runtime/evidence//README.md` containing: - - Summary of changes. - - Exact files changed. - - Commands run and their real results. - - Performance/correctness evidence. - - Known limitations and deferred work. - - Compatibility or migration notes. - - Clear handoff for dependent stories. -4. Save machine-readable evidence beside it when the story produces metrics or schemas. -5. Update the source issue status to `done` only after all gates pass. -6. Preserve failures honestly. Never fabricate model, benchmark, test or hardware output. - -## Authoritative references - -Active decisions: - -- `.scratch/distributed-gguf-runtime/README.md` -- `.scratch/distributed-gguf-runtime/implementation-strategy.md` -- `.scratch/distributed-gguf-runtime/architecture.md` -- `docs/adr/0024-distributed-gguf-runtime.md` -- `.scratch/distributed-gguf-runtime/PRD.md` -- `.scratch/distributed-gguf-runtime/prd.json` - -Source research: - -- `docs/research/distributed-gguf-landscape.md` -- `docs/research/distributed-gguf-github-followup.md` -- `docs/research/vllm-distributed-gguf-assessment.md` - -If historical notes conflict with these files, the active decisions above win. +Each story writes `/run/media/popov/d/DEV/repos/d-popov.com/AI/.claude/worktrees/distributed-gguf-runtime/.scratch/distributed-gguf-runtime/evidence//README.md` with exact files, commands/results, limitations, identities, and dependent-story handoff. Only `prd.json` may record `passes`; DGR-017 and DGR-018 are complete and DGR-019 onward remain false. Generated Markdown and Gitea issues cannot override it. One scoped commit per story is expected during future execution. diff --git a/.scratch/distributed-gguf-runtime/README.md b/.scratch/distributed-gguf-runtime/README.md index 27d3deb..0875ee7 100644 --- a/.scratch/distributed-gguf-runtime/README.md +++ b/.scratch/distributed-gguf-runtime/README.md @@ -1,46 +1,32 @@ -# Performant concurrent distributed GGUF runtime +# Distributed GGUF Runtime planning workspace -Status: active benchmark-gated implementation program. +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -## Objective -Serve top open models across consumer machines with useful performance and concurrent Route Sessions while keeping the runtime lean. +## Locked scope -## Critical path +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. -```text -Meshnet control plane - -> versioned gRPC/Protobuf Shard protocol - -> project-owned standalone C++ worker - -> small pinned llama.cpp patch stack -``` +## Target identities -Transformers/safetensors remains the correctness baseline. vLLM remains an optional complete managed provider and a design donor; it is not forked into the public mesh. +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. -## Planning artifacts +## Navigation -- **[Mandatory Ralph context](RALPH-CONTEXT.md)** — read first in every fresh iteration -- [Task evidence contract](evidence/README.md) -- [Implementation strategy](implementation-strategy.md) -- [Current architecture](architecture.md) -- [PRD](PRD.md) -- [Ralph backlog](prd.json) -- [ADR-0024](../../docs/adr/0024-distributed-gguf-runtime.md) -- [Milestones](milestones.md) -- [Issues](issues/) -- [Distributed GGUF research](../../docs/research/distributed-gguf-landscape.md) -- [GitHub follow-up](../../docs/research/distributed-gguf-github-followup.md) -- [vLLM assessment](../../docs/research/vllm-distributed-gguf-assessment.md) - -## Ralph execution - -Use supervised one-story iterations for this high-risk runtime: - -```bash -ralph-tui run \ - --prd .scratch/distributed-gguf-runtime/prd.json \ - --agent claude --model opus \ - --iterations 1 --no-tui --no-setup --verify -``` - -Inspect the diff, run the story gates, and commit one verified story before the next iteration. Real-model stories require the explicit environment gate and mounted-drive model storage. +- [`prd.json`](prd.json) — sole authoritative 55-story backlog, DGR-017..071. +- [`PRD.md`](PRD.md) — human-readable projection of goals, gates, and all stories. +- [`RALPH-CONTEXT.md`](RALPH-CONTEXT.md) — mandatory fresh-session context. +- [`architecture.md`](architecture.md), [`implementation-strategy.md`](implementation-strategy.md), [`milestones.md`](milestones.md) — design and execution sequence. +- [`issues/`](issues/) — generated story specs; files 01..16 are retained legacy artifacts pending DGR-017. +- [`evidence/`](evidence/) — provenance and future per-story handoffs. diff --git a/.scratch/distributed-gguf-runtime/architecture.md b/.scratch/distributed-gguf-runtime/architecture.md index b431bde..2be43f4 100644 --- a/.scratch/distributed-gguf-runtime/architecture.md +++ b/.scratch/distributed-gguf-runtime/architecture.md @@ -1,259 +1,45 @@ -# Performant Concurrent Distributed GGUF Architecture +# Distributed GGUF Runtime architecture -Status: current target architecture -Last updated: 2026-07-13 +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -## Product invariant -The system exists to serve high-quality models that exceed one consumer node's memory while retaining useful interactive speed and aggregate concurrency. A feature that only produces a distributed demo but is slower, globally serialized, or impossible to operate on consumer hardware is not complete. +## Locked scope -## Existing control plane +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. -Meshnet remains the only public control plane: +## Target identities -- Tracker registration, Coverage Map, route scoring and assignment. -- Contiguous Shards and overlap-safe effective starts. -- Stable Route Sessions and route epochs. -- Local per-Shard Hot KV State in the reference backend. -- Direct/relay transport, cancellation and backpressure. -- Generation Telemetry, billing, validation and per-node attribution. -- Model-agnostic capability admission. +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. -No external engine replaces these responsibilities. - -## Runtime topology +## Topology ```text -OpenAI-compatible client - | -Gateway / Tracker Node - | -ordered Inference Route - | - +-- head Shard: tokenizer/embedding + early layers - | local weights and Hot KV State - | - +-- middle Shard(s): architecture boundary + owned layers - | local weights and Hot KV State - | - +-- tail Shard: final layers + norm/head/sampling - local weights and Hot KV State +existing Meshnet Tracker/control plane + -> existing backend-agnostic route/load-balancing decision + -> direct gRPC or existing opaque relay + -> project-owned standalone C++ Shard worker + -> project-owned ShardEngine + -> pinned upstream llama.cpp + numbered range/boundary/state hook patches + -> GGUF mmap, upstream V4 graph/kernels, local per-shard state ``` -Weights never move in the per-request hot path. Every node opens and verifies its local Model Artifact before becoming routable. +A route is ordered contiguous half-open ranges. Head owns token embedding; tail owns final norm/head/sampling. Compatibility fingerprints bind source/split hashes, tokenizer, architecture adapter, typed boundary, runtime pin/patches, backend, quant, activation/compute/KV layout, range, and certification. -## Primary execution substrate +## V4 boundary and state -```text -project-owned C++ Shard worker - | -small exact-commit llama.cpp patch stack - | -GGUF mmap, quantized kernels, architecture graphs, -KV/sequence operations, CPU/CUDA/HIP/Vulkan/Metal backends -``` +The inter-stage boundary is semantic and versioned: mHC 4×4096 residual, positions, token IDs only where the first three hash-routed layers require them, and cache/schema expectations. CSA/HCA/SWA/indexer/compressor/KV state belongs to upstream layer execution on the owning worker and is isolated by `(route_session_id, route_epoch)`. On loss, return cache miss and re-prefill/restart. Never serialize those caches into the WAN bundle. -The patch stack adds only the missing local execution seam: +## Concurrency, failure, and admission -1. Range-aware tensor registration/loading. -2. Endpoint-specific embedding and final head ownership. -3. Architecture-defined intermediate input. -4. Architecture-defined pre-tail boundary output. -5. Layer-filtered KV and external session mapping. - -The worker owns protocol translation and process lifecycle. llama.cpp never receives Tracker, relay, billing or volunteer-network code. - -## Shard data plane - -Use Protocol Buffers and gRPC over HTTP/2. - -### Service shape - -- Unary capability and health. -- Bidirectional Route Session stream. -- Explicit release and cancellation. -- Metrics suitable for capability admission and route scoring. - -### Session stream - -One long-lived stream represents one Route Session Activation Seam. It amortizes connection setup and inherits HTTP/2 flow control. Every message carries enough identity to reject stale or incompatible work. - -```text -schema version -request/work id -Route Session id -route epoch -Model Artifact hash -runtime recipe fingerprint -Shard begin/end and effective start -prefill/decode/release/cancel phase -position and token range -idempotency step id -cache expectation/result -named tensor bundle -compression/checksum -``` - -Prefill tensors are split into bounded ordered frames. Decode messages carry one-step architecture boundary bundles and remain small. - -Direct nodes use gRPC. Nodes requiring the existing relay carry the same protobuf frames as opaque binary through the relay session. This preserves one semantic protocol instead of maintaining separate direct and relay payload contracts. - -## Architecture boundary - -The public boundary is a versioned named-tensor bundle: - -```text -bundle schema/version -architecture adapter and boundary point -named tensors -per-tensor shape, dtype and byte order -payload fragments -compression/checksum -``` - -Dense Llama may use one residual tensor. Other adapters may require more. vLLM's Llama and Qwen3-MoE PP paths demonstrate a boundary with both `hidden_states` and `residual`; therefore the generic protocol must not assume one anonymous tensor. - -Only the head owns token embedding. Only the tail owns final normalization, LM head and sampling. Middle Shards exchange the architecture-defined pre-tail boundary, not final normalized embeddings. - -## Hot KV State and concurrency - -```text -(Route Session id, route epoch) - -> local llama sequence or bounded context - -> KV for owned layers only - -> lease, memory accounting and lifecycle -``` - -Required operations: - -- Prefill append. -- Decode append. -- Truncate after rejected speculative positions if later enabled. -- Explicit release. -- TTL/LRU eviction. -- Cache-miss response. -- Stale-epoch rejection. - -A node must not clear global KV on a new stream or serialize all requests behind one logical serving sequence. - -## Continuous batching - -Autoregressive dependencies remain sequential inside one Route Session. Aggregate throughput comes from batching compatible decode steps across active sessions: - -```text -time 0: session A token 1 + session B token 8 + session C token 3 - -> one llama batch for this Shard - -time 1: next ready positions from active sessions - -> next llama batch -``` - -The node scheduler: - -- Admits work against weight, KV, scratch and queue budgets. -- Keeps per-session token positions and outputs separate. -- Prevents long prefill from starving decode. -- Applies bounded backpressure. -- Reports active sessions, queue depth, batch occupancy, KV pressure and throughput. - -The initial deterministic gate is four concurrent sessions on a small model without cross-talk. Hardware-specific limits are measured and advertised through capability admission. - -## Parallelism boundaries - -| Mechanism | First-runtime use | -|---|---| -| Layer/pipeline parallelism | Public Inference Route across contiguous Shards | -| Continuous batching | Inside every node across active Route Sessions | -| Data parallelism | Multiple complete routes for independent requests | -| Tensor parallelism | Deferred to a trusted composite node/managed cluster | -| Expert parallelism | Deferred to a trusted composite node/managed cluster | -| Disaggregated prefill | Deferred until core route performance passes | -| Speculative decoding | Deferred optimization | - -Public WAN tensor/expert collectives are rejected for the first runtime because their per-layer communication and static rank assumptions conflict with heterogeneous volunteer nodes. - -## Optional providers - -### Transformers/safetensors - -Remains: - -- Correctness/reference backend. -- Fallback for unsupported architectures. -- Baseline for performance and output quality. - -### vLLM - -May run unmodified as a complete model or managed TP/PP/EP cluster represented as one logical provider. Its internal ranks are not independently routed or rewarded. - -Borrow only concepts such as named bundles, continuous batching, typed compatibility fingerprints, explicit transfer lifecycle and load telemetry. - -### Whole-model llama.cpp - -Provides a local proxy backend, correctness oracle and performance baseline. It is not the native distributed milestone. - -## Artifact and recipe compatibility - -A routable recipe identifies separately: - -- Source Model Artifact hash and optional derivative/slice hash. -- Architecture and adapter version. -- Tokenizer revision and vocabulary. -- Weight quantization. -- Activation interchange dtype/schema. -- Backend compute dtype and backend implementation. -- KV dtype/layout. -- RoPE/context parameters. -- llama.cpp commit and project patch version. -- Shard range and endpoint ownership. - -Compatibility fails closed. Similar quantization labels or model names are not enough. - -## Admission and failure - -A recipe becomes routable only after a real local and distributed forward passes. Synthetic tests remain unit coverage. - -Alpha failure behavior: - -- Deadline or node loss cancels the Route Session. -- Every node releases KV and queued buffers. -- Uncertain mutations are not replayed silently. -- Retry starts from token zero on a newly compatible route. -- No cross-node KV import is trusted until a later signed/compatible snapshot protocol exists. - -## Performance release contract - -Before native development proceeds, compare the current Transformers/safetensors backend with whole-model llama.cpp under controlled model/hardware/quality lanes. - -Final release compares distributed GGUF with distributed safetensors using thresholds locked before seeing final results. - -Required measurements: - -- TTFT. -- Prefill and decode tokens/sec. -- Aggregate concurrency throughput. -- p50/p95 latency. -- Seam bytes and latency. -- Queue/batch occupancy. -- RSS, VRAM and KV pressure. -- Output-quality drift. -- Cancellation/failure cleanup. - -The GGUF path ships only if it is faster at acceptable quality or enables a larger otherwise-unroutable model at useful measured speed. - -## Implementation sequence - -1. Lock benchmark/performance contract. -2. Define gRPC/protobuf and exact recipe identity. -3. Pin llama.cpp and create the minimal patch stack. -4. Implement dense-Llama range loading and boundary parity. -5. Implement concurrent local KV. -6. Build and integrate the standalone worker. -7. Pass local two-process real-model acceptance. -8. Pass real heterogeneous two-machine acceptance. -9. Add continuous batching and failure hardening. -10. Enforce the GGUF-versus-safetensors release gate. -11. Add Qwen3/Qwen3-MoE as a separately certified adapter. -12. Prepare narrow upstream collaboration patches/tests. - -See [the Ralph backlog](prd.json) and [implementation strategy](implementation-strategy.md). +Compatible sessions may be continuously batched within a worker while retaining isolated positions/state. Admission bounds weights, local state/KV, scratch, fragments, and queues. Uncertain cross-route mutation is not replayed. Registration can show an uncertified lane, but existing admission keeps it unroutable until signed/versioned real-hardware evidence exists. diff --git a/.scratch/distributed-gguf-runtime/decision-framework.md b/.scratch/distributed-gguf-runtime/decision-framework.md index 4b1098c..8787567 100644 --- a/.scratch/distributed-gguf-runtime/decision-framework.md +++ b/.scratch/distributed-gguf-runtime/decision-framework.md @@ -1,270 +1,40 @@ -# Distributed GGUF Decision Framework - -> **Superseded for active implementation decisions.** The grill was resolved on 2026-07-13. Use [implementation-strategy.md](implementation-strategy.md), [architecture.md](architecture.md), [ADR-0024](../../docs/adr/0024-distributed-gguf-runtime.md), and [prd.json](prd.json). This file remains as historical decision rationale. - -This framework is for grilling open decisions. It keeps decisions tied to project vocabulary and implementation gates instead of vague "distributed inference" language. - -## Core Vocabulary - -Use the existing domain terms this way: - -- **Shard**: contiguous transformer layer range. This is the compute, routing, cache, and reward unit. -- **Shard Swarm**: storage/download group for artifacts needed by a shard. -- **Inference Route**: ordered node sequence that covers all layers for one request. -- **Route Session**: one active request bound to one inference route and stable session id. -- **Hot KV State**: live per-shard cache held by the route node during a route session. -- **Prefix Snapshot**: persisted route-session state used for reuse or failover, not the hot decode path. -- **Artifact Manifest**: canonical mapping from model artifacts to semantic model parts and runtime support. -- **Generation Telemetry**: realtime progress for a route session, including phase and tokens/sec, independent of whether token deltas are streamed. - -## The Five Planes - -### 1. Control Plane - -Owner: Tracker. - -Responsibilities: - -- node registry -- coverage map -- route selection -- rebalance directives -- route-session creation -- health and telemetry -- client-visible Generation Telemetry -- billing/audit records - -Must not do: - -- serve hot KV during every token -- become the only place model artifacts can be fetched - -### 2. Artifact Plane - -Owner: Shard Swarms, local node storage, optional CDN/bootstrap mirrors. - -Responsibilities: - -- GGUF/safetensors/tokenizer download -- content-addressed verification -- local artifact inventory -- artifact-to-layer mapping -- cache eviction - -Must not do: - -- define execution order by file split alone -- imply that a downloaded file chunk equals a Shard - -### 3. Execution Plane - -Owner: active Inference Route. - -Responsibilities: - -- chunked prefill -- one-step decode -- hidden-state transfer across activation seams -- start-layer handling for overlapping shards -- backpressure - -Must not do: - -- resend full context activations during decode -- require cross-node tensor parallel all-reduce for public v1 - -### 4. Session State Plane - -Owner: route nodes for hot KV; cache servers only for snapshots. - -Responsibilities: - -- per-shard local KV ownership -- cache allocation and eviction -- cache ABI compatibility -- session close/release -- optional prefix snapshots - -Must not do: - -- centralize hot KV in a remote service -- let a replacement node continue from incompatible state - -### 5. Economics And Trust Plane - -Owner: tracker plus settlement/validation components. - -Responsibilities: - -- distinguish storage/seeding work from inference work -- account for prefill and decode separately -- record route participation -- sample validation events -- slash proven fraud - -Must not do: - -- pay a node for merely holding files as if it generated tokens -- hide public-swarm privacy limits from clients - -## Hard Invariants - -These are the framework rules unless we deliberately write a new ADR: - -1. Public-network Shards are contiguous layer ranges. -2. Hot KV State is local to the node serving that Shard in that Route Session. -3. Artifact distribution and route execution are separate systems. -4. Decode seam payload must be `O(hidden_size)`. -5. Prefill may be `O(sequence_length * hidden_size)`, but only in bounded chunks. -6. The tracker chooses routes; nodes do not negotiate route topology peer-to-peer. -7. Model/backend-specific cache internals stay behind backend capability reports. -8. PyTorch remains the correctness/reference backend while llama.cpp/GGUF becomes the performance backend. -9. Streaming responses are preferred when feasible; Generation Telemetry is always required. - -## Resolved Gates - -### Gate 1: Public Shard Semantics - -Decision: public-network Shards are contiguous transformer layer ranges. Tensor-parallel or ring-style execution is allowed only inside one trusted node, one colocated pod, or a future composite node abstraction. - -Rationale: - -- Layer ranges match the existing `Shard`, `Coverage Map`, `Inference Route`, billing, and fraud vocabulary. -- Public volunteer nodes should not require cross-node all-reduce or tight per-layer synchronization in v1. -- Existing projects such as prima.cpp and Distributed Llama can still inform local-cluster/backend execution without becoming the public routing primitive. - -Consequences: - -- Artifact Manifests must map files/tensors to semantic layer ranges. -- Route selection remains ordered layer coverage. -- Rewards can be attributed to layer-range work. -- Hot KV State is naturally owned by the node serving that layer range for the Route Session. - -### Gate 2: Hot KV Strategy - -Decision: v1 rejects centralized hot KV. Hot KV State is local to the node serving the relevant Shard in the active Route Session. Cache servers may store Prefix Snapshots for reuse, retry, or failover, but they are not in the per-token decode path. - -Rationale: - -- Decode is the tight loop; adding remote cache I/O there makes latency and bandwidth worse at the worst point. -- Local KV naturally follows layer-range Shard ownership. -- Centralized hot KV increases privacy exposure and creates consistency problems. -- Prefix Snapshots preserve the useful part of central storage without making it mandatory for every generated token. - -Consequences: - -- Route Session must be sticky. -- Failover is limited in alpha unless a compatible Prefix Snapshot exists. -- Cache servers are optimization infrastructure, not required runtime infrastructure. -- Route repair requires compatible model revision, layer range, backend cache ABI, and snapshot position. - -### Gate 3: First Runtime Proof - -Decision: prove distributed Route Session and Hot KV State semantics in the existing PyTorch route before modifying llama.cpp/GGUF. - -Rationale: - -- PyTorch exposes model internals and cache objects more directly, so it is the fastest way to validate the distributed protocol. -- The current distributed PyTorch route already has the right high-level shape but disables cache and recomputes full prompts. -- Fixing that path gives us a reference implementation for correctness tests, telemetry, session lifecycle, and wire protocol behavior. -- llama.cpp/GGUF should receive a clear target ABI rather than becoming both the protocol experiment and the performance backend at once. - -Consequences: - -- Issue 02 precedes issue 05. -- llama.cpp collaboration has a concrete target ABI. -- The PyTorch route remains the architecture-coverage/reference backend even after GGUF becomes the preferred performance path. -- The first success metric is eliminating full-prompt recompute in distributed decode. - -### Gate 3A: Client Feedback During Latency - -Decision: streaming responses are preferred when feasible, and realtime Generation Telemetry is required regardless of streaming support. - -Rationale: - -- The product optimizes for access to large capable models, so some latency is acceptable. -- Users still need confidence that the route is alive and roughly how fast it is generating. -- Streaming token deltas give the best user experience when the backend exposes them cleanly. -- Tokens/sec remains useful during prefill, queueing, and any backend that cannot stream token deltas. - -Consequences: - -- The gateway should stream token deltas through an OpenAI-compatible response when possible. -- The gateway must expose progress through SSE, WebSocket, or polling. -- The final answer can be delivered after completion only as a fallback. -- Telemetry must include route phase, generated token count, and rolling tokens/sec. -- Non-streaming clients still need realtime telemetry. - -### Gate 4: llama.cpp Collaboration Shape - -Decision: target upstreamable `libllama`/ggml hooks instead of planning around a permanent fork. - -Rationale: - -- llama.cpp changes quickly across model support, quantization, kernels, and hardware backends. -- A permanent fork would become expensive to maintain and would lag upstream improvements. -- A short-lived prototype branch is acceptable if it proves the API and makes upstream collaboration concrete. -- Keeping tracker/routing logic outside llama.cpp makes the upstream ask smaller and cleaner. - -Consequences: - -- Need a minimal reproducible localhost demo before asking upstream to carry the design. -- Need to separate "what llama.cpp should expose" from "what our tracker does". -- Desired upstream surface is layer-range execution, hidden-state boundary I/O, partial loading/introspection, and per-session KV ownership. -- If upstream rejects the shape, we revisit whether to carry a narrow adapter fork or keep GGUF distributed execution as experimental. - -### Gate 5: First Model Target - -Decision: use a two-tier model target. Use a small, boring, llama.cpp-supported GGUF model for the first protocol smoke test. Use `deepseek-ai/DeepSeek-V4-Flash` as the first serious large-model target. Keep GLM-5.2 and Ornith as later support audits. - -Rationale: - -- The first protocol proof should isolate route/session/KV bugs from model-architecture bugs. -- DeepSeek-V4-Flash is a strong first serious target because it is much smaller than 1.6T-class models while still being large enough to validate the product thesis. -- DeepSeek-V4-Flash still has architecture-specific risks, so it should not be the first smoke test. -- GLM-5.2 and Ornith remain valuable targets, but they add DSA/MLA/hybrid attention uncertainty. - -Consequences: - -- 128K cache accounting can be modeled now. -- The first "real" target-model audit is DeepSeek-V4-Flash support in PyTorch, vLLM/SGLang, and any available GGUF/llama.cpp quantization path. -- Production support waits for backend capability reports and exact cache ABI support. - -### Gate 6: Failure Semantics - -Decision: alpha fails Route Sessions on route-node loss instead of attempting automatic route repair. - -Rationale: - -- Route repair requires compatible Prefix Snapshots, cache ABI checks, replacement-node selection, billing correction, and client stream/error recovery. -- Local Hot KV State means a replacement node cannot continue unless it has compatible state at the same position. -- Fail-fast keeps the first implementation correct while the session/KV protocol is still being proven. - -Consequences: - -- Better observability and explicit errors are required. -- Snapshotting becomes a later feature, not a blocker for first inference. -- Generation Telemetry must report the last known phase and failure reason. -- Client or gateway retry starts a new Route Session from scratch. - -### Gate 7: Transport - -Decision: keep binary HTTP for v1 activation transfer instead of jumping immediately to QUIC, WebRTC, or a custom transport. - -Rationale: - -- ADR-0008 already defines binary activation bodies with HTTP headers. -- HTTP keeps the first implementation debuggable with the existing server stack and tooling. -- The core risk is route/session/KV correctness, not transport optimization. -- QUIC/WebRTC can be introduced later behind the same activation protocol once semantics are proven. - -Consequences: - -- Focus benchmark work on payload shape, chunking, and cache behavior first. -- QUIC/WebRTC can be introduced as an optimization behind the same activation protocol. -- v1 implementation can reuse the current HTTP routing, relay, and observability infrastructure. -- Transport abstraction should be kept narrow enough that HTTP can be replaced later without changing backend cache semantics. - -## Grilling Progress - -Gates 1, 2, 3, 3A, 4, 5, 6, and 7 are resolved. The remaining work is to convert the resolved framework into implementation-ready issue briefs and prototype milestones. +# Distributed GGUF Runtime decision framework + +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. + +## Decision order + +1. DGR-019 locks comparable lanes and thresholds before results. +2. DGR-020 runs safetensors and whole-model llama.cpp only, then returns `go`, `optimize baseline`, or `stop`. +3. Dense and V4 work must prove parity, independent per-stage execution, local-state isolation, bounded failure, and measured resources. +4. DGR-054 returns `alpha`, `optimize measured bottleneck`, or `stop`; MTP is explicitly off. +5. Post-alpha optimizations must be selected from profiles, not assumptions. +6. DGR-070 returns `beta`, `targeted optimization`, or `stop/rollback`, and requires MTP and the exact certified hardware/recipe matrix. + +## Interpretation rules + +- Quant/model-fit gains are separate from runtime/kernel/transport gains. +- Fixture, real-model, real-hardware, and release evidence are never interchangeable. +- 2–4 and 10+ stages are certification scenarios only. +- Existing routing policy is certified, not redesigned. +- Build success is not hardware certification; dark lanes remain unroutable. +- Route loss uses cache miss and re-prefill/restart, never WAN cache migration. + +## Locked scope + +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. + +## Target identities + +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md deleted file mode 100644 index a29623d..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md +++ /dev/null @@ -1,127 +0,0 @@ -# DGR-001 — performance contract baseline - -## Files changed - -- `packages/node/meshnet_node/performance_contract.py` -- `tests/test_performance_contract.py` -- `.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md` -- `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json` - -## What this slice does - -- Locks the DGR-001 benchmark contract in code. -- Pins the architecture-aligned baseline to **DeepSeek-V2-Lite-Chat** (`deepseek2`). -- Uses the same model on both sides of the comparison: - - **safetensors:** `deepseek-ai/DeepSeek-V2-Lite-Chat` in **BF16** - - **GGUF:** `second-state/DeepSeek-V2-Lite-Chat-GGUF` in **Q2_K** -- Exposes a machine-readable JSON contract with: - - benchmark lanes for `transformers` safetensors and `llama.cpp` GGUF on **CPU** and **GPU** - - concurrency levels `1` and `4` - - the required metrics list - - an explicit stop condition for “no meaningful speed or fit benefit” -- Adds a deterministic stub benchmark report so the contract now has an executable report shape end to end. - -## Recent benchmark runner slice - -The runner currently uses a deterministic stub backend to exercise the comparison matrix without downloading a model. It emits: - -- `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json` -- `.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json` - -The report includes per-device comparisons for: - -- `transformers-safetensors-cpu` vs `llama-cpp-gguf-cpu` -- `transformers-safetensors-gpu` vs `llama-cpp-gguf-gpu` - -and records the memory metric (`rss_bytes` on CPU, `vram_bytes` on GPU), decode speedup, artifact ratio, and output drift. - -## Live endpoint CLI wiring - -The contract CLI can now drive the live endpoint runner. Passing one `--live-endpoint LANE_ID=URL` mapping per contract lane (plus `--live-benchmark-out`) invokes `run_real_model_endpoint_benchmark` against already-running OpenAI-compatible servers and writes the report using the same schema as the stub: - -```bash -PYTHONPATH=packages/node python -m meshnet_node.performance_contract \ - --live-endpoint transformers-safetensors-cpu=http://127.0.0.1:8001 \ - --live-endpoint llama-cpp-gguf-cpu=http://127.0.0.1:8002 \ - --live-endpoint transformers-safetensors-gpu=http://127.0.0.1:8003 \ - --live-endpoint llama-cpp-gguf-gpu=http://127.0.0.1:8004 \ - --live-benchmark-out .scratch/distributed-gguf-runtime/evidence/DGR-001/live-benchmark-report.json -``` - -`--live-model` overrides the model name sent in requests (defaults to the contract's safetensors repo). Without any `--live-endpoint` flags the CLI behaves exactly as before: it writes the contract JSON and, with `--benchmark-out`, the deterministic stub report. - -## Exact commands and real results - -### Targeted tests - -```bash -PYTHONPATH=packages/node pytest -q tests/test_performance_contract.py tests/test_route_session_benchmark.py -``` - -Result: `19 passed in 0.11s` - -### Contract artifact generation - -```bash -PYTHONPATH=packages/node python -m meshnet_node.performance_contract --json-out .scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json -``` - -Result: wrote `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json` - -### Python compile check - -```bash -python -m compileall packages/node/meshnet_node/performance_contract.py tests/test_performance_contract.py -``` - -Result: passed - -## Public relay smoke benchmark (2026-07-15) - -A real streamed request was run through the public tracker — **not** by connecting directly to the private node address: - -```text -https://meshnet.2.d-popov.com/v1/chat/completions - -> wss://meshnet.2.d-popov.com/ws - -> wss://meshnet.2.d-popov.com/rpc/7j77FsPY1evV8tuf-7000 - -> local CUDA node, Qwen/Qwen2.5-0.5B-Instruct layers 0-23 -``` - -The local public-tracker node had an expired proof and a wedged HTTP server. A graceful restart refreshed its CUDA capability proof in `336 ms`, restored `admitted`/`routable` status, and reconnected its relay endpoint. - -Measured streaming results after recovery: - -| metric | result | -| --- | ---: | -| warm-up TTFT | 420.80 ms | -| warm-up elapsed | 610.23 ms | -| p50 TTFT (3 runs) | 288.26 ms | -| p50 elapsed (3 runs) | 363.20 ms | -| tracker-recorded relay throughput | 58.18-65.25 tok/s | -| HTTP status | 200 for all runs | - -The tracker recorded `relay: true` and the local node ID `7j77FsPY-b32476219492` for each completion. Full redacted evidence is in `public-relay-smoke-benchmark.json`. - -The other connected node is still alive but **not routable** because its capability proof is stale. It must revalidate before a multi-node shard/relay test can run. - -## Limitations - -- This slice still uses a deterministic stub backend for the core comparison matrix. -- It now also includes a live endpoint runner, reachable from the CLI via `--live-endpoint`/`--live-benchmark-out`, that fans out one OpenAI-compatible request per lane when the caller provides endpoints; the CLI does not start those servers. -- It does **not** download or run a real model from within the repo. -- Real safetensors vs GGUF execution, TTFT/prefill/decode measurements, RSS/VRAM capture, and output-drift comparison are still to be implemented against the contract. - -## Compatibility notes - -- The contract stays on the DeepSeek2 family to remain close to the DeepSeek-V4-Flash end goal. -- A smaller non-DeepSeek model can still be used later for loader-plumbing smoke tests, but it does not replace this baseline. -- Model artifacts must stay on the mounted drive and not under `/home`. - -## Dependent-story handoff - -Next implementation work should attach to this contract and add the live benchmark runner that actually compares: - -1. current Transformers/safetensors recipe -2. whole-model llama.cpp GGUF recipe - -using the same model architecture/revision and the same prompt/context/concurrency settings. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json deleted file mode 100644 index 02d1562..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "benchmark_lanes": [ - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "cpu", - "id": "transformers-safetensors-cpu", - "recipe": "current safetensors recipe", - "runtime": "transformers" - }, - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "cpu", - "id": "llama-cpp-gguf-cpu", - "recipe": "whole-model GGUF recipe", - "runtime": "llama.cpp" - }, - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "gpu", - "id": "transformers-safetensors-gpu", - "recipe": "current safetensors recipe", - "runtime": "transformers" - }, - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "gpu", - "id": "llama-cpp-gguf-gpu", - "recipe": "whole-model GGUF recipe", - "runtime": "llama.cpp" - } - ], - "metrics": [ - "ttft_ms", - "prefill_tok_per_sec", - "decode_tok_per_sec", - "p50_latency_ms", - "p95_latency_ms", - "aggregate_throughput_tok_per_sec", - "rss_bytes", - "vram_bytes", - "artifact_bytes", - "failure_count", - "output_drift" - ], - "model_target": { - "architecture": "deepseek2", - "comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF", - "gguf_quant": "Q2_K", - "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", - "gguf_size_gb": 6.43, - "name": "DeepSeek-V2-Lite-Chat", - "rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.", - "safetensors_precision": "bfloat16", - "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat" - }, - "notes": [ - "Real model execution stays opt-in and must keep model artifacts on the mounted drive.", - "Use the tiny fallback only for loader plumbing smoke tests; it does not replace the architecture-aligned baseline." - ], - "schema_version": 1, - "stop_condition": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.", - "story_id": "DGR-001" -} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json deleted file mode 100644 index 7f3e272..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json +++ /dev/null @@ -1,247 +0,0 @@ -{ - "comparisons": { - "cpu": { - "artifact_bytes_ratio": 0.2048, - "decode_speedup": 2.3333, - "gguf_benefit": true, - "gguf_lane": "llama-cpp-gguf-cpu", - "memory_bytes_ratio": 0.2152, - "memory_metric": "rss_bytes", - "output_drift": 0.0, - "safetensors_lane": "transformers-safetensors-cpu", - "ttft_speedup": 1.8947 - }, - "gpu": { - "artifact_bytes_ratio": 0.2048, - "decode_speedup": 1.5294, - "gguf_benefit": true, - "gguf_lane": "llama-cpp-gguf-gpu", - "memory_bytes_ratio": 0.2273, - "memory_metric": "vram_bytes", - "output_drift": 0.0, - "safetensors_lane": "transformers-safetensors-gpu", - "ttft_speedup": 1.6154 - } - }, - "lanes": [ - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "cpu", - "id": "transformers-safetensors-cpu", - "output_tokens": [ - "mesh", - "activation", - "seam", - "baseline" - ], - "recipe": "current safetensors recipe", - "results": [ - { - "concurrency": 1, - "metrics": { - "aggregate_throughput_tok_per_sec": 6.0, - "artifact_bytes": 33715493273, - "decode_tok_per_sec": 6.0, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 166.6667, - "p95_latency_ms": 208.3334, - "prefill_tok_per_sec": 45.0, - "rss_bytes": 35433480192, - "ttft_ms": 1800.0, - "vram_bytes": 0 - } - }, - { - "concurrency": 4, - "metrics": { - "aggregate_throughput_tok_per_sec": 20.4, - "artifact_bytes": 33715493273, - "decode_tok_per_sec": 5.1, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 196.0784, - "p95_latency_ms": 245.098, - "prefill_tok_per_sec": 38.25, - "rss_bytes": 35433480192, - "ttft_ms": 2340.0, - "vram_bytes": 0 - } - } - ], - "runtime": "transformers" - }, - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "cpu", - "id": "llama-cpp-gguf-cpu", - "output_tokens": [ - "mesh", - "activation", - "seam", - "baseline" - ], - "recipe": "whole-model GGUF recipe", - "results": [ - { - "concurrency": 1, - "metrics": { - "aggregate_throughput_tok_per_sec": 14.0, - "artifact_bytes": 6904159928, - "decode_tok_per_sec": 14.0, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 71.4286, - "p95_latency_ms": 89.2858, - "prefill_tok_per_sec": 90.0, - "rss_bytes": 7623566950, - "ttft_ms": 950.0, - "vram_bytes": 0 - } - }, - { - "concurrency": 4, - "metrics": { - "aggregate_throughput_tok_per_sec": 47.6, - "artifact_bytes": 6904159928, - "decode_tok_per_sec": 11.9, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 84.0336, - "p95_latency_ms": 105.042, - "prefill_tok_per_sec": 76.5, - "rss_bytes": 7623566950, - "ttft_ms": 1235.0, - "vram_bytes": 0 - } - } - ], - "runtime": "llama.cpp" - }, - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "gpu", - "id": "transformers-safetensors-gpu", - "output_tokens": [ - "mesh", - "activation", - "seam", - "baseline" - ], - "recipe": "current safetensors recipe", - "results": [ - { - "concurrency": 1, - "metrics": { - "aggregate_throughput_tok_per_sec": 34.0, - "artifact_bytes": 33715493273, - "decode_tok_per_sec": 34.0, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 29.4118, - "p95_latency_ms": 36.7647, - "prefill_tok_per_sec": 850.0, - "rss_bytes": 4294967296, - "ttft_ms": 420.0, - "vram_bytes": 35433480192 - } - }, - { - "concurrency": 4, - "metrics": { - "aggregate_throughput_tok_per_sec": 115.6, - "artifact_bytes": 33715493273, - "decode_tok_per_sec": 28.9, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 34.6021, - "p95_latency_ms": 43.2526, - "prefill_tok_per_sec": 722.5, - "rss_bytes": 4294967296, - "ttft_ms": 546.0, - "vram_bytes": 35433480192 - } - } - ], - "runtime": "transformers" - }, - { - "concurrency_levels": [ - 1, - 4 - ], - "device": "gpu", - "id": "llama-cpp-gguf-gpu", - "output_tokens": [ - "mesh", - "activation", - "seam", - "baseline" - ], - "recipe": "whole-model GGUF recipe", - "results": [ - { - "concurrency": 1, - "metrics": { - "aggregate_throughput_tok_per_sec": 52.0, - "artifact_bytes": 6904159928, - "decode_tok_per_sec": 52.0, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 19.2308, - "p95_latency_ms": 24.0385, - "prefill_tok_per_sec": 640.0, - "rss_bytes": 1610612736, - "ttft_ms": 260.0, - "vram_bytes": 8053063680 - } - }, - { - "concurrency": 4, - "metrics": { - "aggregate_throughput_tok_per_sec": 176.8, - "artifact_bytes": 6904159928, - "decode_tok_per_sec": 44.2, - "failure_count": 0, - "output_drift": 0.0, - "p50_latency_ms": 22.6244, - "p95_latency_ms": 28.2805, - "prefill_tok_per_sec": 544.0, - "rss_bytes": 1610612736, - "ttft_ms": 338.0, - "vram_bytes": 8053063680 - } - } - ], - "runtime": "llama.cpp" - } - ], - "model_target": { - "architecture": "deepseek2", - "comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF", - "gguf_quant": "Q2_K", - "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", - "gguf_size_gb": 6.43, - "name": "DeepSeek-V2-Lite-Chat", - "rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.", - "safetensors_precision": "bfloat16", - "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat" - }, - "schema_version": 1, - "source": "stub-backend", - "stop_condition": { - "gguf_benefit": true, - "text": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.", - "triggered": false - }, - "story_id": "DGR-001" -} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-002/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-002/README.md deleted file mode 100644 index 15da8c8..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-002/README.md +++ /dev/null @@ -1,176 +0,0 @@ -# DGR-002 — Versioned gRPC Shard protocol: evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **synthetic-unit** (schema round-trip + cross-language protobuf -compatibility). No model download, no GPU, no network, no API credits. - -## Summary - -Added the versioned Protocol Buffers schema that is the semantic contract between -Python and C++ Shards (ADR-0024), plus reproducible Python and C++ code -generation/build wiring and generated-schema round-trip + compatibility tests in -**both** languages. The schema defines one long-lived bidirectional gRPC stream -per Route Session Activation Seam, bounded prefill chunking, a small decode fast -path, and a versioned named-tensor bundle carrying every required identifier. - -No existing runtime code was modified — this story is purely additive (a new -`.proto`, a `native_protocol` loader package, C++ build wiring, and one new test -module). Generated stubs are produced on demand into gitignored `build/` -directories, so nothing generated is committed. - -## Files changed (all new) - -- `packages/node/native/proto/shard_runtime.proto` — the schema (package - `meshnet.shard.v1`, proto3). Service `ShardRuntime` with `GetCapability`, - `Health`, `ActivateSession` (bidi stream), `Release`, `Cancel`. -- `packages/node/meshnet_node/native_protocol/__init__.py` — reproducible - on-demand `grpc_tools.protoc` codegen + loader (`load()`, `load_grpc()`) and - shared bundle helpers (`compute_checksum`, `verify_checksum`, `fragment_tensor`, - `reassemble_tensor`). -- `packages/node/native/scripts/generate_python.py` — standalone reproducible - Python generation (self-contained; does not import `meshnet_node`). -- `packages/node/native/scripts/generate_cpp.sh` — reproducible C++ generation - (message stubs always; gRPC service stubs when `grpc_cpp_plugin` is present). -- `packages/node/native/CMakeLists.txt` — C++ build wiring; works with both - CONFIG-mode (`protobuf::libprotobuf`/`protobuf::protoc`) and CMake's - `FindProtobuf` module. -- `packages/node/native/tests/roundtrip_test.cpp` — C++ round-trip / compat test - (`--selftest`, `--read`, `--write`). -- `tests/test_native_shard_protocol.py` — Python round-trip + compatibility tests - and the Python↔C++ cross-language driver. - -## Acceptance criteria → evidence - -- **Capability/health/session-stream/release/cancellation schema** — the - `ShardRuntime` service's five RPCs; `test_capability_and_health_round_trip`, - `test_session_stream_carries_open_prefill_decode_release_cancel`. -- **One long-lived bidi stream per Activation Seam with deadlines, cancellation, - flow control, structured errors** — `rpc ActivateSession (stream ...) returns - (stream ...)`. Deadlines: gRPC call deadline on direct transport, plus - `SessionOpen.deadline_unix_nanos` for relay-carried frames. Cancellation: - `Cancel` RPC and in-stream `CancelRequest`/`PHASE_CANCEL`. Flow control: - `FlowControl` frames (credits + in-flight byte/message caps). Structured errors: - `Status` (canonical code, message, `RetryClass`, details). Verified by - `test_session_response_carries_structured_status_and_results`. -- **Bounded prefill chunking + small decode fast path** — `PrefillChunk` - (`chunk_index`/`chunk_count`/`final_chunk`, `SessionOpen.max_prefill_tokens_per_chunk`) - and `DecodeStep` (minimal single-bundle path). Bounded fragments via - `SessionOpen.max_fragment_bytes` and `fragment_tensor(...)`. -- **Carries schema version, work ID, Route Session ID, route epoch, - artifact/recipe fingerprint, shard range/effective start, phase, position, - idempotency step, cache expectation, compression, checksum** — all on - `MessageHeader` (+ `ArtifactFingerprint.runtime_recipe_fingerprint`, - `ShardRange.effective_start_layer`). Verified field-by-field by - `test_message_header_carries_every_required_field`. -- **Versioned named-tensor bundle (name, shape, dtype, byte order, fragments)** — - `TensorBundle`/`NamedTensor`/`TensorFragment`; - `test_named_tensor_bundle_describes_shape_dtype_byteorder_and_fragments`, - `test_fragment_and_reassemble_round_trip_with_checksums`. -- **Round-trip + compatibility tests in Python and C++** — Python: - `tests/test_native_shard_protocol.py` (11 tests). C++: `roundtrip_test.cpp` - built via CMake; cross-language driver `test_cross_language_roundtrip_python_and_cpp` - exercises Python→C++ and C++→Python in both directions. -- **Targeted pytest** — `11 passed, 1 skipped` (default env); `12 passed` with the - C++ toolchain on PATH. -- **compileall packages tests** — exit 0. -- **git diff --check** — clean. -- **Deterministic / download-free / credit-free / GPU-free** — all tests are pure - protobuf serialization; the C++ path uses only local compilers. -- **Full deterministic pytest** — `704 passed, 14 skipped, 11 failed`. The 11 - failures are pre-existing and unrelated (see below). - -## Commands and real results - -See `commands.txt` for the exact command list. Key results: - -- `python packages/node/native/scripts/generate_python.py` → - `shard_runtime_pb2.py: ok`, `shard_runtime_pb2_grpc.py: ok`. -- `pytest tests/test_native_shard_protocol.py -q` → **11 passed, 1 skipped** - (skip reason: `C++ toolchain unavailable: cmake not found on PATH`). -- With `/tmp/pbsrc/install/bin` (protoc 33.1) and `.venv/bin` (cmake) on PATH and - `CMAKE_PREFIX_PATH=/tmp/pbsrc/install`: - - `generate_cpp.sh` → `shard_runtime.pb.cc`, `shard_runtime.pb.h` - (grpc service stubs skipped: `grpc_cpp_plugin` absent). - - `cmake -S ... -B ...` + `cmake --build ...` → build OK. - - `shard_protocol_roundtrip_test --selftest` → `selftest ok (128 bytes)`, exit 0. - - `ctest` → `1/1 Test #1: shard_protocol_roundtrip ... Passed`. - - `pytest ...::test_cross_language_roundtrip_python_and_cpp -q` → **1 passed** - (Python serializes → C++ parses & verifies → C++ serializes → Python parses - & verifies). -- `compileall -q packages tests` → exit 0. -- `git diff --check` → clean. - -## Pre-existing unrelated failures (full-suite) - -`pytest -q` on the full tree reports 11 failures, all in tracker routing / -dynamic routing / manual route benchmark / toploc calibration — none import the -Shard protocol. Clean-tree reproduction: with **all DGR-002 files moved aside** -(`git status` shows only the pre-existing `.ralph-tui/config.toml` deletion), -re-running exactly these tests gives `11 failed, 3 passed` — identical failures. -They exist on the `ralph/distributed-gguf-runtime` branch independent of this -story. The full list is in `results.json.preexisting_unrelated_failures`. - -Note: the earlier `progress.md` (RCR-001, on master) recorded a different set of -6 optional-dependency failures (zstandard, langchain_openai). Those did **not** -recur here; this environment has those deps. The 11 above are branch-local -routing/benchmark failures, not environmental. - -## Limitations and deferred work - -- **C++ toolchain is host-provided, not vendored.** The default test env has no - `protoc`/`cmake`/protobuf C++ headers on PATH, so the C++ cross-language test - **skips** by default (explicit skip reason). It was executed for this evidence - using an ephemeral from-source protobuf 33.1 install at `/tmp/pbsrc/install` - plus the `.venv` cmake. DGR-004/DGR-008 should pin the C++ protobuf/gRPC - toolchain (upstream commit + reproducible fetch/build) so this test runs in CI - without relying on an ad-hoc `/tmp` install. -- **gRPC C++ service stubs not built here.** `grpc_cpp_plugin` is absent, so - `generate_cpp.sh` produced message stubs only. The round-trip test needs only - message serialization; the service stubs are DGR-008's concern. -- **No live gRPC transport yet.** This story delivers the schema + serialization - contract and generation/build wiring only. Channel setup, the bidi stream - server/client, deadlines/cancellation propagation over a real HTTP/2 channel, - and relay framing are DGR-008/DGR-009. -- **Protobuf runtime version skew.** Python runtime is pip protobuf 7.35.1; the - C++ side used protoc 33.1. Protobuf wire format is stable across these, and the - cross-language round-trip confirms interop; version pinning is deferred to the - toolchain-pinning stories. - -## Compatibility / migration notes - -- proto3 with a 0-valued `*_UNSPECIFIED` member on every enum and never-reused - field numbers. Forward compatibility (unknown-field preservation) is verified - behaviourally by `test_unknown_fields_are_preserved_for_forward_compatibility` - — note protobuf 7.x's upb backend does not implement the `UnknownFields()` - introspection accessor, so the test asserts the observable re-serialization - outcome instead. Backward defaults verified by - `test_defaults_are_stable_for_backward_compatibility`. -- Wire schema version is `SchemaVersion.SCHEMA_VERSION_1` (int 1), also exposed as - `meshnet_node.native_protocol.SCHEMA_VERSION`. - -## Handoff for dependent stories - -- **DGR-003 (recipe/fingerprint):** populate `ArtifactFingerprint` - (`model_id`, `revision`, `artifact_hash`, `quantization`, - `runtime_recipe_fingerprint`). Admission compares these before activation; a - mismatch is a fatal `Status` (`RetryClass.RETRY_CLASS_FATAL`). -- **DGR-004 (llama.cpp pin) / DGR-008 (C++ worker):** pin the C++ - protobuf + gRPC toolchain and add `grpc_cpp_plugin`; then `generate_cpp.sh` - emits service stubs and the CMake target can link gRPC. Implement the - `ShardRuntime` servicer; map `(route_session_id, route_epoch)` to an isolated - llama sequence. Use `SessionOpen` for stream-scoped bounds and `FlowControl` - for backpressure. -- **DGR-009 (Meshnet integration/relay):** the relay may carry serialized - `SessionActivation`/`SessionResponse` frames as opaque binary; use the in-message - `deadline_unix_nanos`, `CancelRequest`, and `FlowControl` since gRPC call - metadata is lost over relay. -- **Loader usage:** `from meshnet_node import native_protocol as proto; - pb2 = proto.load()`. Stubs regenerate automatically when the `.proto` changes - (mtime check). `proto.load_grpc()` returns the service stubs (needs the `grpc` - runtime). -- **Gotcha:** the `.venv` installs the meshnet packages editable via a PEP 660 - meta-path finder pointing at the **main** checkout. Import the worktree copy by - ensuring the worktree `packages/node` is on `sys.path` first (conftest already - does this for pytest); standalone tooling must derive paths from `__file__` and - not `import meshnet_node` (why `generate_python.py` is self-contained). diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-002/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-002/commands.txt deleted file mode 100644 index b3b75c7..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-002/commands.txt +++ /dev/null @@ -1,40 +0,0 @@ -# DGR-002 reproduction commands (run from repo root, project .venv = Python 3.14). - -# 1. Generate Python stubs (reproducible; writes to gitignored build/ dir). -.venv/bin/python packages/node/native/scripts/generate_python.py - -# 2. Python round-trip + compatibility tests (default env; C++ test skips if -# cmake/protoc absent). -.venv/bin/python -m pytest tests/test_native_shard_protocol.py -q -# => 11 passed, 1 skipped - -# 3. Quality gates. -.venv/bin/python -m compileall -q packages tests # exit 0 -git diff --check # clean - -# 4. Full deterministic suite (records pre-existing unrelated failures). -.venv/bin/python -m pytest -q -# => 704 passed, 14 skipped, 11 failed (all pre-existing, unrelated; see below) - -# 5. Clean-tree reproduction of the 11 pre-existing failures (DGR-002 files moved -# aside): same 11 fail => not caused by this story. - -# --- C++ / cross-language (requires protoc + protobuf C++ dev + cmake) -------- -# On this host a from-source protobuf 33.1 toolchain lives under /tmp/pbsrc/install -# and cmake ships in the .venv. To execute the C++ test instead of skipping it: -export PATH="/tmp/pbsrc/install/bin:$PWD/.venv/bin:$PATH" -export CMAKE_PREFIX_PATH="/tmp/pbsrc/install:$CMAKE_PREFIX_PATH" - -# 6. Generate C++ stubs (message stubs always; gRPC service stubs if -# grpc_cpp_plugin present). -packages/node/native/scripts/generate_cpp.sh - -# 7. Standalone C++ build + selftest + ctest. -cmake -S packages/node/native -B packages/node/native/build/cpp -cmake --build packages/node/native/build/cpp --target shard_protocol_roundtrip_test -packages/node/native/build/cpp/shard_protocol_roundtrip_test --selftest # "selftest ok (128 bytes)" -(cd packages/node/native/build/cpp && ctest --output-on-failure) # 1/1 passed - -# 8. Cross-language Python<->C++ round-trip via the pytest driver (now runs, not skips). -.venv/bin/python -m pytest tests/test_native_shard_protocol.py::test_cross_language_roundtrip_python_and_cpp -q -# => 1 passed diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-002/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-002/results.json deleted file mode 100644 index 9e9725c..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-002/results.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "task": "DGR-002", - "title": "Adopt the versioned gRPC Shard protocol", - "schema": { - "proto": "packages/node/native/proto/shard_runtime.proto", - "package": "meshnet.shard.v1", - "syntax": "proto3", - "schema_version": 1, - "service": "ShardRuntime", - "rpcs": ["GetCapability", "Health", "ActivateSession", "Release", "Cancel"], - "streaming_seam": "ActivateSession (bidirectional stream)" - }, - "toolchain": { - "python": "3.14.6", - "protobuf_runtime_python": "7.35.1", - "grpcio": "1.82.1", - "grpcio_tools": "1.82.1", - "cpp_protoc": "libprotoc 33.1", - "cpp_protobuf_toolchain": "/tmp/pbsrc/install (from-source protobuf 33.1, ephemeral host build)", - "cmake": "4.4.0 (.venv)", - "cxx": "g++ (system)" - }, - "generation": { - "python_cmd": "python packages/node/native/scripts/generate_python.py", - "python_out": "packages/node/native/build/python/shard_runtime_pb2{,_grpc}.py (gitignored)", - "cpp_cmd": "packages/node/native/scripts/generate_cpp.sh", - "cpp_out": "packages/node/native/build/cpp-gen/shard_runtime.pb.{h,cc} (gitignored)", - "cpp_build": "cmake -S packages/node/native -B && cmake --build " - }, - "tests": { - "python_default_env": {"passed": 11, "skipped": 1, "note": "C++ cross-language test skips when cmake/protoc absent"}, - "python_with_cpp_toolchain": {"passed": 12, "skipped": 0}, - "cpp_selftest_bytes": 128, - "cpp_ctest": "1/1 passed", - "cross_language": "Python->C++ and C++->Python round-trip verified in both directions" - }, - "quality_gates": { - "targeted_pytest": "11 passed, 1 skipped (default); 12 passed with C++ toolchain", - "compileall_packages_tests": "exit 0", - "git_diff_check": "clean", - "full_pytest": { - "passed": 704, - "skipped": 14, - "failed": 11, - "failed_are_preexisting_unrelated": true, - "clean_tree_reproduction": "same 11 fail with all DGR-002 files removed (11 failed, 3 passed)" - } - }, - "preexisting_unrelated_failures": [ - "tests/test_dynamic_routing.py::test_admin_can_replace_a_served_model_and_release_it", - "tests/test_manual_route_benchmark.py::test_pinned_route_uses_named_node", - "tests/test_manual_route_benchmark.py::test_unknown_route_node_is_400", - "tests/test_manual_route_benchmark.py::test_invalid_route_shape_is_400", - "tests/test_manual_route_benchmark.py::test_clients_without_route_are_unaffected", - "tests/test_manual_route_benchmark.py::test_benchmark_records_one_and_two_node_routes", - "tests/test_toploc_calibration_dispatch.py::test_calibration_run_dispatches_only_solo_capable_nodes", - "tests/test_toploc_calibration_dispatch.py::test_calibration_run_persists_corpus_and_results_endpoint_reports_it", - "tests/test_toploc_calibration_dispatch.py::test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed", - "tests/test_tracker_routing.py::test_torch_node_applies_tracker_load_shard_directive", - "tests/test_tracker_routing.py::test_shard_heal_cycle_surviving_node_covers_dead_peers_gap" - ], - "evidence_kind": "synthetic-unit (schema round-trip + cross-language protobuf; no model, no GPU, no network, no API credits)" -} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-003/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-003/README.md deleted file mode 100644 index c7a67ae..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-003/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# DGR-003 — Exact artifact and runtime-recipe identity: evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **synthetic-unit + repo checks**. No model download, no GPU, no network, no API credits. - -## Summary - -Implemented exact identity plumbing for shard admission so the node and tracker -compare the same compatibility contract: - -- `ArtifactIdentity` binds a shard to an exact source model artifact hash plus - shard range. -- `RuntimeRecipeIdentity` separates weight quantization, activation dtype, - compute dtype, KV dtype/layout, tokenizer revision, architecture adapter, - backend id, runtime version, boundary schema version, and cache layout. -- `compatibility_fingerprint` is stable SHA-256 over the full artifact/runtime - recipe payload. -- Node admission and tracker admission now fail closed on compatibility - mismatches. -- Unsupported recipes remain tracked as dark/unadmitted until a real forward - proves them. - -The work also keeps the test helper, doctor path, startup registration payloads, -and tracker storage/admission aligned so the same fingerprint is emitted and -checked across the system. - -## Files changed - -- `packages/node/meshnet_node/runtime_recipe.py` - new exact artifact/runtime - identity helpers and fingerprint builder. -- `packages/node/meshnet_node/capability.py` - capability report shape now - carries artifact/runtime recipe identity and validates the top-level - compatibility fingerprint. -- `packages/node/meshnet_node/admission.py` - fail-closed admission on - compatibility fingerprint mismatch. -- `packages/node/meshnet_node/doctor.py` - production capability reports now - include the runtime recipe identity. -- `packages/node/meshnet_node/testing.py` - test report builder now mirrors the - production fingerprint fields. -- `packages/node/meshnet_node/startup.py` - registration payload now includes - the compatibility fingerprint. -- `packages/tracker/meshnet_tracker/capability.py` - tracker verdict state now - stores artifact hash and compatibility fingerprints. -- `packages/tracker/meshnet_tracker/server.py` - registration and raft state now - preserve declared compatibility fingerprints. -- `tests/test_node_capability.py` - identity shape and fingerprint regression - tests. -- `tests/test_node_admission.py` - fail-closed admission regression tests. -- `tests/test_tracker_capability_admission.py` - tracker compatibility mismatch - regression tests. - -## Commands and real results - -- `python -m compileall packages tests` -> exit 0. -- `pytest -q tests/test_node_capability.py` -> `48 passed in 0.09s`. -- `pytest -q tests/test_node_admission.py` -> `20 passed in 0.11s`. -- `pytest -q tests/test_tracker_capability_admission.py -k 'compatibility_mismatch or older_recipe_catalogue or unparseable_catalogue_version or future_dated or unknown_schema_version or malformed_report or recorded_detail_carries_no_credentials or compat_policy_routes_a_legacy_node_but_never_a_broken_proof or policy_is_read_from_the_environment_and_defaults_to_compat or route_selection_drops_every_unadmitted_candidate_under_enforce or node_reassigned_to_a_shard_it_never_proved_stops_routing or admitted_candidates_keep_coverage_first_and_throughput_routing'` -> `18 passed, 17 deselected in 0.11s`. -- `git diff --check` -> exit 0. -- `pytest -q` -> not green in this sandbox. Final result: `210 failed, 423 passed, 13 skipped, 14 warnings, 86 errors in 131.34s`. - -## Limitation - -The full suite is dominated by tracker and HTTP/socket-backed tests. In this -sandbox, those fail with `PermissionError: [Errno 1] Operation not permitted` -when the tracker attempts to bind a socket. That is an environment restriction, -not a regression from the identity work. The pure unit slices above pass. - -## Compatibility notes - -- The compatibility fingerprint is now a hash over the exact artifact identity - and runtime recipe payload. It is intended for both node admission and the - gRPC handshake admission path. -- Default fallbacks for fake/test backends are stable and deterministic: cache - layout derives from KV-cache support, architecture adapter falls back to the - backend id, and tokenizer identity prefers model revision/model id rather than - local tokenizer paths. - -## Handoff for dependent stories - -- DGR-004 / DGR-008 can reuse `runtime_recipe.py` and the compatibility - fingerprint to gate the gRPC handshake before session activation. -- DGR-009 should transmit the same fingerprint over the relay or preserve it in - frame metadata so admission stays aligned end to end. -- Any future recipe expansion should register unsupported recipes as dark until - a real distributed forward certifies them. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-004/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-004/README.md deleted file mode 100644 index 0f01d63..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-004/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# DGR-004 — reproducible pinned llama.cpp patch stack evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **synthetic-build + repo checks**. No model download, no GPU, -no network fetch during validation, no API credits. - -## Summary - -Implemented the reproducible source-dependency boundary for llama.cpp and kept -the fork seam narrow and auditable: - -- exact pinned upstream commit and repository metadata -- numbered patch stack isolated under `packages/node/native/llama/patches/` -- build script that verifies the pin, applies the patch stack, stages notices, - and compiles a standalone worker scaffold without manual source copying -- upstream file assumptions and fail-closed pin checking -- license/attribution preservation by staging upstream `LICENSE` and `AUTHORS` -- clean rebuild smoke test that only uses a fake local checkout and does not - download a model - -The native smoke path is intentionally minimal in this story. It proves the -reproducible source dependency and build seam without pulling Meshnet protocol -code into llama.cpp. - -## Files changed - -- `packages/node/native/llama/UPSTREAM_COMMIT` -- `packages/node/native/llama/UPSTREAM_REPOSITORY` -- `packages/node/native/llama/UPSTREAM_ASSUMPTIONS.md` -- `packages/node/native/llama/README.md` -- `packages/node/native/llama/patches/0001-add-meshnet-worker-scaffold.patch` -- `packages/node/native/llama/templates/meshnet_worker.cpp` -- `packages/node/native/scripts/build_llama_worker.sh` -- `tests/test_llama_worker_build.py` - -## Exact commands and real results - -### Native smoke build against a fake pinned checkout - -```bash -tmpdir=$(mktemp -d) -mkdir -p "$tmpdir/llama.cpp" -printf 'MIT\n' > "$tmpdir/llama.cpp/LICENSE" -printf 'AUTHORS\n' > "$tmpdir/llama.cpp/AUTHORS" -printf '# placeholder\n' > "$tmpdir/llama.cpp/CMakeLists.txt" -printf '%s\n' 'b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac' > "$tmpdir/llama.cpp/.meshnet-upstream-commit" -git init -q "$tmpdir/llama.cpp" -packages/node/native/scripts/build_llama_worker.sh \ - --source-dir "$tmpdir/llama.cpp" \ - --build-dir "$tmpdir/build" -``` - -Result: - -- `meshnet worker scaffold ok` -- `upstream commit: b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac` -- `patchset version: 0001` -- `build ok: /tmp/.../build/meshnet_worker` - -### Targeted pytest - -```bash -python -m pytest -q tests/test_llama_worker_build.py -``` - -Result: `1 passed in 0.53s` - -### Python compile check - -```bash -python -m compileall -q packages tests -``` - -Result: exit 0 - -### Diff hygiene - -```bash -git diff --check -``` - -Result: exit 0 - -### Full deterministic pytest - -```bash -python -m pytest -q -``` - -Result: `424 passed, 13 skipped, 210 failed, 86 errors in 131.04s` - -The failures are pre-existing sandbox socket failures in tracker/HTTP-backed -tests. Representative error: - -- `PermissionError: [Errno 1] Operation not permitted` when the tracker tries - to bind a socket. - -This matches the previously observed environment limitation in the DGR-002 and -DGR-003 evidence and is unrelated to the llama.cpp pin/build scaffold. - -## Limitations - -- The sandbox does not provide `cmake`, so the smoke build uses the available - direct C++ compiler path (`g++` here) instead of a CMake-generated target. -- The pinned upstream source was not fetched from GitHub during validation. - The script supports fetching the exact commit when network access is - available, but the validation run used a fake local checkout to keep the test - deterministic and model-free. -- The patch stack in this story is deliberately narrow and additive. It creates - a worker scaffold and build seam, not the final llama.cpp runtime patches. - -## Compatibility notes - -- The exact upstream pin is `b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac`. -- The build script fails closed if the checkout pin differs from that commit or - if the expected upstream files (`LICENSE`, `AUTHORS`, `CMakeLists.txt`) are - missing. -- The patch stack is isolated from Meshnet networking code and can be applied - to a clean pinned checkout before later worker stories extend the scaffold. -- Upstream attribution notices are preserved in the build output by copying the - staged `LICENSE` and `AUTHORS` files into `build/.../upstream-notices/`. - -## Dependent-story handoff - -- DGR-008 can replace the scaffold source with the real supervised C++ worker - while keeping the same pin metadata, patch stack, and build script boundary. -- DGR-005 and later native stories should keep using the same exact pin so the - worker seam remains reproducible while range-loading and session logic are - added. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-005/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-005/README.md deleted file mode 100644 index 9c24446..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-005/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# DGR-005 — dense-Llama range-aware GGUF ownership evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **synthetic-unit + repo checks**. No model download, no GPU, no network, no API credits. - -## Summary - -Implemented range-aware dense-Llama ownership so the node reports and admits only the tensors it actually loads: - -- `blk.N.*` tensors are selected strictly by assigned layer range. -- Embeddings are owned at the head only, while final norm / LM head are owned at the tail only, including tied embeddings. -- Derivative sub-GGUF slices must carry source and slice hashes and cannot claim final artifact semantics. -- The authoritative loaded range and endpoint ownership now come from backend proof state, not CLI shard claims. -- Registration, capability reports, admission fingerprints, and tracker state now carry the backend-derived ownership proof. - -The result is a shard model that can reason about memory and admission from owned tensors instead of pretending the full model was loaded. - -## Files changed - -- `packages/node/meshnet_node/gguf_ownership.py` - dense-Llama tensor selection and authoritative ownership helpers. -- `packages/node/meshnet_node/capability.py` - shard reports now carry endpoint ownership and parse it round-trip. -- `packages/node/meshnet_node/doctor.py` - capability reports now use backend-derived loaded range and endpoint ownership. -- `packages/node/meshnet_node/testing.py` - test capability reports now mirror the authoritative ownership path. -- `packages/node/meshnet_node/admission.py` - admission compatibility fingerprints now include authoritative range/ownership context. -- `packages/node/meshnet_node/model_backend.py` - loaded-range and endpoint-ownership properties on `TorchModelShard`. -- `packages/node/meshnet_node/startup.py` - registration payloads now use the proof-driven shard range. -- `packages/tracker/meshnet_tracker/capability.py` - tracker capability state preserves endpoint ownership. -- `tests/test_gguf_ownership.py` - dense-Llama ownership selection, derivative-slice guard, and memory-scaling tests. -- `tests/test_node_capability.py` - capability report ownership round-trip tests. -- `tests/test_node_admission.py` - backend-loaded range beats CLI claim regression tests. -- `tests/test_tracker_capability_admission.py` - tracker capability proof parsing tests. - -## Exact commands and real results - -### Targeted pytest slices - -```bash -python -m pytest -q tests/test_gguf_ownership.py tests/test_node_capability.py tests/test_node_admission.py -``` - -Result: `73 passed` - -```bash -python -m pytest -q tests/test_tracker_capability_admission.py -k 'test_a_passing_report_that_covers_the_registration_is_admitted or test_a_missing_report_is_absent_not_admitted or test_a_failed_report_is_recorded_as_failed or test_a_report_for_a_different_model_is_a_model_mismatch or test_a_report_for_a_different_shard_is_a_shard_mismatch or test_a_report_for_a_different_recipe_than_the_node_declares_is_a_recipe_mismatch or test_a_report_for_a_different_compatibility_fingerprint_is_a_compatibility_mismatch or test_an_older_recipe_catalogue_is_incompatible or test_an_unparseable_catalogue_version_is_incompatible or test_a_stale_report_is_not_admitted or test_a_future_dated_report_is_not_admitted or test_a_report_from_an_unknown_schema_version_is_invalid or test_a_malformed_report_is_invalid_and_never_admitted or test_recorded_detail_carries_no_credentials_from_node_diagnostics or test_compat_policy_routes_a_legacy_node_but_never_a_broken_proof or test_the_policy_is_read_from_the_environment_and_defaults_to_compat' -``` - -Result: `22 passed, 13 deselected` - -### Python compile check - -```bash -python -m compileall -q packages tests -``` - -Result: exit 0 - -### Diff hygiene - -```bash -git diff --check -``` - -Result: exit 0 - -### Full deterministic pytest - -```bash -python -m pytest -q -``` - -Result: `211 failed, 428 passed, 13 skipped, 14 warnings, 86 errors in 135.03s` - -The failing set is not caused by this story. The dominant environment issues were: - -- tracker and HTTP/socket-backed tests fail with `PermissionError: [Errno 1] Operation not permitted` when the tracker tries to bind sockets in this sandbox -- native protocol tests fail early with a protobuf runtime/gencode mismatch: generated code expects protobuf 7.35.0 while the installed runtime is 6.33.6 - -## Limitations - -- This evidence is intentionally deterministic and model-free. -- The memory-scaling check is synthetic: it validates that owned tensor bytes scale with selected tensors, not a live GGUF download. -- Native C++ code was not changed by this story, so the pinned llama.cpp build validation remains covered by DGR-004 rather than repeated here. - -## Compatibility notes - -- Dense-Llama ownership is range-first: the shard interior is `blk.N.*`, and endpoint tensors are only attributed to the head or tail owner as appropriate. -- Derivative GGUF slices are explicitly not final artifacts; they must preserve source and slice hashes if used as a temporary compatibility bridge. -- The model proof path is authoritative for reported range and endpoint ownership, so operator CLI claims no longer control what the node advertises. -- Admission and tracker state now consume the same proof-derived ownership shape, keeping capability reports aligned end to end. - -## Handoff for dependent stories - -- DGR-006 can reuse `gguf_ownership.py` and the new capability fields to wire the shard protocol to proof-derived ownership without re-deriving tensor names. -- DGR-008 and later routing work should continue to treat endpoint ownership as metadata and `blk.N.*` ownership as the core range contract. -- If a future temporary slice path is needed, it should keep source/slice hashes visible and avoid claiming final-artifact semantics until a real proof exists. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-006/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-006/README.md deleted file mode 100644 index bfbfdb2..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-006/README.md +++ /dev/null @@ -1,203 +0,0 @@ -# DGR-006 — Architecture-defined boundary input/output: evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **synthetic-unit** (pure-numpy dense-Llama reference + boundary -contract). No model download, no GPU, no torch, no network, no API credit. - -## Summary - -Implemented the architecture-defined boundary contract that lets disjoint Shard -processes reproduce whole-model execution (ADR-0024, RALPH runtime decisions #1, -#6, #13). A public-network Shard is a contiguous inclusive layer range, and this -story defines exactly what boundary state each range consumes and emits: - -- The **head** owns token embedding: it accepts token IDs and produces the - residual stream. It refuses an upstream boundary bundle. -- **Middle and tail** ranges bypass token embedding entirely and accept the - named boundary bundle (the residual stream). They refuse token IDs. -- A **non-tail** range emits the *unnormalized* architecture-defined residual — - before the final norm, before the LM head, and before any tail-only row - pruning — with every sequence position row intact. -- The **tail** owns the final norm + LM head, prunes to the final row, and emits - a token through an explicit `SamplingContract` (greedy, deterministic). -- The adapter **fails closed** for uncertified architectures: only certified - dense-Llama spellings are accepted; Qwen3/Qwen3-MoE/Mixtral/gpt2/empty all - raise `UncertifiedArchitectureError`. - -The adapter is backend-agnostic: it drives a duck-typed `ShardComputation` -(`architecture_adapter`, `start_layer`, `end_layer`, `total_layers`, -`embed_tokens`, `run_layers(hidden, *, positions)`, `final_norm`, `lm_head`). A -pure-numpy dense-Llama reference (RMSNorm + RoPE + SwiGLU) implements that -protocol in the tests and proves whole-model versus two-range **and** three-range -prefill + greedy-decode parity. torch/transformers are not installed in the -default `.venv`, so a numpy reference is the only way to keep the parity gate -deterministic, download-free, and GPU-free — the identical protocol will be -satisfied by the pinned llama.cpp worker (DGR-008) and the PyTorch backend. - -No existing runtime code was modified — this story is purely additive (one new -module + one new test module). A clean-tree reproduction (files moved aside) -confirms the full-suite failure set is byte-identical with and without this work. - -## Files changed (all new) - -- `packages/node/meshnet_node/boundary_adapter.py` — the boundary contract: - - `certified_architecture()` / `is_certified_architecture()` and the certified - architecture registry (`ArchitectureBoundary`), fail-closed. - - `ShardRole` + `role_for_range()` (head/middle/tail/full). - - `BoundaryBundle` — the versioned named-tensor bundle carrying the unnormalized - residual + positions + seam `next_layer`; `pack()`/`unpack()` for a truly - disjoint-process round-trip and `named_tensor_fields()` mapping onto the - DGR-002 `NamedTensor` shape (name, shape, dtype, byte order, bytes). - - `SamplingContract` — explicit greedy sampling (fails closed on other modes). - - `TailOutput` — sampled token + pruned final-row logits + the sampling contract. - - `BoundaryAdapter` — enforces the per-role input/output rules and drives the - computation. -- `tests/test_boundary_adapter.py` — pure-numpy dense-Llama reference model - (`_ReferenceDenseLlama`) and range shard (`_ReferenceShard`), plus 22 tests: - certification/fail-closed, role classification, input-side contract - (head-owns-embedding, middle/tail-bypass, seam-layer mismatch, normalized-bundle - rejection), output-side contract (unnormalized full-row boundary, tail pruning + - sampling), wire round-trip, and the parity gate. - -## Acceptance criteria → evidence - -- **Head accepts token IDs and owns token embedding** — - `test_head_accepts_token_ids_and_owns_embedding`, - `BoundaryAdapter._ingest_tokens` (head requires token IDs, refuses a bundle). -- **Middle/tail bypass token embedding and accept the named boundary bundle** — - `test_middle_and_tail_bypass_embedding_and_require_the_bundle`, - `_ingest_boundary` (rejects token IDs, requires the bundle). -- **Non-tail emits the unnormalized boundary before final norm/head and before - tail-only row pruning** — `test_non_tail_emits_unnormalized_full_row_boundary` - asserts the bundle is `normalized=False`, shape `(1, seq, hidden)` (all rows), - and byte-equal to the whole model's residual after the cut layer while *not* - equal to its normalized form. `_emit_boundary`. -- **Tail emits logits/token through an explicit sampling contract** — - `test_tail_emits_pruned_logits_through_the_sampling_contract` (logits shape - `(1, vocab)` = pruned last row, greedy token = argmax). `_emit_tail`, - `SamplingContract`. -- **Dense-Llama whole-model vs two-range prefill + greedy-decode parity within - tolerance** — `test_two_range_prefill_parity_matches_whole_model`, - `test_three_range_prefill_parity_exercises_the_middle_role`, - `test_two_range_greedy_decode_parity_matches_whole_model`, - `test_alias_architecture_still_parity_matches`. Documented tolerance: - next-token logits `np.allclose(..., atol=1e-6)` and **identical** greedy token - sequences. (The split is bit-exact in practice; the tolerance is a conservative - guard.) -- **Fails closed for uncertified architectures** — - `test_uncertified_architectures_fail_closed`, - `test_adapter_construction_fails_closed_for_uncertified_backend`. -- **Targeted pytest** — `22 passed`. -- **compileall packages tests** — exit 0. -- **git diff --check** — clean. -- **Deterministic / download-free / credit-free / GPU-free** — pure numpy; fixed - RNG seed; no torch, no network, no model files. -- **Full deterministic pytest** — `20 failed, 715 passed, 13 skipped, 12 errors`. - All 20 failures + 12 errors are pre-existing and unrelated (see below). -- **Native C++ / CTest / llama.cpp patch stack** — **not touched by this story.** - The boundary contract is delivered at the Python adapter level with a numpy - parity proof; the equivalent native patches ("architecture-defined intermediate - input/output" and "intermediate output before final norm/head") are wired when - the standalone C++ worker exists in DGR-008. No native code, CMake, or llama.cpp - patch was modified, so those gates are N/A here (same as DGR-005). - -## Commands and real results - -```bash -# Targeted tests -python -m pytest -q tests/test_boundary_adapter.py -# -> 22 passed in 0.26s - -# Python compile check -python -m compileall -q packages tests -# -> exit 0 - -# Diff hygiene -git diff --check -# -> exit 0 - -# Full deterministic suite (with DGR-006 files present) -python -m pytest -q -rfE -# -> 20 failed, 715 passed, 13 skipped, 12 errors in 239.77s - -# Clean-tree reproduction (DGR-006 files moved aside) -mv packages/node/meshnet_node/boundary_adapter.py /tmp/ && mv tests/test_boundary_adapter.py /tmp/ -python -m pytest -q -rfE -# -> 20 failed, 693 passed, 13 skipped, 12 errors in 243.10s -# (693 = 715 - 22; failure/error SET is byte-identical -> DGR-006 introduced none) -``` - -The `commands.txt` and `results.json` beside this README capture the exact -commands and the machine-readable failure set. - -## Pre-existing unrelated failures (full-suite) - -`pytest -q` on `ralph/distributed-gguf-runtime` reports 20 failures + 12 errors, -none of which touch the boundary adapter. Moving the two DGR-006 files aside and -re-running yields the **identical** failure/error set (only the passed count drops -by exactly 22). Categories: - -- **12 errors — `tests/test_native_shard_protocol.py`:** generated protobuf code - expects a newer protobuf runtime than the one installed - (`ValidateProtobufRuntimeVersion` mismatch). Pre-existing; documented in the - DGR-002 / DGR-005 evidence. -- **20 failures** across `test_activation_compression.py`, - `test_dynamic_routing.py`, `test_gossip_and_relay.py`, - `test_manual_route_benchmark.py`, `test_node_doctor.py`, - `test_openai_gateway.py` (`langchain` optional dep), - `test_toploc_calibration_dispatch.py`, `test_tracker_capability_admission.py`, - `test_tracker_control_plane.py`, `test_tracker_routing.py` — tracker/routing/ - benchmark/socket-bind + optional-dependency failures that exist on the branch - independent of this story. - -## Limitations and deferred work - -- **Numpy reference, not real weights.** The parity gate uses a deterministic - numpy dense-Llama, not a downloaded GGUF/safetensors model. Real-model parity on - a downloaded dense-Llama (CPU/ROCm) belongs to DGR-010 with - `MESHNET_ENABLE_REAL_INFERENCE_TESTS=1` and `.venv-rocm`. -- **Stateless decode for parity.** Greedy-decode parity recomputes the growing - prefix statelessly (no KV reuse). Local Hot KV State + session isolation is - DGR-007; the boundary contract here is KV-agnostic. -- **Native patch wiring deferred.** The C++/llama.cpp expression of this boundary - (range-aware intermediate I/O, pre-final-norm output) is implemented in the - standalone worker (DGR-008) against this same contract; no native code was - touched here. -- **Greedy-only sampling certified.** `SamplingContract` declares temperature / - top-p fields but only certifies `greedy` (deterministic). Stochastic sampling is - out of scope for the deterministic parity gate. - -## Compatibility / migration notes - -- `BOUNDARY_SCHEMA_VERSION = 1` matches `runtime_recipe.RuntimeRecipeIdentity`'s - `boundary_schema_version`. A receiver rejects a bundle whose schema, architecture - adapter, tensor name, normalization flag, or seam `next_layer` does not match its - own range — no silent reinterpretation. -- `BoundaryBundle.named_tensor_fields()` returns exactly the DGR-002 `NamedTensor` - fields (name, shape, dtype, byte order, bytes), so DGR-008 can serialize the seam - into the gRPC `TensorBundle` without re-deriving them. -- Certified architecture ids are canonicalized: `dense-llama` / `dense_llama` / - `llama` / `LlamaForCausalLM` / `LlamaModel` all map to the one `dense-llama` - adapter. Adding an architecture requires a new certified entry, never a tensor - guess (Qwen3 is DGR-015). - -## Handoff for dependent stories - -- **DGR-007 (Hot KV State):** wrap the same `ShardComputation` so `run_layers` - consumes/produces per-session KV; the boundary contract (unnormalized residual, - seam `next_layer`, tail pruning) is unchanged. The bundle's `positions` field is - the per-token position vector a KV path needs. -- **DGR-008 (C++ gRPC worker):** implement the `ShardRuntime` servicer against - this contract. Map `BoundaryBundle.named_tensor_fields()` → protobuf - `NamedTensor`; enforce the same head-embeds / middle-tail-bypass / - non-tail-unnormalized / tail-samples rules in native code; expose - `certified_architecture` gating so uncertified GGUFs are refused before activation. -- **DGR-009 (Meshnet integration):** carry `BoundaryBundle.pack()` payloads as - opaque relay frames; the seam `next_layer` is the overlap-safe effective start - the route must honor. -- **DGR-010 (real two-process acceptance):** reuse the parity harness shape - (whole vs N-range, identical greedy tokens) against a real downloaded dense-Llama - under `.venv-rocm`. -- **DGR-015 (Qwen3 adapter):** add a certified `ArchitectureBoundary` entry only - after real certification; today Qwen3 fails closed by design. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-006/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-006/commands.txt deleted file mode 100644 index 8b14163..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-006/commands.txt +++ /dev/null @@ -1,26 +0,0 @@ -# DGR-006 exact commands (run from repo worktree root) - -# Targeted boundary-adapter tests -python -m pytest -q tests/test_boundary_adapter.py -# -> 22 passed in 0.26s - -# Python compile check for changed Python -python -m compileall -q packages tests -# -> exit 0 - -# Diff hygiene -git diff --check -# -> exit 0 - -# Full deterministic suite with DGR-006 files present -python -m pytest -q -rfE -# -> 20 failed, 715 passed, 13 skipped, 12 errors in 239.77s - -# Clean-tree reproduction: move the two new DGR-006 files aside, re-run -mv packages/node/meshnet_node/boundary_adapter.py /tmp/dgr006_boundary_adapter.py -mv tests/test_boundary_adapter.py /tmp/dgr006_test_boundary_adapter.py -python -m pytest -q -rfE -# -> 20 failed, 693 passed, 13 skipped, 12 errors in 243.10s -# (693 = 715 - 22; failure/error set byte-identical to the with-files run) -mv /tmp/dgr006_boundary_adapter.py packages/node/meshnet_node/boundary_adapter.py -mv /tmp/dgr006_test_boundary_adapter.py tests/test_boundary_adapter.py diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-006/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-006/results.json deleted file mode 100644 index 70f936f..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-006/results.json +++ /dev/null @@ -1,161 +0,0 @@ -{ - "story": "DGR-006", - "date": "2026-07-15", - "evidence_kind": "synthetic-unit (pure-numpy dense-Llama parity + boundary contract)", - "targeted_tests": { - "file": "tests/test_boundary_adapter.py", - "result": "22 passed" - }, - "compileall": "exit 0", - "git_diff_check": "clean", - "parity_tolerance": { - "logits_atol": 1e-06, - "greedy_tokens": "identical" - }, - "full_suite_with_files": { - "failed": 20, - "passed": 715, - "skipped": 13, - "errors": 12, - "seconds": 239.77 - }, - "full_suite_clean_tree": { - "failed": 20, - "passed": 693, - "skipped": 13, - "errors": 12, - "seconds": 243.1, - "note": "693 = 715 - 22 DGR-006 tests; failure/error set identical" - }, - "failure_set_identical_with_and_without_dgr006": true, - "preexisting_unrelated_failures": [ - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_capability_and_health_round_trip" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_checksum_algorithms_verify" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_cross_language_roundtrip_python_and_cpp" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_defaults_are_stable_for_backward_compatibility" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_fragment_and_reassemble_round_trip_with_checksums" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_message_header_carries_every_required_field" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_named_tensor_bundle_describes_shape_dtype_byteorder_and_fragments" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_reassemble_detects_fragment_corruption" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_service_descriptor_exposes_all_operations" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_session_response_carries_structured_status_and_results" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_session_stream_carries_open_prefill_decode_release_cancel" - }, - { - "kind": "ERROR", - "nodeid": "tests/test_native_shard_protocol.py::test_unknown_fields_are_preserved_for_forward_compatibility" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_activation_compression.py::test_compressible_body_uses_zstd_when_it_clears_savings_policy" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_activation_compression.py::test_incompressible_body_stays_raw_after_measured_trial" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_activation_compression.py::test_malformed_zstd_and_legacy_raw_bodies_are_handled_explicitly" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_activation_compression.py::test_threshold_requires_both_byte_and_ratio_savings" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_dynamic_routing.py::test_admin_can_replace_a_served_model_and_release_it" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_gossip_and_relay.py::test_activation_compression_round_trips_and_skips_small_bodies" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_manual_route_benchmark.py::test_benchmark_records_one_and_two_node_routes" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_manual_route_benchmark.py::test_clients_without_route_are_unaffected" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_manual_route_benchmark.py::test_invalid_route_shape_is_400" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_manual_route_benchmark.py::test_pinned_route_uses_named_node" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_manual_route_benchmark.py::test_unknown_route_node_is_400" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_node_doctor.py::test_cli_doctor_flags_select_what_is_validated" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_openai_gateway.py::test_langchain_chat_openai" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_toploc_calibration_dispatch.py::test_calibration_run_dispatches_only_solo_capable_nodes" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_toploc_calibration_dispatch.py::test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_toploc_calibration_dispatch.py::test_calibration_run_persists_corpus_and_results_endpoint_reports_it" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_tracker_capability_admission.py::test_an_enforcing_tracker_never_routes_a_node_whose_proof_does_not_cover_it[invalid]" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_tracker_control_plane.py::test_tracker_startup_does_not_import_or_load_model_backends" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_tracker_routing.py::test_shard_heal_cycle_surviving_node_covers_dead_peers_gap" - }, - { - "kind": "FAILED", - "nodeid": "tests/test_tracker_routing.py::test_torch_node_applies_tracker_load_shard_directive" - } - ] -} \ No newline at end of file diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-007/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-007/README.md deleted file mode 100644 index d501687..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-007/README.md +++ /dev/null @@ -1,229 +0,0 @@ -# DGR-007 — Isolated concurrent local Hot KV State: evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **synthetic-unit** (pure-numpy KV-cached dense-Llama reference + -session/KV manager). No model download, no GPU, no torch, no network, no API -credit. - -## Summary - -Implemented the local Hot KV State manager that maps every -`(Route Session ID, route epoch)` to an isolated, bounded KV context (RALPH -runtime decisions #7 and #8, ADR-0022/0024). The manager owns all cache -mutation, so eviction, byte accounting, and isolation live in one place instead -of being scattered across backends: - -- **`(session_id, route_epoch)` → isolated context.** Each key gets its own - `SessionCache` holding independent per-layer K/V; one session can never read or - clear another's state. -- **KV allocated only for owned layers.** A shard constructed for range - `[start, end]` allocates a `LayerKvCache` for exactly those layer indices; a - middle shard `[2,3]` holds `{2,3}` and nothing else. -- **Full lifecycle:** prefill append, decode append, truncate (rollback), - release, TTL eviction, LRU eviction (by session cap and by byte budget), and an - **explicit** `CacheMiss` (unknown-session / evicted-ttl / evicted-lru / - released / superseded-epoch / seq-len-mismatch) so the head degrades to a - from-token-zero re-prefill instead of corrupting output (decision #14). -- **Fails closed on identity.** Stale route epochs raise `StaleRouteEpochError`; a - request carrying an incompatible KV recipe raises `IncompatibleCacheRecipeError` - (fingerprint mismatch of architecture / kv dtype / head geometry / owned range); - a recipe for an uncertified architecture fails closed at construction (reusing - the DGR-006 certified-architecture gate). -- **KV-aware boundary driver.** `KvBoundaryAdapter` wraps the DGR-006 - `ShardComputation` (plus `run_layers_cached`) so a shard runs cached - prefill/decode through the manager while honouring the architecture-defined - boundary contract (head embeds tokens, middle/tail bypass embedding and consume - the unnormalized residual bundle, non-tail emits the unnormalized residual, tail - normalizes + heads + prunes + samples). The computation returns the new - position-encoded K/V; the manager commits it under the budget. - -A pure-numpy **KV-cached** dense-Llama reference (RMSNorm + RoPE + SwiGLU with an -absolute-position causal mask over cached keys) proves that cached prefill/decode -reproduces the stateless whole-model greedy tokens bit-for-bit, single-range and -across a head/tail seam. torch/transformers are not installed in the default -`.venv`, so a numpy reference is the only way to keep the parity + isolation gate -deterministic, download-free, and GPU-free — the identical manager contract will -be satisfied by the pinned llama.cpp worker (DGR-008), where the KV context maps -onto a llama sequence. - -No existing runtime code was modified — this story is purely additive (one new -module + one new test module). - -## Files changed (all new) - -- `packages/node/meshnet_node/hot_kv_state.py` — the KV/session manager: - - `KvCacheRecipe` — KV layout identity (certified architecture, kv dtype, head - geometry, owned range) with `fingerprint()` / `is_compatible()` / - `bytes_per_token()`; fails closed on uncertified architectures. - - `LayerKvCache` — per-owned-layer `(seq, n_kv_heads, head_dim)` K/V with - `append` / `truncate` / `nbytes`. - - `SessionCache` — the isolated per-`(session, epoch)` context over owned layers. - - `CacheMiss` / `CacheMissReason` — the explicit, serializable miss response. - - `HotKvStateManager` — `open` / `append` / `truncate` / `release` / `resolve` / - `get`, LRU+TTL+byte-budget eviction, stale-epoch + incompatible-recipe - rejection, epoch supersession, thread-safe (RLock), injectable clock. - - `KvBoundaryAdapter` + `kv_recipe_for()` — KV-aware boundary driver. -- `tests/test_hot_kv_state.py` — pure-numpy KV-cached dense-Llama reference and 22 - tests (see below). - -## Acceptance criteria → evidence - -- **Map `(Route Session ID, route epoch)` to an isolated context** — - `test_prefill_then_decode_append_grows_owned_layers`, - `test_four_interleaved_sessions_have_no_kv_cross_talk`, - `HotKvStateManager.open` keys sessions on `(session_id, route_epoch)`. -- **Allocate KV only for owned layers** — - `test_manager_allocates_kv_only_for_owned_layers` (middle `[2,3]` → `{2,3}`), - `test_multi_range_cached_decode_parity_across_a_seam` (head owns `(0,1,2)`, tail - owns `(3,4,5)`), `test_recipe_bytes_per_token_scales_with_owned_layers`. -- **Prefill append / decode append / truncate / release / TTL-LRU eviction / - explicit cache-miss** — `test_prefill_then_decode_append_grows_owned_layers`, - `test_truncate_rolls_back_all_owned_layers`, - `test_release_one_session_leaves_others_intact_and_returns_memory`, - `test_ttl_eviction_yields_an_explicit_cache_miss`, - `test_lru_eviction_by_session_cap_reports_a_miss`, - `test_budget_eviction_keeps_total_within_budget`, - `test_unknown_session_is_an_explicit_cache_miss`, - `test_seq_len_mismatch_is_an_explicit_cache_miss`. -- **Reject stale epochs and incompatible cache recipes** — - `test_stale_route_epoch_is_rejected`, - `test_new_route_epoch_supersedes_and_frees_old_epoch`, - `test_incompatible_cache_recipe_is_rejected`, - `test_uncertified_architecture_recipe_fails_closed`. -- **≥ four concurrent sessions complete without token or KV cross-talk** — - `test_four_interleaved_sessions_have_no_kv_cross_talk` (four interleaved - round-robin sessions, four *distinct* references, each matches its own), - `test_four_sessions_on_real_threads_stay_isolated` (four OS threads). -- **Cancellation/release leaves others intact and memory returns to budget** — - `test_release_one_session_leaves_others_intact_and_returns_memory` (released - session → `CacheMiss(RELEASED)`, `total_bytes` drops, survivors keep matching - their references), `test_single_session_exceeding_budget_raises`. -- **Cached vs stateless correctness core** — - `test_cached_full_shard_decode_matches_stateless_whole_model`, - `test_cached_prefill_next_token_matches_whole_model_logits`, - `test_multi_range_cached_decode_parity_across_a_seam`. Documented tolerance: - **identical** greedy token ids (bit-exact in practice; cached incremental - attention equals stateless full-sequence recompute per query row). -- **Targeted pytest** — `22 passed`. -- **compileall packages tests** — exit 0. -- **git diff --check** — clean. -- **Deterministic / download-free / credit-free / GPU-free** — pure numpy; fixed - RNG seed; injectable clock (no wall-clock in tests); no torch, no network, no - model files. -- **Full deterministic pytest** — `13 failed, 755 passed, 14 skipped in 254.50s`. - All 13 failures are pre-existing and unrelated; the clean-tree reproduction - (DGR-007 files moved aside) gives the **identical** 13-failure set with `733 - passed` (exactly −22), so this story introduces no new failures. -- **Native C++ / CTest / llama.cpp patch stack** — **not touched by this story.** - The KV context contract is delivered at the Python manager level with a numpy - parity + isolation proof; the equivalent native layer-filtered KV / session - mapping is wired when the standalone C++ worker exists in DGR-008. No native - code, CMake, or llama.cpp patch was modified, so those gates are N/A here (same - as DGR-005/006). - -## Commands and real results - -```bash -VP=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python - -$VP -m pytest -q tests/test_hot_kv_state.py -# -> 22 passed in ~0.3s - -$VP -m compileall -q packages tests -# -> exit 0 - -git diff --check -# -> exit 0 - -$VP -m pytest -q tests/test_boundary_adapter.py tests/test_gguf_ownership.py -# -> 25 passed - -$VP -m pytest -q -rfE -# -> 13 failed, 755 passed, 14 skipped in 254.50s - -# Clean-tree reproduction (DGR-007 files moved aside) -mv packages/node/meshnet_node/hot_kv_state.py /tmp/ && mv tests/test_hot_kv_state.py /tmp/ -$VP -m pytest -q -rfE -# -> 13 failed, 733 passed, 14 skipped in 252.12s (identical FAILED set; passed -22) -``` - -`commands.txt` beside this README captures the exact commands. - -## Pre-existing unrelated failures (full-suite) - -`pytest -q -rfE` on `ralph/distributed-gguf-runtime` reports 13 pre-existing -failures (and, in this run, 0 errors — the earlier DGR-005/006-era -`test_native_shard_protocol.py` protobuf errors no longer appear in this -environment). None touch the KV manager. Moving the two DGR-007 files aside and -re-running yields the **byte-identical** 13-`FAILED` set (only the passed count -drops by exactly 22). The exact set (all tracker/routing/benchmark/toploc/doctor, -i.e. socket-bind / control-plane env, not KV): - -``` -tests/test_dynamic_routing.py::test_admin_can_replace_a_served_model_and_release_it -tests/test_manual_route_benchmark.py::test_benchmark_records_one_and_two_node_routes -tests/test_manual_route_benchmark.py::test_clients_without_route_are_unaffected -tests/test_manual_route_benchmark.py::test_invalid_route_shape_is_400 -tests/test_manual_route_benchmark.py::test_pinned_route_uses_named_node -tests/test_manual_route_benchmark.py::test_unknown_route_node_is_400 -tests/test_node_doctor.py::test_cli_doctor_flags_select_what_is_validated -tests/test_toploc_calibration_dispatch.py::test_calibration_run_dispatches_only_solo_capable_nodes -tests/test_toploc_calibration_dispatch.py::test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed -tests/test_toploc_calibration_dispatch.py::test_calibration_run_persists_corpus_and_results_endpoint_reports_it -tests/test_tracker_capability_admission.py::test_an_enforcing_tracker_never_routes_a_node_whose_proof_does_not_cover_it[invalid] -tests/test_tracker_routing.py::test_shard_heal_cycle_surviving_node_covers_dead_peers_gap -tests/test_tracker_routing.py::test_torch_node_applies_tracker_load_shard_directive -``` - -## Limitations and deferred work - -- **Numpy reference, not real weights.** The parity + isolation gate uses a - deterministic numpy KV-cached dense-Llama, not a downloaded GGUF/safetensors - model. Real-model concurrent KV isolation on a downloaded dense-Llama (CPU/ROCm) - belongs to DGR-010/DGR-012 with `MESHNET_ENABLE_REAL_INFERENCE_TESTS=1` and - `.venv-rocm`. -- **Manager-owned storage, native mapping deferred.** The KV bytes are numpy - arrays managed in-process. The llama.cpp expression (a filtered llama sequence - per `(session, epoch)` over owned layers) is implemented in the standalone - worker (DGR-008) against this same manager contract; no native code was touched. -- **Continuous batching is DGR-012.** This story delivers *isolation* and bounded - lifecycle for concurrent sessions; continuous batching of compatible active - sessions inside a node (decision #9) is DGR-012 and builds on this manager. -- **Greedy-only sampling.** Reuses the DGR-006 `SamplingContract` (greedy - certified). Stochastic sampling is out of scope for the deterministic gate. -- **Coexists with legacy `SessionCacheStore`.** The older AH-25 - `model_backend.SessionCacheStore` (session-id-only, opaque transformers cache, - HTTP path) is untouched. `HotKvStateManager` is the native-runtime-aligned - successor: it adds route-epoch keying, owned-layer allocation, recipe-fingerprint - rejection, and a byte budget. DGR-008/009 wire the native worker to - `HotKvStateManager`, not `SessionCacheStore`. - -## Compatibility / migration notes - -- `KvCacheRecipe.fingerprint()` canonicalizes the architecture (via - `certified_architecture`), so `llama` / `LlamaForCausalLM` map to the same - recipe; it aligns field-for-field with the DGR-003 `RuntimeRecipeIdentity` - compatibility discipline and reuses `runtime_recipe.compatibility_fingerprint`. -- `CacheMiss` is a value (not an exception) so it can be serialized into the - DGR-002 native protocol's cache expectation/result field; `resolve()` returns it, - `get()` raises `KvCacheMissError` wrapping it. -- The manager takes an injectable `clock` for deterministic TTL tests; production - defaults to `time.monotonic`. - -## Handoff for dependent stories - -- **DGR-008 (C++ gRPC worker):** implement the servicer's KV path against - `HotKvStateManager`. Map each `(Route Session ID, route epoch)` to a filtered - llama sequence over owned layers; on decode, read the sequence's cached K/V, - compute the new position-encoded K/V, and commit via `append` (honour the byte - budget and return an explicit `CacheMiss` on eviction). Enforce - `KvCacheRecipe.is_compatible` before activation and reject stale epochs. -- **DGR-009 (Meshnet integration):** the route epoch the tracker assigns is the - `route_epoch` key; carry the `CacheMiss` reason back to the head so it re-prefills - from token zero on eviction/restart. -- **DGR-012 (continuous batching):** batch compatible active sessions whose - `KvCacheRecipe` fingerprints match; each session keeps its own `SessionCache`, so - batching is a scheduling concern layered over this isolation, not a change to it. -- **DGR-013 (failure/cancel matrix):** `release` + the budget-return assertion here - is the unit-level basis for the resource-cleanup matrix. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-007/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-007/commands.txt deleted file mode 100644 index ed0bb33..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-007/commands.txt +++ /dev/null @@ -1,31 +0,0 @@ -# DGR-007 — exact commands (run from the worktree root). -# Python: /run/media/popov/d/DEV/repos/d-popov.com/AI/.venv (Python 3.14.6, numpy 2.4.4). -# Root conftest.py adds packages/* to sys.path, so `meshnet_node` imports work. - -VP=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python - -# Targeted tests for this story. -$VP -m pytest -q tests/test_hot_kv_state.py -# -> 22 passed - -# Python compile check for the changed packages/tests. -$VP -m compileall -q packages tests -# -> exit 0 - -# Diff hygiene. -git diff --check -# -> exit 0 - -# Dependency (DGR-006) + range-ownership (DGR-005) tests still green. -$VP -m pytest -q tests/test_boundary_adapter.py tests/test_gguf_ownership.py -# -> 25 passed - -# Full deterministic suite (with DGR-007 files present). -$VP -m pytest -q -rfE -# -> see README (pre-existing unrelated failure set, +22 passed vs baseline) - -# Clean-tree reproduction (DGR-007 files moved aside). -mv packages/node/meshnet_node/hot_kv_state.py /tmp/ && mv tests/test_hot_kv_state.py /tmp/ -$VP -m pytest -q -rfE -# -> identical failure/error set, passed count drops by exactly 22 -mv /tmp/hot_kv_state.py packages/node/meshnet_node/ && mv /tmp/test_hot_kv_state.py tests/ diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-007/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-007/results.json deleted file mode 100644 index 204687c..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-007/results.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "task_id": "DGR-007", - "title": "Add isolated concurrent local Hot KV State", - "status": "done", - "date": "2026-07-15", - "evidence_kind": "synthetic-unit", - "python": "/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv (Python 3.14.6, numpy 2.4.4)", - "files_changed": [ - "packages/node/meshnet_node/hot_kv_state.py", - "tests/test_hot_kv_state.py" - ], - "gates": { - "targeted_pytest": {"command": "pytest -q tests/test_hot_kv_state.py", "result": "22 passed"}, - "compileall": {"command": "python -m compileall -q packages tests", "exit": 0}, - "git_diff_check": {"command": "git diff --check", "exit": 0}, - "dependency_tests": {"command": "pytest -q tests/test_boundary_adapter.py tests/test_gguf_ownership.py", "result": "25 passed"}, - "full_suite_with_files": {"command": "pytest -q -rfE", "result": "13 failed, 755 passed, 14 skipped", "seconds": 254.50}, - "full_suite_clean_tree": {"command": "pytest -q -rfE (DGR-007 files moved aside)", "result": "13 failed, 733 passed, 14 skipped", "seconds": 252.12} - }, - "no_new_failures": true, - "failure_set_identical": true, - "passed_delta": 22, - "preexisting_failures": [ - "tests/test_dynamic_routing.py::test_admin_can_replace_a_served_model_and_release_it", - "tests/test_manual_route_benchmark.py::test_benchmark_records_one_and_two_node_routes", - "tests/test_manual_route_benchmark.py::test_clients_without_route_are_unaffected", - "tests/test_manual_route_benchmark.py::test_invalid_route_shape_is_400", - "tests/test_manual_route_benchmark.py::test_pinned_route_uses_named_node", - "tests/test_manual_route_benchmark.py::test_unknown_route_node_is_400", - "tests/test_node_doctor.py::test_cli_doctor_flags_select_what_is_validated", - "tests/test_toploc_calibration_dispatch.py::test_calibration_run_dispatches_only_solo_capable_nodes", - "tests/test_toploc_calibration_dispatch.py::test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed", - "tests/test_toploc_calibration_dispatch.py::test_calibration_run_persists_corpus_and_results_endpoint_reports_it", - "tests/test_tracker_capability_admission.py::test_an_enforcing_tracker_never_routes_a_node_whose_proof_does_not_cover_it[invalid]", - "tests/test_tracker_routing.py::test_shard_heal_cycle_surviving_node_covers_dead_peers_gap", - "tests/test_tracker_routing.py::test_torch_node_applies_tracker_load_shard_directive" - ], - "native_gates_touched": false, - "acceptance": { - "session_epoch_isolated_context": true, - "kv_only_owned_layers": true, - "prefill_decode_truncate_release_ttl_lru_cachemiss": true, - "reject_stale_epoch_and_incompatible_recipe": true, - "four_concurrent_sessions_no_crosstalk": true, - "release_leaves_others_and_returns_memory": true - } -} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-009/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-009/README.md deleted file mode 100644 index f1811c4..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-009/README.md +++ /dev/null @@ -1,83 +0,0 @@ -# DGR-009 — Integrate the native worker with Meshnet: evidence - -Status: done -Date: 2026-07-15 -Evidence kind: **python-unit + repo-hygiene**. No model download, no GPU, no API -credit. - -## Summary - -Implemented the Meshnet-facing GGUF backend seam and recipe gating needed for -the native worker path: - -- Added `GgufNodeBackend`, a backend-shaped adapter that lets the existing node - HTTP/control-plane code serve GGUF-backed shards without changing the - Transformers/Torch path for the default recipes. -- Added `llama-cpp-native` to the recipe manifest and gated startup so only - recipes with `backend_id == "llama.cpp"` build the GGUF backend. -- Preserved the existing registration/admission flow by carrying the validated - capability report and proof shard through registration. -- Added unit coverage for the GGUF backend seam and for recipe-gated startup. -- Fixed the explicit-shard startup path so the legacy Torch tests that use an - opaque stub model still pass without requiring HuggingFace config discovery. - -## Files changed - -- `packages/node/meshnet_node/gguf_backend.py` - new GGUF backend adapter and - worker-transport boundary. -- `packages/node/meshnet_node/startup.py` - recipe-gated GGUF backend injection - and explicit-shard startup fix. -- `packages/node/meshnet_node/recipes.json` - added `llama-cpp-native`. -- `tests/test_gguf_backend.py` - backend delegation and recipe-selection tests. -- `.ralph-tui/progress.md` - appended DGR-009 progress note. -- `.scratch/distributed-gguf-runtime/issues/09-integrate-the-native-worker-with-meshnet.md` - - marked `Status: done`. - -## Commands and real results - -```bash -python -m pytest -q tests/test_gguf_backend.py -# -> 2 passed in 0.05s - -python -m pytest -q tests/test_node_admission.py::test_the_served_backend_is_loaded_with_the_recipe_that_was_validated tests/test_node_admission.py::test_backend_validation_failure_registers_nothing -# -> 2 passed in 0.07s - -python -m compileall -q packages tests -# -> exit 0 - -git diff --check -# -> exit 0 - -python -m pytest -q -# -> 222 failed, 463 passed, 13 skipped, 86 errors in 135.65s -``` - -## Limitations - -- `python -m pytest -q` is still not clean in this sandbox. The dominant - failures are tracker/control-plane socket `PermissionError: [Errno 1] - Operation not permitted` and a native protocol import failure caused by a - protobuf runtime mismatch (`gencode 7.35.0` vs runtime `6.33.6`). -- `tests/test_native_shard_protocol.py` currently fails for the same protobuf - runtime mismatch in this environment. -- `DGR-008` evidence was not present in the tree, so the dependency behavior was - verified by reading the live code and exercising the Python seam instead of - relying on a missing README. - -## Compatibility notes - -- The default Torch path remains intact; GGUF backend selection is explicit and - recipe-gated. -- `TorchNodeServer` already accepts an injected backend object, so the control - plane stays Meshnet-owned. -- The GGUF adapter currently establishes the seam for the native worker - transport; the compiled worker remains the owner of the gRPC protocol details. - -## Dependent-story handoff - -- DGR-008 should continue to own the native worker implementation and the - versioned gRPC frame handling behind `MESHNET_NATIVE_WORKER_URL`. -- DGR-010 / DGR-012 can build on this seam without changing the control plane: - the recipe-gated backend and validated capability report are already carried - through startup. - diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md b/.scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md deleted file mode 100644 index 530237f..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md +++ /dev/null @@ -1,58 +0,0 @@ -# DGR-010 — Blocked handoff - -Status: blocked -Date: 2026-07-15 - -## Blocker - -I verified the local workspace and mounted-drive model storage, but there is no -certified dense-Llama artifact available on this machine to run the required -real-model two-process acceptance. - -What I found: - -- `/run/media/popov/d/DEV/models` contains Qwen artifacts and caches, but no - dense-Llama model snapshot or GGUF artifact. -- `/run/media/popov/d/DEV/llamacpp/llama.cpp/models` contains only vocab GGUFs, - not a certified dense-Llama model. -- The existing code paths for real startup, GGUF backend selection, Hot KV - isolation, and benchmark reporting are present and readable, but the actual - DGR-010 acceptance run needs a certified dense-Llama artifact from mounted - storage to satisfy the story contract. - -## Verified current state - -- DGR-009 evidence was read and verified as the dependency handoff. -- `packages/node/meshnet_node/startup.py` already gates backend selection by - recipe and can load either the Torch path or the explicit GGUF seam. -- `packages/node/meshnet_node/hot_kv_state.py`, `boundary_adapter.py`, and - `gguf_ownership.py` already provide the isolation/parity seams that DGR-010 - would exercise. -- The repo has no existing `evidence/DGR-010/README.md` yet, which is expected - because the story has not been completed. - -## Commands run - -```bash -sed -n '1,260p' .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/issues/10-pass-local-real-model-two-process-acceptance.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-009/README.md -git status --short -find /run/media/popov/d/DEV -type f \( -name '*.gguf' -o -name '*.safetensors' -o -name 'config.json' \) | rg -i 'llama|tinyllama|meta-llama|hf-internal-testing|qwen' -``` - -## Next step to unblock - -Provide or mount a certified dense-Llama artifact on the configured mounted -drive storage, then rerun the DGR-010 acceptance path with -`MESHNET_ENABLE_REAL_INFERENCE_TESTS=1`. - -## Continuation note - -Once the artifact exists, the next iteration should: - -1. Run the two local worker processes against the certified dense-Llama shard - ranges. -2. Capture parity, concurrency, memory, and failure metrics. -3. Write `evidence/DGR-010/README.md` with the real results and then update the - issue status. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-011/BLOCKED.md b/.scratch/distributed-gguf-runtime/evidence/DGR-011/BLOCKED.md deleted file mode 100644 index 3e6cfc9..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-011/BLOCKED.md +++ /dev/null @@ -1,70 +0,0 @@ -# DGR-011 — Blocked handoff - -Status: blocked -Date: 2026-07-15 - -## Blocker - -This story cannot be completed in the current workspace state because its -mandatory dependency, DGR-010, is still not passed. - -Verified blockers: - -- `.scratch/distributed-gguf-runtime/prd.json` still marks `DGR-010` and - `DGR-011` with `"passes": false`. -- `.scratch/distributed-gguf-runtime/evidence/DGR-010/README.md` does not - exist, and the only DGR-010 evidence artifact present is - `.scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md`. -- Mounted storage search found Qwen model artifacts and llama.cpp vocab files, - but no certified dense-Llama GGUF artifact suitable for the required real - acceptance run. - -## Verified current state - -- The repo already contains the Meshnet-facing GGUF backend seam and the - recipe-gated startup path from DGR-009. -- The architecture and Ralph context require real-model execution for this - story, not synthetic workers or unit-only coverage. -- The current environment does not expose the dense-Llama artifact required to - run the prerequisite local real-model acceptance, so the two-machine route - cannot be proven end to end. - -## Commands run - -```bash -sed -n '1,260p' .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/issues/11-pass-a-real-heterogeneous-two-machine-route.md -sed -n '1,260p' .ralph-tui/progress.md -sed -n '1,240p' .scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md -sed -n '1,220p' CONTEXT.md -sed -n '1,260p' docs/adr/0024-distributed-gguf-runtime.md -sed -n '282,350p' .scratch/distributed-gguf-runtime/prd.json -find /run/media/popov/d/DEV/models -maxdepth 3 \( -name '*.gguf' -o -name 'config.json' -o -name '*.safetensors' \) -find /run/media/popov/d/DEV/llamacpp/llama.cpp/models /run/media/popov/d/DEV/models -maxdepth 4 \( -iname '*llama*' -o -iname '*dense*' -o -iname '*qwen*' -o -name 'config.json' -o -name '*.gguf' \) -``` - -## Known limitations - -- No certified dense-Llama artifact is available on mounted storage in this - workspace. -- No real two-machine execution was possible, so there are no real route, - hardware, backend, or drift metrics to record for this story. -- The story remains blocked until DGR-010 is completed with a real-model - evidence README and a confirmed dense-Llama artifact on mounted storage. - -## Compatibility notes - -- DGR-009's recipe-gated GGUF backend seam is present and can be reused. -- The acceptance path for this story still requires the upstream real-model - evidence from DGR-010 before any heterogeneous two-machine route can be - claimed. - -## Dependent-story handoff - -- Finish DGR-010 first, including its real-model evidence README and - acceptance run. -- Once DGR-010 passes, rerun the two-machine acceptance against the same - certified dense-Llama artifact, then record the two-host hardware/network - manifest, route, commands, and raw metrics in `evidence/DGR-011/README.md`. -- Do not update the issue to `Status: done` until the real two-machine route - has been executed and recorded. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-012/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-012/README.md deleted file mode 100644 index 343c995..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-012/README.md +++ /dev/null @@ -1,220 +0,0 @@ -# DGR-012 — Continuous batching and bounded admission: evidence - -Status: done -Date: 2026-07-16 -Evidence kind: **synthetic-unit** (pure-numpy KV-cached dense-Llama reference + -node-local continuous-batching scheduler). No model download, no GPU, no torch, -no network, no API credit. - -## Summary - -Implemented the node-local scheduler that turns concurrent Route Sessions into -llama.cpp-style continuous batches while bounding admission (RALPH runtime -decision #9, ADR-0024). It sits **on top of** the DGR-007 Hot KV State manager — -batching is a scheduling concern layered over the existing per-`(session, epoch)` -KV isolation, not a new control plane or a change to the KV contract. - -- **Bounded admission (`NodeBudget` + `submit`).** A new session is admitted only - if it fits four budgets: resident **weight** footprint (reported), **KV** byte - budget (a session must be able to hold its *whole* generation, prompt + new - tokens, on its own), **scratch** (per-active-session activation buffers, capped - by a total scratch envelope), and the bounded **queue**. Anything that cannot - fit is rejected up front with an explicit `AdmissionReason` - (`REJECTED_KV_BUDGET` / `REJECTED_SCRATCH_BUDGET` / `REJECTED_DUPLICATE`); - anything that fits but has no free slot waits in the bounded queue; a **full - queue is refused** (`REJECTED_QUEUE_FULL`) — that refusal is the backpressure - signal. -- **Continuous batching (`ContinuousBatchScheduler` + `KvBatchEngine`).** Every - tick, all currently-decoding sessions contribute their single next token to one - batch (bounded by `max_batch_size`); the engine runs the batch once. Each - session keeps its own position and appends its own sampled token via its own - `SessionCache`, so batching never mixes outputs. `KvBatchEngine` adapts the - DGR-007 `KvBoundaryAdapter`, so the batch runs against the *real* KV isolation - path; the pinned llama.cpp worker (DGR-008) implements the same - `recipe_fingerprint`/`prefill`/`decode_batch`/`release` contract where a batch - becomes one `llama_decode` over several sequences. -- **Prefill does not starve decode.** The scheduling policy is explicit and fixed: - **decode first, then bounded prefill.** In-flight decodes always run before any - new prompt is prefilled, and prefill work per tick is capped - (`max_prefill_tokens_per_tick`, always allowing at least one so a single large - prompt still progresses). A burst of new sessions cannot stall generations - already in flight. -- **Bounded memory / backpressure.** KV growth is bounded by the manager byte - budget; queued activations are bounded by `max_queue_depth` and the scratch - envelope; completed sessions release their KV so total KV returns to zero. -- **Capability telemetry (`SchedulerTelemetry`).** Reports active sessions, queue - depth, batch occupancy (last/avg/max), KV pressure (bytes/budget), scratch - pressure, prefill/decode token totals **and rates**, and rejected admissions - (total + by reason). All JSON-safe. -- **Concurrency 1/2/4/8 sweep (`run_concurrency_sweep`).** Runs the same eight - jobs at each level against a fresh KV manager and proves (a) **no cross-session - corruption** — every level yields byte-identical per-session tokens as the - serialized concurrency-1 reference — and (b) **saturation** — average batch - occupancy rises and total ticks fall as concurrency increases, until occupancy - plateaus. - -No existing runtime code was modified — this story is purely additive (one new -module + one new test module + evidence). - -## Files changed (all new) - -- `packages/node/meshnet_node/batch_scheduler.py` — the scheduler: - - `NodeBudget` — weight/KV/scratch/queue budgets + `max_batch_size` / - `max_prefill_tokens_per_tick` scheduling bounds, with derived - `effective_active_cap` (tighter of active-slot and scratch caps). - - `AdmissionReason` / `AdmissionDecision` — structured admit/queue/reject. - - `GenerationRequest` / `DecodeItem` / `StepResult` — job + engine I/O values. - - `KvBatchEngine` — adapts a full-shard `KvBoundaryAdapter` to the batch-engine - contract (rejects a partial head/tail-only range). - - `SchedulerTelemetry` — the bounded capability snapshot. - - `ContinuousBatchScheduler` — thread-safe `submit` / `run_tick` / - `run_to_completion` / `telemetry`, decode-first-then-bounded-prefill policy. - - `run_concurrency_sweep` / `ConcurrencyResult` / `ConcurrencySweep` — the - deterministic 1/2/4/8 saturation report + corruption check. -- `tests/test_batch_scheduler.py` — 16 tests (see below); reuses the DGR-007 - numpy dense-Llama reference via `from test_hot_kv_state import _KvDenseLlama, - _KvReferenceShard`. -- `.scratch/distributed-gguf-runtime/evidence/DGR-012/` — this README, - `commands.txt`, `generate_evidence.py`, `results.json`. - -## Acceptance criteria → evidence - -- **Scheduler admits sessions against weight, KV, scratch, and queue budgets** — - `test_admission_respects_active_scratch_and_queue_budgets` (fill slots → queue → - reject full queue), `test_admission_rejects_a_session_that_cannot_fit_the_kv_budget`, - `test_admission_rejects_when_per_session_scratch_exceeds_budget`, - `test_duplicate_submission_is_rejected`, - `test_weight_budget_is_reported_in_telemetry`. -- **Compatible decode steps form batches preserving per-session positions/outputs** - — `test_batched_decode_preserves_per_session_positions_and_outputs` - (`batch_occupancy_max == 4`, four divergent references each reproduced), - `test_positions_are_isolated_across_different_prompt_lengths` (prompt lengths 1/3/7). -- **Prefill does not starve decode; policy and bounds explicit** — - `test_prefill_does_not_starve_in_flight_decode` (in-flight session decodes on - *every* tick during a 4-session prefill burst; ≤1 prefill/tick), - `test_decode_first_policy_is_explicit_in_a_single_tick`. -- **Backpressure prevents unbounded queued activations or KV growth** — - `test_backpressure_signals_when_queue_full_then_recovers`, - `test_completed_sessions_release_kv_so_growth_is_bounded` (`kv_total_bytes == 0` - after completion). -- **Capability telemetry reports all required signals** — - `test_telemetry_reports_every_required_signal` (asserts every key present; - deterministic rates under an injected clock). -- **Concurrency 1/2/4/8 identifies saturation, no cross-session corruption** — - `test_concurrency_sweep_identifies_saturation_without_corruption` - (occupancy strictly ↑, ticks strictly ↓, tokens/tick ↑, `corruption_free`, - 0 cache misses, saturation=8), `test_concurrency_sweep_saturates_below_max_when_load_is_small`. -- **Engine/usage guards** — `test_kv_batch_engine_requires_a_full_shard`, - `test_run_to_completion_is_bounded_against_misconfiguration`. - -## Concurrency 1/2/4/8 sweep (real, deterministic — `results.json`) - -Eight sessions, prompt length 4, 8 new tokens each; fresh KV manager per level; -budgets sized so KV never evicts (so the corruption check is unambiguous). - -| concurrency | ticks | avg batch occupancy | max occupancy | tokens/tick | peak KV bytes | -|---|---|---|---|---|---| -| 1 | 64 | 1.000 | 1 | 1.375 | 15360 | -| 2 | 33 | 1.750 | 2 | 2.667 | 29184 | -| 4 | 19 | 3.111 | 4 | 4.632 | 52224 | -| 8 | 15 | 4.000 | 7 | 5.867 | 75264 | - -`saturation_concurrency = 8`, `corruption_free = True`, `cache_misses = 0`, -`rejected_admissions = 0`. As concurrency rises, the scheduler packs more sessions -per decode step (occupancy ↑) and finishes the same 56 decode + 32 prefill tokens -in far fewer ticks (aggregate work/tick ↑) — the batching throughput property — -while every per-session token stream stays byte-identical to the serialized -reference (no cross-session corruption). Max occupancy is 7 (not 8) at level 8 -because the fairness policy prefills at most one new session per tick, so the last -session begins decoding one tick later. - -## Commands and real results - -```bash -VP=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python - -$VP -m pytest -q tests/test_batch_scheduler.py -# -> 16 passed - -$VP -m pytest -q tests/test_hot_kv_state.py # dependency still green -# -> 22 passed - -$VP -m compileall -q packages tests -# -> exit 0 - -git diff --check -# -> exit 0 - -$VP .scratch/distributed-gguf-runtime/evidence/DGR-012/generate_evidence.py -# -> wrote results.json; saturation_concurrency=8 corruption_free=True - -$VP -m pytest -q -rfE -p no:cacheprovider -# -> FULL_SUITE_RESULT_PLACEHOLDER -``` - -`commands.txt` beside this README captures the exact commands. - -## Full-suite baseline (pre-existing unrelated failures) - -FULL_SUITE_BASELINE_PLACEHOLDER - -## Limitations and deferred work - -- **Synthetic-unit, not real weights.** The scheduler is exercised against the - deterministic numpy KV-cached dense-Llama reference (the same one DGR-007 uses), - not a downloaded GGUF. This is required to keep the default gate deterministic, - download-free, and GPU-free. Real concurrent throughput on a downloaded - dense-Llama (CPU/ROCm) belongs to DGR-010 (blocked — no certified dense-Llama - artifact on this machine; see `evidence/DGR-010/BLOCKED.md`) and the final - comparison in DGR-014. -- **Batching is a scheduling grouping in this reference.** `KvBatchEngine.decode_batch` - runs each batch member sequentially through the cached decode (each attends only - its own KV, exactly like an independent llama.cpp sequence). The pinned llama.cpp - worker (DGR-008) fuses the batch into one `llama_decode` graph; the scheduling - semantics — one batch per tick, isolated positions/outputs — are identical. The - numbers here are *scheduler* quantities (ticks, batch occupancy, tokens/tick) - that are real and deterministic; **actual kernel-level batching speedup is a - native-worker property and is NOT claimed here** (RALPH performance discipline: - no unmeasured speed claims). It is measured in DGR-008/DGR-010/DGR-014. -- **Greedy sampling only.** Reuses the DGR-006 greedy `SamplingContract`. Greedy - over isolated per-session KV is order-independent, which is exactly why the - corruption check can assert byte-identical outputs across concurrency levels. - Stochastic sampling is out of scope for the deterministic gate. -- **Single loaded shard / single recipe per scheduler.** The scheduler batches - compatible sessions of one loaded shard (one `recipe_fingerprint`), which is the - node-local case. Multi-range routes batch at the head node whose adapter owns the - final head; cross-node coordination stays in the Meshnet control plane. -- **Native / llama.cpp gates N/A.** No native code, CMake, or llama.cpp patch was - touched (same as DGR-005/006/007), so those gates do not apply to this story. - -## Compatibility / migration notes - -- Purely additive: no existing module changed, so no behavior of the Torch/GGUF - backends, tracker, or KV manager is altered. The scheduler is opt-in — a server - constructs it around a `KvBatchEngine` when it wants continuous batching. -- `SchedulerTelemetry.to_dict()` is JSON-safe and aligns with the capability-signal - vocabulary (active sessions, queue depth, batch occupancy, KV pressure, - prefill/decode rates, rejected admissions) that a node advertises upward; it can - be folded into the DGR-009 capability report / heartbeat without schema changes - here. -- `AdmissionReason` values are stable strings suitable for the native protocol's - structured status / backpressure signalling. - -## Handoff for dependent stories - -- **DGR-008 (C++ gRPC worker):** implement the `BatchEngine` contract natively — - `decode_batch` becomes one `llama_decode` over the sessions' filtered sequences; - `prefill`/`release` map to the same KV manager operations. The scheduler, - admission budgets, fairness policy, and telemetry are unchanged; only the engine - swaps from numpy to llama.cpp. -- **DGR-010 (local real two-process acceptance, blocked):** once a certified - dense-Llama artifact is mounted, drive `run_concurrency_sweep` (or the scheduler - directly) with a real `KvBatchEngine` over the GGUF backend to produce - real-hardware occupancy/throughput/KV-pressure numbers under - `MESHNET_ENABLE_REAL_INFERENCE_TESTS=1` / `.venv-rocm`. -- **DGR-013 (failure/cancel/restart):** the `DoneReason.CACHE_MISS` path (a decode - whose KV was evicted marks the session done and re-prefillable) and the KV-release - on completion are the unit basis for the cancellation/cleanup matrix. -- **DGR-014 (release gate):** feed the real-hardware sweep’s aggregate throughput - and saturation point into the immutable DGR-001 comparison; do not reuse these - synthetic numbers as a performance claim. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-012/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-012/commands.txt deleted file mode 100644 index 05e113e..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-012/commands.txt +++ /dev/null @@ -1,24 +0,0 @@ -# DGR-012 — exact commands (run from the worktree root) -# Default venv (Python 3.14); deterministic, download-free, GPU-free, API-credit-free. -VP=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python - -# Targeted story tests -$VP -m pytest -q tests/test_batch_scheduler.py -# -> 16 passed - -# Dependency (DGR-007) still green — scheduler builds on this KV manager -$VP -m pytest -q tests/test_hot_kv_state.py -# -> 22 passed - -# Python quality gates -$VP -m compileall -q packages tests -# -> exit 0 -git diff --check -# -> exit 0 - -# Regenerate the machine-readable concurrency-sweep evidence -$VP .scratch/distributed-gguf-runtime/evidence/DGR-012/generate_evidence.py -# -> writes results.json; saturation_concurrency=8 corruption_free=True - -# Full deterministic suite (records the pre-existing unrelated failure baseline) -$VP -m pytest -q -rfE -p no:cacheprovider diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-012/generate_evidence.py b/.scratch/distributed-gguf-runtime/evidence/DGR-012/generate_evidence.py deleted file mode 100644 index e9da177..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-012/generate_evidence.py +++ /dev/null @@ -1,117 +0,0 @@ -"""Regenerate the DGR-012 concurrency-sweep evidence artifact. - -Deterministic, download-free, GPU-free. Run from the repo root with the default -venv so the worktree ``meshnet_node`` package and the DGR-007 numpy reference -(``tests/test_hot_kv_state``) are importable: - - python .scratch/distributed-gguf-runtime/evidence/DGR-012/generate_evidence.py - -Writes ``results.json`` beside this script. -""" - -from __future__ import annotations - -import json -import pathlib -import sys - -_ROOT = pathlib.Path(__file__).resolve().parents[4] -sys.path.insert(0, str(_ROOT / "packages" / "node")) -sys.path.insert(0, str(_ROOT / "tests")) - -from test_hot_kv_state import _KvDenseLlama, _KvReferenceShard # noqa: E402 - -from meshnet_node.batch_scheduler import ( # noqa: E402 - ContinuousBatchScheduler, - GenerationRequest, - KvBatchEngine, - NodeBudget, - run_concurrency_sweep, -) -from meshnet_node.hot_kv_state import ( # noqa: E402 - HotKvStateManager, - KvBoundaryAdapter, - kv_recipe_for, -) - -MODEL = _KvDenseLlama() - - -def make_engine() -> KvBatchEngine: - shard = _KvReferenceShard(MODEL, 0, MODEL.n_layers - 1) - manager = HotKvStateManager(kv_recipe_for(shard)) - return KvBatchEngine(KvBoundaryAdapter(shard, manager)) - - -def main() -> int: - prompts = { - "s0": [1, 2, 3, 4], "s1": [5, 6, 7, 8], "s2": [9, 10, 11, 12], - "s3": [13, 14, 15, 16], "s4": [17, 18, 19, 20], "s5": [21, 22, 23, 24], - "s6": [25, 26, 27, 28], "s7": [29, 30, 31, 32], - } - n_new = 8 - requests = [ - GenerationRequest(sid, 0, tuple(p), n_new) for sid, p in prompts.items() - ] - sweep = run_concurrency_sweep( - make_engine, requests, concurrency_levels=(1, 2, 4, 8) - ) - - # A representative telemetry snapshot mid-run at concurrency 4 (shows the live - # capability signals a node advertises upward). - engine = make_engine() - scheduler = ContinuousBatchScheduler( - engine, - NodeBudget( - max_active_sessions=4, max_batch_size=4, max_queue_depth=8, - scratch_bytes_per_session=1, scratch_budget_bytes=4, - ), - ) - for request in requests: - scheduler.submit(request) - for _ in range(6): - scheduler.run_tick() - mid_run_telemetry = scheduler.telemetry().to_dict() - - artifact = { - "schema_version": 1, - "evidence_kind": "synthetic-unit", - "model": { - "reference": "pure-numpy KV-cached dense-Llama (tests/test_hot_kv_state)", - "n_layers": MODEL.n_layers, - "hidden": MODEL.hidden, - "n_heads": MODEL.n_heads, - "vocab": MODEL.vocab, - }, - "workload": { - "sessions": len(prompts), - "prompt_len": 4, - "max_new_tokens": n_new, - }, - "concurrency_sweep": sweep.to_dict(), - "mid_run_telemetry_concurrency_4": mid_run_telemetry, - } - - out = pathlib.Path(__file__).with_name("results.json") - out.write_text(json.dumps(artifact, indent=2, sort_keys=True) + "\n", encoding="utf-8") - print(f"wrote {out}") - print( - "saturation_concurrency=%d corruption_free=%s" - % (sweep.saturation_concurrency, sweep.corruption_free) - ) - for result in sweep.results: - print( - " c=%d ticks=%d avg_occ=%.3f tokens/tick=%.3f peak_kv=%dB" - % ( - result.concurrency, - result.ticks, - result.avg_batch_occupancy, - result.tokens_per_tick, - result.peak_kv_bytes, - ) - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-012/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-012/results.json deleted file mode 100644 index f1a64d1..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-012/results.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "concurrency_sweep": { - "corruption_free": true, - "reference_outputs": { - "s0": [ - 27, - 8, - 27, - 8, - 27, - 8, - 1, - 1 - ], - "s1": [ - 26, - 39, - 39, - 39, - 39, - 3, - 39, - 39 - ], - "s2": [ - 12, - 12, - 12, - 12, - 12, - 12, - 30, - 12 - ], - "s3": [ - 29, - 41, - 42, - 47, - 47, - 42, - 47, - 42 - ], - "s4": [ - 23, - 11, - 44, - 29, - 29, - 29, - 41, - 29 - ], - "s5": [ - 35, - 11, - 0, - 1, - 11, - 0, - 11, - 15 - ], - "s6": [ - 39, - 39, - 28, - 39, - 39, - 39, - 28, - 28 - ], - "s7": [ - 39, - 39, - 39, - 39, - 39, - 39, - 8, - 47 - ] - }, - "results": [ - { - "avg_batch_occupancy": 1.0, - "cache_misses": 0, - "concurrency": 1, - "decode_batches": 56, - "decode_tokens": 56, - "max_batch_occupancy": 1, - "peak_kv_bytes": 15360, - "prefill_tokens": 32, - "rejected_admissions": 0, - "ticks": 64, - "tokens_per_tick": 1.375 - }, - { - "avg_batch_occupancy": 1.75, - "cache_misses": 0, - "concurrency": 2, - "decode_batches": 32, - "decode_tokens": 56, - "max_batch_occupancy": 2, - "peak_kv_bytes": 29184, - "prefill_tokens": 32, - "rejected_admissions": 0, - "ticks": 33, - "tokens_per_tick": 2.6667 - }, - { - "avg_batch_occupancy": 3.1111, - "cache_misses": 0, - "concurrency": 4, - "decode_batches": 18, - "decode_tokens": 56, - "max_batch_occupancy": 4, - "peak_kv_bytes": 52224, - "prefill_tokens": 32, - "rejected_admissions": 0, - "ticks": 19, - "tokens_per_tick": 4.6316 - }, - { - "avg_batch_occupancy": 4.0, - "cache_misses": 0, - "concurrency": 8, - "decode_batches": 14, - "decode_tokens": 56, - "max_batch_occupancy": 7, - "peak_kv_bytes": 75264, - "prefill_tokens": 32, - "rejected_admissions": 0, - "ticks": 15, - "tokens_per_tick": 5.8667 - } - ], - "saturation_concurrency": 8, - "schema_version": 1 - }, - "evidence_kind": "synthetic-unit", - "mid_run_telemetry_concurrency_4": { - "active_sessions": 4, - "batch_occupancy_avg": 4.0, - "batch_occupancy_last": 4, - "batch_occupancy_max": 4, - "completed_sessions": 0, - "decode_tokens_per_sec": 1637.355, - "decode_tokens_total": 20, - "kv_budget_bytes": 67108864, - "kv_pressure": 0.0008, - "kv_total_bytes": 55296, - "prefill_tokens_per_sec": 1309.884, - "prefill_tokens_total": 16, - "queue_depth": 4, - "rejected_admissions_total": 0, - "rejected_by_reason": {}, - "scratch_budget_bytes": 4, - "scratch_pressure": 1.0, - "scratch_used_bytes": 4, - "ticks": 6, - "weight_bytes": 0 - }, - "model": { - "hidden": 32, - "n_heads": 4, - "n_layers": 6, - "reference": "pure-numpy KV-cached dense-Llama (tests/test_hot_kv_state)", - "vocab": 48 - }, - "schema_version": 1, - "workload": { - "max_new_tokens": 8, - "prompt_len": 4, - "sessions": 8 - } -} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-013/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-013/README.md deleted file mode 100644 index 4258c21..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-013/README.md +++ /dev/null @@ -1,223 +0,0 @@ -# DGR-013 — Harden failure, cancellation, and restart semantics: evidence - -Status: done -Date: 2026-07-16 -Evidence kind: **synthetic-unit** (pure-numpy KV-cached dense-Llama reference + -node-local hardened stream). No model download, no GPU, no torch, no network, no -API credit. - -## Summary - -Implemented bounded, explicit failure/cancellation/restart semantics for the -per-Route-Session decode stream, layered on the DGR-007 Hot KV State manager -(isolated `(session, epoch)` KV) and the DGR-012 continuous-batch scheduler. The -goal (RALPH product objective) is that distributed speed never comes with hanging -or corrupted generations: every blocked op is bounded, every cancel frees state, -duplicate steps are idempotent, uncertain mutations are never silently replayed, -alpha failover restarts from token zero, and billing distinguishes what actually -completed. - -Everything runs against the same deterministic numpy dense-Llama reference the -default gate uses (`tests/test_hot_kv_state.py::_KvDenseLlama` / `_KvReferenceShard`), -so the whole failure matrix is deterministic, download-free, GPU-free, and -API-credit-free while exercising the **real** KV isolation path -(`KvBoundaryAdapter` + `HotKvStateManager`). The pinned llama.cpp worker (DGR-008) -implements the identical adapter contract, so the semantics carry over to native -execution unchanged. - -### What was built (`packages/node/meshnet_node/failure_semantics.py`, new) - -- **`DeadlineGuard` + `StreamTerminated`** — bounds every step against an absolute - deadline and a heartbeat-timeout on an injected clock. A reached deadline or a - lost heartbeat (peer health loss) raises `StreamTerminated(kind)` so a blocked - stream terminates instead of hanging. (**AC: deadlines/heartbeat terminate - blocked ops.**) -- **`CancellationToken`, `ShardCancellationGroup`, `CancellationOutcome`** — one - cancel fans across **every** node-local Shard of a Route Session, releasing the - `(session, epoch)` KV on each shard's manager and invoking every queued-buffer - release callback (the pending activation bundles). Idempotent. The DGR-012 - scheduler also gains a `cancel()` that drops queued/active work on this node and - frees its KV. (**AC: cancellation propagates across every Shard, releases KV + - queued buffers.**) -- **`IdempotencyLedger`, `StepKey`, `StepDisposition`, `UncertainMutationError`** — - records each committed `(session, epoch, step)`; a duplicate delivery returns the - recorded token with no re-mutation. A step whose mutation outcome is *uncertain* - (worker died mid-step) is marked uncertain and can **never** be replayed - silently — `begin()` on an uncertain (or still in-flight) step raises - `UncertainMutationError`, forcing verify-or-restart. (**AC: duplicate steps - idempotent; uncertain mutations never replayed silently.**) -- **`RestartController`** — alpha failover: opens the *next* route epoch, releases - every shard's prior-epoch KV, and `assert_fresh_start` fails closed if any shard - still holds new-epoch KV. The restart re-prefills the whole prompt from token - zero; the failed epoch becomes stale (KV manager rejects it). Unverified KV is - never migrated (RALPH runtime decision #14). (**AC: alpha failover restarts from - token zero rather than importing unverified KV.**) -- **`WorkStatus`, `WorkRecord`, `WorkLedger`** — a typed per-attempt work record - with four distinct statuses: `completed`, `cancelled`, `failed`, `unverified`. - Only `completed` records are billable; cancelled/failed/unverified tokens are - recorded for observability but never charged. JSON-safe for the tracker billing - handoff (`packages/tracker/meshnet_tracker/billing.py` charges only completed, - verified work). (**AC: billing/work records distinguish completed/cancelled/ - failed/unverified.**) -- **`HardenedSessionRunner`** — composes all of the above to drive one session's - prefill+decode through the adapter under a deadline/heartbeat guard + cancel - token, records the typed outcome, and `run_with_failover` restarts a transient - failure from token zero on a fresh epoch. -- **`FailureKind` + `classify_exception` + `work_status_for`** — stable-string - classification of worker death, stream reset, malformed bundle, stale epoch, - cache miss, deadline, heartbeat loss, and cancel, plus the failure→billing-status - mapping. Suitable for the native protocol's structured status. - -### Scheduler extension (`packages/node/meshnet_node/batch_scheduler.py`, DGR-012 file, additive) - -Purely additive so the DGR-012 gate stays green (16/16): -- `DoneReason.CANCELLED` / `DoneReason.FAILED` terminal reasons. -- `ContinuousBatchScheduler.cancel(session_id, *, reason)` — drops a queued - session from the bounded queue or releases an active session's KV, moving it to - the done set with a non-completed reason (never counted as completed work). -- `SchedulerTelemetry.cancelled_sessions` / `failed_sessions` counters. - -## Files changed - -- `packages/node/meshnet_node/failure_semantics.py` — new module (the whole - failure/cancel/restart layer above). -- `packages/node/meshnet_node/batch_scheduler.py` — additive `cancel()` + two - `DoneReason` members + two telemetry counters (DGR-012 file; its 16 tests still - pass unchanged). -- `tests/test_failure_semantics.py` — new, 22 tests (matrix below); reuses the - DGR-007 numpy reference via `from test_hot_kv_state import _KvDenseLlama, - _KvReferenceShard`. -- `.scratch/distributed-gguf-runtime/evidence/DGR-013/` — this README, - `commands.txt`, `generate_evidence.py`, `results.json`. -- `.ralph-tui/progress.md` — appended the DGR-013 note. -- `.scratch/distributed-gguf-runtime/issues/13-...md` — set `Status: done`. - -## Acceptance criteria → evidence - -| Criterion | Tests (`tests/test_failure_semantics.py`) | -|---|---| -| Deadlines/heartbeat loss terminate blocked stream ops | `test_deadline_terminates_a_blocked_stream_and_releases_kv`, `test_heartbeat_loss_terminates_a_blocked_stream`, `test_deadline_guard_reports_remaining_and_resets_on_heartbeat` | -| Cancellation propagates across every Shard, releases KV + queued buffers | `test_cancellation_token_terminates_stream_and_releases_kv`, `test_shard_cancellation_group_releases_every_shard_and_queued_buffers`, `test_scheduler_cancel_drains_queue_and_releases_active_kv`, `test_scheduler_cancel_rejects_a_completed_reason` | -| Duplicate steps idempotent; uncertain mutations never replayed silently | `test_duplicate_step_delivery_is_idempotent_no_remutation`, `test_idempotent_run_replays_tokens_without_advancing_kv`, `test_uncertain_mutation_is_never_replayed_silently`, `test_in_flight_duplicate_is_treated_as_uncertain` | -| Alpha failover restarts from token zero, no unverified KV import | `test_alpha_failover_restarts_from_token_zero_and_completes`, `test_failover_refuses_to_import_unverified_kv`, `test_non_restartable_failure_is_not_retried` | -| Worker death, stream reset, malformed bundle, stale epoch, cache miss | `test_worker_death_midstream_is_unverified_and_marks_step_uncertain`, `test_stream_reset_is_restartable_failure`, `test_malformed_bundle_is_classified_and_does_not_corrupt_kv`, `test_stale_epoch_reference_is_rejected_and_classified`, `test_cache_miss_midstream_is_restartable` | -| Billing/work records distinguish completed/cancelled/failed/unverified | `test_work_ledger_distinguishes_all_four_statuses`, `test_work_status_and_classification_mapping`, plus the clean-run billability check `test_clean_run_matches_stateless_reference_and_is_billable` | - -## Failure matrix (real, deterministic — `results.json`) - -Generated by `generate_evidence.py` against the numpy dense-Llama (prompt `[7,3,9,1]`, -8 new tokens): - -| scenario | status | failure_kind | tokens | restartable | KV released | -|---|---|---|---|---|---| -| clean | completed | — | 8 | — | (held, then reaped) | -| deadline | failed | deadline-exceeded | 2 | no | yes | -| heartbeat_loss | failed | heartbeat-lost | 3 | no | yes | -| cancel | cancelled | cancelled | 3 | no | yes | -| worker_death | unverified | worker-death | 3 | yes | yes | -| stream_reset | failed | stream-reset | — | yes | yes | -| stale_epoch | failed | stale-epoch | — | no | (never opened) | -| cache_miss | failed | cache-miss | 4 | yes | (already evicted) | -| alpha_failover | **completed** (epoch 1) | — | 8 | — | old epoch stale | - -Alpha failover: attempt 0 (epoch 0) dies mid-step → `unverified`; the controller -advances to epoch 1, drops epoch-0 KV, and the restart re-prefills from token zero -→ `completed`, reproducing the byte-identical stateless reference. The old epoch is -now stale (a reference to it raises `StaleRouteEpochError`). Work ledger: -`{completed: 2, cancelled: 1, failed: 0, unverified: 2}`, `billable_tokens = 16` -(only the two completed streams — the failover restart and the clean run — are -billed; the cancelled and the two unverified attempts are not). - -## Commands and real results - -See `commands.txt`. Key results: - -``` -tests/test_failure_semantics.py -> 22 passed -tests/test_batch_scheduler.py -> 16 passed (DGR-012 unchanged) -tests/test_hot_kv_state.py -> 22 passed (DGR-007) -tests/test_gguf_backend.py -> 2 passed (DGR-009) -python -m compileall -q packages tests -> exit 0 -git diff --check -> exit 0 -python -m pytest -q -> 16 failed, 792 passed, 14 skipped in 253.93s -``` - -## Full-suite baseline (pre-existing, unrelated failures) - -The 16 failures are **pre-existing and unrelated to DGR-013**. None import -`failure_semantics` or `batch_scheduler`; they live in the tracker/control-plane, -node-startup, doctor, calibration, and route-benchmark suites and fail on the -model-download / control-plane / recipe-admission paths (e.g. -`UnsupportedRecipeParam: worker_transport` from the DGR-009 native recipe against -the Torch backend, and Torch/HF-model startup that this deterministic sandbox does -not provide). Removing the two DGR-013 files and re-running the failing tests -reproduces the identical failures (see `commands.txt`, 4-test spot check → same -4 failures), so DGR-013 introduces no new failure. - -Exact failing set (16): - -``` -tests/test_dynamic_routing.py::test_admin_can_replace_a_served_model_and_release_it -tests/test_manual_route_benchmark.py::test_pinned_route_uses_named_node -tests/test_manual_route_benchmark.py::test_unknown_route_node_is_400 -tests/test_manual_route_benchmark.py::test_invalid_route_shape_is_400 -tests/test_manual_route_benchmark.py::test_clients_without_route_are_unaffected -tests/test_manual_route_benchmark.py::test_benchmark_records_one_and_two_node_routes -tests/test_node_doctor.py::test_the_shipped_recipes_are_all_applicable_by_the_backend -tests/test_node_doctor.py::test_cli_doctor_flags_select_what_is_validated -tests/test_node_startup.py::test_preset_model_with_hf_repo_loads_torch_backend -tests/test_node_startup.py::test_real_model_startup_registers_downloaded_inventory_without_checksum -tests/test_toploc_calibration_dispatch.py::test_calibration_run_dispatches_only_solo_capable_nodes -tests/test_toploc_calibration_dispatch.py::test_calibration_run_persists_corpus_and_results_endpoint_reports_it -tests/test_toploc_calibration_dispatch.py::test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed -tests/test_tracker_capability_admission.py::test_an_enforcing_tracker_never_routes_a_node_whose_proof_does_not_cover_it[invalid] -tests/test_tracker_routing.py::test_torch_node_applies_tracker_load_shard_directive -tests/test_tracker_routing.py::test_shard_heal_cycle_surviving_node_covers_dead_peers_gap -``` - -## Limitations and deferred work - -- **Synthetic-unit, not real weights.** Semantics are exercised against the - deterministic numpy dense-Llama, not a downloaded GGUF, to keep the default gate - deterministic/download-free/GPU-free. Real worker-death/stream-reset behavior on - a live llama.cpp worker over gRPC belongs to DGR-008/DGR-010 (DGR-010 is blocked - — no certified dense-Llama artifact on this machine; see - `evidence/DGR-010/BLOCKED.md`). -- **Single-node per-session stream.** `HardenedSessionRunner` drives one full-shard - session (the node-local case); multi-node cancellation is modelled by - `ShardCancellationGroup` fanning across each node's KV manager. The cross-node - propagation *transport* (cancel frames over gRPC/relay) is the native protocol's - job (DGR-002/008); this story owns the local release + record semantics the - transport triggers. -- **Fault injection is deterministic.** Worker death is a shard that raises on the - Nth step; stream reset / deadline / heartbeat are injected via an explicit clock - and hook. This is what makes the matrix reproducible; live fault behavior is a - native/real-hardware property. -- **Greedy sampling only.** Reuses the DGR-006 greedy `SamplingContract`; the - idempotent-replay equality check depends on order-independent greedy decode. -- **Native / llama.cpp gates N/A.** No native code, CMake, or llama.cpp patch was - touched (same as DGR-005/006/007/012), so those gates do not apply. - -## Compatibility / migration notes - -- `failure_semantics.py` is a new, additive module — no existing behavior changes. -- `batch_scheduler.py` changes are additive (new enum members, one method, two - telemetry fields); the DGR-012 contract and its 16 tests are unchanged. -- `WorkRecord.to_dict()` / `WorkLedger.to_dict()` are JSON-safe and map cleanly to - the tracker `BillingLedger.charge_request` inputs: report `node_work` only for - `billable` (completed) records so cancelled/failed/unverified work is never - charged. `FailureKind` / `WorkStatus` are stable strings suitable for the native - protocol's structured status and the capability/heartbeat report. - -## Handoff for dependent stories - -- **DGR-008 (C++ gRPC worker):** implement the same contract natively — the worker - maps a transport deadline/heartbeat to `StreamTerminated`, a dropped stream to a - restartable failure, and a mid-`llama_decode` crash to an *uncertain* step - (mark-uncertain, never silent replay). `RestartController.failover` maps to - opening a fresh llama sequence under the new `(session, epoch)`; the failed - sequence's KV is dropped, never migrated. -- **DGR-010/DGR-014 (real acceptance / release gate):** drive the same failure - scenarios against the live worker to produce real cleanup/latency numbers, and - feed the `WorkLedger` status split into the billing/attribution comparison — - only `completed` work is charged. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-013/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-013/commands.txt deleted file mode 100644 index b1547d9..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-013/commands.txt +++ /dev/null @@ -1,36 +0,0 @@ -# DGR-013 — exact commands and real results (worktree venv) -VP=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python - -# Targeted story tests (this story) -$VP -m pytest -q tests/test_failure_semantics.py -# -> 22 passed - -# Dependency gates stay green -$VP -m pytest -q tests/test_batch_scheduler.py # DGR-012 -# -> 16 passed -$VP -m pytest -q tests/test_hot_kv_state.py # DGR-007 -# -> 22 passed -$VP -m pytest -q tests/test_gguf_backend.py # DGR-009 -# -> 2 passed - -# Quality gates -$VP -m compileall -q packages tests -# -> exit 0 -git diff --check -# -> exit 0 - -# Machine-readable evidence -$VP .scratch/distributed-gguf-runtime/evidence/DGR-013/generate_evidence.py -# -> wrote results.json; work statuses {'completed':2,'cancelled':1,'failed':0,'unverified':2} billable_tokens=16 - -# Full deterministic suite -$VP -m pytest -q -p no:cacheprovider -# -> 16 failed, 792 passed, 14 skipped in 253.93s - -# Clean-tree reproduction of the 16 pre-existing failures (DGR-013 files removed) -# rm packages/node/meshnet_node/failure_semantics.py tests/test_failure_semantics.py -$VP -m pytest -q tests/test_dynamic_routing.py::test_admin_can_replace_a_served_model_and_release_it \ - tests/test_node_doctor.py::test_the_shipped_recipes_are_all_applicable_by_the_backend \ - tests/test_tracker_routing.py::test_torch_node_applies_tracker_load_shard_directive \ - tests/test_node_startup.py::test_preset_model_with_hf_repo_loads_torch_backend -# -> 4 failed (same failures reproduce without any DGR-013 change) diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-013/generate_evidence.py b/.scratch/distributed-gguf-runtime/evidence/DGR-013/generate_evidence.py deleted file mode 100644 index ea4496f..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-013/generate_evidence.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python -"""Generate deterministic DGR-013 failure/cancel/restart evidence (results.json). - -Runs the real hardened per-session stream (``HardenedSessionRunner`` over the -DGR-007 ``KvBoundaryAdapter`` + ``HotKvStateManager``) through each failure mode -with the same pure-numpy dense-Llama reference the default gate uses. No model -download, no GPU, no torch, no network, no API credit. - -Run from the repo root with the worktree venv: - - .venv/bin/python .scratch/distributed-gguf-runtime/evidence/DGR-013/generate_evidence.py -""" - -from __future__ import annotations - -import json -import os -import sys - -import numpy as np - -# Make the worktree packages and the DGR-007 numpy reference importable, exactly -# as pytest's prepend-import + conftest do. -ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) -sys.path.insert(0, os.path.join(ROOT, "packages", "node")) -sys.path.insert(0, os.path.join(ROOT, "tests")) - -from meshnet_node.hot_kv_state import ( # noqa: E402 - HotKvStateConfig, - HotKvStateManager, - KvBoundaryAdapter, - StaleRouteEpochError, - kv_recipe_for, -) -from meshnet_node.batch_scheduler import GenerationRequest # noqa: E402 -from meshnet_node.failure_semantics import ( # noqa: E402 - CancellationToken, - FailureKind, - HardenedSessionRunner, - RestartController, - StreamTerminated, - WorkLedger, - WorkStatus, -) - -from test_hot_kv_state import _KvDenseLlama, _KvReferenceShard # noqa: E402 - - -class _FaultyShard(_KvReferenceShard): - def __init__(self, model, start, end, *, fail_at_call=None): - super().__init__(model, start, end) - self._fail_at_call = fail_at_call - self.calls = 0 - - def run_layers_cached(self, hidden, *, positions, past_kv): - self.calls += 1 - if self._fail_at_call is not None and self.calls == self._fail_at_call: - raise RuntimeError("worker died mid-step") - return super().run_layers_cached(hidden, positions=positions, past_kv=past_kv) - - -class _Clock: - def __init__(self): - self.now = 0.0 - - def __call__(self): - return self.now - - def advance(self, d): - self.now += d - - -def _adapter(model, *, config=None, shard=None): - shard = shard or _KvReferenceShard(model, 0, model.n_layers - 1) - manager = HotKvStateManager(kv_recipe_for(shard), config=config) - return KvBoundaryAdapter(shard, manager) - - -def _gen(sid, prompt, n, epoch=0): - return GenerationRequest( - session_id=sid, route_epoch=epoch, - prompt_token_ids=tuple(prompt), max_new_tokens=n, - ) - - -def _kv_released(manager, sid, epoch): - from meshnet_node.hot_kv_state import CacheMiss - return isinstance(manager.resolve(sid, epoch), CacheMiss) - - -def main() -> None: - model = _KvDenseLlama() - prompt = [7, 3, 9, 1] - n_new = 8 - ledger = WorkLedger() - scenarios = [] - - # 1. Clean baseline. - ad = _adapter(model) - r = HardenedSessionRunner(ad, work_ledger=ledger).run(_gen("clean", prompt, n_new)) - scenarios.append({ - "scenario": "clean", - "status": r.status.value, - "tokens": r.token_count, - "matches_reference": list(r.tokens) == model.stateless_greedy(prompt, n_new), - "kv_released": _kv_released(ad.manager, "clean", 0), - }) - - # 2. Deadline terminates a blocked stream. - clk = _Clock() - ad = _adapter(model) - r = HardenedSessionRunner(ad, clock=clk).run( - _gen("deadline", prompt, 50), deadline=3.0, - before_step=lambda _s: clk.advance(1.0), - ) - scenarios.append({ - "scenario": "deadline", "status": r.status.value, - "failure_kind": r.failure_kind.value, "tokens": r.token_count, - "kv_released": _kv_released(ad.manager, "deadline", 0), - }) - - # 3. Heartbeat/health loss terminates a blocked stream. - clk = _Clock() - ad = _adapter(model) - r = HardenedSessionRunner(ad, clock=clk).run( - _gen("heartbeat", prompt, 50), heartbeat_timeout=1.5, - heartbeat=lambda step: step < 2, - before_step=lambda _s: clk.advance(1.0), - ) - scenarios.append({ - "scenario": "heartbeat_loss", "status": r.status.value, - "failure_kind": r.failure_kind.value, "tokens": r.token_count, - "kv_released": _kv_released(ad.manager, "heartbeat", 0), - }) - - # 4. Explicit client cancellation releases KV. - ad = _adapter(model) - tok = CancellationToken() - r = HardenedSessionRunner(ad, work_ledger=ledger).run( - _gen("cancel", prompt, 50), cancel_token=tok, - before_step=lambda step: tok.cancel("client-hangup") if step == 3 else None, - ) - scenarios.append({ - "scenario": "cancel", "status": r.status.value, - "failure_kind": r.failure_kind.value, "tokens": r.token_count, - "kv_released": _kv_released(ad.manager, "cancel", 0), - }) - - # 5. Worker death mid-step -> unverified. - ad = _adapter(model, shard=_FaultyShard(model, 0, model.n_layers - 1, fail_at_call=4)) - r = HardenedSessionRunner(ad, work_ledger=ledger).run(_gen("worker", prompt, n_new)) - scenarios.append({ - "scenario": "worker_death", "status": r.status.value, - "failure_kind": r.failure_kind.value, "tokens": r.token_count, - "restartable": r.restartable, "kv_released": _kv_released(ad.manager, "worker", 0), - }) - - # 6. Stream reset -> failed, restartable. - ad = _adapter(model) - def reset(step): - if step == 2: - raise StreamTerminated(FailureKind.STREAM_RESET, "peer reset") - r = HardenedSessionRunner(ad).run(_gen("reset", prompt, n_new), before_step=reset) - scenarios.append({ - "scenario": "stream_reset", "status": r.status.value, - "failure_kind": r.failure_kind.value, "restartable": r.restartable, - }) - - # 7. Stale epoch -> failed. - ad = _adapter(model) - ad.manager.open("stale", 5) - r = HardenedSessionRunner(ad).run(_gen("stale", prompt, n_new, epoch=3)) - scenarios.append({ - "scenario": "stale_epoch", "status": r.status.value, - "failure_kind": r.failure_kind.value, - }) - - # 8. Cache miss mid-stream -> restartable. - ad = _adapter(model) - mgr = ad.manager - r = HardenedSessionRunner(ad).run( - _gen("miss", prompt, 12), - before_step=lambda step: mgr.release("miss", 0) if step == 4 else None, - ) - scenarios.append({ - "scenario": "cache_miss", "status": r.status.value, - "failure_kind": r.failure_kind.value, "tokens": r.token_count, - "restartable": r.restartable, - }) - - # 9. Alpha failover: restart from token zero, no unverified KV import. - faulty = _FaultyShard(model, 0, model.n_layers - 1, fail_at_call=3) - ad = _adapter(model, shard=faulty) - runner = HardenedSessionRunner(ad, work_ledger=ledger) - controller = RestartController([ad.manager]) - fo = runner.run_with_failover(_gen("failover", prompt, n_new, epoch=0), controller, - max_restarts=2) - old_epoch_stale = False - try: - ad.manager.resolve("failover", 0) - except StaleRouteEpochError: - old_epoch_stale = True - scenarios.append({ - "scenario": "alpha_failover", - "final_status": fo.outcome.status.value, - "final_epoch": fo.outcome.route_epoch, - "restarts": fo.restarts, - "restarted_from_token_zero": list(fo.outcome.tokens) == model.stateless_greedy(prompt, n_new), - "old_epoch_stale": old_epoch_stale, - "attempt_statuses": [a.status.value for a in fo.attempts], - }) - - result = { - "schema_version": 1, - "evidence_kind": "synthetic-unit", - "model": { - "architecture": model.architecture_adapter, - "n_layers": model.n_layers, "vocab": model.vocab, "hidden": model.hidden, - }, - "scenarios": scenarios, - "work_ledger": ledger.to_dict(), - } - - out_path = os.path.join(os.path.dirname(__file__), "results.json") - with open(out_path, "w") as fh: - json.dump(result, fh, indent=2) - fh.write("\n") - counts = ledger.counts_by_status() - print(f"wrote {out_path}") - print(f"work statuses: {counts} billable_tokens={ledger.billable_tokens()}") - - -if __name__ == "__main__": - main() diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-013/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-013/results.json deleted file mode 100644 index 2070d3e..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-013/results.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "schema_version": 1, - "evidence_kind": "synthetic-unit", - "model": { - "architecture": "dense-llama", - "n_layers": 6, - "vocab": 48, - "hidden": 32 - }, - "scenarios": [ - { - "scenario": "clean", - "status": "completed", - "tokens": 8, - "matches_reference": true, - "kv_released": false - }, - { - "scenario": "deadline", - "status": "failed", - "failure_kind": "deadline-exceeded", - "tokens": 2, - "kv_released": true - }, - { - "scenario": "heartbeat_loss", - "status": "failed", - "failure_kind": "heartbeat-lost", - "tokens": 3, - "kv_released": true - }, - { - "scenario": "cancel", - "status": "cancelled", - "failure_kind": "cancelled", - "tokens": 3, - "kv_released": true - }, - { - "scenario": "worker_death", - "status": "unverified", - "failure_kind": "worker-death", - "tokens": 3, - "restartable": true, - "kv_released": true - }, - { - "scenario": "stream_reset", - "status": "failed", - "failure_kind": "stream-reset", - "restartable": true - }, - { - "scenario": "stale_epoch", - "status": "failed", - "failure_kind": "stale-epoch" - }, - { - "scenario": "cache_miss", - "status": "failed", - "failure_kind": "cache-miss", - "tokens": 4, - "restartable": true - }, - { - "scenario": "alpha_failover", - "final_status": "completed", - "final_epoch": 1, - "restarts": 1, - "restarted_from_token_zero": true, - "old_epoch_stale": true, - "attempt_statuses": [ - "unverified", - "completed" - ] - } - ], - "work_ledger": { - "schema_version": 1, - "records": [ - { - "session_id": "clean", - "route_epoch": 0, - "status": "completed", - "tokens": 8, - "failure_kind": null, - "detail": "", - "billable": true - }, - { - "session_id": "cancel", - "route_epoch": 0, - "status": "cancelled", - "tokens": 3, - "failure_kind": "cancelled", - "detail": "operation cancelled: client-hangup", - "billable": false - }, - { - "session_id": "worker", - "route_epoch": 0, - "status": "unverified", - "tokens": 3, - "failure_kind": "worker-death", - "detail": "worker died mid-step", - "billable": false - }, - { - "session_id": "failover", - "route_epoch": 0, - "status": "unverified", - "tokens": 2, - "failure_kind": "worker-death", - "detail": "worker died mid-step", - "billable": false - }, - { - "session_id": "failover", - "route_epoch": 1, - "status": "completed", - "tokens": 8, - "failure_kind": null, - "detail": "", - "billable": true - } - ], - "counts_by_status": { - "completed": 2, - "cancelled": 1, - "failed": 0, - "unverified": 2 - }, - "billable_tokens": 16 - } -} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-014/BLOCKED.md b/.scratch/distributed-gguf-runtime/evidence/DGR-014/BLOCKED.md deleted file mode 100644 index f855943..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-014/BLOCKED.md +++ /dev/null @@ -1,55 +0,0 @@ -# DGR-014 — Blocked handoff - -Status: blocked -Date: 2026-07-16 - -## Blocker - -This release-gate story cannot be completed in the current workspace state because the prerequisite real-model comparison chain is still missing its certified dense-Llama artifact on mounted storage. - -Verified blockers: - -- `DGR-011` is still not passed in `.scratch/distributed-gguf-runtime/prd.json`. -- `DGR-011` is explicitly blocked in `.scratch/distributed-gguf-runtime/evidence/DGR-011/BLOCKED.md`. -- `DGR-011` depends on `DGR-010`, and `DGR-010` is blocked because there is no certified dense-Llama artifact available on the mounted drive. -- Current mounted-model storage still only shows Qwen artifacts and llama.cpp vocab GGUFs, not the certified dense-Llama GGUF/safetensors pair needed for a comparable real run. - -## Verified current state - -- The DGR-001 performance contract exists and defines the benchmark lanes, metrics, and stop condition that later release gates must keep unchanged. -- The DGR-012 scheduler and DGR-013 failure semantics evidence are present and usable as supporting context, but they do not satisfy the real final comparison required here. -- `packages/node/meshnet_node/performance_contract.py` already contains the contract metadata and a live endpoint benchmark shim, but there is no recorded DGR-014 release-gate run and no final immutable comparison artifact. -- `evidence/DGR-014/README.md` does not exist yet because the acceptance criteria could not be completed. - -## Commands run - -```bash -sed -n '1,260p' .claude/memory/MEMORY.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/issues/14-enforce-the-gguf-versus-safetensors-release-gate.md -sed -n '1,260p' .ralph-tui/progress.md -git status --short -sed -n '1,260p' .scratch/distributed-gguf-runtime/prd.json -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-001/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-012/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-013/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-011/BLOCKED.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md -find /run/media/popov/d/DEV/models /run/media/popov/d/DEV/llamacpp/llama.cpp/models -maxdepth 4 \( -iname '*llama*' -o -iname '*deepseek*' -o -iname '*dense*' -o -name '*.gguf' -o -name '*.safetensors' -o -name 'config.json' \) -``` - -## Known limitations - -- No certified dense-Llama artifact is mounted, so the real distributed safetensors-versus-GGUF comparison cannot be executed. -- No immutable release-gate evidence can be produced without that artifact and the completed DGR-011 route comparison. -- No code was changed in this iteration. - -## Compatibility notes - -- The DGR-001 contract remains the source of truth for thresholds and metric names. -- Any future DGR-014 run must keep those thresholds unchanged and compare the same certified model/hardware/network scenario for both routes. - -## Dependent-story handoff - -- Finish `DGR-010` and `DGR-011` first with a certified dense-Llama artifact on mounted storage. -- Then run the current distributed safetensors and distributed GGUF routes on the same comparable scenario, record the final numbers in `evidence/DGR-014/README.md`, and update the issue status only after the gate passes. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-015/BLOCKED.md b/.scratch/distributed-gguf-runtime/evidence/DGR-015/BLOCKED.md deleted file mode 100644 index 8be09f6..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-015/BLOCKED.md +++ /dev/null @@ -1,78 +0,0 @@ -# DGR-015 — Blocked handoff - -Status: blocked -Date: 2026-07-16 - -## Blocker - -This story cannot be completed in the current workspace state because its -mandatory prerequisite, DGR-014, is still not passed. - -Verified blocker chain: - -- `.scratch/distributed-gguf-runtime/prd.json` still marks `DGR-014` as - `"passes": false`, so DGR-015 is not released for completion. -- `.scratch/distributed-gguf-runtime/evidence/DGR-014/BLOCKED.md` records the - release-gate blocker: the certified dense-Llama artifact required for the - comparable real-model comparison is not mounted on this machine. -- `DGR-014` depends on `DGR-011`, which is also blocked because `DGR-010` - cannot run without that same certified dense-Llama artifact. -- The current codebase still fails closed for `qwen3` / `qwen3-moe` in - `packages/node/meshnet_node/boundary_adapter.py`, which is correct for the - current state but means no Qwen3 family recipe is certified yet. - -## Verified current state - -- Dense-Llama boundary semantics, Hot KV isolation, batching, and failure - semantics are already implemented and covered by prior stories. -- Qwen3 strings are present in tracker/model metadata, but they are not yet - backed by a certified architecture adapter or real-model acceptance evidence. -- No `evidence/DGR-015/README.md` exists yet because the acceptance criteria - could not be completed. - -## Commands run - -```bash -sed -n '1,260p' .claude/memory/MEMORY.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/issues/15-add-and-certify-a-qwen3-qwen3-moe-adapter.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/architecture.md -sed -n '1,260p' CONTEXT.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/prd.json -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-014/BLOCKED.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-013/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-012/README.md -sed -n '1,260p' packages/node/meshnet_node/boundary_adapter.py -sed -n '1,260p' packages/node/meshnet_node/model_catalog.py -sed -n '1,220p' packages/node/meshnet_node/model_metadata.json -sed -n '1,260p' packages/tracker/meshnet_tracker/capability.py -sed -n '1,260p' packages/tracker/meshnet_tracker/server.py -rg -n "qwen3|qwen3-moe|Qwen3|MoE|router|top-k|shared expert|shared_expert|expert" packages/node/meshnet_node packages/tracker/meshnet_tracker tests -g '!**/__pycache__/**' -git status --short -``` - -## Known limitations - -- No certified dense-Llama artifact is mounted, so DGR-014 cannot complete and - DGR-015 remains blocked behind it. -- No real consumer-hardware Qwen3 acceptance run was possible in this workspace. -- No code was changed in this iteration. - -## Compatibility notes - -- The current boundary adapter intentionally fails closed for uncertified - architectures. That is the correct behavior until a dedicated Qwen3 adapter is - implemented and certified. -- Existing dense-Llama coverage and Hot KV semantics remain the source of truth - for the shared protocol and cache behavior. - -## Dependent-story handoff - -- Finish `DGR-010`, `DGR-011`, and `DGR-014` first with a certified dense-Llama - artifact on mounted storage. -- Once the release gate passes, implement the Qwen3 family adapter as a separate - certified architecture rather than by extending dense-Llama with unchecked name - substitutions. -- Record the real-model Qwen3 parity, admission, memory, and communication - evidence in `evidence/DGR-015/README.md`, then update the issue status only - after the gate passes. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-016/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-016/README.md deleted file mode 100644 index 49ada7f..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-016/README.md +++ /dev/null @@ -1,145 +0,0 @@ -# DGR-016 — Upstream llama.cpp collaboration package - -Status: partial, blocked by DGR-010 -Date: 2026-07-16 - -## Summary - -Assembled the upstream-facing collaboration package for llama.cpp without -pulling Meshnet routing or control-plane logic into the upstream ask. - -Durable outputs created for this story: - -- `api-note.md` with the generic hook split and patch-per-concern proposal -- `outreach.md` with a maintainer-facing draft for Georgi/llama.cpp - -The package is grounded in the existing research artifacts and the already -implemented deterministic tests for: - -- range-aware GGUF ownership and introspection -- architecture boundary input/output -- layer-filtered KV/session ownership -- reproducible pinned worker build wiring - -The story itself remains blocked because DGR-010 is still marked `passes: false` -and only has a blocked handoff, not a completed real-model acceptance README. - -## Files changed - -- `.scratch/distributed-gguf-runtime/evidence/DGR-016/README.md` -- `.scratch/distributed-gguf-runtime/evidence/DGR-016/api-note.md` -- `.scratch/distributed-gguf-runtime/evidence/DGR-016/outreach.md` - -## Commands run and real results - -### Dependency and context review - -```bash -sed -n '1,260p' .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/issues/16-produce-the-upstream-llama-cpp-collaboration-package.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-010/BLOCKED.md -sed -n '1,260p' docs/adr/0024-distributed-gguf-runtime.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/architecture.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/decision-framework.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/implementation-strategy.md -sed -n '1,260p' CONTEXT.md -``` - -Result: - -- confirmed the runtime target is a small pinned llama.cpp worker with Meshnet - kept outside upstream -- confirmed DGR-010 is still blocked because there is no certified dense-Llama - artifact on mounted storage - -### Package-relevant targeted pytest - -```bash -python -m pytest -q tests/test_llama_worker_build.py tests/test_gguf_backend.py tests/test_gguf_ownership.py tests/test_boundary_adapter.py tests/test_hot_kv_state.py -``` - -Result: - -- `50 passed in 0.90s` - -### Broader focused pytest slice - -```bash -python -m pytest -q tests/test_llama_worker_build.py tests/test_native_shard_protocol.py tests/test_gguf_backend.py tests/test_boundary_adapter.py tests/test_gguf_ownership.py tests/test_hot_kv_state.py tests/test_kv_cache_distributed.py -``` - -Result: - -- `58 passed, 1 skipped, 9 failed, 12 errors in 1.27s` -- failures were pre-existing environment issues, not this documentation-only - package: - - `tests/test_native_shard_protocol.py` imported generated protobuf code built - against gencode 7.35.0 while the active runtime is 6.33.6 - - `tests/test_kv_cache_distributed.py` hit sandbox socket `PermissionError` - when trying to bind localhost servers - -### Research evidence review - -```bash -sed -n '1,260p' docs/research/distributed-gguf-landscape.md -sed -n '1,260p' docs/research/distributed-gguf-github-followup.md -sed -n '1,220p' .scratch/distributed-gguf-runtime/evidence/DGR-004/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-006/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-007/README.md -sed -n '1,260p' .scratch/distributed-gguf-runtime/evidence/DGR-009/README.md -``` - -Result: - -- confirmed Nakshatra and prima.cpp are the right source/test donors for the - upstream ask -- confirmed the generic API surface is range loading, boundary I/O, and KV - ownership, not Meshnet policy - -### Package assembly - -No code generation, downloads, or model execution were required for this story. -The package is documentation-only and deterministic. - -```bash -python -m compileall -q packages tests -git diff --check -``` - -Result: - -- both commands exited 0 - -## Correctness / performance / hardware classification - -- Correctness evidence: research-only, no live model execution -- Performance evidence: none in this story -- Hardware evidence: none in this story - -## Known limitations and deferred work - -- DGR-010 remains blocked, so this package cannot be treated as the final - release-ready upstream handoff. -- The outreach draft is human-ready but not sent. -- The doc package does not change llama.cpp source code; it only prepares the - upstream ask and test mapping. - -## Compatibility / migration notes - -- Exact upstream pin for the eventual patch series: `b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac` -- The proposed patch split is: - 1. range-aware loading and ownership introspection - 2. boundary input/output and named tensor bundles - 3. layer-filtered KV and local sequence ownership -- Meshnet routing, billing, relay transport, and volunteer-network policy stay - outside llama.cpp. -- The deterministic examples already exist in the tree and can be trimmed into - upstream-facing MREs when the human maintainer sends the package. - -## Dependent-story handoff - -- DGR-010 must clear before any real-model validation can be cited as the final - end-to-end proof for this upstream package. -- Once DGR-010 has a completed evidence README, the package can be refreshed - with the real-model context and sent to the llama.cpp maintainers as a - smaller review bundle. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-016/api-note.md b/.scratch/distributed-gguf-runtime/evidence/DGR-016/api-note.md deleted file mode 100644 index ab73302..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-016/api-note.md +++ /dev/null @@ -1,90 +0,0 @@ -# DGR-016 API note: narrow llama.cpp hooks, no Meshnet policy - -This note is the upstream-facing shape for the collaboration package. - -## Goal - -Keep the llama.cpp ask small: - -- expose generic model-layer hooks that are useful to any local or remote - layer-worker setup; -- keep Meshnet routing, session ownership, billing, and relay transport out of - llama.cpp; -- preserve one patch per concern so the series rebases cleanly on the pinned - upstream commit. - -## Concern 1: range-aware loading and authoritative tensor ownership - -Requested surface: - -- accept a contiguous `[start_layer, end_layer)` range; -- expose whether the worker owns embeddings, final norm, and final head; -- make the loaded range authoritative from the model state, not from CLI - claims; -- allow unowned tensors to be absent rather than fabricated. - -Why this is upstreamable: - -- it is generic loader and introspection plumbing; -- it helps any local partitioned inference mode; -- it does not require any Meshnet identity, route, or transport type. - -Minimal examples/tests: - -- `tests/test_gguf_ownership.py` -- `tests/test_llama_worker_build.py` - -## Concern 2: architecture boundary input/output - -Requested surface: - -- accept a versioned boundary bundle carrying one or more named tensors; -- support an unnormalized residual stream as the intermediate handoff; -- keep final norm, LM head, and sampling on the tail shard only; -- keep the bundle format explicit about name, shape, dtype, byte order, and - fragments. - -Why this is upstreamable: - -- it matches both dense Llama and other certified adapter families; -- it does not assume Meshnet or any specific wire protocol; -- it gives a stable ABI for a layer-worker boundary. - -Minimal examples/tests: - -- `tests/test_boundary_adapter.py` -- `tests/test_native_shard_protocol.py` - -## Concern 3: layer-filtered KV and session mapping - -Requested surface: - -- let the worker own KV only for its layer range; -- map a stable session/context identifier to the local sequence; -- allow cache miss, stale epoch, truncate, release, and eviction semantics; -- reject incompatible cache recipes rather than trying to heal them silently. - -Why this is upstreamable: - -- it is a local sequence/KV API, not a network scheduler; -- it is useful to any supervisor that needs one process per layer range; -- it keeps session semantics outside llama.cpp while still making the worker - stateful in a controlled way. - -Minimal examples/tests: - -- `tests/test_hot_kv_state.py` -- `tests/test_kv_cache_distributed.py` - -## Suggested patch split - -Keep the series narrow and independently reviewable against the exact pinned -commit `b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac`: - -1. `range-aware-loading` and ownership introspection. -2. `boundary-input-output` and named tensor bundle handoff. -3. `layer-filtered-kv` and sequence ownership. - -The current Meshnet worker scaffold remains a project-owned wrapper and is not -part of the upstream ask. - diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-016/outreach.md b/.scratch/distributed-gguf-runtime/evidence/DGR-016/outreach.md deleted file mode 100644 index 95ed9d7..0000000 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-016/outreach.md +++ /dev/null @@ -1,43 +0,0 @@ -# DGR-016 outreach draft - -Subject: Narrow llama.cpp hooks for range loading, boundary I/O, and local KV ownership - -Hi Georgi and llama.cpp maintainers, - -We have been building a distributed GGUF route on top of a Meshnet control -plane, and the narrow upstreamable seam is now clear enough to summarize. - -We are not asking llama.cpp to own Meshnet routing, billing, relay transport, -or any volunteer-network policy. The upstream ask is limited to generic local -hooks that make partitioned inference easier to implement and easier to review: - -1. Range-aware loading and ownership introspection for contiguous layer ranges. -2. Architecture-defined boundary input/output using an explicit named-tensor - bundle. -3. Layer-filtered KV ownership and stable local sequence mapping. - -Why we think this is generally useful: - -- Nakshatra already demonstrates the value of a narrow layer-worker seam and - partial GGUF loading. -- prima.cpp shows the same idea from a different angle with selective loading, - local KV, and boundary residual transport. -- Both projects suggest the same conclusion: the missing API is not Meshnet - specific, it is a local runtime seam that any layer-partitioned supervisor can - use. - -The package we would upstream is intentionally split into one concern per patch -so review stays small: - -- range-aware loading and tensor ownership; -- boundary I/O for intermediate residual state; -- layer-filtered KV and sequence ownership. - -If useful, we can send the concrete MRE/test mapping next. We already have -deterministic examples covering the loader, boundary contract, and KV/session -semantics in the Meshnet tree, and we can trim them into upstream-focused test -cases. - -Thanks, -Meshnet maintainers - diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-017/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-017/README.md new file mode 100644 index 0000000..a1de3ff --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-017/README.md @@ -0,0 +1,114 @@ +# DGR-017 evidence — superseded backlog cleanup + +**Completed:** 2026-07-16 +**Branch:** `ralph/distributed-gguf-runtime` +**Planning checkpoint before cleanup:** `81b1fa6` +**Authority:** `.scratch/distributed-gguf-runtime/prd.json` + +## Outcome + +The old DGR-001…016 completion claims and active artifacts were reconciled against the live branch. No old pass state transferred to the new implementation roadmap. + +The active `packages/` and `tests/` trees were restored exactly to `origin/master`. The branch therefore no longer exposes a nominal GGUF startup path backed by unimplemented transport methods, a protobuf-only native scaffold, or isolated synthetic scheduler/cache/failure modules as if they were a working distributed GGUF runtime. + +## Classification and disposition + +### Retained + +- Accepted ADRs and repository research, including `docs/research/colibri-implementation-audit.md`. +- The authoritative 55-story roadmap `DGR-017…071` and its generated issue specifications. +- The real public-relay smoke benchmark, moved with provenance to `legacy-public-relay-smoke-benchmark.json`. +- Git history containing the complete superseded implementation/reference work. + +### Removed from the active tree + +- Legacy issue specifications DGR-001…016 and their stale/blocked/synthetic evidence directories. +- The nonfunctional `gguf_backend` startup path whose gRPC execution methods raised not-implemented errors. +- Synthetic/reference-only boundary, Hot KV, scheduler, failure, recipe, ownership, and native-protocol modules that were not a real llama.cpp Shard runtime. +- The protobuf round-trip-only native scaffold, placeholder llama.cpp patch, generated bindings/build workspace, and associated tests. +- Tracker/admission/source modifications coupled to that superseded scaffold. + +### Confirmed absent and still required + +- Real standalone C++ gRPC Shard worker. +- Exact pinned llama.cpp manifest and verified patch stack. +- Range-aware GGUF tensor ownership and real ranged execution. +- Real Shard-local llama.cpp KV/V4 auxiliary state. +- DeepSeek V4 boundary adapter and ranged parity. +- Real multi-machine DeepSeek V4 alpha or beta acceptance. + +These remain `passes: false` in DGR-018…071. + +## Before-cleanup baseline + +Command: + +```bash +.venv-rocm/bin/python -m pytest -q \ + tests/test_performance_contract.py tests/test_native_shard_protocol.py \ + tests/test_gguf_ownership.py tests/test_boundary_adapter.py \ + tests/test_hot_kv_state.py tests/test_gguf_backend.py \ + tests/test_batch_scheduler.py tests/test_failure_semantics.py \ + tests/test_llama_worker_build.py tests/test_node_admission.py \ + tests/test_node_capability.py tests/test_tracker_capability_admission.py +``` + +Result: + +```text +216 passed, 2 skipped, 1 failed, 1 warning +``` + +The failure was a synthetic capability-test helper `KeyError: 'compatibility_fingerprint'`. The warning was a pre-existing heartbeat-thread `SystemExit` warning. + +## Cleanup verification + +### Source equality + +Command: + +```bash +git diff --quiet origin/master -- packages tests +``` + +Result: + +```text +packages_tests_match_origin_master=yes +``` + +The staged cleanup removes approximately 15.3k obsolete source/test/evidence lines from the active branch. + +### Cleanup-relevant regression suite + +Command: + +```bash +.venv-rocm/bin/python -m pytest -q \ + tests/test_node_admission.py tests/test_node_capability.py \ + tests/test_tracker_capability_admission.py \ + tests/test_kv_cache_distributed.py tests/test_real_distributed_inference.py +``` + +Result: + +```text +119 passed, 2 skipped, 1 warning in 15.90s +``` + +The warning is the same pre-existing heartbeat-thread `SystemExit` warning. + +### Known `origin/master` limitations + +The wider routing run produced `210 passed, 2 skipped, 4 failed, 1 warning`. Each failure reproduced individually while `packages/` and `tests/` matched `origin/master` exactly: + +- `test_tracker_models_endpoint_lists_registered_hf_repo_and_short_name_alias` +- `test_torch_node_applies_tracker_load_shard_directive` +- `test_shard_heal_cycle_surviving_node_covers_dead_peers_gap` +- `test_a_node_with_an_unusable_precision_covers_no_layers` + +They are recorded as pre-existing baseline defects and were not repaired or hidden by this cleanup story. + +## Dependency handoff + +DGR-018 and later stories must start from the cleaned upstream-equivalent runtime tree. Reuse concepts from superseded commits only by explicitly porting the smallest verified slice under the new story’s contracts, tests, and evidence gates. Git history is provenance, not completion evidence. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/public-relay-smoke-benchmark.json b/.scratch/distributed-gguf-runtime/evidence/DGR-017/legacy-public-relay-smoke-benchmark.json similarity index 100% rename from .scratch/distributed-gguf-runtime/evidence/DGR-001/public-relay-smoke-benchmark.json rename to .scratch/distributed-gguf-runtime/evidence/DGR-017/legacy-public-relay-smoke-benchmark.json diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-018/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-018/README.md new file mode 100644 index 0000000..43cd1b0 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-018/README.md @@ -0,0 +1,204 @@ +# DGR-018 evidence — canonical Ralph and Gitea metadata schema + +**Completed:** 2026-07-16 +**Branch:** `ralph/distributed-gguf-runtime` +**Authority:** `.scratch/distributed-gguf-runtime/prd.json` +**Dependency:** DGR-017 (`evidence/DGR-017/README.md`) — cleaned backlog reconciled to `origin/master`; no old pass state transferred. + +## Objective + +Make `prd.json` the validated source from which Markdown (and, later, Gitea) issues +can be generated losslessly, per +`.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md`. + +## Pre-existing state found (not caused by this story) + +Before any change in this session, `git status` showed `.scratch/distributed-gguf-runtime/prd.json` +already modified in the working tree relative to `HEAD` (commit `369b207`), with no corresponding +progress-log entry. Diffing against `HEAD` showed the working copy had **dropped** prd.json's +top-level `sourceOfTruth`, `qualityGates`, `metadataSchema`, `milestones`, and `supersededStories` +objects, while `userStories` itself was byte-identical to `HEAD`. This looked like an abandoned, +uncommitted partial edit from a prior session, not intentional current work — those fields are +exactly the schema/quality-gate/audit-provenance content this story depends on, and their loss +wasn't explained by any acceptance criterion. They were restored (see "Changes" below) rather than +silently accepted or discarded, per the instruction to investigate unexplained working-tree state +before building on top of it. + +## Changes + +### `scripts/ralph_prd_schema.py` (new) + +Single module providing: + +- **Parse:** `load_prd(path)` — JSON load with clear `PrdValidationError`s for missing file / + invalid JSON / non-object document. +- **Canonical schema registry:** `STORY_FIELDS` (name → required/type), `EXECUTION_MODES`, + `EVIDENCE_CLASSES`, `HARDWARE_FLAGS`, `UPSTREAM_FLAGS`, `TRIAGE_VALUES`. Covers every field named + in the acceptance criteria: stable `id`/`title`, `labels`, `milestone`, derived `type` + (`derive_type`), `dependsOn`, derived `blocks`, `triage`, `evidenceClass`, and + `hardware`/`model`/`upstream` flags. +- **Structural validation:** `validate_schema(data)` — required fields, types, enum membership, + ID convention, `type:`/`priority:` label cardinality, non-empty `acceptanceCriteria`. +- **Semantic validation:** `validate_semantics(data)` — unique IDs, unique titles, `dependsOn` + resolves to known stories (no self-dependency), dependency graph is acyclic (with a reported + cycle path on failure), `blocks` matches the dependency graph exactly (sorted set equality, not + superset), and `evidencePath` matches the per-story convention. +- **Fresh vs. in-progress backlog:** `validate_fresh_backlog(data)` additionally requires every + story to start `passes: false` (for a backlog that hasn't started execution yet); + `validate_backlog(data)` is the composed check for a real, in-flight backlog where some stories + have legitimately completed. +- **Self-consistency check:** `validate_metadata_schema_consistency(data)` — when prd.json declares + its own `metadataSchema`/`qualityGates` (as this one now does), verifies that self-documentation + hasn't drifted from what the validator actually enforces (enum sets, required/optional field + lists, presence of `qualityGates` and `generatedArtifactDisclaimer`). This is a no-op for minimal + fixture PRDs that don't carry that documentation. +- **Generation (one-directional, prd.json → artifact):** `render_issue_markdown(story, data)` + renders the exact Markdown convention already used by + `.scratch/distributed-gguf-runtime/issues/*.md`, sourcing the "Shared quality gates" bullets from + `data["qualityGates"]` and the leading disclaimer from + `data["metadataSchema"]["generatedArtifactDisclaimer"]` (falling back to a module default only + when `data` omits them) — not from a duplicated Python string literal. + `to_gitea_issue_payload(story, data)` wraps the same body into a Gitea create-issue-shaped payload + (`title`, `body`, `labels`, `milestone`). +- **Authority guard:** `check_generated_markdown_authority(text, disclaimer=...)` rejects generated + Markdown that's missing the disclaimer or that contains a conflicting authority claim (e.g. "this + file is authoritative"). There is deliberately no Markdown → prd.json parser, so a generated + artifact structurally cannot feed `passes` (or anything else) back into the authoritative source. +- CLI: `python scripts/ralph_prd_schema.py validate [--fresh]` and + `... render `. + +### `.scratch/distributed-gguf-runtime/prd.json` + +- Restored the top-level `sourceOfTruth`, `qualityGates`, `milestones`, and `supersededStories` + objects to their `HEAD` content (see "Pre-existing state" above); `userStories` was already + identical to `HEAD` and is unchanged in content. +- Extended `metadataSchema` (previously incomplete for this story's own acceptance criteria) with: + `requiredStoryFields` now also lists `notes` and `blocks` (present on all 55 stories); new + `optionalStoryFields: ["completionNotes"]`; new `hardwareValues`/`upstreamValues` enums (`model` + is documented as an open convention, not a closed enum, since quantization/model targets are + dynamic recipe inputs per `RALPH-CONTEXT.md`); new `typeDerivation` and `labelConventions` + (reserved prefixes, cardinality); new `generatedArtifactDisclaimer` (the exact string generated + artifacts must start with); extended `dependencyRules`/`authorityRule` prose to match what the + validator enforces. +- Reworded `sourceOfTruth`'s stale "All stories are unimplemented ... passes=false" clause, which + was no longer accurate once DGR-017 completed. +- Marked `DGR-018.passes = true` with `completionNotes` recording this story's outcome. + +### `.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md` + +Regenerated via `render_issue_markdown` to reflect `passes: true` (checked acceptance criteria, +"completed" status line, "Verified evidence" handoff line) — matching the same convention DGR-017's +issue file already used for a completed story. + +### `tests/test_ralph_prd_schema.py` (new) + +108 deterministic, model-download-free, GPU-free tests: + +- **Parse** (4 tests): real backlog parses to 55 stories; missing file, invalid JSON, and + non-object documents raise `PrdValidationError`. +- **Structural/semantic validation against the real backlog** (7 tests): passes `validate_schema`, + `validate_semantics`, and the composed `validate_backlog`; unique IDs/titles; all `dependsOn` + resolve; `blocks` matches the derived dependency graph for all 55 stories; no cycle; every + `passes: true` story carries `completionNotes` and an existing evidence README (a durable + invariant, not a hardcoded list of which stories have completed — that list will keep growing). +- **Structural/semantic failure-mode fixtures** (13 tests): missing required field, bad enum, wrong + type, empty `acceptanceCriteria`, multiple `type:` labels, duplicate ID, duplicate title, unknown + dependency, self-dependency, dependency cycle, mismatched `blocks`, bad `evidencePath`. +- **Fresh-backlog invariant** (3 tests): accepts all-`false`, rejects a premature `passes: true`, + and confirms `validate_backlog` (the in-progress variant) permits completed stories. +- **prd.json-as-source-of-truth for boilerplate** (9 tests): `qualityGates`/`metadataSchema` + self-consistency checks (no-op without them, catches a drifted enum, catches a missing + `qualityGates`), `quality_gate_bullets` flattening order, `authority_disclaimer` precedence and + fallback, and 3 tests asserting the real backlog's declared schema matches the code, its 7 + quality-gate bullets are intact, and its disclaimer matches the module default. +- **`derive_type`** (4 tests): label-derived type, release-gate synthetic type for HITL gate + stories, `None` when absent, and confirmation that the real backlog's two release-gate stories + (`DGR-054`, `DGR-070`) derive `release-gate`. +- **Markdown generation round trips** (55 parametrized + 6 tests): `render_issue_markdown` for + every story `DGR-017`..`DGR-071` is byte-for-byte identical to the corresponding file already in + `.scratch/distributed-gguf-runtime/issues/`; determinism; leading disclaimer; `Blocks (derived)` + rendering (`None` vs. listed); checkbox reflects `passes`; filename convention. +- **Authority-claim rejection** (4 tests): accepts real generated text, rejects a missing + disclaimer, rejects an overriding claim, and confirms every committed issue file in + `.scratch/distributed-gguf-runtime/issues/` passes the check. +- **Gitea payload generation** (3 tests): payload shape, body carries no information beyond what's + in prd.json, and every real story's payload is well-formed and authority-clean. + +## Commands and results + +```bash +python3 -m pytest -q tests/test_ralph_prd_schema.py +``` +```text +108 passed in 0.16s +``` + +```bash +python3 -m compileall -q packages tests +``` +Exit code 0, no output (all files compile). + +```bash +git diff --check +``` +Exit code 0 (no whitespace errors). + +```bash +python3 scripts/ralph_prd_schema.py validate .scratch/distributed-gguf-runtime/prd.json +``` +```text +OK: 55 stories validated. +``` + +```bash +python3 scripts/ralph_prd_schema.py validate .scratch/distributed-gguf-runtime/prd.json --fresh +``` +```text +ERROR: DGR-017: fresh backlog requires passes=false, got True +ERROR: DGR-018: fresh backlog requires passes=false, got True +2 validation error(s). +``` +Expected: `--fresh` is the invariant for a backlog that hasn't started execution; this backlog has +legitimately completed two stories, so it correctly fails that stricter check while passing the +plain (in-progress) `validate` command above. + +### Baseline: full repository suite (ad hoc `python3`, not a project venv) + +```bash +python3 -m pytest -q +``` +```text +20 failed, 776 passed, 13 skipped, 2 warnings in 244.17s (0:04:04) +``` +None of the failures touch `scripts/ralph_prd_schema.py` or `tests/test_ralph_prd_schema.py` +(neither file existed before this story; this story adds no changes to `packages/`). Four of the +20 failures reproduce exactly the pre-existing baseline defects DGR-017's evidence already recorded +(`test_tracker_models_endpoint_lists_registered_hf_repo_and_short_name_alias`, +`test_torch_node_applies_tracker_load_shard_directive`, +`test_shard_heal_cycle_surviving_node_covers_dead_peers_gap`, +`test_a_node_with_an_unusable_precision_covers_no_layers`). The remaining 16 (activation +compression, dynamic routing, gossip/relay, manual route benchmark, openai gateway, TOPLoC +calibration dispatch, tracker control plane) include a `ModuleNotFoundError: langchain` failure, +indicating this ad hoc `python3` lacks the project's `dev` extras (`langchain-openai`, etc.) rather +than a real regression; this environment has no project virtualenv (e.g. no `.venv-rocm`) to run +against instead. Not investigated further as out of scope for this story. + +## Limitations + +- No real Gitea instance or API integration exists; `to_gitea_issue_payload` defines the payload + shape (title/body/labels/milestone) only. Creating issues against a live Gitea server is future + work, not claimed here. +- `model` is intentionally validated as an open string, not a closed enum, per + `RALPH-CONTEXT.md`'s "Quantization and placement are dynamic recipe inputs" constraint; the schema + documents (`metadataSchema.modelConvention`) but does not restrict its value set. +- Validation and generation were exercised only against this feature's `prd.json` + (`.scratch/distributed-gguf-runtime/prd.json`); `docs/prd.json` and other `.scratch/*/prd.json` + files in this repo use a materially different (simpler) shape and are out of scope. + +## Dependency handoff + +DGR-021 and DGR-025 (this story's derived `blocks`) may treat `prd.json`'s `metadataSchema`, +`qualityGates`, and this validator/generator as stable. Any future field addition to a story shape +must extend `STORY_FIELDS` in `scripts/ralph_prd_schema.py` and the corresponding +`metadataSchema.requiredStoryFields`/`optionalStoryFields` in `prd.json` together — +`validate_metadata_schema_consistency` fails closed if they drift apart. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-021/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-021/README.md new file mode 100644 index 0000000..a1e1bcc --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-021/README.md @@ -0,0 +1,101 @@ +# DGR-021 evidence — versioned named-tensor activation envelope + +**Completed:** 2026-07-17 +**Branch:** `distributed-gguf-runtime` +**Authority:** `.scratch/distributed-gguf-runtime/prd.json` +**Dependency:** DGR-018 (`evidence/DGR-018/README.md`) — canonical backlog schema / issue projection contract + +## Objective + +Establish the backend-neutral activation envelope used by direct and relayed Shard traffic, with stable versioning, named tensors, bounded fragmentation, checksum validation, and reserved extensibility for future state. + +## Changes + +### `packages/node/meshnet_node/protocol.py` (new) + +Added a self-contained activation-envelope module with: + +- `SCHEMA_NAME = "meshnet.activation-stream"` and `SCHEMA_VERSION = 1` +- `TensorFragment` + - bounded byte fragments with offset, compression tag, checksum, and extension preservation + - deterministic `to_dict()` / `from_dict()` round-trip +- `NamedTensor` + - named tensor metadata: `name`, `shape`, `dtype`, `byte_order`, `compression`, `checksum`, `fragments` + - fragmentation via `from_bytes(..., max_fragment_bytes=...)` + - checksum validation over reconstructed tensor bytes + - unknown-field preservation via `extensions` +- `ActivationEnvelope` + - top-level fields for `request_id`, `work_id`, `route_session`, `route_epoch`, `shard_start`, `effective_start`, `phase`, `position`, and `idempotency_step` + - reserved extension fields for `token_id_sideband`, `architecture_state`, `recurrent_state`, and `mtp` + - deterministic canonical serialization (`to_bytes`) and round-trip parsing (`from_bytes`) + - size-limit enforcement (`to_bytes(max_bytes=...)`) + - conversion from a live `TensorPayload` into the envelope and back again + +### `packages/node/meshnet_node/model_backend.py` + +Extended `TensorPayload` with envelope conversion helpers: + +- `TensorPayload.to_envelope(...)` +- `TensorPayload.from_envelope(...)` + +These keep the existing activation payload interface intact while exposing the new versioned envelope as the shared protocol layer. + +### `tests/test_activation_envelope.py` (new) + +Added focused deterministic tests covering: + +- deterministic envelope serialization and round-trip parsing +- tensor fragmentation and checksum validation +- unknown-field preservation at both envelope and tensor levels +- size-limit rejection +- `TensorPayload` ↔ envelope round-trip + +### `.scratch/distributed-gguf-runtime/prd.json` + +Marked `DGR-021.passes = true` and added completion notes recording the envelope implementation and verification commands. + +## Commands and results + +```bash +pytest -q tests/test_activation_envelope.py +``` + +```text +5 passed in 0.06s +``` + +```bash +pytest -q tests/test_activation_envelope.py tests/test_kv_cache_distributed.py -k 'session_is_stable_and_decode_payloads_are_single_token or large_prefill_activation_survives_zstd_compressed_hop' +``` + +```text +.. [100%] +2 passed, 21 deselected in 1.84s +``` + +```bash +python3 -m compileall packages/node/meshnet_node tests/test_activation_envelope.py +``` + +```text +Listing 'packages/node/meshnet_node'... +Listing 'packages/node/meshnet_node/native_protocol'... +Compiling 'tests/test_activation_envelope.py'... +``` + +```bash +git diff --check +``` + +```text +No whitespace errors +``` + +## Limitations + +- The envelope is implemented as a canonical deterministic JSON contract with dataclasses and conversion hooks, not generated `.proto` classes. The environment had `protobuf` available but not the `grpc_tools` generation toolchain, so I did not materialize a compiled proto artifact here. +- The direct/relayed HTTP/WebSocket transports remain byte-oriented; the envelope is the shared structured contract layered above those transports. + +## Dependency handoff + +DGR-022 and later shard-control stories can reuse the envelope contract and its `TensorPayload` conversion hooks as the stable activation metadata layer. Future work that requires generated protobuf code can replace the JSON serialization with a generated wire codec without changing the top-level field contract defined here. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-022/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-022/README.md new file mode 100644 index 0000000..0277eda --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-022/README.md @@ -0,0 +1,46 @@ +# DGR-022 evidence — Shard lifecycle and structured status RPC contract + +**Completed:** 2026-07-17 +**Branch:** `ralph/distributed-gguf-runtime` +**Authority:** `.scratch/distributed-gguf-runtime/prd.json` + +## Outcome + +Implemented the versioned, backend-neutral lifecycle/status contract consumed by a future generated gRPC binding. The contract keeps Meshnet routing, identity, authentication policy, billing, and llama.cpp ownership outside the worker contract. + +## Implemented + +- `packages/node/meshnet_node/shard_lifecycle.py` + - capability, health, session, cancellation, release, and metrics RPC names + - schema version negotiation and fail-closed unsupported-version handling + - structured status/error taxonomy with retryability and details + - lifecycle state machine for prefill/decode/cancel/release transitions + - monotonic idempotency-step enforcement and duplicate rejection + - bounded frame/byte flow control with cancellation-aware waits + - explicit cache expectation/result types + - deadline policy and TLS/auth transport hooks + - deterministic contract serialization round-trip +- `tests/test_shard_lifecycle.py` + - contract round-trip and RPC coverage + - unsupported-version rejection + - malformed transition and idempotency rejection + - cancellation/release behavior + - bounded flow-control behavior + - TLS hook and incomplete-contract fail-closed behavior + +## Verification + +```text +$ PYTHONPATH=packages/node pytest -q tests/test_shard_lifecycle.py tests/test_activation_envelope.py +17 passed in 0.10s +``` + +The existing DGR-021 activation-envelope tests remain green alongside DGR-022. + +## Scope limitation + +This story defines the lifecycle/status contract only. Generated Python/C++ protobuf bindings and the concrete `shard_runtime.proto` generation pipeline are DGR-023 and remain separate. + +## Dependency handoff + +DGR-023 may consume the RPC names, status taxonomy, version identity, deadlines, flow-control limits, and TLS/auth hooks when the canonical `.proto` schema and toolchain are provisioned. diff --git a/.scratch/distributed-gguf-runtime/evidence/README.md b/.scratch/distributed-gguf-runtime/evidence/README.md index c2a4e1e..e0cc243 100644 --- a/.scratch/distributed-gguf-runtime/evidence/README.md +++ b/.scratch/distributed-gguf-runtime/evidence/README.md @@ -1,15 +1,15 @@ -# Ralph task evidence +# Distributed GGUF Runtime evidence -Each completed story creates `evidence//README.md`. Fresh dependent iterations must read it before coding. +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -Required README sections: +## Authority and classes -1. Summary and acceptance decision. -2. Exact files changed. -3. Commands run and real exit/results. -4. Correctness, performance and hardware evidence classification. -5. Known limitations and deferred work. -6. Compatibility/migration notes. -7. Explicit handoff for each dependent story. +Evidence supports but never overrides `prd.json`. Valid classes are `model-free`, `fixture`, `real-model`, `real-hardware`, and `release`; lower classes cannot satisfy higher-class acceptance. Legacy DGR-001..016 directories remain unchanged for DGR-017 provenance audit and confer no completion credit. -Store raw machine-readable metrics, manifests and protocol artifacts beside the README. Never store secrets, model weights, build outputs or Ralph iteration logs here. +## Future story layout + +Each DGR-017..071 story writes `evidence//README.md` with summary, exact changed files, exact commands and real outputs, limitations, compatibility/migration notes, and dependent-story handoff. Machine-readable contracts, manifests, metrics, and raw logs live beside it. Never fabricate output. + +Real runs record exact model SHA and all split hashes, tokenizer, quant/recipe, llama.cpp pin+patch identity, backend/driver/toolchain, host/hardware/network, commands/environment (without secrets), raw parity/performance/resource results, and evidence class. Models live on configured mounted-drive storage, never `/home`. + +Routing certification records prove only the exact exercised backend/model/recipe lane. Compile-only, fixture, failed, or unavailable lanes remain registered-dark. V4 cache/state evidence must show KV and CSA/HCA/SWA/indexer/compressor data remain shard-local/session-keyed; route recovery evidence must show cache miss plus re-prefill/restart, not migration. diff --git a/.scratch/distributed-gguf-runtime/gitea-issues.json b/.scratch/distributed-gguf-runtime/gitea-issues.json new file mode 100644 index 0000000..26a7d9a --- /dev/null +++ b/.scratch/distributed-gguf-runtime/gitea-issues.json @@ -0,0 +1,335 @@ +{ + "repository": "https://git.d-popov.com/popov/neuron-tai", + "stories": { + "DGR-017": { + "number": 1, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/1", + "state": "closed", + "status": "completed" + }, + "DGR-018": { + "number": 2, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/2", + "state": "closed", + "status": "completed" + }, + "DGR-019": { + "number": 3, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/3", + "state": "open", + "status": "ready" + }, + "DGR-020": { + "number": 4, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/4", + "state": "open", + "status": "blocked" + }, + "DGR-021": { + "number": 5, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/5", + "state": "closed", + "status": "completed" + }, + "DGR-022": { + "number": 6, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/6", + "state": "open", + "status": "ready" + }, + "DGR-023": { + "number": 7, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/7", + "state": "open", + "status": "ready" + }, + "DGR-024": { + "number": 8, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/8", + "state": "open", + "status": "blocked" + }, + "DGR-025": { + "number": 9, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/9", + "state": "open", + "status": "ready" + }, + "DGR-026": { + "number": 10, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/10", + "state": "open", + "status": "blocked" + }, + "DGR-027": { + "number": 11, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/11", + "state": "open", + "status": "ready" + }, + "DGR-028": { + "number": 12, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/12", + "state": "open", + "status": "blocked" + }, + "DGR-029": { + "number": 13, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/13", + "state": "open", + "status": "blocked" + }, + "DGR-030": { + "number": 14, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/14", + "state": "open", + "status": "blocked" + }, + "DGR-031": { + "number": 15, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/15", + "state": "open", + "status": "blocked" + }, + "DGR-032": { + "number": 16, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/16", + "state": "open", + "status": "blocked" + }, + "DGR-033": { + "number": 17, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/17", + "state": "open", + "status": "blocked" + }, + "DGR-034": { + "number": 18, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/18", + "state": "open", + "status": "blocked" + }, + "DGR-035": { + "number": 19, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/19", + "state": "open", + "status": "blocked" + }, + "DGR-036": { + "number": 20, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/20", + "state": "open", + "status": "blocked" + }, + "DGR-037": { + "number": 21, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/21", + "state": "open", + "status": "blocked" + }, + "DGR-038": { + "number": 22, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/22", + "state": "open", + "status": "blocked" + }, + "DGR-039": { + "number": 23, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/23", + "state": "open", + "status": "blocked" + }, + "DGR-040": { + "number": 24, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/24", + "state": "open", + "status": "blocked" + }, + "DGR-041": { + "number": 25, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/25", + "state": "open", + "status": "blocked" + }, + "DGR-042": { + "number": 26, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/26", + "state": "open", + "status": "blocked" + }, + "DGR-043": { + "number": 27, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/27", + "state": "open", + "status": "blocked" + }, + "DGR-044": { + "number": 28, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/28", + "state": "open", + "status": "blocked" + }, + "DGR-045": { + "number": 29, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/29", + "state": "open", + "status": "blocked" + }, + "DGR-046": { + "number": 30, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/30", + "state": "open", + "status": "blocked" + }, + "DGR-047": { + "number": 31, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/31", + "state": "open", + "status": "blocked" + }, + "DGR-048": { + "number": 32, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/32", + "state": "open", + "status": "blocked" + }, + "DGR-049": { + "number": 33, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/33", + "state": "open", + "status": "blocked" + }, + "DGR-050": { + "number": 34, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/34", + "state": "open", + "status": "blocked" + }, + "DGR-051": { + "number": 35, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/35", + "state": "open", + "status": "blocked" + }, + "DGR-052": { + "number": 36, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/36", + "state": "open", + "status": "blocked" + }, + "DGR-053": { + "number": 37, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/37", + "state": "open", + "status": "blocked" + }, + "DGR-054": { + "number": 38, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/38", + "state": "open", + "status": "blocked" + }, + "DGR-055": { + "number": 39, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/39", + "state": "open", + "status": "blocked" + }, + "DGR-056": { + "number": 40, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/40", + "state": "open", + "status": "blocked" + }, + "DGR-057": { + "number": 41, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/41", + "state": "open", + "status": "blocked" + }, + "DGR-058": { + "number": 42, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/42", + "state": "open", + "status": "blocked" + }, + "DGR-059": { + "number": 43, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/43", + "state": "open", + "status": "blocked" + }, + "DGR-060": { + "number": 44, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/44", + "state": "open", + "status": "blocked" + }, + "DGR-061": { + "number": 45, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/45", + "state": "open", + "status": "blocked" + }, + "DGR-062": { + "number": 46, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/46", + "state": "open", + "status": "blocked" + }, + "DGR-063": { + "number": 47, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/47", + "state": "open", + "status": "blocked" + }, + "DGR-064": { + "number": 48, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/48", + "state": "open", + "status": "blocked" + }, + "DGR-065": { + "number": 49, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/49", + "state": "open", + "status": "blocked" + }, + "DGR-066": { + "number": 50, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/50", + "state": "open", + "status": "blocked" + }, + "DGR-067": { + "number": 51, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/51", + "state": "open", + "status": "blocked" + }, + "DGR-068": { + "number": 52, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/52", + "state": "open", + "status": "blocked" + }, + "DGR-069": { + "number": 53, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/53", + "state": "open", + "status": "blocked" + }, + "DGR-070": { + "number": 54, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/54", + "state": "open", + "status": "blocked" + }, + "DGR-071": { + "number": 55, + "url": "https://git.d-popov.com/popov/neuron-tai/issues/55", + "state": "open", + "status": "blocked" + } + } +} diff --git a/.scratch/distributed-gguf-runtime/implementation-strategy.md b/.scratch/distributed-gguf-runtime/implementation-strategy.md index 1801065..3050425 100644 --- a/.scratch/distributed-gguf-runtime/implementation-strategy.md +++ b/.scratch/distributed-gguf-runtime/implementation-strategy.md @@ -1,241 +1,40 @@ -# Focused implementation strategy: performant concurrent distributed inference +# Distributed GGUF Runtime implementation strategy -Status: Accepted planning direction -Last updated: 2026-07-13 +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -## Product objective +## Execution model -Enable clients to run top open models that do not fit on one consumer machine by combining independently owned model Shards into performant, concurrent Inference Routes. +Execute one numerically ordered, dependency-ready story per fresh Ralph context. Read `RALPH-CONTEXT.md`, source issue, and dependency evidence first; use TDD/fixture-first verification; finish with exact evidence. `prd.json` is the only state authority. -The project is not trying to reproduce every vLLM feature or support every inference engine. It is optimizing for: +## Sequence -1. Models larger than one node's RAM/VRAM. -2. Useful interactive decode speed on consumer CPU, AMD, NVIDIA, Vulkan, and mixed routes where certified. -3. Multiple concurrent Route Sessions without cache corruption or global serialization. -4. A lean runtime with one control plane and one primary GGUF engine. -5. Measured improvement over the existing Transformers/safetensors implementation. +1. **M0 DGR-017..020:** reconcile legacy reality, lock metadata/performance contracts, and run the independent whole-model baseline. +2. **M1 DGR-021..033:** protocol/lifecycle/codegen, exact identities and split artifacts, pinned upstream/patches, CPU then accelerator builds, `ShardEngine`, fixtures, fake worker. +3. **M2 DGR-034..043:** dense ranged ownership/boundary/parity/local state, worker integration, supervision/direct-relay, and measured GGUF inputs to unchanged routing. +4. **M3 DGR-044..054:** pin/inventory V4, adapt upstream boundary/local state/MoE/hash execution, pass parity and real 2–4 scenario, then enforce alpha with MTP off. +5. **M4 DGR-055..067:** batching/backpressure/failure/recovery/long-context, existing-routing 10+ certification, real scale, measured optimization/compression, MTP contract+implementation, hardware certification. +6. **M5 DGR-068..071:** packages, human upstream collaboration, beta gate (including MTP), and pin/patch/certification maintenance. -## Current reality +## Guardrails -The existing project already owns the differentiating distributed control plane: -- Tracker-selected contiguous Shards. -- Stable Route Sessions. -- Local per-Shard Hot KV State in the Transformers reference backend. -- Binary Activation Seams. -- Relay/direct routing, cancellation, telemetry, billing, and capability admission. -- Persistent relay and direct transport optimizations. +## Locked scope -The missing production path is a native GGUF execution worker that can load and execute only an assigned layer range while retaining local Hot KV State for concurrent Route Sessions. +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. -Whole-model llama.cpp, vLLM, and existing Transformers serving remain baselines or optional route kinds. They are not substitutes for native distributed Shards. +## Target identities -## Performance hypothesis—not an assumption +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. -GGUF itself is a format. Performance comes from llama.cpp/GGML's quantized kernels, memory layout, mmap, backend scheduling, and reduced working set. - -Quantized GGUF may be faster or may merely fit a larger model. Comparisons against safetensors must report both speed and quality because BF16 safetensors and Q4/Q8 GGUF are not numerically equivalent. - -Before expensive native work, establish controlled lanes: - -- Same model architecture and upstream revision. -- Same machine, prompt set, context, output length, sampling policy, and concurrency. -- Transformers/safetensors BF16 or the current production recipe. -- llama.cpp GGUF F16/BF16 or Q8 correctness lane where available. -- Q4_K_M or selected production quantization performance/fit lane. -- TTFT, prefill tok/s, decode tok/s, p50/p95 latency, RSS, VRAM, artifact size, energy where available, and output-quality drift. - -The program proceeds only if llama.cpp/GGUF provides at least one meaningful advantage recorded in a machine-readable performance contract: - -- Better decode or aggregate throughput at acceptable quality; or -- Materially lower memory that makes the target model routable while preserving useful throughput. - -## Parallelism we will use - -### Public Inference Route: layer/pipeline parallelism - -Each node independently executes one contiguous Shard. Activations cross seams; weights and Hot KV State remain local. - -This is the only public cross-machine model-parallel primitive in the first runtime. - -### Per-node continuous batching - -Autoregressive tokens remain sequential within one generation. Throughput comes from batching decode steps from multiple active Route Sessions inside each node using llama.cpp batches and sequence IDs or bounded context pools. - -This is essential. A worker that globally serializes sessions is not production-ready. - -### Multiple complete routes: data parallelism - -The Tracker may select multiple complete routes for independent requests. This increases network throughput and availability without requiring collectives between routes. - -### Trusted composite node: optional tensor/expert parallelism - -Tensor parallelism and expert parallelism require frequent collectives and tight compatibility. They may be used later inside one operator-controlled composite node or managed cluster exposed as one logical provider. They are not public WAN routing primitives. - -### Deferred mechanisms - -- Disaggregated prefill and KV transfer. -- Speculative decoding. -- Cross-route prefix snapshots. -- Route repair with KV migration. -- Public tensor/expert parallel collectives. - -They remain out of the critical path until the native layer route passes performance and concurrency gates. - -## Reuse decisions - -### llama.cpp/GGML: primary runtime substrate - -Reuse: - -- GGUF parsing and mmap. -- Quantized kernels. -- CPU, CUDA, HIP/ROCm, Vulkan, Metal, and other supported backends. -- Tokenizer and model architecture implementations. -- KV and sequence operations. -- Backend scheduler and graph execution. - -Maintain a small exact-commit fork only for the missing local seam: - -- Range-aware tensor ownership/loading. -- Architecture-defined boundary input/output. -- Intermediate boundary output without tail normalization. -- Layer-filtered KV and sequence mapping. - -Keep networking, Tracker logic, billing, and public protocol outside llama.cpp. Upstream generic hooks where possible. - -### vLLM: concepts and optional managed backend - -Use unmodified vLLM only as: - -- A whole-model node backend. -- A managed TP/PP/EP cluster represented as one logical provider. -- A performance/correctness baseline. - -Adapt concepts, not runtime code: - -- Named intermediate tensor bundles. -- Continuous batching and request-owner maps. -- Versioned KV-transfer compatibility fingerprints. -- Explicit send/receive/abort/failure lifecycle. -- Load telemetry and unbiased route selection. - -Do not fork vLLM for public Shards and do not transplant PagedAttention, Torch process groups, or GGUF-plugin kernels into the llama.cpp worker. - -### Nakshatra, prima.cpp, llama-gguf, LiGGUF, GPUStack - -Use as source and test donors only: - -- Nakshatra: partial-GGUF patches, daemon concepts, replay cases. -- prima.cpp: selected tensor ownership and local-layer KV evidence. -- llama-gguf: small protocol and integration-test patterns. -- LiGGUF: Q8 activation transport and tensor-reduction reference. -- historical GPUStack: resource preflight and role-oriented placement. - -Do not adopt or fork their repositories wholesale. - -## Battle-proven transport decision - -Use gRPC over HTTP/2 with Protocol Buffers for the native C++ Shard worker protocol. - -Why: - -- Mature Python and C++ implementations. -- Bidirectional streaming. -- HTTP/2 flow control and connection reuse. -- Deadlines, cancellation, status codes, TLS, authentication interceptors, and generated schemas. -- Avoids inventing a socket protocol. - -Scope boundary: - -- OpenAI-compatible client/Gateway APIs remain HTTP/SSE. -- Tracker/control APIs remain existing project interfaces. -- One long-lived bidirectional gRPC stream serves one Route Session Activation Seam. -- Existing relay/WebSocket infrastructure may carry the same versioned protobuf frames as opaque binary when direct gRPC reachability is unavailable. -- Large prefill tensors are chunked into bounded frames; decode bundles stay small. -- No QUIC/WebRTC/custom transport in this milestone. - -The public boundary uses a versioned named-tensor bundle rather than one anonymous tensor because architecture boundaries can require more than `hidden_states`. - -Minimum identity: - -```text -schema version -request/work id -Route Session id and route epoch -Model Artifact and runtime recipe fingerprint -Shard range and effective start -phase: prefill/decode/release/cancel -position/token range -named tensors with shape/dtype/byte order -compression and checksum -idempotency step id -cache expectation/result -``` - -## Concurrency model - -A native worker must not use one global serving sequence or one lock around all model execution. - -Required ownership: - -```text -(Route Session id, route epoch) - -> local sequence/context - -> Shard-local Hot KV State - -> bounded lease and memory accounting -``` - -The node scheduler: - -- Admits sessions against model memory and KV budget. -- Forms compatible decode batches from active sessions. -- Preserves per-session position and route order. -- Applies bounded queues and backpressure. -- Cancels/releases independently. -- Reports queue, batch, KV, prefill, decode, and seam telemetry. - -Initial deterministic gate: at least four concurrent sessions on a small certified model with no token/KV cross-talk. Final concurrency targets are hardware/recipe-specific and recorded by capability admission rather than hardcoded globally. - -## Stage gates - -### Gate A: performance hypothesis - -Controlled safetensors-versus-GGUF benchmark produces a signed/reproducible report and locks thresholds. Stop native work if there is no meaningful speed or fit benefit. - -### Gate B: local range parity - -Two local processes own disjoint GGUF ranges and match whole-model llama.cpp within the certified numerical tolerance for prefill and greedy decode. - -### Gate C: concurrent KV - -Multiple Route Sessions prefill/decode concurrently with isolated local KV, bounded memory, cancellation, and release. - -### Gate D: real distributed route - -Two physical machines execute one model that uses both Shards. Synthetic activation tests do not satisfy this gate. - -### Gate E: consumer-hardware performance - -On certified consumer hardware, the GGUF route beats the current distributed safetensors route under the locked performance contract or enables a larger otherwise-unroutable model at useful measured speed. - -### Gate F: architecture expansion - -Only after dense Llama-family gates pass, add an explicit Qwen3/Qwen3-MoE adapter and certify it independently. - -## Scope discipline - -The following do not block the first production candidate: - -- New cryptocurrency/economics work. -- New artifact P2P protocol. -- QUIC or WebRTC. -- vLLM fork. -- Whole-repository Nakshatra/prima adoption. -- Every GGUF architecture. -- Automatic route repair. -- Prefix snapshot migration. -- Speculative decoding. -- A large-model marketing demo before small-model parity and concurrency pass. - -Every optimization must preserve output contract, session isolation, cancellation, resource cleanup, capability admission, and per-node attribution. +DGR-020 cannot use distributed results. DGR-054 does not depend on MTP. DGR-070 depends on DGR-066. Compile support and scenario success never imply general routability. diff --git a/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md b/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md deleted file mode 100644 index 35904ba..0000000 --- a/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md +++ /dev/null @@ -1,68 +0,0 @@ -# 01 — Lock the safetensors-versus-GGUF performance contract - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-001` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a runtime engineer, I need a controlled baseline so that GGUF work proceeds from measured speed, memory, and quality rather than reputation. - -## Baseline model target - -Use the same model on both sides of the comparison, with the closest practical low-footprint precision pair: - -- **safetensors:** `deepseek-ai/DeepSeek-V2-Lite-Chat` in **BF16** -- **GGUF:** `second-state/DeepSeek-V2-Lite-Chat-GGUF` in **Q2_K** (~6.5GB) - -Keep the benchmark matrix explicit for **CPU** and **GPU** runs. Reserve smaller non-DeepSeek fallback models only for loader plumbing smoke tests if needed; they do not count as the DGR-001 architecture-aligned baseline. - -## Expected durable outputs - -- Benchmark harness and deterministic tests -- evidence/DGR-001/performance-contract.json -- Raw and summarized safetensors/GGUF benchmark evidence - -## Acceptance criteria - -- [ ] Benchmark the same model architecture/revision, machine, prompts, context lengths, output lengths, sampling policy, and concurrency across the current Transformers/safetensors recipe and whole-model llama.cpp recipes. -- [ ] Separate correctness/quality lanes from quantized performance/fit lanes instead of claiming BF16 and Q4 are numerically equivalent. -- [ ] Report TTFT, prefill tok/s, decode tok/s, p50/p95 latency, aggregate throughput, RSS, VRAM, artifact size, failures, and output drift in machine-readable JSON. -- [ ] Add concurrency levels 1 and 4 where memory permits. -- [ ] Write a versioned performance contract consumed by later release gates, including an explicit stop condition when llama.cpp/GGUF has no meaningful speed or fit benefit. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence -- [ ] Model artifacts remain on the configured mounted-drive storage and never under /home -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-001/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- None. This story may start immediately. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md b/.scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md new file mode 100644 index 0000000..586fbb7 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md @@ -0,0 +1,39 @@ + +# DGR-017: Reconcile and clean the superseded DGR backlog + +- **Status / triage:** completed; `passes: true` +- **Execution mode:** `AFK` +- **Milestone:** `M0` +- **Dependencies:** None +- **Blocks (derived):** `DGR-018`, `DGR-019`, `DGR-027`, `DGR-054` +- **Labels:** `area:provenance`, `area:cleanup`, `type:audit`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md`, and evidence READMEs for dependencies (none) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Audit implementation reality, void inherited completion credit, and clean misleading backlog/stub baggage while preserving attributable evidence and accepted research. + +## Acceptance criteria + +- [x] Compare the branch, old DGR-001..016 issue/pass states, evidence, and actual runtime sources; classify each output as reusable, reference-only, blocked, obsolete, or absent. +- [x] Record an authoritative old-to-new disposition and provenance; explicitly give no completion credit to any new story and note absent implementation/evidence. +- [x] Remove or archive only artifacts the audit proves obsolete while preserving accepted ADRs, useful research, raw benchmark evidence, and attributable reusable work. +- [x] Protect ignored build workspaces, generated protobuf outputs, Ralph logs, and model artifacts from accidental commits, and document every retained legacy artifact. +- [x] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Verified evidence: `.scratch/distributed-gguf-runtime/evidence/DGR-017/README.md`. Legacy evidence remains provenance only and grants no implementation completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md b/.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md new file mode 100644 index 0000000..eff3eb2 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md @@ -0,0 +1,39 @@ + +# DGR-018: Define canonical Ralph and Gitea metadata schema + +- **Status / triage:** completed; `passes: true` +- **Execution mode:** `AFK` +- **Milestone:** `M0` +- **Dependencies:** `DGR-017` +- **Blocks (derived):** `DGR-021`, `DGR-025` +- **Labels:** `area:planning`, `area:gitea`, `type:infrastructure`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Make `prd.json` the validated source from which Markdown and Gitea issues can later be generated losslessly. + +## Acceptance criteria + +- [x] Define fields for stable ID/title, labels, milestone, type, `dependsOn`, derived `blocks`, triage, evidence class, and hardware/model/upstream flags. +- [x] Validate that all stories start `passes: false`, use known dependencies, and have unique stable IDs. +- [x] Reject cycles, missing dependencies, mismatched generated `blocks`, duplicate titles/IDs, and generated artifacts claiming authority over `prd.json`. +- [x] Add deterministic model-free tests for parse, validation, and generation round trips. +- [x] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Verified evidence: `.scratch/distributed-gguf-runtime/evidence/DGR-018/README.md`. Legacy evidence remains provenance only and grants no implementation completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md b/.scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md new file mode 100644 index 0000000..c9fc876 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md @@ -0,0 +1,40 @@ + +# DGR-019: Lock alpha and beta performance contracts + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M0` +- **Dependencies:** `DGR-017` +- **Blocks (derived):** `DGR-020`, `DGR-044`, `DGR-054` +- **Labels:** `area:performance`, `type:contract`, `priority:p0`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `release` +- **Hardware:** `required` +- **Model:** `generic+deepseek-v4` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Freeze useful speed, correctness, memory-fit, and stop/go thresholds before implementation results are visible. + +## Acceptance criteria + +- [ ] Define controlled safetensors, whole-model GGUF, dense distributed GGUF, and V4 Flash distributed lanes with fixed prompts, context/output lengths, sampling, concurrency, hardware, and metrics. +- [ ] Alpha requires correctness plus a human-approved useful-speed threshold; beta adds concurrency, long-context, failure, and sustained-throughput thresholds. +- [ ] Separate quantization/model-fit gains from runtime, transport, batching, and kernel gains. +- [ ] Treat quants and 2–4/10+ stage counts only as named certification scenarios; no product logic may hardcode them. +- [ ] Lock thresholds and stop conditions in versioned machine-readable data before benchmark result ingestion. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-019/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/02-adopt-the-versioned-grpc-shard-protocol.md b/.scratch/distributed-gguf-runtime/issues/02-adopt-the-versioned-grpc-shard-protocol.md deleted file mode 100644 index f7130df..0000000 --- a/.scratch/distributed-gguf-runtime/issues/02-adopt-the-versioned-grpc-shard-protocol.md +++ /dev/null @@ -1,59 +0,0 @@ -# 02 — Adopt the versioned gRPC Shard protocol - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-002` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a node developer, I need a battle-proven streaming protocol so that Python and C++ Shards communicate without a custom socket protocol. - -## Expected durable outputs - -- packages/node/native/proto/shard_runtime.proto -- Reproducible Python/C++ schema generation and build wiring -- Protocol round-trip and compatibility tests -- evidence/DGR-002/README.md - -## Acceptance criteria - -- [x] Add a Protocol Buffers schema for capability, health, session stream, release, and cancellation operations. -- [x] Define one long-lived bidirectional gRPC stream per Route Session Activation Seam with deadlines, cancellation, flow control, and structured errors. -- [x] Define bounded chunking for prefill and a small decode fast path. -- [x] Carry schema version, request/work ID, Route Session ID, route epoch, artifact/recipe fingerprint, Shard range/effective start, phase, position, idempotency step, cache expectation, compression, and checksum. -- [x] Define a versioned named-tensor bundle with per-tensor name, shape, dtype, byte order, and payload fragments. -- [x] Add generated-schema round-trip and compatibility tests in Python and C++. -- [x] Targeted pytest tests pass -- [x] python -m compileall packages tests passes for Python changes -- [x] git diff --check passes -- [x] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [x] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [x] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [x] Read and verify every dependency evidence README before relying on dependency behavior -- [x] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [x] Write .scratch/distributed-gguf-runtime/evidence/DGR-002/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [x] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- None. This story may start immediately. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md b/.scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md new file mode 100644 index 0000000..af0330e --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md @@ -0,0 +1,39 @@ + +# DGR-020: Run the controlled whole-model GGUF baseline + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M0` +- **Dependencies:** `DGR-019` +- **Blocks (derived):** `DGR-054` +- **Labels:** `area:performance`, `type:benchmark`, `priority:p0`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md`, and evidence READMEs for dependencies (DGR-019) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Execute the locked safetensors and whole-model llama.cpp lanes before distributed implementation results can influence the decision. + +## Acceptance criteria + +- [ ] Run the exact DGR-019 safetensors and whole-model llama.cpp benchmark lanes with locked prompts, lengths, sampling, concurrency, hardware, and artifact/runtime identities. +- [ ] Record raw machine-readable correctness, TTFT, prefill/decode, throughput, latency, memory, artifact-size, failure, and quality-drift metrics without ingesting distributed implementation results. +- [ ] Separate quantization/model-fit effects from runtime/kernel effects and preserve failed or unavailable lanes honestly. +- [ ] Publish a threshold-based `go`, `optimize baseline`, or `stop` decision without changing the locked contract. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-020/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md b/.scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md new file mode 100644 index 0000000..2db4fe5 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md @@ -0,0 +1,39 @@ + +# DGR-021: Define the versioned named-tensor stream envelope + +- **Status / triage:** completed; `passes: true` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-018` +- **Blocks (derived):** `DGR-022`, `DGR-023`, `DGR-025`, `DGR-031`, `DGR-035`, `DGR-046` +- **Labels:** `area:protocol`, `type:infrastructure`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md`, and evidence READMEs for dependencies (DGR-018) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish the backend-neutral protobuf envelope used by direct and relayed Shard activation traffic. + +## Acceptance criteria + +- [x] Define schema version, request/work ID, route session/epoch, shard range/effective start, phase, position, and idempotency step. +- [x] Define named tensors with shape, dtype, byte order, bounded fragments, compression identity, and checksum. +- [x] Reserve extensible fields for token-ID sidebands, architecture state, recurrent state, and MTP without claiming implementations. +- [x] Add deterministic serialization, fragmentation, checksum, unknown-field, and size-limit tests. +- [x] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Verified evidence: `.scratch/distributed-gguf-runtime/evidence/DGR-021/README.md`. Legacy evidence remains provenance only and grants no implementation completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/022-define-shard-lifecycle-and-structured-status-rpcs.md b/.scratch/distributed-gguf-runtime/issues/022-define-shard-lifecycle-and-structured-status-rpcs.md new file mode 100644 index 0000000..0736910 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/022-define-shard-lifecycle-and-structured-status-rpcs.md @@ -0,0 +1,39 @@ + +# DGR-022: Define Shard lifecycle and structured status RPCs + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-021` +- **Blocks (derived):** `DGR-024`, `DGR-033`, `DGR-037` +- **Labels:** `area:protocol`, `area:lifecycle`, `type:infrastructure`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/022-define-shard-lifecycle-and-structured-status-rpcs.md`, and evidence READMEs for dependencies (DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Complete the gRPC contract for worker capability, health, sessions, cancellation, release, and metrics. + +## Acceptance criteria + +- [ ] Define capability, health, bidirectional session stream, cancellation, release, and metrics RPCs. +- [ ] Specify deadlines, cancellation propagation, bounded flow control, cache expectations/results, and structured error taxonomy. +- [ ] Specify TLS/auth hooks without moving Meshnet authentication or billing into the worker. +- [ ] Add compatibility tests for supported versions and fail-closed tests for unsupported versions and malformed lifecycle transitions. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-022/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md b/.scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md new file mode 100644 index 0000000..2dc8d1d --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md @@ -0,0 +1,39 @@ + +# DGR-023: Make Python and C++ protobuf generation reproducible + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-021` +- **Blocks (derived):** `DGR-024`, `DGR-037` +- **Labels:** `area:protocol`, `area:build`, `type:tooling`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md`, and evidence READMEs for dependencies (DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Generate identical Python/C++ protocol bindings without manual copying or checked-in build debris. + +## Acceptance criteria + +- [ ] Pin protoc, gRPC, and plugin versions or declare a verified compatible range. +- [ ] Generate Python and C++ bindings into out-of-tree build/package locations through documented commands. +- [ ] Add Python↔C++ round-trip and descriptor compatibility tests. +- [ ] A clean checkout regenerates bindings deterministically or fails with an actionable toolchain error. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-023/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md b/.scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md new file mode 100644 index 0000000..ddb3e68 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md @@ -0,0 +1,39 @@ + +# DGR-024: Implement in-memory fake gRPC seam transport + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-022`, `DGR-023` +- **Blocks (derived):** `DGR-033`, `DGR-042` +- **Labels:** `area:protocol`, `area:testing`, `type:vertical-slice`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `fake` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md`, and evidence READMEs for dependencies (DGR-022, DGR-023) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Exercise the complete streaming protocol deterministically before a real model or worker exists. + +## Acceptance criteria + +- [ ] Provide a fake bidirectional stream supporting prefill fragments, decode fast-path frames, release, cancel, and structured errors. +- [ ] Test flow-control blocking, deadlines, malformed fragments, checksum failure, duplicates, and stale epochs. +- [ ] Verify direct and opaque-relay framing preserve identical protobuf bytes. +- [ ] Tests require no sockets outside localhost, model downloads, or native accelerator. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-024/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md b/.scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md new file mode 100644 index 0000000..71a13ae --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md @@ -0,0 +1,39 @@ + +# DGR-025: Define exact artifact and runtime recipe identity + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-018`, `DGR-021` +- **Blocks (derived):** `DGR-026`, `DGR-031`, `DGR-041`, `DGR-044` +- **Labels:** `area:identity`, `area:admission`, `type:domain`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md`, and evidence READMEs for dependencies (DGR-018, DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure the tracker and worker only combine numerically and operationally compatible shards. + +## Acceptance criteria + +- [ ] Fingerprint source artifact SHA, tokenizer revision, architecture adapter/version, boundary schema, runtime pin/patch stack, backend, quant, activation/compute dtype, and KV/state layout. +- [ ] Bind each shard to an exact half-open range without hardcoding a topology or quant. +- [ ] Fail closed on any artifact, adapter, boundary, cache, backend, or runtime mismatch. +- [ ] Unsupported recipes remain registered-but-dark until real-hardware evidence certifies them. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-025/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md b/.scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md new file mode 100644 index 0000000..e864f50 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md @@ -0,0 +1,39 @@ + +# DGR-026: Provision exact split-GGUF artifacts outside /home + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-025` +- **Blocks (derived):** `DGR-044`, `DGR-045` +- **Labels:** `area:artifacts`, `area:provenance`, `type:tooling`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md`, and evidence READMEs for dependencies (DGR-025) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Make exact split-GGUF inputs reproducibly available from mounted-drive storage without embedding a quantization or topology assumption in product code. + +## Acceptance criteria + +- [ ] Create an exact manifest that binds the source artifact, tokenizer/revision, every split file name, size, range/role, and cryptographic hash. +- [ ] Provide resumable, hash-verifying download/provision tooling targeting configured mounted-drive storage; refuse paths under `/home` and incomplete or mismatched splits. +- [ ] Keep quantization and split topology as manifest/recipe inputs with no hardcoded quant, node count, or range layout. +- [ ] Add deterministic model-download-free tests using tiny local split fixtures, including interrupted resume, missing split, hash mismatch, and `/home` rejection. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-026/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md b/.scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md new file mode 100644 index 0000000..ee282c9 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md @@ -0,0 +1,39 @@ + +# DGR-027: Add exact llama.cpp provenance manifest and fetch workspace + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-017` +- **Blocks (derived):** `DGR-028`, `DGR-029`, `DGR-044` +- **Labels:** `area:upstream`, `area:build`, `type:provenance`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Pin llama.cpp exactly through an in-repo manifest while fetching source only into an ignored build workspace. + +## Acceptance criteria + +- [ ] Manifest records upstream URL, exact commit, expected source archive/tree hash, license, and retrieval method. +- [ ] Fetch tooling verifies identity before use and refuses an unpinned branch/tag. +- [ ] Source is fetched into an ignored build workspace; no submodule, vendored source tree, or permanent fork is introduced. +- [ ] Offline reuse is supported only after the cached tree’s exact identity is verified. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-027/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md b/.scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md new file mode 100644 index 0000000..edf3496 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md @@ -0,0 +1,39 @@ + +# DGR-028: Implement numbered patch-stack apply and verification + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-027` +- **Blocks (derived):** `DGR-029`, `DGR-034`, `DGR-069` +- **Labels:** `area:upstream`, `area:patches`, `type:tooling`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md`, and evidence READMEs for dependencies (DGR-027) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Maintain a minimal auditable llama.cpp delta with one numbered patch per concern. + +## Acceptance criteria + +- [ ] Add deterministic apply/check/reverse verification against the exact manifest pin. +- [ ] Separate range loading, boundary I/O, filtered state, and worker hooks into scoped patches. +- [ ] Record upstream file/API assumptions and fail with the first incompatible patch when the pin changes. +- [ ] Verify license/attribution and prove no Meshnet routing, billing, relay, or authentication code enters the patch stack. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-028/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md b/.scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md new file mode 100644 index 0000000..90076c8 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md @@ -0,0 +1,39 @@ + +# DGR-029: Create the native CMake skeleton and deterministic CPU lane + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-027`, `DGR-028` +- **Blocks (derived):** `DGR-030`, `DGR-034` +- **Labels:** `area:build`, `type:toolchain`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md`, and evidence READMEs for dependencies (DGR-027, DGR-028) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish an out-of-tree standalone native build with a deterministic CPU lane before accelerator matrix work. + +## Acceptance criteria + +- [ ] Create the standalone native CMake target/skeleton and isolated out-of-tree configure/build preset for CPU. +- [ ] Build and run a deterministic model-free CPU smoke/CTest lane from a clean checkout with actionable toolchain failures. +- [ ] Keep fetched upstream sources, generated bindings, and all build outputs ignored and out of tree. +- [ ] Ensure build success alone does not advertise any backend/model/recipe capability. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-029/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md b/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md deleted file mode 100644 index c69cfd8..0000000 --- a/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md +++ /dev/null @@ -1,57 +0,0 @@ -# 03 — Define exact Artifact and runtime recipe identity - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-003` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As the Tracker, I need exact compatibility identity so that only numerically and operationally compatible Shards form an Inference Route. - -## Expected durable outputs - -- Exact runtime recipe/fingerprint implementation -- Tracker/node fail-closed admission tests -- evidence/DGR-003/README.md - -## Acceptance criteria - -- [ ] Separate weight quantization, activation dtype, compute dtype, KV dtype/layout, tokenizer revision, architecture adapter, backend, and runtime version. -- [ ] Bind derivative or split artifacts to an exact source Model Artifact hash and Shard range. -- [ ] Produce a stable compatibility fingerprint used by capability admission and the gRPC handshake. -- [ ] Fail closed on mismatched artifact, tokenizer, architecture, range, boundary schema, activation recipe, or cache layout. -- [ ] Keep unsupported recipes registered-but-dark until a real distributed forward certifies them. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-003/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-002` must have `passes: true`; read `../evidence/DGR-002/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md b/.scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md new file mode 100644 index 0000000..c66f15e --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md @@ -0,0 +1,39 @@ + +# DGR-030: Add accelerator build presets and native CI matrix + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-029` +- **Blocks (derived):** `DGR-053`, `DGR-067`, `DGR-068` +- **Labels:** `area:build`, `area:hardware`, `type:toolchain`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `optional` +- **Model:** `generic` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md`, and evidence READMEs for dependencies (DGR-029) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Compile all required accelerator lanes reproducibly while keeping untested real-hardware capabilities registered-dark. + +## Acceptance criteria + +- [ ] Add isolated out-of-tree presets for CUDA, ROCm, Vulkan, and Metal without changing the deterministic CPU default. +- [ ] Add a native CI/build matrix that reports unavailable SDKs as explicit unavailable/skipped lanes rather than false success. +- [ ] Compile each available lane and preserve exact compiler, SDK, upstream pin, patch-stack, and build-option evidence. +- [ ] Keep every backend/model/recipe lane registered-dark until a separate real-hardware certification record exists. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-030/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md b/.scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md new file mode 100644 index 0000000..a96a09e --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md @@ -0,0 +1,39 @@ + +# DGR-031: Introduce the project-owned `ShardEngine` interface + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-021`, `DGR-025` +- **Blocks (derived):** `DGR-032`, `DGR-034`, `DGR-035`, `DGR-037` +- **Labels:** `area:engine`, `type:architecture`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md`, and evidence READMEs for dependencies (DGR-021, DGR-025) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Isolate worker/protocol code from llama.cpp internals behind a stable project-owned engine contract. + +## Acceptance criteria + +- [ ] Define load, capabilities, prefill/decode, boundary/logits result, cancel, release, health, and metrics operations. +- [ ] Use project-owned request/result/state types; expose no `ggml_tensor`, llama context, scheduler, or ABI-owned structure. +- [ ] Reserve typed MTP and architecture auxiliary-state hooks without enabling them. +- [ ] Add contract tests proving fake and future llama implementations obey identical lifecycle semantics. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-031/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md b/.scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md new file mode 100644 index 0000000..b22d2e0 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md @@ -0,0 +1,39 @@ + +# DGR-032: Implement deterministic fake `ShardEngine` + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-031` +- **Blocks (derived):** `DGR-033` +- **Labels:** `area:engine`, `area:testing`, `type:fixture`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `fake` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md`, and evidence READMEs for dependencies (DGR-031) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Provide an engine fixture that deterministically transforms typed boundary bundles and session state. + +## Acceptance criteria + +- [ ] Support head, middle, tail, prefill, decode, cancellation, and release with deterministic outputs. +- [ ] Model isolated session/epoch state and deterministic cache-miss/stale-epoch failures. +- [ ] Support configurable delay, memory pressure, malformed output, and crash injection. +- [ ] Contract tests distinguish fixture evidence from real-model certification. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-032/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md b/.scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md new file mode 100644 index 0000000..7ce3989 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md @@ -0,0 +1,39 @@ + +# DGR-033: Build a standalone fake C++ gRPC Shard worker + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M1` +- **Dependencies:** `DGR-022`, `DGR-024`, `DGR-032` +- **Blocks (derived):** `DGR-036`, `DGR-040` +- **Labels:** `area:worker`, `area:grpc`, `type:vertical-slice`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `fake` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md`, and evidence READMEs for dependencies (DGR-022, DGR-024, DGR-032) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove the standalone worker process, stream, lifecycle, and supervision shape before llama.cpp integration. + +## Acceptance criteria + +- [ ] A standalone C++ executable serves the complete lifecycle and stream RPC contract using the fake engine. +- [ ] Python integration tests cover startup, health, capability, fragmented prefill, decode, release, cancellation, and graceful shutdown. +- [ ] Bounded messages, deadlines, flow control, and independent session cancellation are enforced. +- [ ] The worker exposes neither llama.cpp RPC nor arbitrary graph execution. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-033/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md b/.scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md new file mode 100644 index 0000000..391cef1 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md @@ -0,0 +1,39 @@ + +# DGR-034: Implement dense-Llama range-aware GGUF ownership + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-028`, `DGR-029`, `DGR-031` +- **Blocks (derived):** `DGR-035`, `DGR-037`, `DGR-051` +- **Labels:** `area:llama`, `area:weights`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `dense-llama` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md`, and evidence READMEs for dependencies (DGR-028, DGR-029, DGR-031) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Load only the dense-Llama tensors owned by a shard range. + +## Acceptance criteria + +- [ ] Load only `blk.N.*` tensors in the assigned range, embeddings only at the head, and norm/output or tied output only at the tail. +- [ ] Derive authoritative range and endpoint ownership from the loaded engine state. +- [ ] Reject invalid/gapped/out-of-model ranges and unexpected required tensors. +- [ ] Real-model evidence shows mapped/resident memory scales with owned tensors rather than full artifact size. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-034/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md b/.scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md new file mode 100644 index 0000000..9b03003 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md @@ -0,0 +1,39 @@ + +# DGR-035: Implement dense architecture boundary input/output + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-021`, `DGR-031`, `DGR-034` +- **Blocks (derived):** `DGR-036`, `DGR-037`, `DGR-069` +- **Labels:** `area:llama`, `area:boundary`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `dense-llama` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md`, and evidence READMEs for dependencies (DGR-021, DGR-031, DGR-034) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Execute dense head/middle/tail ranges through an architecture-defined residual boundary. + +## Acceptance criteria + +- [ ] Head accepts token IDs and owns embedding; middle/tail bypass embedding and accept a named boundary bundle. +- [ ] Non-tail returns the unnormalized residual before final norm/head and before tail-only row pruning. +- [ ] Tail returns logits or sampled-token output under an explicit contract. +- [ ] Uncertified architectures and incompatible boundary schemas fail closed. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-035/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md b/.scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md new file mode 100644 index 0000000..9c5b492 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md @@ -0,0 +1,39 @@ + +# DGR-036: Prove dense fixture and real-model range parity + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-033`, `DGR-035` +- **Blocks (derived):** `DGR-039` +- **Labels:** `area:parity`, `area:testing`, `type:certification`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `dense-llama` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md`, and evidence READMEs for dependencies (DGR-033, DGR-035) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove the engine/protocol seam before adding session concurrency. + +## Acceptance criteria + +- [ ] Model-free two-stage tests pass through two fake worker processes with disjoint ranges. +- [ ] A small real dense GGUF passes whole-model versus two-range prefill parity. +- [ ] At least 32 greedy decode tokens match the locked tolerance. +- [ ] Evidence distinguishes deterministic fixture proof from opt-in real-model proof. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-036/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md b/.scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md new file mode 100644 index 0000000..f615059 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md @@ -0,0 +1,39 @@ + +# DGR-037: Bind llama.cpp to the standalone worker + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-022`, `DGR-023`, `DGR-031`, `DGR-034`, `DGR-035` +- **Blocks (derived):** `DGR-038`, `DGR-039`, `DGR-040`, `DGR-051` +- **Labels:** `area:worker`, `area:llama`, `type:vertical-slice`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `dense-llama` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md`, and evidence READMEs for dependencies (DGR-022, DGR-023, DGR-031, DGR-034, DGR-035) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Replace the fake engine with the pinned llama.cpp `ShardEngine` implementation without changing the public worker contract. + +## Acceptance criteria + +- [ ] Worker loads exactly one artifact/recipe/range identity and rejects mismatched stream requests. +- [ ] All execution passes through `ShardEngine`; llama.cpp implementation types remain private. +- [ ] Health and metrics expose loaded identity, authoritative ownership, memory, and execution state. +- [ ] Graceful shutdown releases model/session resources; injected process death is observable and bounded. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-037/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md b/.scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md new file mode 100644 index 0000000..8505bbd --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md @@ -0,0 +1,39 @@ + +# DGR-038: Implement isolated shard-local Hot KV State + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-037` +- **Blocks (derived):** `DGR-039`, `DGR-052`, `DGR-055`, `DGR-069` +- **Labels:** `area:kv-cache`, `area:concurrency`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `dense-llama` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md`, and evidence READMEs for dependencies (DGR-037) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Map each route session and epoch to isolated state containing only locally owned layers. + +## Acceptance criteria + +- [ ] Map `(route_session_id, route_epoch)` to an isolated llama sequence or bounded context. +- [ ] Support prefill/decode append, truncate, release, TTL/LRU eviction, cache miss, and stale-epoch rejection. +- [ ] Four concurrent sessions complete without token, KV, position, or cancellation cross-talk. +- [ ] Release/eviction returns memory to the configured budget without affecting other sessions. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-038/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md b/.scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md new file mode 100644 index 0000000..dbe997a --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md @@ -0,0 +1,39 @@ + +# DGR-039: Pass local two-process dense acceptance + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-036`, `DGR-037`, `DGR-038` +- **Blocks (derived):** `DGR-054` +- **Labels:** `area:integration`, `area:parity`, `type:certification`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `dense-llama` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md`, and evidence READMEs for dependencies (DGR-036, DGR-037, DGR-038) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish the first real, independently executing distributed GGUF route on one host. + +## Acceptance criteria + +- [ ] Two worker processes open disjoint dense ranges and both execute real prefill/decode work. +- [ ] Whole-model parity, 32-token greedy decode, four-session isolation, cancellation, and cleanup pass. +- [ ] Record TTFT, prefill/decode rates, seam bytes/latency, RSS/VRAM, KV, queue, and failure metrics. +- [ ] Killing one worker returns a bounded structured failure rather than hanging. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-039/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/04-create-the-reproducible-pinned-llama-cpp-patch-stack.md b/.scratch/distributed-gguf-runtime/issues/04-create-the-reproducible-pinned-llama-cpp-patch-stack.md deleted file mode 100644 index 5c388a1..0000000 --- a/.scratch/distributed-gguf-runtime/issues/04-create-the-reproducible-pinned-llama-cpp-patch-stack.md +++ /dev/null @@ -1,61 +0,0 @@ -# 04 — Create the reproducible pinned llama.cpp patch stack - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-004` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a maintainer, I need a small auditable fork boundary so that upstream updates do not turn the runtime into an unmaintainable stitched codebase. - -## Expected durable outputs - -- Exact llama.cpp upstream pin -- Numbered minimal patch stack -- Reproducible fetch/apply/build smoke -- evidence/DGR-004/README.md - -## Acceptance criteria - -- [ ] Pin one exact llama.cpp commit through a reproducible source dependency mechanism. -- [ ] Store a numbered minimal patch stack separately from Meshnet networking code. -- [ ] Add a build script that applies/checks patches and builds the standalone worker without manual source copying. -- [ ] Record upstream file/ABI assumptions and fail clearly when the pin changes. -- [ ] Preserve upstream license and attribution notices. -- [ ] Add a clean rebuild smoke test that does not download a model. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-004/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-001` must have `passes: true`; read `../evidence/DGR-001/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md b/.scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md new file mode 100644 index 0000000..e184a1e --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md @@ -0,0 +1,39 @@ + +# DGR-040: Add node-side native worker supervision + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-033`, `DGR-037` +- **Blocks (derived):** `DGR-041`, `DGR-042`, `DGR-055`, `DGR-058` +- **Labels:** `area:node`, `area:supervision`, `type:integration`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md`, and evidence READMEs for dependencies (DGR-033, DGR-037) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Let the existing node service safely start, probe, stop, and restart the standalone worker. + +## Acceptance criteria + +- [ ] Supervision owns process startup, readiness, log capture, graceful shutdown, and bounded forced termination. +- [ ] Startup verifies worker binary, artifact identity, recipe, and range before registration. +- [ ] Crashes or health loss make the capability unavailable without corrupting the Transformers backend. +- [ ] Tests use the fake worker and deterministic crash injection. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-040/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md b/.scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md new file mode 100644 index 0000000..cd16f52 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md @@ -0,0 +1,39 @@ + +# DGR-041: Register native Shard capabilities without redesigning Meshnet + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-025`, `DGR-040` +- **Blocks (derived):** `DGR-043` +- **Labels:** `area:meshnet`, `area:admission`, `type:integration`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md`, and evidence READMEs for dependencies (DGR-025, DGR-040) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Adapt native worker identity and capacity to existing backend-agnostic capability admission. + +## Acceptance criteria + +- [ ] Registration carries exact recipe fingerprint, authoritative range, backend, memory/KV capacity, concurrency, and certification status. +- [ ] Existing tracker, billing, routing, telemetry, and provider semantics remain backend-agnostic. +- [ ] Uncertified backend/model/recipe combinations are visible but unroutable. +- [ ] Existing Transformers registration and route tests remain unchanged in behavior. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-041/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md b/.scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md new file mode 100644 index 0000000..816f8c7 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md @@ -0,0 +1,39 @@ + +# DGR-042: Carry native frames through direct and existing relay seams + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-024`, `DGR-040` +- **Blocks (derived):** `DGR-054`, `DGR-058` +- **Labels:** `area:meshnet`, `area:relay`, `area:grpc`, `type:integration`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md`, and evidence READMEs for dependencies (DGR-024, DGR-040) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Use direct gRPC where available and the existing relay as opaque protobuf transport without redesigning relay behavior. + +## Acceptance criteria + +- [ ] Direct paths use the long-lived gRPC activation stream. +- [ ] Relayed paths carry byte-identical versioned protobuf frames through the existing relay contract. +- [ ] Request/work identity, cancellation, deadlines, telemetry, billing correlation, and per-node attribution survive both paths. +- [ ] Fake-worker tests cover direct, relay, disconnect, cancellation, and bounded buffering. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-042/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md b/.scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md new file mode 100644 index 0000000..e44e8ad --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md @@ -0,0 +1,39 @@ + +# DGR-043: Expose GGUF compatibility and measured cost inputs to existing routing + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M2` +- **Dependencies:** `DGR-041` +- **Blocks (derived):** `DGR-053`, `DGR-054`, `DGR-059`, `DGR-061` +- **Labels:** `area:tracker`, `area:routing`, `type:integration`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md`, and evidence READMEs for dependencies (DGR-041) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Adapt exact GGUF recipe/range compatibility and measured capacity, queue, seam-cost, health, and reliability data into the existing backend-agnostic routing inputs without redesigning routing or load balancing. + +## Acceptance criteria + +- [ ] Expose exact recipe, range coverage, capacity, queue/load, seam-cost, health, reliability, backend, and certification measurements through existing tracker input contracts. +- [ ] Prove existing routing forms complete compatible coverage and excludes dark or mismatched candidates using its current backend-agnostic mechanisms. +- [ ] Regression-test unchanged Transformers behavior and unchanged tracker routing, load-balancing, billing, relay, and provider semantics. +- [ ] Regression-test that no quant, stage count, fixed split, architecture, backend sequence, or DeepSeek-specific policy is hardcoded. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-043/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md b/.scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md new file mode 100644 index 0000000..6cbb816 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md @@ -0,0 +1,39 @@ + +# DGR-044: Pin the DeepSeek V4 Flash target contract + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-019`, `DGR-025`, `DGR-026`, `DGR-027` +- **Blocks (derived):** `DGR-045` +- **Labels:** `area:deepseek-v4`, `area:provenance`, `type:contract`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md`, and evidence READMEs for dependencies (DGR-019, DGR-025, DGR-026, DGR-027) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Fix the alpha/beta model identity and upstream maturity assumptions before implementing the adapter. + +## Acceptance criteria + +- [ ] Pin Hugging Face target SHA `60d8d70770c6776ff598c94bb586a859a38244f1`. +- [ ] Record the selected GGUF provenance and exact source binding without assuming one quant. +- [ ] Record that llama.cpp support began with PR 24162/commit `8c146a8` and remains maturing; the build manifest still pins one exact validated upstream commit. +- [ ] Record 43 main layers plus MTP and all known architecture-specific state obligations. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-044/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md b/.scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md new file mode 100644 index 0000000..20f09c5 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md @@ -0,0 +1,39 @@ + +# DGR-045: Inventory V4 GGUF tensors and layer ownership + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-026`, `DGR-044` +- **Blocks (derived):** `DGR-046`, `DGR-047`, `DGR-050` +- **Labels:** `area:deepseek-v4`, `area:weights`, `type:research-spike`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `none` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md`, and evidence READMEs for dependencies (DGR-026, DGR-044) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Derive exact head, repeated-layer, tail, expert, auxiliary-state, and MTP ownership from the pinned artifact. + +## Acceptance criteria + +- [ ] Produce a machine-readable inventory tied to the target SHA and exact GGUF artifact. +- [ ] Classify embeddings, 43 main layers, final head, MTP, mHC, attention variants, indexer/compressor, and MoE tensors. +- [ ] Validate half-open range ownership and fail on unclassified required tensors. +- [ ] Inventory tooling works across quants without embedding quant names in architecture logic. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-045/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md b/.scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md new file mode 100644 index 0000000..8105352 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md @@ -0,0 +1,39 @@ + +# DGR-046: Define the V4 typed architecture boundary schema + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-021`, `DGR-045` +- **Blocks (derived):** `DGR-047`, `DGR-048`, `DGR-049` +- **Labels:** `area:deepseek-v4`, `area:boundary`, `type:protocol`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `model-free` +- **Hardware:** `none` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md`, and evidence READMEs for dependencies (DGR-021, DGR-045) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Define the exact cross-stage V4 architecture boundary while keeping per-layer attention and auxiliary caches shard-local. + +## Acceptance criteria + +- [ ] Define a versioned named bundle for the mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations. +- [ ] Explicitly exclude per-layer CSA, HCA, SWA, indexer, compressor, KV, and MTP caches/state from the WAN boundary; those remain local to the owning shard and session/epoch. +- [ ] Reserve typed MTP boundary fields but mark MTP execution unsupported and unroutable for alpha. +- [ ] Fingerprint independently of quant/topology and fail closed on missing, incompatible, incorrectly shaped, or stale boundary/cache expectations. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-046/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md b/.scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md new file mode 100644 index 0000000..39cc728 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md @@ -0,0 +1,39 @@ + +# DGR-047: Adapt the upstream V4 mHC boundary for ranged ownership + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-045`, `DGR-046` +- **Blocks (derived):** `DGR-048`, `DGR-049`, `DGR-050`, `DGR-051` +- **Labels:** `area:deepseek-v4`, `area:mhc`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md`, and evidence READMEs for dependencies (DGR-045, DGR-046) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Add range-boundary adapters around upstream llama.cpp V4 mHC execution without reimplementing the V4 graph or kernels. + +## Acceptance criteria + +- [ ] Represent and validate the upstream V4 4×4096 mHC boundary without flattening semantic axes. +- [ ] Add only head/intermediate/tail range ownership and boundary conversion hooks around the pinned upstream llama.cpp graph. +- [ ] Compare deterministic fixture vectors and single-process ranged outputs with upstream whole-model execution. +- [ ] Document that llama.cpp owns V4 mHC graph/kernels and that quantized storage does not alter the logical boundary schema. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-047/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md b/.scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md new file mode 100644 index 0000000..4cc121f --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md @@ -0,0 +1,39 @@ + +# DGR-048: Carry token-ID sideband through the first three hash-routed layers + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-046`, `DGR-047` +- **Blocks (derived):** `DGR-051` +- **Labels:** `area:deepseek-v4`, `area:routing-state`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md`, and evidence READMEs for dependencies (DGR-046, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure any stage containing the first three hash-routed layers receives exact token identities. + +## Acceptance criteria + +- [ ] Head emits token IDs in the typed sideband for boundaries that require them. +- [ ] Intermediate stages preserve or consume the sideband according to their layer range. +- [ ] Routes splitting before, within, and after the first three layers pass deterministic partition tests. +- [ ] Missing or mismatched token IDs fail closed rather than silently selecting incorrect experts. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-048/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md b/.scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md new file mode 100644 index 0000000..1eac77b --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md @@ -0,0 +1,39 @@ + +# DGR-049: Keep V4 attention and auxiliary state shard-local + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-046`, `DGR-047` +- **Blocks (derived):** `DGR-051` +- **Labels:** `area:deepseek-v4`, `area:attention`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md`, and evidence READMEs for dependencies (DGR-046, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Bind V4 CSA/HCA/SWA/indexer/compressor and KV state to the owning shard session/epoch rather than transferring caches over the WAN. + +## Acceptance criteria + +- [ ] Map CSA, HCA, SWA, indexer, compressor, and KV ownership from the pinned upstream graph to each owned layer range. +- [ ] Key all such caches/state by route session and epoch with bounded lifecycle, memory accounting, stale-epoch rejection, release, and eviction. +- [ ] Prove cross-stage messages carry only the typed architecture boundary and cache expectations, never per-layer cache contents. +- [ ] On route change or state loss, use explicit cache miss and re-prefill/restart; do not migrate V4 caches across workers. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-049/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/05-implement-dense-llama-range-aware-gguf-ownership.md b/.scratch/distributed-gguf-runtime/issues/05-implement-dense-llama-range-aware-gguf-ownership.md deleted file mode 100644 index 5a4112b..0000000 --- a/.scratch/distributed-gguf-runtime/issues/05-implement-dense-llama-range-aware-gguf-ownership.md +++ /dev/null @@ -1,61 +0,0 @@ -# 05 — Implement dense-Llama range-aware GGUF ownership - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-005` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a node, I need to map only my assigned dense-Llama Shard so that aggregate consumer memory can hold a model larger than one node. - -## Expected durable outputs - -- Dense-Llama range-aware ownership implementation -- Authoritative loaded-range introspection -- Mapped/resident memory evidence -- evidence/DGR-005/README.md - -## Acceptance criteria - -- [ ] Register and allocate only `blk.N.*` tensors in the assigned range. -- [ ] Load embeddings only for the head and final norm/LM head only for the tail, including tied embeddings. -- [ ] Prefer range-aware mapping from one exact source GGUF; if derivative sub-GGUFs are used temporarily, verify source/slice hashes and avoid claiming final artifact semantics. -- [ ] Report authoritative loaded range and endpoint ownership from the model, not operator CLI claims. -- [ ] Demonstrate mapped/resident memory scales with owned tensors rather than full model size. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-005/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-003` must have `passes: true`; read `../evidence/DGR-003/README.md` and verify its referenced files/commands. -- `DGR-004` must have `passes: true`; read `../evidence/DGR-004/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md b/.scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md new file mode 100644 index 0000000..8e644ed --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md @@ -0,0 +1,39 @@ + +# DGR-050: Validate upstream V4 MoE and hash-routing execution under ranged ownership + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-045`, `DGR-047` +- **Blocks (derived):** `DGR-051` +- **Labels:** `area:deepseek-v4`, `area:moe`, `type:runtime`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md`, and evidence READMEs for dependencies (DGR-045, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Validate the pinned upstream llama.cpp V4 MoE and hash-routing graph when layers are range-owned, without reimplementing routing or expert kernels in Meshnet. + +## Acceptance criteria + +- [ ] Load range-owned tensors for 256 routed plus one shared expert and validate upstream execution activates six routed experts where specified. +- [ ] Validate the first three upstream hash-routed layers receive exact token-ID sideband across splits before, within, and after them. +- [ ] Compare upstream whole-model and ranged router/expert outputs, shared-expert contribution, and malformed sideband failures. +- [ ] Document that llama.cpp owns V4 MoE, attention, hash routing, graph, and kernels; no expert-parallel WAN collective or Meshnet reimplementation is introduced. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-050/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md b/.scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md new file mode 100644 index 0000000..544f2ad --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md @@ -0,0 +1,39 @@ + +# DGR-051: Assemble the DeepSeek V4 Flash `ShardEngine` adapter + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-034`, `DGR-037`, `DGR-047`, `DGR-048`, `DGR-049`, `DGR-050` +- **Blocks (derived):** `DGR-052`, `DGR-060`, `DGR-065`, `DGR-069` +- **Labels:** `area:deepseek-v4`, `area:engine`, `type:vertical-slice`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md`, and evidence READMEs for dependencies (DGR-034, DGR-037, DGR-047, DGR-048, DGR-049, DGR-050) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Assemble range ownership hooks and boundary/local-state adapters around upstream llama.cpp V4 execution behind the stable ShardEngine interface. + +## Acceptance criteria + +- [ ] Support valid head, middle, tail, and whole-model ranges over all 43 main layers using the pinned upstream llama.cpp V4 graph/kernels. +- [ ] Integrate exact range ownership, mHC boundary, positions, required token-ID sideband, and shard-local session/epoch state adapters. +- [ ] Report MTP reserved-but-not-executable for alpha and reject unsupported split points or cache expectations. +- [ ] Include exact adapter/schema/runtime/artifact identity and valid split restrictions without reimplementing V4 MoE, attention, mHC, or hash routing. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-051/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md b/.scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md new file mode 100644 index 0000000..53d3534 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md @@ -0,0 +1,39 @@ + +# DGR-052: Pass local V4 whole-model versus ranged parity + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M3` +- **Dependencies:** `DGR-038`, `DGR-051` +- **Blocks (derived):** `DGR-053`, `DGR-054` +- **Labels:** `area:deepseek-v4`, `area:parity`, `type:certification`, `priority:p0`, `ready-for-agent` +- **Evidence class:** `real-model` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md`, and evidence READMEs for dependencies (DGR-038, DGR-051) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove V4 prefill and decode correctness locally before network certification. + +## Acceptance criteria + +- [ ] Compare pinned whole-model llama.cpp with at least two valid ranged partitions. +- [ ] Prefill intermediates and locked-length greedy decode pass architecture-specific tolerances. +- [ ] Test splits around the first three hash-routed layers and at least one CSA/HCA/SWA/indexer/compressor transition. +- [ ] Four concurrent sessions pass KV/state isolation and cleanup. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-052/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md b/.scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md new file mode 100644 index 0000000..eaf56fc --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md @@ -0,0 +1,40 @@ + +# DGR-053: Certify a real 2–4-stage V4 route + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M3` +- **Dependencies:** `DGR-030`, `DGR-043`, `DGR-052` +- **Blocks (derived):** `DGR-054`, `DGR-062`, `DGR-067` +- **Labels:** `area:deepseek-v4`, `area:hardware`, `type:certification`, `priority:p0`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md`, and evidence READMEs for dependencies (DGR-030, DGR-043, DGR-052) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove real Tracker-selected V4 execution across physical machines before alpha. + +## Acceptance criteria + +- [ ] Run one documented 2–4-stage certification scenario using exact compatible artifacts/recipes; the count and chosen quant are evidence inputs, not product constants. +- [ ] Actual CPU/GPU work executes on every stage; fake workers do not satisfy acceptance. +- [ ] Record parity, TTFT, prefill/decode speed, seam cost, memory, cache/state isolation, cancellation, and cleanup. +- [ ] Tracker selection remains dynamic and rejects an injected incompatible backend/recipe. +- [ ] Only the exact real-hardware lanes exercised become eligible for advertisement. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-053/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md b/.scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md new file mode 100644 index 0000000..9f589f0 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md @@ -0,0 +1,39 @@ + +# DGR-054: Enforce the DeepSeek V4 Flash alpha gate + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M3` +- **Dependencies:** `DGR-017`, `DGR-019`, `DGR-020`, `DGR-039`, `DGR-042`, `DGR-043`, `DGR-052`, `DGR-053` +- **Blocks (derived):** `DGR-055`, `DGR-065`, `DGR-066`, `DGR-068` +- **Labels:** `area:deepseek-v4`, `area:release`, `milestone:alpha`, `priority:p0`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `release` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md`, and evidence READMEs for dependencies (DGR-017, DGR-019, DGR-020, DGR-039, DGR-042, DGR-043, DGR-052, DGR-053) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Release alpha only when V4 is correct and clears the pre-locked useful-speed requirement. + +## Acceptance criteria + +- [ ] Evaluate correctness, model fit, TTFT, prefill/decode speed, seam cost, memory, and failure rate against DGF-004 without changing thresholds. +- [ ] Separate quant/model-fit gains from runtime/transport gains. +- [ ] Decision is `alpha`, `optimize measured bottleneck`, or `stop`; failures cannot be relabeled as certification. +- [ ] Alpha documentation states MTP is reserved but not implemented and identifies every dark hardware lane. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-054/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md b/.scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md new file mode 100644 index 0000000..2b4c646 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md @@ -0,0 +1,39 @@ + +# DGR-055: Implement compatible continuous decode batching + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-038`, `DGR-040`, `DGR-054` +- **Blocks (derived):** `DGR-056`, `DGR-057` +- **Labels:** `area:scheduler`, `area:batching`, `type:performance`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md`, and evidence READMEs for dependencies (DGR-038, DGR-040, DGR-054) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Batch compatible active sessions without violating route, recipe, position, or architecture state. + +## Acceptance criteria + +- [ ] Batch only sessions with compatible engine, recipe, phase, boundary schema, and state layout. +- [ ] Preserve per-session positions, sequence IDs, outputs, cancellation, and accounting. +- [ ] Decode receives bounded service while prefill remains schedulable. +- [ ] Deterministic 1/2/4/8 concurrency tests show no cross-session corruption. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-055/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md b/.scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md new file mode 100644 index 0000000..47b120f --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md @@ -0,0 +1,39 @@ + +# DGR-056: Enforce bounded admission, queues, and backpressure + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-055` +- **Blocks (derived):** `DGR-057`, `DGR-058`, `DGR-060` +- **Labels:** `area:scheduler`, `area:backpressure`, `type:performance`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md`, and evidence READMEs for dependencies (DGR-055) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prevent weight, KV, state, scratch, and stream buffers from exceeding admitted capacity. + +## Acceptance criteria + +- [ ] Admission accounts separately for weights, KV/state, scratch, in-flight fragments, and queue slots. +- [ ] Queue and stream limits produce structured retryable or terminal responses. +- [ ] Prefill cannot starve decode and one route session cannot monopolize queue capacity. +- [ ] Telemetry reports active sessions, queue depth/time, batch occupancy, memory pressure, and rejects. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-056/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md b/.scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md new file mode 100644 index 0000000..6af3514 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md @@ -0,0 +1,39 @@ + +# DGR-057: Benchmark batching saturation and fairness + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-055`, `DGR-056` +- **Blocks (derived):** `DGR-061`, `DGR-063`, `DGR-070` +- **Labels:** `area:performance`, `area:batching`, `type:benchmark`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `dense-llama+deepseek-v4` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md`, and evidence READMEs for dependencies (DGR-055, DGR-056) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Identify useful concurrency and saturation points without assuming one global batch size. + +## Acceptance criteria + +- [ ] Run controlled concurrency 1/2/4/8 and record aggregate throughput, per-request p50/p95, TTFT, queue time, occupancy, and memory. +- [ ] Measure dense and V4 lanes separately where hardware permits. +- [ ] Verify bounded fairness and no prefill/decode starvation. +- [ ] Produce machine-readable saturation recommendations consumed by capability reporting, not hardcoded constants. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-057/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md b/.scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md new file mode 100644 index 0000000..0fe2efe --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md @@ -0,0 +1,39 @@ + +# DGR-058: Harden worker and stream failure semantics + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-040`, `DGR-042`, `DGR-056` +- **Blocks (derived):** `DGR-059`, `DGR-060`, `DGR-061` +- **Labels:** `area:reliability`, `area:failure`, `type:hardening`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md`, and evidence READMEs for dependencies (DGR-040, DGR-042, DGR-056) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure malformed input, process death, transport reset, and cancellation terminate cleanly. + +## Acceptance criteria + +- [ ] Test worker death, stream reset, malformed bundle, checksum failure, deadline, cache miss, stale epoch, and partial fragment loss. +- [ ] Cancellation propagates across all stages and releases KV/state, queue entries, and fragment buffers. +- [ ] Duplicate steps are idempotent; uncertain mutations are never silently replayed. +- [ ] Billing/work outcomes distinguish completed, cancelled, failed, and unverified work. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-058/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md b/.scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md new file mode 100644 index 0000000..20e72fa --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md @@ -0,0 +1,39 @@ + +# DGR-059: Implement alpha-safe route recovery + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-043`, `DGR-058` +- **Blocks (derived):** `DGR-070` +- **Labels:** `area:recovery`, `area:routing`, `type:hardening`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md`, and evidence READMEs for dependencies (DGR-043, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Recover from route loss without importing or continuing unverified state. + +## Acceptance criteria + +- [ ] On route loss or local-state cache miss, restart from token zero or re-prefill on a newly compatible route according to the locked alpha contract. +- [ ] Never import KV, CSA, HCA, SWA, indexer, compressor, recurrent, or MTP cache/state across workers. +- [ ] Keep exact same-route duplicate delivery idempotent while cross-route continuation fails closed. +- [ ] Test direct/relay loss, cache miss, replacement selection, client cancellation, and billing reconciliation. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-059/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/06-implement-architecture-defined-boundary-input-output.md b/.scratch/distributed-gguf-runtime/issues/06-implement-architecture-defined-boundary-input-output.md deleted file mode 100644 index 385f710..0000000 --- a/.scratch/distributed-gguf-runtime/issues/06-implement-architecture-defined-boundary-input-output.md +++ /dev/null @@ -1,61 +0,0 @@ -# 06 — Implement architecture-defined boundary input/output - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-006` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a Shard, I need to consume and emit the correct transformer boundary state so that disjoint processes reproduce whole-model execution. - -## Expected durable outputs - -- Architecture boundary adapter -- Whole-model/two-range parity tests and results -- evidence/DGR-006/README.md - -## Acceptance criteria - -- [ ] Head accepts token IDs and owns token embedding. -- [ ] Middle/tail bypass token embedding and accept the named boundary bundle. -- [ ] Non-tail emits the unnormalized architecture-defined residual/boundary before final norm/head and before tail-only row pruning. -- [ ] Tail emits logits or token output through an explicit sampling contract. -- [ ] Dense-Llama whole-model versus two-range prefill and greedy-decode parity passes the documented tolerance. -- [ ] The adapter interface fails closed for uncertified architectures. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-006/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-002` must have `passes: true`; read `../evidence/DGR-002/README.md` and verify its referenced files/commands. -- `DGR-005` must have `passes: true`; read `../evidence/DGR-005/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md b/.scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md new file mode 100644 index 0000000..3384151 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md @@ -0,0 +1,39 @@ + +# DGR-060: Certify V4 long-context state correctness + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-051`, `DGR-056`, `DGR-058` +- **Blocks (derived):** `DGR-062`, `DGR-064`, `DGR-065`, `DGR-066`, `DGR-070` +- **Labels:** `area:deepseek-v4`, `area:long-context`, `type:certification`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md`, and evidence READMEs for dependencies (DGR-051, DGR-056, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove V4’s KV and auxiliary state remain correct and bounded at long contexts. + +## Acceptance criteria + +- [ ] Exercise pre-locked context lengths covering multiple prefill chunks and sustained decode. +- [ ] Validate KV plus CSA/HCA/SWA/indexer/compressor state positions across every stage. +- [ ] Measure memory growth, fragment sizes, TTFT, decode degradation, cancellation latency, and cleanup. +- [ ] Cache mismatch, truncation, and context-limit behavior fail deterministically. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-060/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md b/.scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md new file mode 100644 index 0000000..1ce9344 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md @@ -0,0 +1,39 @@ + +# DGR-061: Certify existing routing with 10+ GGUF stage candidates + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-043`, `DGR-057`, `DGR-058` +- **Blocks (derived):** `DGR-062`, `DGR-063` +- **Labels:** `area:tracker`, `area:scale`, `type:simulation`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `generic` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md`, and evidence READMEs for dependencies (DGR-043, DGR-057, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Certify that existing backend-agnostic routing consumes GGUF compatibility and measured-cost inputs for a 10+ candidate scenario without algorithm or policy redesign. + +## Acceptance criteria + +- [ ] Generate deterministic pools with 10+ compatible GGUF stage candidates plus slower, overloaded, unhealthy, and incompatible alternatives. +- [ ] Run the existing routing implementation unchanged and verify complete coverage, compatibility, acyclicity, and use of measured inputs. +- [ ] Regression-test that no stage count, quant, fixed split, architecture, or backend sequence is encoded in production routing policy. +- [ ] Verify existing route recomputation reacts to measured load/failure while tracker, load-balancing, billing, relay, and provider semantics remain unchanged. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-061/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md b/.scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md new file mode 100644 index 0000000..689a939 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md @@ -0,0 +1,40 @@ + +# DGR-062: Certify a dynamic real 10+ stage V4 scenario + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M4` +- **Dependencies:** `DGR-053`, `DGR-060`, `DGR-061` +- **Blocks (derived):** `DGR-067`, `DGR-070` +- **Labels:** `area:deepseek-v4`, `area:scale`, `area:hardware`, `type:certification`, `priority:p1`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md`, and evidence READMEs for dependencies (DGR-053, DGR-060, DGR-061) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Demonstrate that a large real route can be dynamically selected and usefully execute V4. + +## Acceptance criteria + +- [ ] Run one real 10+-stage V4 certification scenario; stage count, quant, and split remain scenario metadata only. +- [ ] Every selected stage proves real work, exact compatible identity, local state ownership, and bounded resources. +- [ ] Record end-to-end and per-stage TTFT, prefill/decode, seam cost, queueing, memory, failures, and cleanup. +- [ ] Inject at least one incompatible or degraded candidate and verify dynamic exclusion/reselection. +- [ ] The scenario must meet its pre-locked usefulness threshold or remain failed evidence. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-062/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md b/.scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md new file mode 100644 index 0000000..2cf1955 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md @@ -0,0 +1,39 @@ + +# DGR-063: Profile and optimize the critical seam + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-057`, `DGR-061` +- **Blocks (derived):** `DGR-064` +- **Labels:** `area:performance`, `type:optimization`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `generic` +- **Upstream:** `conditional` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md`, and evidence READMEs for dependencies (DGR-057, DGR-061) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Optimize only the measured dominant compute, serialization, copy, or transport bottleneck. + +## Acceptance criteria + +- [ ] Profile worker compute, tensor conversion, copies, protobuf framing, compression, relay/direct transport, queueing, and synchronization. +- [ ] Select one dominant bottleneck from evidence and lock its before/after benchmark. +- [ ] Implement one bounded optimization without weakening correctness or protocol identity. +- [ ] Report whether the optimization changes TTFT, prefill, decode, throughput, or memory and preserve raw data. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-063/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md b/.scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md new file mode 100644 index 0000000..877aaed --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md @@ -0,0 +1,39 @@ + +# DGR-064: Add negotiated activation compression + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-060`, `DGR-063` +- **Blocks (derived):** `DGR-066`, `DGR-070` +- **Labels:** `area:protocol`, `area:compression`, `type:optimization`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `no` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md`, and evidence READMEs for dependencies (DGR-060, DGR-063) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Reduce V4 seam cost only when a certified compression recipe improves end-to-end performance acceptably. + +## Acceptance criteria + +- [ ] Negotiate compression through recipe identity; unnegotiated or mismatched compression fails closed. +- [ ] Preserve names, shapes, semantic axes, token-ID sideband, and checksums. +- [ ] Measure quality drift, encode/decode cost, bytes, TTFT, and throughput against uncompressed boundaries. +- [ ] Compression remains dark unless it meets pre-locked quality and speed thresholds. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-064/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md b/.scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md new file mode 100644 index 0000000..ed34fe6 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md @@ -0,0 +1,39 @@ + +# DGR-065: Define V4 MTP ownership and distributed state contract + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-051`, `DGR-054`, `DGR-060` +- **Blocks (derived):** `DGR-066` +- **Labels:** `area:deepseek-v4`, `area:mtp`, `type:contract`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `fixture` +- **Hardware:** `none` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md`, and evidence READMEs for dependencies (DGR-051, DGR-054, DGR-060) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Lock MTP tensor, execution, boundary, acceptance/rejection, and session-state ownership before implementation while keeping MTP disabled for alpha. + +## Acceptance criteria + +- [ ] Derive exact MTP tensor and execution ownership from the pinned V4 target and upstream llama.cpp graph, distinguishing main-layer, tail, and MTP responsibilities. +- [ ] Define typed MTP inputs/outputs plus accepted-token, rejection, rollback/truncation, position, cache, and session/epoch lifecycle semantics. +- [ ] Specify which MTP state remains shard-local and which typed boundary values cross a seam; prohibit WAN migration of KV or V4 auxiliary caches. +- [ ] Keep every MTP recipe disabled and unroutable for alpha, with fail-closed capability negotiation and deterministic contract fixtures. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-065/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md b/.scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md new file mode 100644 index 0000000..3d4348f --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md @@ -0,0 +1,39 @@ + +# DGR-066: Implement and benchmark V4 MTP + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M4` +- **Dependencies:** `DGR-054`, `DGR-060`, `DGR-064`, `DGR-065` +- **Blocks (derived):** `DGR-067`, `DGR-070` +- **Labels:** `area:deepseek-v4`, `area:mtp`, `type:runtime`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md`, and evidence READMEs for dependencies (DGR-054, DGR-060, DGR-064, DGR-065) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Implement the locked MTP contract around upstream llama.cpp before beta, without making MTP an alpha prerequisite. + +## Acceptance criteria + +- [ ] Implement the exact DGR-065 MTP ownership and typed state contract against the pinned upstream V4 graph while preserving shard-local caches. +- [ ] Pass whole-model versus ranged MTP correctness, accepted-token, rejection, rollback/truncation, and session-isolation tests. +- [ ] Benchmark MTP-off versus MTP-on quality, accepted-token rate, TTFT, decode speed, seam bytes, and memory on the locked lane. +- [ ] Distinguish implemented, certified, disabled, and unsupported MTP recipes in fail-closed capability advertisement. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-066/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md b/.scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md new file mode 100644 index 0000000..6918a93 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md @@ -0,0 +1,40 @@ + +# DGR-067: Certify the backend capability matrix + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M4` +- **Dependencies:** `DGR-030`, `DGR-053`, `DGR-062`, `DGR-066` +- **Blocks (derived):** `DGR-068`, `DGR-070` +- **Labels:** `area:hardware`, `area:admission`, `type:certification`, `priority:p1`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `real-hardware` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md`, and evidence READMEs for dependencies (DGR-030, DGR-053, DGR-062, DGR-066) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Publish only backend/model/recipe lanes proven on actual CPU, CUDA, ROCm, Vulkan, or Metal hardware. + +## Acceptance criteria + +- [ ] Maintain independent build and certification status for CPU, CUDA, ROCm, Vulkan, and Metal. +- [ ] Each advertised lane includes real hardware/driver/runtime/model evidence for parity, concurrency, cancellation, cleanup, and useful speed. +- [ ] Build success alone never makes a lane routable; unavailable hardware remains explicitly dark. +- [ ] Tracker admission consumes signed/versioned certification records rather than backend-name allowlists. +- [ ] Mixed-backend routes require exact compatibility evidence and fail closed otherwise. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-067/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md b/.scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md new file mode 100644 index 0000000..9ccd3f2 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md @@ -0,0 +1,39 @@ + +# DGR-068: Package reproducible native worker releases + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M5` +- **Dependencies:** `DGR-030`, `DGR-054`, `DGR-067` +- **Blocks (derived):** `DGR-070` +- **Labels:** `area:packaging`, `area:release`, `type:delivery`, `priority:p1`, `ready-for-agent` +- **Evidence class:** `release` +- **Hardware:** `optional` +- **Model:** `generic` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md`, and evidence READMEs for dependencies (DGR-030, DGR-054, DGR-067) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Produce installable worker packages that preserve exact source, patch, protocol, and certification identity. + +## Acceptance criteria + +- [ ] Package worker binaries/configuration for supported host/backend combinations without bundling an unverified llama.cpp tree. +- [ ] Embed protocol, engine, upstream pin, patch-stack, build-toolchain, and certification fingerprints. +- [ ] Installation/startup refuses mismatched artifacts, runtime recipes, or unsupported hardware lanes. +- [ ] Produce checksums, SBOM/license attribution, reproducible build instructions, and smoke tests. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-068/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md b/.scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md new file mode 100644 index 0000000..d665e1c --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md @@ -0,0 +1,39 @@ + +# DGR-069: Prepare narrow upstream llama.cpp collaboration patches + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M5` +- **Dependencies:** `DGR-028`, `DGR-035`, `DGR-038`, `DGR-051` +- **Blocks (derived):** `DGR-071` +- **Labels:** `area:upstream`, `area:patches`, `type:collaboration`, `priority:p2`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `release` +- **Hardware:** `none` +- **Model:** `generic+deepseek-v4` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md`, and evidence READMEs for dependencies (DGR-028, DGR-035, DGR-038, DGR-051) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Reduce long-term patch burden through generic upstreamable hooks without asking llama.cpp to own Meshnet. + +## Acceptance criteria + +- [ ] Separate generic range loading, boundary I/O, and filtered state hooks from project worker/protocol code. +- [ ] Provide one scoped patch, focused test, and minimal reproducer per concern against the exact upstream pin. +- [ ] Compare the proposal with prior-art evidence and current upstream V4 support/maturity. +- [ ] Prepare human-ready design/outreach text; actual issue/PR submission remains a human action. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-069/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md b/.scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md deleted file mode 100644 index 5fcfd40..0000000 --- a/.scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md +++ /dev/null @@ -1,60 +0,0 @@ -# 07 — Add isolated concurrent local Hot KV State - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-007` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a client, I need concurrent Route Sessions to retain independent per-Shard cache so that one request cannot clear or corrupt another. - -## Expected durable outputs - -- Concurrent local KV/session manager -- Isolation, eviction, cancellation and cleanup tests -- evidence/DGR-007/README.md - -## Acceptance criteria - -- [ ] Map `(Route Session ID, route epoch)` to an isolated llama sequence or bounded context. -- [ ] Allocate KV only for owned layers. -- [ ] Support prefill append, decode append, truncate, release, TTL/LRU eviction, and explicit cache-miss response. -- [ ] Reject stale epochs and incompatible cache recipes. -- [ ] At least four concurrent sessions on a small model complete without token or KV cross-talk. -- [ ] Cancellation/release of one session leaves other sessions intact and memory returns to the configured budget. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-007/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-006` must have `passes: true`; read `../evidence/DGR-006/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md b/.scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md new file mode 100644 index 0000000..c833514 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md @@ -0,0 +1,39 @@ + +# DGR-070: Enforce the DeepSeek V4 Flash beta gate + +- **Status / triage:** specification only; `ready-for-human`; `passes: false` +- **Execution mode:** `HITL` +- **Milestone:** `M5` +- **Dependencies:** `DGR-057`, `DGR-059`, `DGR-060`, `DGR-062`, `DGR-064`, `DGR-066`, `DGR-067`, `DGR-068` +- **Blocks (derived):** `DGR-071` +- **Labels:** `area:deepseek-v4`, `area:release`, `milestone:beta`, `priority:p0`, `gate:hitl`, `ready-for-human` +- **Evidence class:** `release` +- **Hardware:** `required` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md`, and evidence READMEs for dependencies (DGR-057, DGR-059, DGR-060, DGR-062, DGR-064, DGR-066, DGR-067, DGR-068) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Promote beta only after concurrency, resilience, long-context, MTP, scale, packaging, and speed gates pass. + +## Acceptance criteria + +- [ ] Evaluate all beta metrics against DGF-004 without changing thresholds after observing results. +- [ ] Require V4 parity, batching/fairness, bounded backpressure, failure recovery, long-context correctness, MTP evidence, and dynamic 10+ stage evidence. +- [ ] Publish the exact certified hardware/backend/recipe matrix; all other lanes remain dark. +- [ ] Decision is `beta`, `targeted optimization`, or `stop/rollback`, with unresolved failures listed explicitly. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-070/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md b/.scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md new file mode 100644 index 0000000..8070865 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md @@ -0,0 +1,39 @@ + +# DGR-071: Establish upstream pin, patch, and certification maintenance + +- **Status / triage:** specification only; `ready-for-agent`; `passes: false` +- **Execution mode:** `AFK` +- **Milestone:** `M5` +- **Dependencies:** `DGR-069`, `DGR-070` +- **Blocks (derived):** None +- **Labels:** `area:maintenance`, `area:upstream`, `area:certification`, `type:operations`, `priority:p2`, `ready-for-agent` +- **Evidence class:** `release` +- **Hardware:** `optional` +- **Model:** `deepseek-v4-flash` +- **Upstream:** `yes` + +## Objective / description + +Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md`, and evidence READMEs for dependencies (DGR-069, DGR-070) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Keep beta reproducible and detect upstream/model/hardware drift before it reaches routable nodes. + +## Acceptance criteria + +- [ ] Add a documented update procedure for llama.cpp pin, numbered patches, protocol/engine versions, V4 target revision, and certification records. +- [ ] A candidate pin update must apply/build/test in isolation and cannot replace the production pin automatically. +- [ ] Any changed graph, tensor inventory, boundary, cache/state layout, backend behavior, or performance invalidates affected certification. +- [ ] Add scheduled model-free checks plus opt-in real-hardware recertification instructions and rollback procedure. +- [ ] Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff. + +## Shared quality gates + +- Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`. +- `git diff --check` passes. +- Default tests are model-download-free, API-credit-free, and GPU-free. +- Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit. +- Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin. +- Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`. +- Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed. + +## Evidence handoff + +Write and verify `.scratch/distributed-gguf-runtime/evidence/DGR-071/README.md`. Until every criterion and applicable gate has real evidence, this story remains `passes: false`. Legacy evidence is provenance only, not completion credit. diff --git a/.scratch/distributed-gguf-runtime/issues/08-build-the-standalone-c-grpc-shard-worker.md b/.scratch/distributed-gguf-runtime/issues/08-build-the-standalone-c-grpc-shard-worker.md deleted file mode 100644 index 0faa6a9..0000000 --- a/.scratch/distributed-gguf-runtime/issues/08-build-the-standalone-c-grpc-shard-worker.md +++ /dev/null @@ -1,65 +0,0 @@ -# 08 — Build the standalone C++ gRPC Shard worker - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-008` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a node runtime, I need one supervised native process so that llama.cpp internals remain behind a stable project-owned protocol. - -## Expected durable outputs - -- Standalone C++ gRPC worker -- Fake-model Python/C++ integration tests -- Lifecycle and bounded-failure evidence -- evidence/DGR-008/README.md - -## Acceptance criteria - -- [ ] Worker exposes capability, health, session stream, release, cancellation, and metrics services from DGR-002. -- [ ] Worker loads one exact Artifact/recipe/Shard identity and refuses mismatched requests. -- [ ] Streaming path enforces bounded messages, flow control, deadlines, idempotency, and independent session cancellation. -- [ ] Worker does not expose raw llama.cpp RPC or arbitrary GGML graph execution. -- [ ] Graceful shutdown releases sessions; crash behavior is bounded and observable. -- [ ] Python integration tests run against a fake model mode without model downloads. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-008/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-002` must have `passes: true`; read `../evidence/DGR-002/README.md` and verify its referenced files/commands. -- `DGR-003` must have `passes: true`; read `../evidence/DGR-003/README.md` and verify its referenced files/commands. -- `DGR-004` must have `passes: true`; read `../evidence/DGR-004/README.md` and verify its referenced files/commands. -- `DGR-006` must have `passes: true`; read `../evidence/DGR-006/README.md` and verify its referenced files/commands. -- `DGR-007` must have `passes: true`; read `../evidence/DGR-007/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/09-integrate-the-native-worker-with-meshnet.md b/.scratch/distributed-gguf-runtime/issues/09-integrate-the-native-worker-with-meshnet.md deleted file mode 100644 index 7d3a8fe..0000000 --- a/.scratch/distributed-gguf-runtime/issues/09-integrate-the-native-worker-with-meshnet.md +++ /dev/null @@ -1,61 +0,0 @@ -# 09 — Integrate the native worker with Meshnet - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-009` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As the existing node service, I need a GGUF Shard backend adapter so that the Tracker, relay, billing, telemetry, and capability admission remain the sole control plane. - -## Expected durable outputs - -- Meshnet GGUF backend adapter -- Registration, routing, relay, telemetry and billing tests -- evidence/DGR-009/README.md - -## Acceptance criteria - -- [ ] Implement the existing model-backend surface without changing Transformers behavior. -- [ ] Registration carries exact validated GGUF recipe, Shard, backend and concurrency/KV capacity. -- [ ] Tracker forms only complete compatible routes and keeps uncertified recipes dark. -- [ ] Direct routes use gRPC streams; relayed routes carry the same versioned protobuf frames as opaque binary through the existing relay seam. -- [ ] Existing request/work IDs, cancellation, Generation Telemetry, billing, and per-node attribution remain correlated. -- [ ] No vLLM, Nakshatra, prima.cpp, or custom-engine control plane becomes a core dependency. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-009/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-003` must have `passes: true`; read `../evidence/DGR-003/README.md` and verify its referenced files/commands. -- `DGR-008` must have `passes: true`; read `../evidence/DGR-008/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/10-pass-local-real-model-two-process-acceptance.md b/.scratch/distributed-gguf-runtime/issues/10-pass-local-real-model-two-process-acceptance.md deleted file mode 100644 index fa2eb63..0000000 --- a/.scratch/distributed-gguf-runtime/issues/10-pass-local-real-model-two-process-acceptance.md +++ /dev/null @@ -1,62 +0,0 @@ -# 10 — Pass local real-model two-process acceptance - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-010` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a release engineer, I need real local distributed parity before involving network variability. - -## Expected durable outputs - -- Real local two-process commands and configuration -- Raw parity, memory and performance results -- evidence/DGR-010/README.md - -## Acceptance criteria - -- [ ] Two local worker processes open disjoint dense-Llama ranges from the certified Artifact. -- [ ] Prefill and at least 32 greedy decode tokens match whole-model llama.cpp within the certified tolerance. -- [ ] Each worker retains only its own tensors and Hot KV State. -- [ ] Four concurrent Route Sessions pass isolation and cleanup checks. -- [ ] Report TTFT, prefill/decode throughput, seam bytes/latency, worker RSS/VRAM, KV memory, batch size, and queue time. -- [ ] Killing one worker produces a bounded structured failure rather than a deadlock. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence -- [ ] Model artifacts remain on the configured mounted-drive storage and never under /home -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-010/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-009` must have `passes: true`; read `../evidence/DGR-009/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/11-pass-a-real-heterogeneous-two-machine-route.md b/.scratch/distributed-gguf-runtime/issues/11-pass-a-real-heterogeneous-two-machine-route.md deleted file mode 100644 index 8d23aca..0000000 --- a/.scratch/distributed-gguf-runtime/issues/11-pass-a-real-heterogeneous-two-machine-route.md +++ /dev/null @@ -1,62 +0,0 @@ -# 11 — Pass a real heterogeneous two-machine route - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-011` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a consumer-hardware operator, I need two physical machines to execute one GGUF model so that the distributed claim is real. - -## Expected durable outputs - -- Two-machine hardware/network/runtime manifest -- Raw real-route metrics and output evidence -- evidence/DGR-011/README.md - -## Acceptance criteria - -- [ ] Tracker selects two physical nodes with disjoint Shards and one exact certified recipe/compatibility class. -- [ ] Actual CPU/GPU execution occurs on both nodes; synthetic workers do not satisfy acceptance. -- [ ] Prefill/decode, concurrent-session isolation, telemetry, cancellation, and cleanup pass over the real transport/relay path. -- [ ] Exact hardware, network, backend, model hash, route, commands, and raw metrics are recorded. -- [ ] A model or recipe larger than one participating node's admitted memory is exercised when available. -- [ ] Output drift is measured and incompatible mixed backends fail closed. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence -- [ ] Model artifacts remain on the configured mounted-drive storage and never under /home -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-011/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-010` must have `passes: true`; read `../evidence/DGR-010/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/12-implement-continuous-batching-and-bounded-admission.md b/.scratch/distributed-gguf-runtime/issues/12-implement-continuous-batching-and-bounded-admission.md deleted file mode 100644 index 182c20b..0000000 --- a/.scratch/distributed-gguf-runtime/issues/12-implement-continuous-batching-and-bounded-admission.md +++ /dev/null @@ -1,63 +0,0 @@ -# 12 — Implement continuous batching and bounded admission - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-012` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a node operator, I need active sessions batched safely so that concurrency increases aggregate throughput rather than serializing every request. - -## Expected durable outputs - -- Continuous batching/admission scheduler -- Concurrency 1/2/4/8 report -- Queue, batch and KV-pressure evidence -- evidence/DGR-012/README.md - -## Acceptance criteria - -- [ ] Node scheduler admits sessions against weight, KV, scratch, and queue budgets. -- [ ] Compatible decode steps from multiple sessions form llama.cpp batches while preserving per-session positions and outputs. -- [ ] Prefill does not starve decode; scheduling policy and bounds are explicit. -- [ ] Backpressure prevents unbounded queued activations or KV growth. -- [ ] Capability telemetry reports active sessions, queue depth, batch occupancy, KV pressure, prefill/decode rates, and rejected admissions. -- [ ] Concurrency 1/2/4/8 benchmark identifies saturation and shows no cross-session corruption. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-012/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-007` must have `passes: true`; read `../evidence/DGR-007/README.md` and verify its referenced files/commands. -- `DGR-009` must have `passes: true`; read `../evidence/DGR-009/README.md` and verify its referenced files/commands. -- `DGR-010` must have `passes: true`; read `../evidence/DGR-010/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md b/.scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md deleted file mode 100644 index 0aca94f..0000000 --- a/.scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md +++ /dev/null @@ -1,62 +0,0 @@ -# 13 — Harden failure, cancellation, and restart semantics - -Status: done - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-013` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a client, I need failures to be bounded and explicit so that distributed speed does not come with hanging or corrupted generations. - -## Expected durable outputs - -- Failure/cancel/restart test matrix -- Resource cleanup and billing-state evidence -- evidence/DGR-013/README.md - -## Acceptance criteria - -- [ ] Deadlines and heartbeat/health loss terminate blocked stream operations. -- [ ] Cancellation propagates across every Shard and releases local KV and queued buffers. -- [ ] Duplicate steps are idempotent; uncertain mutations are never replayed silently. -- [ ] Alpha failover restarts from token zero on a newly compatible route rather than importing unverified KV. -- [ ] Worker death, stream reset, malformed bundle, stale epoch, and cache miss tests pass. -- [ ] Billing/work records distinguish completed, cancelled, failed, and unverified work. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-013/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-008` must have `passes: true`; read `../evidence/DGR-008/README.md` and verify its referenced files/commands. -- `DGR-009` must have `passes: true`; read `../evidence/DGR-009/README.md` and verify its referenced files/commands. -- `DGR-012` must have `passes: true`; read `../evidence/DGR-012/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/14-enforce-the-gguf-versus-safetensors-release-gate.md b/.scratch/distributed-gguf-runtime/issues/14-enforce-the-gguf-versus-safetensors-release-gate.md deleted file mode 100644 index ee55f1c..0000000 --- a/.scratch/distributed-gguf-runtime/issues/14-enforce-the-gguf-versus-safetensors-release-gate.md +++ /dev/null @@ -1,65 +0,0 @@ -# 14 — Enforce the GGUF-versus-safetensors release gate - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-014` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As the product owner, I need an end-to-end comparison so that the native runtime ships only if it advances model access or performance. - -## Expected durable outputs - -- Immutable comparison against DGR-001 thresholds -- Machine-readable final report -- Ship/optimize/stop recommendation -- evidence/DGR-014/README.md - -## Acceptance criteria - -- [ ] Run current distributed safetensors and distributed GGUF routes on the same certified model/hardware/network scenario where technically comparable. -- [ ] Report quality, TTFT, prefill/decode throughput, aggregate concurrency throughput, p95 latency, seam cost, memory, KV pressure, failures, and cleanup. -- [ ] Evaluate against the DGR-001 performance contract without changing thresholds after seeing results. -- [ ] Ship recommendation is one of: promote GGUF, optimize a measured bottleneck with a new bounded task, or stop the native track. -- [ ] Results clearly separate quantization gains from transport/runtime gains. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence -- [ ] Model artifacts remain on the configured mounted-drive storage and never under /home -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-014/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-001` must have `passes: true`; read `../evidence/DGR-001/README.md` and verify its referenced files/commands. -- `DGR-011` must have `passes: true`; read `../evidence/DGR-011/README.md` and verify its referenced files/commands. -- `DGR-012` must have `passes: true`; read `../evidence/DGR-012/README.md` and verify its referenced files/commands. -- `DGR-013` must have `passes: true`; read `../evidence/DGR-013/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/15-add-and-certify-a-qwen3-qwen3-moe-adapter.md b/.scratch/distributed-gguf-runtime/issues/15-add-and-certify-a-qwen3-qwen3-moe-adapter.md deleted file mode 100644 index 5240d8d..0000000 --- a/.scratch/distributed-gguf-runtime/issues/15-add-and-certify-a-qwen3-qwen3-moe-adapter.md +++ /dev/null @@ -1,61 +0,0 @@ -# 15 — Add and certify a Qwen3/Qwen3-MoE adapter - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-015` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a client seeking top models, I need a separately certified MoE-capable architecture after the dense runtime proves stable. - -## Expected durable outputs - -- Qwen3-family architecture adapter -- Architecture-specific parity/admission/performance results -- evidence/DGR-015/README.md - -## Acceptance criteria - -- [ ] Implement explicit tensor ownership, router/top-k, expert/shared-expert, Q/K normalization, boundary bundle, and cache semantics for the selected Qwen3 family recipe. -- [ ] Do not reuse the dense-Llama adapter through unchecked name substitutions. -- [ ] Whole-model versus distributed prefill/decode parity passes the architecture-specific tolerance. -- [ ] Expert memory ownership and communication are measured. -- [ ] Real consumer-hardware acceptance and capability admission pass before the recipe becomes routable. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence -- [ ] Model artifacts remain on the configured mounted-drive storage and never under /home -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-015/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-014` must have `passes: true`; read `../evidence/DGR-014/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/issues/16-produce-the-upstream-llama-cpp-collaboration-package.md b/.scratch/distributed-gguf-runtime/issues/16-produce-the-upstream-llama-cpp-collaboration-package.md deleted file mode 100644 index b8d5de0..0000000 --- a/.scratch/distributed-gguf-runtime/issues/16-produce-the-upstream-llama-cpp-collaboration-package.md +++ /dev/null @@ -1,60 +0,0 @@ -# 16 — Produce the upstream llama.cpp collaboration package - -Status: ready-for-agent - -## Mandatory fresh-session context - -- Read [RALPH-CONTEXT.md](../RALPH-CONTEXT.md) completely before changing code. -- This issue is `DGR-016` in [prd.json](../prd.json). -- Read the evidence README for every dependency listed below. -- Inspect current code and `git status`; historical text and previous agent claims are not evidence. - -## Description - -As a maintainer, I need narrow upstreamable proposals so that our patch burden can shrink without asking llama.cpp to own Meshnet networking. - -## Expected durable outputs - -- Narrow upstream patches/tests -- Generic API design note -- Human-ready llama.cpp outreach package -- evidence/DGR-016/README.md - -## Acceptance criteria - -- [ ] Separate generic llama.cpp hooks from Meshnet protocol/control-plane code. -- [ ] Prepare minimal reproducible examples and tests for range-aware loading, boundary input/output, and layer-filtered KV. -- [ ] Compare the proposal with Nakshatra and prima.cpp evidence and explain why the API is generally useful. -- [ ] Preserve one scoped commit/patch per concern against the exact upstream pin. -- [ ] Produce an outreach document suitable for Georgi/llama.cpp maintainers; actual sending remains a human action. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched -- [ ] llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-016/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes - -## Dependency handoff - -- `DGR-010` must have `passes: true`; read `../evidence/DGR-010/README.md` and verify its referenced files/commands. - -## Finish contract - -- Create the task evidence directory and durable handoff required above. -- Preserve real failures and blockers; never fabricate benchmark, model, test or hardware output. -- Change this issue to `Status: done` only after all criteria pass. -- Emit `COMPLETE` only after the evidence handoff exists. - -## References - -- [Ralph execution context](../RALPH-CONTEXT.md) -- [PRD](../PRD.md) -- [Implementation strategy](../implementation-strategy.md) -- [Current architecture](../architecture.md) -- [Architecture decision](../../docs/adr/0024-distributed-gguf-runtime.md) diff --git a/.scratch/distributed-gguf-runtime/milestones.md b/.scratch/distributed-gguf-runtime/milestones.md index 58b2b35..bc71c66 100644 --- a/.scratch/distributed-gguf-runtime/milestones.md +++ b/.scratch/distributed-gguf-runtime/milestones.md @@ -1,35 +1,37 @@ -# Distributed GGUF runtime milestones +# Distributed GGUF Runtime milestones -## Gate A — measured runtime value +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. -- DGR-001 locks the safetensors-versus-GGUF performance/fit/quality contract. -- DGR-002 can proceed independently and defines the battle-proven backend-neutral wire protocol. -- DGR-003 builds exact recipe identity on DGR-002. -- Expensive native llama.cpp work remains gated by DGR-001. +## M0 — Truth and contracts -## Gate B — minimal native execution seam +- **Stories:** `DGR-017..DGR-020` +- **Outcome:** Reconciled legacy truth, canonical metadata, immutable gates, and a controlled whole-model baseline. -- DGR-004 creates the reproducible pinned fork boundary. -- DGR-005 implements dense-Llama range ownership. -- DGR-006 proves architecture-defined boundary parity. +## M1 — Protocol and native substrate -## Gate C — concurrent production worker +- **Stories:** `DGR-021..DGR-033` +- **Outcome:** Versioned gRPC protocol, exact identities/artifacts, pinned upstream, reproducible builds, ShardEngine, and fake worker. -- DGR-007 isolates concurrent Hot KV State. -- DGR-008 exposes the native worker over gRPC. -- DGR-009 integrates the worker without replacing Meshnet's control plane. -- DGR-010 passes local real-model two-process acceptance. +## M2 — Dense vertical proof -## Gate D — real consumer-hardware route +- **Stories:** `DGR-034..DGR-043` +- **Outcome:** Dense ranged execution, parity, local state, worker integration, and GGUF inputs to existing routing. -- DGR-011 passes two-physical-machine execution. -- DGR-012 adds continuous batching and bounded admission. -- DGR-013 hardens failure and cancellation. +## M3 — DeepSeek V4 Flash alpha -## Gate E — product release decision +- **Stories:** `DGR-044..DGR-054` +- **Outcome:** Pinned V4 adapter around upstream llama.cpp, real route certification, and pre-locked alpha decision with MTP off. -- DGR-014 compares distributed GGUF against the current distributed safetensors route under locked thresholds. -- DGR-015 adds Qwen3/Qwen3-MoE only after the dense runtime passes. -- DGR-016 prepares narrow upstream llama.cpp collaboration material. +## M4 — Performance and beta hardening -No later gate may be claimed from synthetic workers or documentation-only evidence. +- **Stories:** `DGR-055..DGR-067` +- **Outcome:** Batching, backpressure, recovery, scale certification, optimization, MTP, and hardware matrix. + +## M5 — Release and maintenance + +- **Stories:** `DGR-068..DGR-071` +- **Outcome:** Reproducible packages, upstream collaboration, beta decision, and sustainable recertification. + +## Release semantics + +Alpha is DGR-054 and requires correctness plus the immutable useful-speed contract; MTP remains off. Beta is DGR-070 and requires DGR-066 MTP evidence plus concurrency, recovery, long-context, 10+ scenario, compression/optimization where certified, hardware matrix, and packaging. Failed gates remain failed evidence; they are not relabeled as success. diff --git a/.scratch/distributed-gguf-runtime/prd.json b/.scratch/distributed-gguf-runtime/prd.json index 323eaae..5b50528 100644 --- a/.scratch/distributed-gguf-runtime/prd.json +++ b/.scratch/distributed-gguf-runtime/prd.json @@ -1,511 +1,2411 @@ { - "name": "Performant Concurrent Distributed GGUF Runtime", - "branchName": "ralph/performant-concurrent-distributed-gguf", - "description": "Benchmark-gated native llama.cpp/GGUF Shards with gRPC streaming, concurrent local KV, continuous batching, real heterogeneous acceptance, and a measured release gate against Transformers/safetensors.", + "name": "Distributed GGUF Runtime", + "branchName": "ralph/distributed-gguf-runtime", + "description": "Benchmark-gated distributed GGUF Shards using existing Meshnet control-plane routing and a standalone C++ gRPC worker around pinned upstream llama.cpp, targeting DeepSeek V4 Flash without hardcoded quantization or topology.", + "sourceOfTruth": "This prd.json is authoritative. Generated issue Markdown and planning summaries are projections and must not override it. DGR-017 and DGR-018 are complete; all later stories remain unimplemented specifications with passes=false.", + "qualityGates": { + "universal": [ + "Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`.", + "`git diff --check` passes.", + "Default tests are model-download-free, API-credit-free, and GPU-free.", + "Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit." + ], + "native": [ + "Native changes pass focused out-of-tree CMake build and CTest; patch changes verify clean apply/check/reverse against the exact llama.cpp pin." + ], + "realModelHardware": [ + "Runs are opt-in and record exact artifact/split hashes, runtime/upstream pin, backend/driver, hardware, network, commands, and raw metrics. Model artifacts use configured mounted-drive storage and never `/home`." + ], + "scope": [ + "Preserve existing Transformers behavior and backend-agnostic Tracker routing/load balancing/billing/relay semantics unless an explicit versioned contract says otherwise. One scoped story commit is expected during execution, but this specification-materialization change is not committed." + ] + }, + "metadataSchema": { + "requiredStoryFields": [ + "id", + "title", + "description", + "acceptanceCriteria", + "priority", + "passes", + "milestone", + "executionMode", + "labels", + "triage", + "evidenceClass", + "evidencePath", + "hardware", + "model", + "upstream", + "dependsOn", + "notes", + "blocks" + ], + "optionalStoryFields": [ + "completionNotes" + ], + "idRange": "DGR-017..DGR-071 inclusive", + "triageValues": [ + "ready-for-agent", + "ready-for-human" + ], + "executionModeValues": [ + "AFK", + "HITL" + ], + "evidenceClassValues": [ + "model-free", + "fixture", + "real-model", + "real-hardware", + "release" + ], + "hardwareValues": [ + "none", + "optional", + "required" + ], + "upstreamValues": [ + "yes", + "no", + "conditional" + ], + "typeDerivation": "A story type is derived from its type: label; gate: stories derive release-gate.", + "labelConventions": "Reserved prefixes include type:, priority:, area:, gate:, and ready-for-agent/ready-for-human triage labels; at most one type: and one priority: label are allowed.", + "generatedArtifactDisclaimer": "", + "dependencyRules": "Dependencies reference existing numerically earlier IDs; graph is acyclic. blocks is mechanically derived from dependsOn.", + "authorityRule": "Generated issue files state that prd.json is authoritative and cannot independently claim completion or override it." + }, + "milestones": [ + { + "id": "M0", + "name": "Truth and contracts", + "stories": "DGR-017..DGR-020", + "outcome": "Reconciled legacy truth, canonical metadata, immutable gates, and a controlled whole-model baseline." + }, + { + "id": "M1", + "name": "Protocol and native substrate", + "stories": "DGR-021..DGR-033", + "outcome": "Versioned gRPC protocol, exact identities/artifacts, pinned upstream, reproducible builds, ShardEngine, and fake worker." + }, + { + "id": "M2", + "name": "Dense vertical proof", + "stories": "DGR-034..DGR-043", + "outcome": "Dense ranged execution, parity, local state, worker integration, and GGUF inputs to existing routing." + }, + { + "id": "M3", + "name": "DeepSeek V4 Flash alpha", + "stories": "DGR-044..DGR-054", + "outcome": "Pinned V4 adapter around upstream llama.cpp, real route certification, and pre-locked alpha decision with MTP off." + }, + { + "id": "M4", + "name": "Performance and beta hardening", + "stories": "DGR-055..DGR-067", + "outcome": "Batching, backpressure, recovery, scale certification, optimization, MTP, and hardware matrix." + }, + { + "id": "M5", + "name": "Release and maintenance", + "stories": "DGR-068..DGR-071", + "outcome": "Reproducible packages, upstream collaboration, beta decision, and sustainable recertification." + } + ], + "supersededStories": { + "DGR-001": { + "newIds": [ + "DGR-019", + "DGR-020", + "DGR-054", + "DGR-070" + ], + "disposition": "Benchmark scaffold/evidence may be audited; old pass state is void." + }, + "DGR-002": { + "newIds": [ + "DGR-021", + "DGR-022", + "DGR-023", + "DGR-024" + ], + "disposition": "Split protocol, lifecycle, code generation, and fake transport." + }, + "DGR-003": { + "newIds": [ + "DGR-025" + ], + "disposition": "Replaced by exact artifact/runtime compatibility identity." + }, + "DGR-004": { + "newIds": [ + "DGR-027", + "DGR-028", + "DGR-029", + "DGR-030", + "DGR-071" + ], + "disposition": "Split provenance, patch stack, builds, and maintenance." + }, + "DGR-005": { + "newIds": [ + "DGR-034", + "DGR-045" + ], + "disposition": "Dense and V4 ownership separated." + }, + "DGR-006": { + "newIds": [ + "DGR-031", + "DGR-035", + "DGR-036", + "DGR-046", + "DGR-047", + "DGR-048", + "DGR-049" + ], + "disposition": "Engine, dense boundary, V4 typed boundary, and local-state adapters separated." + }, + "DGR-007": { + "newIds": [ + "DGR-038", + "DGR-049" + ], + "disposition": "Replaced by session/epoch-keyed local KV and V4 auxiliary state." + }, + "DGR-008": { + "newIds": [ + "DGR-032", + "DGR-033", + "DGR-037" + ], + "disposition": "Old implementation/evidence absent; no completion credit transfers." + }, + "DGR-009": { + "newIds": [ + "DGR-040", + "DGR-041", + "DGR-042", + "DGR-043" + ], + "disposition": "Supervision, registration, relay, and routing-input integration separated." + }, + "DGR-010": { + "newIds": [ + "DGR-036", + "DGR-039", + "DGR-052" + ], + "disposition": "Fixture, dense real acceptance, and V4 parity separated." + }, + "DGR-011": { + "newIds": [ + "DGR-053", + "DGR-061", + "DGR-062", + "DGR-067" + ], + "disposition": "Replaced by scenario-based real 2–4, existing-routing 10+, real 10+, and backend certification." + }, + "DGR-012": { + "newIds": [ + "DGR-055", + "DGR-056", + "DGR-057" + ], + "disposition": "Batching, admission/backpressure, and benchmarking separated." + }, + "DGR-013": { + "newIds": [ + "DGR-058", + "DGR-059" + ], + "disposition": "Failure semantics and restart/re-prefill recovery separated." + }, + "DGR-014": { + "newIds": [ + "DGR-019", + "DGR-054", + "DGR-070" + ], + "disposition": "Replaced by immutable performance, alpha, and beta gates." + }, + "DGR-015": { + "newIds": [ + "DGR-044", + "DGR-045", + "DGR-046", + "DGR-047", + "DGR-048", + "DGR-049", + "DGR-050", + "DGR-051", + "DGR-052", + "DGR-053", + "DGR-054", + "DGR-060", + "DGR-065", + "DGR-066", + "DGR-067" + ], + "disposition": "Qwen target superseded by DeepSeek V4 Flash; no old completion transfers." + }, + "DGR-016": { + "newIds": [ + "DGR-069", + "DGR-071" + ], + "disposition": "Upstream collaboration and ongoing maintenance separated." + } + }, "userStories": [ { - "id": "DGR-001", - "title": "Lock the safetensors-versus-GGUF performance contract", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a runtime engineer, I need a controlled baseline so that GGUF work proceeds from measured speed, memory, and quality rather than reputation.", - "acceptanceCriteria": [ - "Benchmark the same model architecture/revision, machine, prompts, context lengths, output lengths, sampling policy, and concurrency across the current Transformers/safetensors recipe and whole-model llama.cpp recipes.", - "Separate correctness/quality lanes from quantized performance/fit lanes instead of claiming BF16 and Q4 are numerically equivalent.", - "Report TTFT, prefill tok/s, decode tok/s, p50/p95 latency, aggregate throughput, RSS, VRAM, artifact size, failures, and output drift in machine-readable JSON.", - "Add concurrency levels 1 and 4 where memory permits.", - "Write a versioned performance contract consumed by later release gates, including an explicit stop condition when llama.cpp/GGUF has no meaningful speed or fit benefit.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence", - "Model artifacts remain on the configured mounted-drive storage and never under /home", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-001/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], - "priority": 2, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md", - "dependsOn": [] - }, - { - "id": "DGR-002", - "title": "Adopt the versioned gRPC Shard protocol", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/02-adopt-the-versioned-grpc-shard-protocol.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a node developer, I need a battle-proven streaming protocol so that Python and C++ Shards communicate without a custom socket protocol.", - "acceptanceCriteria": [ - "Add a Protocol Buffers schema for capability, health, session stream, release, and cancellation operations.", - "Define one long-lived bidirectional gRPC stream per Route Session Activation Seam with deadlines, cancellation, flow control, and structured errors.", - "Define bounded chunking for prefill and a small decode fast path.", - "Carry schema version, request/work ID, Route Session ID, route epoch, artifact/recipe fingerprint, Shard range/effective start, phase, position, idempotency step, cache expectation, compression, and checksum.", - "Define a versioned named-tensor bundle with per-tensor name, shape, dtype, byte order, and payload fragments.", - "Add generated-schema round-trip and compatibility tests in Python and C++.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-002/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-017", + "title": "Reconcile and clean the superseded DGR backlog", "priority": 1, + "milestone": "M0", + "executionMode": "AFK", + "labels": [ + "area:provenance", + "area:cleanup", + "type:audit", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-017/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md`, and evidence READMEs for dependencies (none) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Audit implementation reality, void inherited completion credit, and clean misleading backlog/stub baggage while preserving attributable evidence and accepted research.", + "acceptanceCriteria": [ + "Compare the branch, old DGR-001..016 issue/pass states, evidence, and actual runtime sources; classify each output as reusable, reference-only, blocked, obsolete, or absent.", + "Record an authoritative old-to-new disposition and provenance; explicitly give no completion credit to any new story and note absent implementation/evidence.", + "Remove or archive only artifacts the audit proves obsolete while preserving accepted ADRs, useful research, raw benchmark evidence, and attributable reusable work.", + "Protect ignored build workspaces, generated protobuf outputs, Ralph logs, and model artifacts from accidental commits, and document every retained legacy artifact.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], "passes": true, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/02-adopt-the-versioned-grpc-shard-protocol.md", - "dependsOn": [] + "completionNotes": "Completed 2026-07-16. Superseded DGR-001..016 issues/evidence and nonfunctional synthetic runtime scaffolding were removed from the active tree; packages/ and tests/ were restored exactly to origin/master. Accepted research and the real public-relay smoke benchmark were retained with provenance. See evidence/DGR-017/README.md.", + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/017-reconcile-and-clean-the-superseded-dgr-backlog.md; prd.json is authoritative.", + "blocks": [ + "DGR-018", + "DGR-019", + "DGR-027", + "DGR-054" + ] }, { - "id": "DGR-003", - "title": "Define exact Artifact and runtime recipe identity", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs the Tracker, I need exact compatibility identity so that only numerically and operationally compatible Shards form an Inference Route.", - "acceptanceCriteria": [ - "Separate weight quantization, activation dtype, compute dtype, KV dtype/layout, tokenizer revision, architecture adapter, backend, and runtime version.", - "Bind derivative or split artifacts to an exact source Model Artifact hash and Shard range.", - "Produce a stable compatibility fingerprint used by capability admission and the gRPC handshake.", - "Fail closed on mismatched artifact, tokenizer, architecture, range, boundary schema, activation recipe, or cache layout.", - "Keep unsupported recipes registered-but-dark until a real distributed forward certifies them.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-003/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" + "id": "DGR-018", + "title": "Define canonical Ralph and Gitea metadata schema", + "priority": 2, + "milestone": "M0", + "executionMode": "AFK", + "labels": [ + "area:planning", + "area:gitea", + "type:infrastructure", + "priority:p0", + "ready-for-agent" ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-018/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-017" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Make `prd.json` the validated source from which Markdown and Gitea issues can later be generated losslessly.", + "acceptanceCriteria": [ + "Define fields for stable ID/title, labels, milestone, type, `dependsOn`, derived `blocks`, triage, evidence class, and hardware/model/upstream flags.", + "Validate that all stories start `passes: false`, use known dependencies, and have unique stable IDs.", + "Reject cycles, missing dependencies, mismatched generated `blocks`, duplicate titles/IDs, and generated artifacts claiming authority over `prd.json`.", + "Add deterministic model-free tests for parse, validation, and generation round trips.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": true, + "completionNotes": "Completed 2026-07-16. Added the canonical schema/validator/generator module and 108 deterministic tests; restored and extended prd.json metadata self-description; regenerated DGR-018. See evidence/DGR-018/README.md.", + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/018-define-canonical-ralph-and-gitea-metadata-schema.md; prd.json is authoritative.", + "blocks": [ + "DGR-021", + "DGR-025" + ] + }, + { + "id": "DGR-019", + "title": "Lock alpha and beta performance contracts", "priority": 3, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md", + "milestone": "M0", + "executionMode": "HITL", + "labels": [ + "area:performance", + "type:contract", + "priority:p0", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "release", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-019/README.md", + "hardware": "required", + "model": "generic+deepseek-v4", + "upstream": "no", "dependsOn": [ - "DGR-002" + "DGR-017" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Freeze useful speed, correctness, memory-fit, and stop/go thresholds before implementation results are visible.", + "acceptanceCriteria": [ + "Define controlled safetensors, whole-model GGUF, dense distributed GGUF, and V4 Flash distributed lanes with fixed prompts, context/output lengths, sampling, concurrency, hardware, and metrics.", + "Alpha requires correctness plus a human-approved useful-speed threshold; beta adds concurrency, long-context, failure, and sustained-throughput thresholds.", + "Separate quantization/model-fit gains from runtime, transport, batching, and kernel gains.", + "Treat quants and 2–4/10+ stage counts only as named certification scenarios; no product logic may hardcode them.", + "Lock thresholds and stop conditions in versioned machine-readable data before benchmark result ingestion.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/019-lock-alpha-and-beta-performance-contracts.md; prd.json is authoritative.", + "blocks": [ + "DGR-020", + "DGR-044", + "DGR-054" ] }, { - "id": "DGR-004", - "title": "Create the reproducible pinned llama.cpp patch stack", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/04-create-the-reproducible-pinned-llama-cpp-patch-stack.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a maintainer, I need a small auditable fork boundary so that upstream updates do not turn the runtime into an unmaintainable stitched codebase.", - "acceptanceCriteria": [ - "Pin one exact llama.cpp commit through a reproducible source dependency mechanism.", - "Store a numbered minimal patch stack separately from Meshnet networking code.", - "Add a build script that applies/checks patches and builds the standalone worker without manual source copying.", - "Record upstream file/ABI assumptions and fail clearly when the pin changes.", - "Preserve upstream license and attribution notices.", - "Add a clean rebuild smoke test that does not download a model.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-004/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-020", + "title": "Run the controlled whole-model GGUF baseline", "priority": 4, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/04-create-the-reproducible-pinned-llama-cpp-patch-stack.md", + "milestone": "M0", + "executionMode": "HITL", + "labels": [ + "area:performance", + "type:benchmark", + "priority:p0", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-020/README.md", + "hardware": "required", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-001" + "DGR-019" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md`, and evidence READMEs for dependencies (DGR-019) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Execute the locked safetensors and whole-model llama.cpp lanes before distributed implementation results can influence the decision.", + "acceptanceCriteria": [ + "Run the exact DGR-019 safetensors and whole-model llama.cpp benchmark lanes with locked prompts, lengths, sampling, concurrency, hardware, and artifact/runtime identities.", + "Record raw machine-readable correctness, TTFT, prefill/decode, throughput, latency, memory, artifact-size, failure, and quality-drift metrics without ingesting distributed implementation results.", + "Separate quantization/model-fit effects from runtime/kernel effects and preserve failed or unavailable lanes honestly.", + "Publish a threshold-based `go`, `optimize baseline`, or `stop` decision without changing the locked contract.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/020-run-the-controlled-whole-model-gguf-baseline.md; prd.json is authoritative.", + "blocks": [ + "DGR-054" ] }, { - "id": "DGR-005", - "title": "Implement dense-Llama range-aware GGUF ownership", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/05-implement-dense-llama-range-aware-gguf-ownership.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a node, I need to map only my assigned dense-Llama Shard so that aggregate consumer memory can hold a model larger than one node.", - "acceptanceCriteria": [ - "Register and allocate only `blk.N.*` tensors in the assigned range.", - "Load embeddings only for the head and final norm/LM head only for the tail, including tied embeddings.", - "Prefer range-aware mapping from one exact source GGUF; if derivative sub-GGUFs are used temporarily, verify source/slice hashes and avoid claiming final artifact semantics.", - "Report authoritative loaded range and endpoint ownership from the model, not operator CLI claims.", - "Demonstrate mapped/resident memory scales with owned tensors rather than full model size.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-005/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-021", + "title": "Define the versioned named-tensor stream envelope", "priority": 5, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/05-implement-dense-llama-range-aware-gguf-ownership.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:protocol", + "type:infrastructure", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-021/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-003", - "DGR-004" + "DGR-018" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md`, and evidence READMEs for dependencies (DGR-018) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish the backend-neutral protobuf envelope used by direct and relayed Shard activation traffic.", + "acceptanceCriteria": [ + "Define schema version, request/work ID, route session/epoch, shard range/effective start, phase, position, and idempotency step.", + "Define named tensors with shape, dtype, byte order, bounded fragments, compression identity, and checksum.", + "Reserve extensible fields for token-ID sidebands, architecture state, recurrent state, and MTP without claiming implementations.", + "Add deterministic serialization, fragmentation, checksum, unknown-field, and size-limit tests.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": true, + "completionNotes": "Added a versioned activation envelope with deterministic JSON serialization, bounded tensor fragmentation, checksum validation, unknown-field preservation, and TensorPayload conversion hooks; verified by targeted pytest and compileall runs.", + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/021-define-the-versioned-named-tensor-stream-envelope.md; prd.json is authoritative.", + "blocks": [ + "DGR-022", + "DGR-023", + "DGR-025", + "DGR-031", + "DGR-035", + "DGR-046" ] }, { - "id": "DGR-006", - "title": "Implement architecture-defined boundary input/output", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/06-implement-architecture-defined-boundary-input-output.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a Shard, I need to consume and emit the correct transformer boundary state so that disjoint processes reproduce whole-model execution.", - "acceptanceCriteria": [ - "Head accepts token IDs and owns token embedding.", - "Middle/tail bypass token embedding and accept the named boundary bundle.", - "Non-tail emits the unnormalized architecture-defined residual/boundary before final norm/head and before tail-only row pruning.", - "Tail emits logits or token output through an explicit sampling contract.", - "Dense-Llama whole-model versus two-range prefill and greedy-decode parity passes the documented tolerance.", - "The adapter interface fails closed for uncertified architectures.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-006/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-022", + "title": "Define Shard lifecycle and structured status RPCs", "priority": 6, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/06-implement-architecture-defined-boundary-input-output.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:protocol", + "area:lifecycle", + "type:infrastructure", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-022/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-002", - "DGR-005" + "DGR-021" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/022-define-shard-lifecycle-and-structured-status-rpcs.md`, and evidence READMEs for dependencies (DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Complete the gRPC contract for worker capability, health, sessions, cancellation, release, and metrics.", + "acceptanceCriteria": [ + "Define capability, health, bidirectional session stream, cancellation, release, and metrics RPCs.", + "Specify deadlines, cancellation propagation, bounded flow control, cache expectations/results, and structured error taxonomy.", + "Specify TLS/auth hooks without moving Meshnet authentication or billing into the worker.", + "Add compatibility tests for supported versions and fail-closed tests for unsupported versions and malformed lifecycle transitions.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": true, + "notes": "Completed from isolated DGR-022 Ralph worktree; verified with 17 focused pytest cases and retained DGR-021 envelope compatibility.", + "blocks": [ + "DGR-024", + "DGR-033", + "DGR-037" ] }, { - "id": "DGR-007", - "title": "Add isolated concurrent local Hot KV State", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a client, I need concurrent Route Sessions to retain independent per-Shard cache so that one request cannot clear or corrupt another.", - "acceptanceCriteria": [ - "Map `(Route Session ID, route epoch)` to an isolated llama sequence or bounded context.", - "Allocate KV only for owned layers.", - "Support prefill append, decode append, truncate, release, TTL/LRU eviction, and explicit cache-miss response.", - "Reject stale epochs and incompatible cache recipes.", - "At least four concurrent sessions on a small model complete without token or KV cross-talk.", - "Cancellation/release of one session leaves other sessions intact and memory returns to the configured budget.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-007/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-023", + "title": "Make Python and C++ protobuf generation reproducible", "priority": 7, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:protocol", + "area:build", + "type:tooling", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-023/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-006" + "DGR-021" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md`, and evidence READMEs for dependencies (DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Generate identical Python/C++ protocol bindings without manual copying or checked-in build debris.", + "acceptanceCriteria": [ + "Pin protoc, gRPC, and plugin versions or declare a verified compatible range.", + "Generate Python and C++ bindings into out-of-tree build/package locations through documented commands.", + "Add Python↔C++ round-trip and descriptor compatibility tests.", + "A clean checkout regenerates bindings deterministically or fails with an actionable toolchain error.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/023-make-python-and-c-protobuf-generation-reproducible.md; prd.json is authoritative.", + "blocks": [ + "DGR-024", + "DGR-037" ] }, { - "id": "DGR-008", - "title": "Build the standalone C++ gRPC Shard worker", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/08-build-the-standalone-c-grpc-shard-worker.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a node runtime, I need one supervised native process so that llama.cpp internals remain behind a stable project-owned protocol.", - "acceptanceCriteria": [ - "Worker exposes capability, health, session stream, release, cancellation, and metrics services from DGR-002.", - "Worker loads one exact Artifact/recipe/Shard identity and refuses mismatched requests.", - "Streaming path enforces bounded messages, flow control, deadlines, idempotency, and independent session cancellation.", - "Worker does not expose raw llama.cpp RPC or arbitrary GGML graph execution.", - "Graceful shutdown releases sessions; crash behavior is bounded and observable.", - "Python integration tests run against a fake model mode without model downloads.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-008/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-024", + "title": "Implement in-memory fake gRPC seam transport", "priority": 8, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/08-build-the-standalone-c-grpc-shard-worker.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:protocol", + "area:testing", + "type:vertical-slice", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-024/README.md", + "hardware": "none", + "model": "fake", + "upstream": "no", "dependsOn": [ - "DGR-002", - "DGR-003", - "DGR-004", - "DGR-006", - "DGR-007" + "DGR-022", + "DGR-023" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md`, and evidence READMEs for dependencies (DGR-022, DGR-023) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Exercise the complete streaming protocol deterministically before a real model or worker exists.", + "acceptanceCriteria": [ + "Provide a fake bidirectional stream supporting prefill fragments, decode fast-path frames, release, cancel, and structured errors.", + "Test flow-control blocking, deadlines, malformed fragments, checksum failure, duplicates, and stale epochs.", + "Verify direct and opaque-relay framing preserve identical protobuf bytes.", + "Tests require no sockets outside localhost, model downloads, or native accelerator.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/024-implement-in-memory-fake-grpc-seam-transport.md; prd.json is authoritative.", + "blocks": [ + "DGR-033", + "DGR-042" ] }, { - "id": "DGR-009", - "title": "Integrate the native worker with Meshnet", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/09-integrate-the-native-worker-with-meshnet.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs the existing node service, I need a GGUF Shard backend adapter so that the Tracker, relay, billing, telemetry, and capability admission remain the sole control plane.", - "acceptanceCriteria": [ - "Implement the existing model-backend surface without changing Transformers behavior.", - "Registration carries exact validated GGUF recipe, Shard, backend and concurrency/KV capacity.", - "Tracker forms only complete compatible routes and keeps uncertified recipes dark.", - "Direct routes use gRPC streams; relayed routes carry the same versioned protobuf frames as opaque binary through the existing relay seam.", - "Existing request/work IDs, cancellation, Generation Telemetry, billing, and per-node attribution remain correlated.", - "No vLLM, Nakshatra, prima.cpp, or custom-engine control plane becomes a core dependency.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-009/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-025", + "title": "Define exact artifact and runtime recipe identity", "priority": 9, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/09-integrate-the-native-worker-with-meshnet.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:identity", + "area:admission", + "type:domain", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-025/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-003", - "DGR-008" + "DGR-018", + "DGR-021" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md`, and evidence READMEs for dependencies (DGR-018, DGR-021) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure the tracker and worker only combine numerically and operationally compatible shards.", + "acceptanceCriteria": [ + "Fingerprint source artifact SHA, tokenizer revision, architecture adapter/version, boundary schema, runtime pin/patch stack, backend, quant, activation/compute dtype, and KV/state layout.", + "Bind each shard to an exact half-open range without hardcoding a topology or quant.", + "Fail closed on any artifact, adapter, boundary, cache, backend, or runtime mismatch.", + "Unsupported recipes remain registered-but-dark until real-hardware evidence certifies them.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/025-define-exact-artifact-and-runtime-recipe-identity.md; prd.json is authoritative.", + "blocks": [ + "DGR-026", + "DGR-031", + "DGR-041", + "DGR-044" ] }, { - "id": "DGR-010", - "title": "Pass local real-model two-process acceptance", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/10-pass-local-real-model-two-process-acceptance.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a release engineer, I need real local distributed parity before involving network variability.", - "acceptanceCriteria": [ - "Two local worker processes open disjoint dense-Llama ranges from the certified Artifact.", - "Prefill and at least 32 greedy decode tokens match whole-model llama.cpp within the certified tolerance.", - "Each worker retains only its own tensors and Hot KV State.", - "Four concurrent Route Sessions pass isolation and cleanup checks.", - "Report TTFT, prefill/decode throughput, seam bytes/latency, worker RSS/VRAM, KV memory, batch size, and queue time.", - "Killing one worker produces a bounded structured failure rather than a deadlock.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence", - "Model artifacts remain on the configured mounted-drive storage and never under /home", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-010/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-026", + "title": "Provision exact split-GGUF artifacts outside /home", "priority": 10, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/10-pass-local-real-model-two-process-acceptance.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:artifacts", + "area:provenance", + "type:tooling", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-026/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-009" + "DGR-025" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md`, and evidence READMEs for dependencies (DGR-025) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Make exact split-GGUF inputs reproducibly available from mounted-drive storage without embedding a quantization or topology assumption in product code.", + "acceptanceCriteria": [ + "Create an exact manifest that binds the source artifact, tokenizer/revision, every split file name, size, range/role, and cryptographic hash.", + "Provide resumable, hash-verifying download/provision tooling targeting configured mounted-drive storage; refuse paths under `/home` and incomplete or mismatched splits.", + "Keep quantization and split topology as manifest/recipe inputs with no hardcoded quant, node count, or range layout.", + "Add deterministic model-download-free tests using tiny local split fixtures, including interrupted resume, missing split, hash mismatch, and `/home` rejection.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/026-provision-exact-split-gguf-artifacts-outside-home.md; prd.json is authoritative.", + "blocks": [ + "DGR-044", + "DGR-045" ] }, { - "id": "DGR-011", - "title": "Pass a real heterogeneous two-machine route", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/11-pass-a-real-heterogeneous-two-machine-route.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a consumer-hardware operator, I need two physical machines to execute one GGUF model so that the distributed claim is real.", - "acceptanceCriteria": [ - "Tracker selects two physical nodes with disjoint Shards and one exact certified recipe/compatibility class.", - "Actual CPU/GPU execution occurs on both nodes; synthetic workers do not satisfy acceptance.", - "Prefill/decode, concurrent-session isolation, telemetry, cancellation, and cleanup pass over the real transport/relay path.", - "Exact hardware, network, backend, model hash, route, commands, and raw metrics are recorded.", - "A model or recipe larger than one participating node's admitted memory is exercised when available.", - "Output drift is measured and incompatible mixed backends fail closed.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence", - "Model artifacts remain on the configured mounted-drive storage and never under /home", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-011/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-027", + "title": "Add exact llama.cpp provenance manifest and fetch workspace", "priority": 11, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/11-pass-a-real-heterogeneous-two-machine-route.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:upstream", + "area:build", + "type:provenance", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-027/README.md", + "hardware": "none", + "model": "generic", + "upstream": "yes", "dependsOn": [ - "DGR-010" + "DGR-017" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md`, and evidence READMEs for dependencies (DGR-017) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Pin llama.cpp exactly through an in-repo manifest while fetching source only into an ignored build workspace.", + "acceptanceCriteria": [ + "Manifest records upstream URL, exact commit, expected source archive/tree hash, license, and retrieval method.", + "Fetch tooling verifies identity before use and refuses an unpinned branch/tag.", + "Source is fetched into an ignored build workspace; no submodule, vendored source tree, or permanent fork is introduced.", + "Offline reuse is supported only after the cached tree’s exact identity is verified.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/027-add-exact-llama-cpp-provenance-manifest-and-fetch-workspace.md; prd.json is authoritative.", + "blocks": [ + "DGR-028", + "DGR-029", + "DGR-044" ] }, { - "id": "DGR-012", - "title": "Implement continuous batching and bounded admission", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/12-implement-continuous-batching-and-bounded-admission.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a node operator, I need active sessions batched safely so that concurrency increases aggregate throughput rather than serializing every request.", - "acceptanceCriteria": [ - "Node scheduler admits sessions against weight, KV, scratch, and queue budgets.", - "Compatible decode steps from multiple sessions form llama.cpp batches while preserving per-session positions and outputs.", - "Prefill does not starve decode; scheduling policy and bounds are explicit.", - "Backpressure prevents unbounded queued activations or KV growth.", - "Capability telemetry reports active sessions, queue depth, batch occupancy, KV pressure, prefill/decode rates, and rejected admissions.", - "Concurrency 1/2/4/8 benchmark identifies saturation and shows no cross-session corruption.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-012/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-028", + "title": "Implement numbered patch-stack apply and verification", "priority": 12, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/12-implement-continuous-batching-and-bounded-admission.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:upstream", + "area:patches", + "type:tooling", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-028/README.md", + "hardware": "none", + "model": "generic", + "upstream": "yes", "dependsOn": [ - "DGR-007", - "DGR-009", - "DGR-010" + "DGR-027" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md`, and evidence READMEs for dependencies (DGR-027) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Maintain a minimal auditable llama.cpp delta with one numbered patch per concern.", + "acceptanceCriteria": [ + "Add deterministic apply/check/reverse verification against the exact manifest pin.", + "Separate range loading, boundary I/O, filtered state, and worker hooks into scoped patches.", + "Record upstream file/API assumptions and fail with the first incompatible patch when the pin changes.", + "Verify license/attribution and prove no Meshnet routing, billing, relay, or authentication code enters the patch stack.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/028-implement-numbered-patch-stack-apply-and-verification.md; prd.json is authoritative.", + "blocks": [ + "DGR-029", + "DGR-034", + "DGR-069" ] }, { - "id": "DGR-013", - "title": "Harden failure, cancellation, and restart semantics", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a client, I need failures to be bounded and explicit so that distributed speed does not come with hanging or corrupted generations.", - "acceptanceCriteria": [ - "Deadlines and heartbeat/health loss terminate blocked stream operations.", - "Cancellation propagates across every Shard and releases local KV and queued buffers.", - "Duplicate steps are idempotent; uncertain mutations are never replayed silently.", - "Alpha failover restarts from token zero on a newly compatible route rather than importing unverified KV.", - "Worker death, stream reset, malformed bundle, stale epoch, and cache miss tests pass.", - "Billing/work records distinguish completed, cancelled, failed, and unverified work.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-013/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-029", + "title": "Create the native CMake skeleton and deterministic CPU lane", "priority": 13, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:build", + "type:toolchain", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-029/README.md", + "hardware": "none", + "model": "generic", + "upstream": "yes", "dependsOn": [ - "DGR-008", - "DGR-009", - "DGR-012" + "DGR-027", + "DGR-028" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md`, and evidence READMEs for dependencies (DGR-027, DGR-028) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish an out-of-tree standalone native build with a deterministic CPU lane before accelerator matrix work.", + "acceptanceCriteria": [ + "Create the standalone native CMake target/skeleton and isolated out-of-tree configure/build preset for CPU.", + "Build and run a deterministic model-free CPU smoke/CTest lane from a clean checkout with actionable toolchain failures.", + "Keep fetched upstream sources, generated bindings, and all build outputs ignored and out of tree.", + "Ensure build success alone does not advertise any backend/model/recipe capability.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/029-create-the-native-cmake-skeleton-and-deterministic-cpu-lane.md; prd.json is authoritative.", + "blocks": [ + "DGR-030", + "DGR-034" ] }, { - "id": "DGR-014", - "title": "Enforce the GGUF-versus-safetensors release gate", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/14-enforce-the-gguf-versus-safetensors-release-gate.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs the product owner, I need an end-to-end comparison so that the native runtime ships only if it advances model access or performance.", - "acceptanceCriteria": [ - "Run current distributed safetensors and distributed GGUF routes on the same certified model/hardware/network scenario where technically comparable.", - "Report quality, TTFT, prefill/decode throughput, aggregate concurrency throughput, p95 latency, seam cost, memory, KV pressure, failures, and cleanup.", - "Evaluate against the DGR-001 performance contract without changing thresholds after seeing results.", - "Ship recommendation is one of: promote GGUF, optimize a measured bottleneck with a new bounded task, or stop the native track.", - "Results clearly separate quantization gains from transport/runtime gains.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence", - "Model artifacts remain on the configured mounted-drive storage and never under /home", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-014/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-030", + "title": "Add accelerator build presets and native CI matrix", "priority": 14, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/14-enforce-the-gguf-versus-safetensors-release-gate.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:build", + "area:hardware", + "type:toolchain", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-030/README.md", + "hardware": "optional", + "model": "generic", + "upstream": "yes", "dependsOn": [ - "DGR-001", - "DGR-011", - "DGR-012", - "DGR-013" + "DGR-029" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md`, and evidence READMEs for dependencies (DGR-029) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Compile all required accelerator lanes reproducibly while keeping untested real-hardware capabilities registered-dark.", + "acceptanceCriteria": [ + "Add isolated out-of-tree presets for CUDA, ROCm, Vulkan, and Metal without changing the deterministic CPU default.", + "Add a native CI/build matrix that reports unavailable SDKs as explicit unavailable/skipped lanes rather than false success.", + "Compile each available lane and preserve exact compiler, SDK, upstream pin, patch-stack, and build-option evidence.", + "Keep every backend/model/recipe lane registered-dark until a separate real-hardware certification record exists.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/030-add-accelerator-build-presets-and-native-ci-matrix.md; prd.json is authoritative.", + "blocks": [ + "DGR-053", + "DGR-067", + "DGR-068" ] }, { - "id": "DGR-015", - "title": "Add and certify a Qwen3/Qwen3-MoE adapter", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/15-add-and-certify-a-qwen3-qwen3-moe-adapter.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a client seeking top models, I need a separately certified MoE-capable architecture after the dense runtime proves stable.", - "acceptanceCriteria": [ - "Implement explicit tensor ownership, router/top-k, expert/shared-expert, Q/K normalization, boundary bundle, and cache semantics for the selected Qwen3 family recipe.", - "Do not reuse the dense-Llama adapter through unchecked name substitutions.", - "Whole-model versus distributed prefill/decode parity passes the architecture-specific tolerance.", - "Expert memory ownership and communication are measured.", - "Real consumer-hardware acceptance and capability admission pass before the recipe becomes routable.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Real-model execution is opt-in through MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 and records exact artifact/runtime/hardware evidence", - "Model artifacts remain on the configured mounted-drive storage and never under /home", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-015/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-031", + "title": "Introduce the project-owned `ShardEngine` interface", "priority": 15, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/15-add-and-certify-a-qwen3-qwen3-moe-adapter.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:engine", + "type:architecture", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-031/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", "dependsOn": [ - "DGR-014" + "DGR-021", + "DGR-025" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md`, and evidence READMEs for dependencies (DGR-021, DGR-025) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Isolate worker/protocol code from llama.cpp internals behind a stable project-owned engine contract.", + "acceptanceCriteria": [ + "Define load, capabilities, prefill/decode, boundary/logits result, cancel, release, health, and metrics operations.", + "Use project-owned request/result/state types; expose no `ggml_tensor`, llama context, scheduler, or ABI-owned structure.", + "Reserve typed MTP and architecture auxiliary-state hooks without enabling them.", + "Add contract tests proving fake and future llama implementations obey identical lifecycle semantics.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/031-introduce-the-project-owned-shardengine-interface.md; prd.json is authoritative.", + "blocks": [ + "DGR-032", + "DGR-034", + "DGR-035", + "DGR-037" ] }, { - "id": "DGR-016", - "title": "Produce the upstream llama.cpp collaboration package", - "description": "MANDATORY FRESH-SESSION CONTEXT: Read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md` and `.scratch/distributed-gguf-runtime/issues/16-produce-the-upstream-llama-cpp-collaboration-package.md` completely before coding. Read the evidence handoff for every dependency. The global goal is performant concurrent inference for models larger than one consumer node, using Meshnet as the sole control plane, gRPC/Protobuf as the Shard protocol, and a small pinned llama.cpp worker\u2014not a stitched collection of runtimes.\n\nAs a maintainer, I need narrow upstreamable proposals so that our patch burden can shrink without asking llama.cpp to own Meshnet networking.", - "acceptanceCriteria": [ - "Separate generic llama.cpp hooks from Meshnet protocol/control-plane code.", - "Prepare minimal reproducible examples and tests for range-aware loading, boundary input/output, and layer-filtered KV.", - "Compare the proposal with Nakshatra and prima.cpp evidence and explain why the API is generally useful.", - "Preserve one scoped commit/patch per concern against the exact upstream pin.", - "Produce an outreach document suitable for Georgi/llama.cpp maintainers; actual sending remains a human action.", - "Targeted pytest tests pass", - "python -m compileall packages tests passes for Python changes", - "git diff --check passes", - "Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free", - "Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction", - "Pinned native C++ target builds and focused CTest/protocol tests pass where native code is touched", - "llama.cpp patch stack applies cleanly to the exact pinned commit where patch code is touched", - "Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code", - "Read and verify every dependency evidence README before relying on dependency behavior", - "Preserve all pre-existing working-tree changes and stage only files belonging to this story", - "Write .scratch/distributed-gguf-runtime/evidence/DGR-016/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff", - "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" - ], + "id": "DGR-032", + "title": "Implement deterministic fake `ShardEngine`", "priority": 16, - "passes": false, - "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/16-produce-the-upstream-llama-cpp-collaboration-package.md", + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:engine", + "area:testing", + "type:fixture", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-032/README.md", + "hardware": "none", + "model": "fake", + "upstream": "no", "dependsOn": [ - "DGR-010" + "DGR-031" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md`, and evidence READMEs for dependencies (DGR-031) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Provide an engine fixture that deterministically transforms typed boundary bundles and session state.", + "acceptanceCriteria": [ + "Support head, middle, tail, prefill, decode, cancellation, and release with deterministic outputs.", + "Model isolated session/epoch state and deterministic cache-miss/stale-epoch failures.", + "Support configurable delay, memory pressure, malformed output, and crash injection.", + "Contract tests distinguish fixture evidence from real-model certification.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/032-implement-deterministic-fake-shardengine.md; prd.json is authoritative.", + "blocks": [ + "DGR-033" ] + }, + { + "id": "DGR-033", + "title": "Build a standalone fake C++ gRPC Shard worker", + "priority": 17, + "milestone": "M1", + "executionMode": "AFK", + "labels": [ + "area:worker", + "area:grpc", + "type:vertical-slice", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-033/README.md", + "hardware": "none", + "model": "fake", + "upstream": "no", + "dependsOn": [ + "DGR-022", + "DGR-024", + "DGR-032" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md`, and evidence READMEs for dependencies (DGR-022, DGR-024, DGR-032) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove the standalone worker process, stream, lifecycle, and supervision shape before llama.cpp integration.", + "acceptanceCriteria": [ + "A standalone C++ executable serves the complete lifecycle and stream RPC contract using the fake engine.", + "Python integration tests cover startup, health, capability, fragmented prefill, decode, release, cancellation, and graceful shutdown.", + "Bounded messages, deadlines, flow control, and independent session cancellation are enforced.", + "The worker exposes neither llama.cpp RPC nor arbitrary graph execution.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/033-build-a-standalone-fake-c-grpc-shard-worker.md; prd.json is authoritative.", + "blocks": [ + "DGR-036", + "DGR-040" + ] + }, + { + "id": "DGR-034", + "title": "Implement dense-Llama range-aware GGUF ownership", + "priority": 18, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:llama", + "area:weights", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-034/README.md", + "hardware": "optional", + "model": "dense-llama", + "upstream": "yes", + "dependsOn": [ + "DGR-028", + "DGR-029", + "DGR-031" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md`, and evidence READMEs for dependencies (DGR-028, DGR-029, DGR-031) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Load only the dense-Llama tensors owned by a shard range.", + "acceptanceCriteria": [ + "Load only `blk.N.*` tensors in the assigned range, embeddings only at the head, and norm/output or tied output only at the tail.", + "Derive authoritative range and endpoint ownership from the loaded engine state.", + "Reject invalid/gapped/out-of-model ranges and unexpected required tensors.", + "Real-model evidence shows mapped/resident memory scales with owned tensors rather than full artifact size.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/034-implement-dense-llama-range-aware-gguf-ownership.md; prd.json is authoritative.", + "blocks": [ + "DGR-035", + "DGR-037", + "DGR-051" + ] + }, + { + "id": "DGR-035", + "title": "Implement dense architecture boundary input/output", + "priority": 19, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:llama", + "area:boundary", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-035/README.md", + "hardware": "optional", + "model": "dense-llama", + "upstream": "yes", + "dependsOn": [ + "DGR-021", + "DGR-031", + "DGR-034" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md`, and evidence READMEs for dependencies (DGR-021, DGR-031, DGR-034) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Execute dense head/middle/tail ranges through an architecture-defined residual boundary.", + "acceptanceCriteria": [ + "Head accepts token IDs and owns embedding; middle/tail bypass embedding and accept a named boundary bundle.", + "Non-tail returns the unnormalized residual before final norm/head and before tail-only row pruning.", + "Tail returns logits or sampled-token output under an explicit contract.", + "Uncertified architectures and incompatible boundary schemas fail closed.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/035-implement-dense-architecture-boundary-input-output.md; prd.json is authoritative.", + "blocks": [ + "DGR-036", + "DGR-037", + "DGR-069" + ] + }, + { + "id": "DGR-036", + "title": "Prove dense fixture and real-model range parity", + "priority": 20, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:parity", + "area:testing", + "type:certification", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-036/README.md", + "hardware": "optional", + "model": "dense-llama", + "upstream": "yes", + "dependsOn": [ + "DGR-033", + "DGR-035" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md`, and evidence READMEs for dependencies (DGR-033, DGR-035) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove the engine/protocol seam before adding session concurrency.", + "acceptanceCriteria": [ + "Model-free two-stage tests pass through two fake worker processes with disjoint ranges.", + "A small real dense GGUF passes whole-model versus two-range prefill parity.", + "At least 32 greedy decode tokens match the locked tolerance.", + "Evidence distinguishes deterministic fixture proof from opt-in real-model proof.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/036-prove-dense-fixture-and-real-model-range-parity.md; prd.json is authoritative.", + "blocks": [ + "DGR-039" + ] + }, + { + "id": "DGR-037", + "title": "Bind llama.cpp to the standalone worker", + "priority": 21, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:worker", + "area:llama", + "type:vertical-slice", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-037/README.md", + "hardware": "optional", + "model": "dense-llama", + "upstream": "yes", + "dependsOn": [ + "DGR-022", + "DGR-023", + "DGR-031", + "DGR-034", + "DGR-035" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md`, and evidence READMEs for dependencies (DGR-022, DGR-023, DGR-031, DGR-034, DGR-035) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Replace the fake engine with the pinned llama.cpp `ShardEngine` implementation without changing the public worker contract.", + "acceptanceCriteria": [ + "Worker loads exactly one artifact/recipe/range identity and rejects mismatched stream requests.", + "All execution passes through `ShardEngine`; llama.cpp implementation types remain private.", + "Health and metrics expose loaded identity, authoritative ownership, memory, and execution state.", + "Graceful shutdown releases model/session resources; injected process death is observable and bounded.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/037-bind-llama-cpp-to-the-standalone-worker.md; prd.json is authoritative.", + "blocks": [ + "DGR-038", + "DGR-039", + "DGR-040", + "DGR-051" + ] + }, + { + "id": "DGR-038", + "title": "Implement isolated shard-local Hot KV State", + "priority": 22, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:kv-cache", + "area:concurrency", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-038/README.md", + "hardware": "optional", + "model": "dense-llama", + "upstream": "yes", + "dependsOn": [ + "DGR-037" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md`, and evidence READMEs for dependencies (DGR-037) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Map each route session and epoch to isolated state containing only locally owned layers.", + "acceptanceCriteria": [ + "Map `(route_session_id, route_epoch)` to an isolated llama sequence or bounded context.", + "Support prefill/decode append, truncate, release, TTL/LRU eviction, cache miss, and stale-epoch rejection.", + "Four concurrent sessions complete without token, KV, position, or cancellation cross-talk.", + "Release/eviction returns memory to the configured budget without affecting other sessions.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/038-implement-isolated-shard-local-hot-kv-state.md; prd.json is authoritative.", + "blocks": [ + "DGR-039", + "DGR-052", + "DGR-055", + "DGR-069" + ] + }, + { + "id": "DGR-039", + "title": "Pass local two-process dense acceptance", + "priority": 23, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:integration", + "area:parity", + "type:certification", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-039/README.md", + "hardware": "optional", + "model": "dense-llama", + "upstream": "yes", + "dependsOn": [ + "DGR-036", + "DGR-037", + "DGR-038" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md`, and evidence READMEs for dependencies (DGR-036, DGR-037, DGR-038) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Establish the first real, independently executing distributed GGUF route on one host.", + "acceptanceCriteria": [ + "Two worker processes open disjoint dense ranges and both execute real prefill/decode work.", + "Whole-model parity, 32-token greedy decode, four-session isolation, cancellation, and cleanup pass.", + "Record TTFT, prefill/decode rates, seam bytes/latency, RSS/VRAM, KV, queue, and failure metrics.", + "Killing one worker returns a bounded structured failure rather than hanging.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/039-pass-local-two-process-dense-acceptance.md; prd.json is authoritative.", + "blocks": [ + "DGR-054" + ] + }, + { + "id": "DGR-040", + "title": "Add node-side native worker supervision", + "priority": 24, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:node", + "area:supervision", + "type:integration", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-040/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-033", + "DGR-037" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md`, and evidence READMEs for dependencies (DGR-033, DGR-037) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Let the existing node service safely start, probe, stop, and restart the standalone worker.", + "acceptanceCriteria": [ + "Supervision owns process startup, readiness, log capture, graceful shutdown, and bounded forced termination.", + "Startup verifies worker binary, artifact identity, recipe, and range before registration.", + "Crashes or health loss make the capability unavailable without corrupting the Transformers backend.", + "Tests use the fake worker and deterministic crash injection.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/040-add-node-side-native-worker-supervision.md; prd.json is authoritative.", + "blocks": [ + "DGR-041", + "DGR-042", + "DGR-055", + "DGR-058" + ] + }, + { + "id": "DGR-041", + "title": "Register native Shard capabilities without redesigning Meshnet", + "priority": 25, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:meshnet", + "area:admission", + "type:integration", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-041/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-025", + "DGR-040" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md`, and evidence READMEs for dependencies (DGR-025, DGR-040) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Adapt native worker identity and capacity to existing backend-agnostic capability admission.", + "acceptanceCriteria": [ + "Registration carries exact recipe fingerprint, authoritative range, backend, memory/KV capacity, concurrency, and certification status.", + "Existing tracker, billing, routing, telemetry, and provider semantics remain backend-agnostic.", + "Uncertified backend/model/recipe combinations are visible but unroutable.", + "Existing Transformers registration and route tests remain unchanged in behavior.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/041-register-native-shard-capabilities-without-redesigning-meshnet.md; prd.json is authoritative.", + "blocks": [ + "DGR-043" + ] + }, + { + "id": "DGR-042", + "title": "Carry native frames through direct and existing relay seams", + "priority": 26, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:meshnet", + "area:relay", + "area:grpc", + "type:integration", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-042/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-024", + "DGR-040" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md`, and evidence READMEs for dependencies (DGR-024, DGR-040) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Use direct gRPC where available and the existing relay as opaque protobuf transport without redesigning relay behavior.", + "acceptanceCriteria": [ + "Direct paths use the long-lived gRPC activation stream.", + "Relayed paths carry byte-identical versioned protobuf frames through the existing relay contract.", + "Request/work identity, cancellation, deadlines, telemetry, billing correlation, and per-node attribution survive both paths.", + "Fake-worker tests cover direct, relay, disconnect, cancellation, and bounded buffering.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/042-carry-native-frames-through-direct-and-existing-relay-seams.md; prd.json is authoritative.", + "blocks": [ + "DGR-054", + "DGR-058" + ] + }, + { + "id": "DGR-043", + "title": "Expose GGUF compatibility and measured cost inputs to existing routing", + "priority": 27, + "milestone": "M2", + "executionMode": "AFK", + "labels": [ + "area:tracker", + "area:routing", + "type:integration", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-043/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-041" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md`, and evidence READMEs for dependencies (DGR-041) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Adapt exact GGUF recipe/range compatibility and measured capacity, queue, seam-cost, health, and reliability data into the existing backend-agnostic routing inputs without redesigning routing or load balancing.", + "acceptanceCriteria": [ + "Expose exact recipe, range coverage, capacity, queue/load, seam-cost, health, reliability, backend, and certification measurements through existing tracker input contracts.", + "Prove existing routing forms complete compatible coverage and excludes dark or mismatched candidates using its current backend-agnostic mechanisms.", + "Regression-test unchanged Transformers behavior and unchanged tracker routing, load-balancing, billing, relay, and provider semantics.", + "Regression-test that no quant, stage count, fixed split, architecture, backend sequence, or DeepSeek-specific policy is hardcoded.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/043-expose-gguf-compatibility-and-measured-cost-inputs-to-existing-routing.md; prd.json is authoritative.", + "blocks": [ + "DGR-053", + "DGR-054", + "DGR-059", + "DGR-061" + ] + }, + { + "id": "DGR-044", + "title": "Pin the DeepSeek V4 Flash target contract", + "priority": 28, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:provenance", + "type:contract", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-044/README.md", + "hardware": "none", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-019", + "DGR-025", + "DGR-026", + "DGR-027" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md`, and evidence READMEs for dependencies (DGR-019, DGR-025, DGR-026, DGR-027) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Fix the alpha/beta model identity and upstream maturity assumptions before implementing the adapter.", + "acceptanceCriteria": [ + "Pin Hugging Face target SHA `60d8d70770c6776ff598c94bb586a859a38244f1`.", + "Record the selected GGUF provenance and exact source binding without assuming one quant.", + "Record that llama.cpp support began with PR 24162/commit `8c146a8` and remains maturing; the build manifest still pins one exact validated upstream commit.", + "Record 43 main layers plus MTP and all known architecture-specific state obligations.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/044-pin-the-deepseek-v4-flash-target-contract.md; prd.json is authoritative.", + "blocks": [ + "DGR-045" + ] + }, + { + "id": "DGR-045", + "title": "Inventory V4 GGUF tensors and layer ownership", + "priority": 29, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:weights", + "type:research-spike", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-045/README.md", + "hardware": "none", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-026", + "DGR-044" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md`, and evidence READMEs for dependencies (DGR-026, DGR-044) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Derive exact head, repeated-layer, tail, expert, auxiliary-state, and MTP ownership from the pinned artifact.", + "acceptanceCriteria": [ + "Produce a machine-readable inventory tied to the target SHA and exact GGUF artifact.", + "Classify embeddings, 43 main layers, final head, MTP, mHC, attention variants, indexer/compressor, and MoE tensors.", + "Validate half-open range ownership and fail on unclassified required tensors.", + "Inventory tooling works across quants without embedding quant names in architecture logic.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/045-inventory-v4-gguf-tensors-and-layer-ownership.md; prd.json is authoritative.", + "blocks": [ + "DGR-046", + "DGR-047", + "DGR-050" + ] + }, + { + "id": "DGR-046", + "title": "Define the V4 typed architecture boundary schema", + "priority": 30, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:boundary", + "type:protocol", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-046/README.md", + "hardware": "none", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-021", + "DGR-045" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md`, and evidence READMEs for dependencies (DGR-021, DGR-045) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Define the exact cross-stage V4 architecture boundary while keeping per-layer attention and auxiliary caches shard-local.", + "acceptanceCriteria": [ + "Define a versioned named bundle for the mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations.", + "Explicitly exclude per-layer CSA, HCA, SWA, indexer, compressor, KV, and MTP caches/state from the WAN boundary; those remain local to the owning shard and session/epoch.", + "Reserve typed MTP boundary fields but mark MTP execution unsupported and unroutable for alpha.", + "Fingerprint independently of quant/topology and fail closed on missing, incompatible, incorrectly shaped, or stale boundary/cache expectations.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/046-define-the-v4-typed-architecture-boundary-schema.md; prd.json is authoritative.", + "blocks": [ + "DGR-047", + "DGR-048", + "DGR-049" + ] + }, + { + "id": "DGR-047", + "title": "Adapt the upstream V4 mHC boundary for ranged ownership", + "priority": 31, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:mhc", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-047/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-045", + "DGR-046" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md`, and evidence READMEs for dependencies (DGR-045, DGR-046) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Add range-boundary adapters around upstream llama.cpp V4 mHC execution without reimplementing the V4 graph or kernels.", + "acceptanceCriteria": [ + "Represent and validate the upstream V4 4×4096 mHC boundary without flattening semantic axes.", + "Add only head/intermediate/tail range ownership and boundary conversion hooks around the pinned upstream llama.cpp graph.", + "Compare deterministic fixture vectors and single-process ranged outputs with upstream whole-model execution.", + "Document that llama.cpp owns V4 mHC graph/kernels and that quantized storage does not alter the logical boundary schema.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/047-adapt-the-upstream-v4-mhc-boundary-for-ranged-ownership.md; prd.json is authoritative.", + "blocks": [ + "DGR-048", + "DGR-049", + "DGR-050", + "DGR-051" + ] + }, + { + "id": "DGR-048", + "title": "Carry token-ID sideband through the first three hash-routed layers", + "priority": 32, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:routing-state", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-048/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-046", + "DGR-047" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md`, and evidence READMEs for dependencies (DGR-046, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure any stage containing the first three hash-routed layers receives exact token identities.", + "acceptanceCriteria": [ + "Head emits token IDs in the typed sideband for boundaries that require them.", + "Intermediate stages preserve or consume the sideband according to their layer range.", + "Routes splitting before, within, and after the first three layers pass deterministic partition tests.", + "Missing or mismatched token IDs fail closed rather than silently selecting incorrect experts.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/048-carry-token-id-sideband-through-the-first-three-hash-routed-layers.md; prd.json is authoritative.", + "blocks": [ + "DGR-051" + ] + }, + { + "id": "DGR-049", + "title": "Keep V4 attention and auxiliary state shard-local", + "priority": 33, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:attention", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-049/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-046", + "DGR-047" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md`, and evidence READMEs for dependencies (DGR-046, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Bind V4 CSA/HCA/SWA/indexer/compressor and KV state to the owning shard session/epoch rather than transferring caches over the WAN.", + "acceptanceCriteria": [ + "Map CSA, HCA, SWA, indexer, compressor, and KV ownership from the pinned upstream graph to each owned layer range.", + "Key all such caches/state by route session and epoch with bounded lifecycle, memory accounting, stale-epoch rejection, release, and eviction.", + "Prove cross-stage messages carry only the typed architecture boundary and cache expectations, never per-layer cache contents.", + "On route change or state loss, use explicit cache miss and re-prefill/restart; do not migrate V4 caches across workers.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/049-keep-v4-attention-and-auxiliary-state-shard-local.md; prd.json is authoritative.", + "blocks": [ + "DGR-051" + ] + }, + { + "id": "DGR-050", + "title": "Validate upstream V4 MoE and hash-routing execution under ranged ownership", + "priority": 34, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:moe", + "type:runtime", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-050/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-045", + "DGR-047" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md`, and evidence READMEs for dependencies (DGR-045, DGR-047) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Validate the pinned upstream llama.cpp V4 MoE and hash-routing graph when layers are range-owned, without reimplementing routing or expert kernels in Meshnet.", + "acceptanceCriteria": [ + "Load range-owned tensors for 256 routed plus one shared expert and validate upstream execution activates six routed experts where specified.", + "Validate the first three upstream hash-routed layers receive exact token-ID sideband across splits before, within, and after them.", + "Compare upstream whole-model and ranged router/expert outputs, shared-expert contribution, and malformed sideband failures.", + "Document that llama.cpp owns V4 MoE, attention, hash routing, graph, and kernels; no expert-parallel WAN collective or Meshnet reimplementation is introduced.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/050-validate-upstream-v4-moe-and-hash-routing-execution-under-ranged-ownership.md; prd.json is authoritative.", + "blocks": [ + "DGR-051" + ] + }, + { + "id": "DGR-051", + "title": "Assemble the DeepSeek V4 Flash `ShardEngine` adapter", + "priority": 35, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:engine", + "type:vertical-slice", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-051/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-034", + "DGR-037", + "DGR-047", + "DGR-048", + "DGR-049", + "DGR-050" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md`, and evidence READMEs for dependencies (DGR-034, DGR-037, DGR-047, DGR-048, DGR-049, DGR-050) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Assemble range ownership hooks and boundary/local-state adapters around upstream llama.cpp V4 execution behind the stable ShardEngine interface.", + "acceptanceCriteria": [ + "Support valid head, middle, tail, and whole-model ranges over all 43 main layers using the pinned upstream llama.cpp V4 graph/kernels.", + "Integrate exact range ownership, mHC boundary, positions, required token-ID sideband, and shard-local session/epoch state adapters.", + "Report MTP reserved-but-not-executable for alpha and reject unsupported split points or cache expectations.", + "Include exact adapter/schema/runtime/artifact identity and valid split restrictions without reimplementing V4 MoE, attention, mHC, or hash routing.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/051-assemble-the-deepseek-v4-flash-shardengine-adapter.md; prd.json is authoritative.", + "blocks": [ + "DGR-052", + "DGR-060", + "DGR-065", + "DGR-069" + ] + }, + { + "id": "DGR-052", + "title": "Pass local V4 whole-model versus ranged parity", + "priority": 36, + "milestone": "M3", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:parity", + "type:certification", + "priority:p0", + "ready-for-agent" + ], + "evidenceClass": "real-model", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-052/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-038", + "DGR-051" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md`, and evidence READMEs for dependencies (DGR-038, DGR-051) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove V4 prefill and decode correctness locally before network certification.", + "acceptanceCriteria": [ + "Compare pinned whole-model llama.cpp with at least two valid ranged partitions.", + "Prefill intermediates and locked-length greedy decode pass architecture-specific tolerances.", + "Test splits around the first three hash-routed layers and at least one CSA/HCA/SWA/indexer/compressor transition.", + "Four concurrent sessions pass KV/state isolation and cleanup.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/052-pass-local-v4-whole-model-versus-ranged-parity.md; prd.json is authoritative.", + "blocks": [ + "DGR-053", + "DGR-054" + ] + }, + { + "id": "DGR-053", + "title": "Certify a real 2–4-stage V4 route", + "priority": 37, + "milestone": "M3", + "executionMode": "HITL", + "labels": [ + "area:deepseek-v4", + "area:hardware", + "type:certification", + "priority:p0", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-053/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-030", + "DGR-043", + "DGR-052" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md`, and evidence READMEs for dependencies (DGR-030, DGR-043, DGR-052) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove real Tracker-selected V4 execution across physical machines before alpha.", + "acceptanceCriteria": [ + "Run one documented 2–4-stage certification scenario using exact compatible artifacts/recipes; the count and chosen quant are evidence inputs, not product constants.", + "Actual CPU/GPU work executes on every stage; fake workers do not satisfy acceptance.", + "Record parity, TTFT, prefill/decode speed, seam cost, memory, cache/state isolation, cancellation, and cleanup.", + "Tracker selection remains dynamic and rejects an injected incompatible backend/recipe.", + "Only the exact real-hardware lanes exercised become eligible for advertisement.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/053-certify-a-real-2-4-stage-v4-route.md; prd.json is authoritative.", + "blocks": [ + "DGR-054", + "DGR-062", + "DGR-067" + ] + }, + { + "id": "DGR-054", + "title": "Enforce the DeepSeek V4 Flash alpha gate", + "priority": 38, + "milestone": "M3", + "executionMode": "HITL", + "labels": [ + "area:deepseek-v4", + "area:release", + "milestone:alpha", + "priority:p0", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "release", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-054/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-017", + "DGR-019", + "DGR-020", + "DGR-039", + "DGR-042", + "DGR-043", + "DGR-052", + "DGR-053" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md`, and evidence READMEs for dependencies (DGR-017, DGR-019, DGR-020, DGR-039, DGR-042, DGR-043, DGR-052, DGR-053) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Release alpha only when V4 is correct and clears the pre-locked useful-speed requirement.", + "acceptanceCriteria": [ + "Evaluate correctness, model fit, TTFT, prefill/decode speed, seam cost, memory, and failure rate against DGF-004 without changing thresholds.", + "Separate quant/model-fit gains from runtime/transport gains.", + "Decision is `alpha`, `optimize measured bottleneck`, or `stop`; failures cannot be relabeled as certification.", + "Alpha documentation states MTP is reserved but not implemented and identifies every dark hardware lane.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/054-enforce-the-deepseek-v4-flash-alpha-gate.md; prd.json is authoritative.", + "blocks": [ + "DGR-055", + "DGR-065", + "DGR-066", + "DGR-068" + ] + }, + { + "id": "DGR-055", + "title": "Implement compatible continuous decode batching", + "priority": 39, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:scheduler", + "area:batching", + "type:performance", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-055/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-038", + "DGR-040", + "DGR-054" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md`, and evidence READMEs for dependencies (DGR-038, DGR-040, DGR-054) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Batch compatible active sessions without violating route, recipe, position, or architecture state.", + "acceptanceCriteria": [ + "Batch only sessions with compatible engine, recipe, phase, boundary schema, and state layout.", + "Preserve per-session positions, sequence IDs, outputs, cancellation, and accounting.", + "Decode receives bounded service while prefill remains schedulable.", + "Deterministic 1/2/4/8 concurrency tests show no cross-session corruption.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/055-implement-compatible-continuous-decode-batching.md; prd.json is authoritative.", + "blocks": [ + "DGR-056", + "DGR-057" + ] + }, + { + "id": "DGR-056", + "title": "Enforce bounded admission, queues, and backpressure", + "priority": 40, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:scheduler", + "area:backpressure", + "type:performance", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-056/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-055" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md`, and evidence READMEs for dependencies (DGR-055) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prevent weight, KV, state, scratch, and stream buffers from exceeding admitted capacity.", + "acceptanceCriteria": [ + "Admission accounts separately for weights, KV/state, scratch, in-flight fragments, and queue slots.", + "Queue and stream limits produce structured retryable or terminal responses.", + "Prefill cannot starve decode and one route session cannot monopolize queue capacity.", + "Telemetry reports active sessions, queue depth/time, batch occupancy, memory pressure, and rejects.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/056-enforce-bounded-admission-queues-and-backpressure.md; prd.json is authoritative.", + "blocks": [ + "DGR-057", + "DGR-058", + "DGR-060" + ] + }, + { + "id": "DGR-057", + "title": "Benchmark batching saturation and fairness", + "priority": 41, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:performance", + "area:batching", + "type:benchmark", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-057/README.md", + "hardware": "required", + "model": "dense-llama+deepseek-v4", + "upstream": "no", + "dependsOn": [ + "DGR-055", + "DGR-056" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md`, and evidence READMEs for dependencies (DGR-055, DGR-056) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Identify useful concurrency and saturation points without assuming one global batch size.", + "acceptanceCriteria": [ + "Run controlled concurrency 1/2/4/8 and record aggregate throughput, per-request p50/p95, TTFT, queue time, occupancy, and memory.", + "Measure dense and V4 lanes separately where hardware permits.", + "Verify bounded fairness and no prefill/decode starvation.", + "Produce machine-readable saturation recommendations consumed by capability reporting, not hardcoded constants.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/057-benchmark-batching-saturation-and-fairness.md; prd.json is authoritative.", + "blocks": [ + "DGR-061", + "DGR-063", + "DGR-070" + ] + }, + { + "id": "DGR-058", + "title": "Harden worker and stream failure semantics", + "priority": 42, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:reliability", + "area:failure", + "type:hardening", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-058/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-040", + "DGR-042", + "DGR-056" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md`, and evidence READMEs for dependencies (DGR-040, DGR-042, DGR-056) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Ensure malformed input, process death, transport reset, and cancellation terminate cleanly.", + "acceptanceCriteria": [ + "Test worker death, stream reset, malformed bundle, checksum failure, deadline, cache miss, stale epoch, and partial fragment loss.", + "Cancellation propagates across all stages and releases KV/state, queue entries, and fragment buffers.", + "Duplicate steps are idempotent; uncertain mutations are never silently replayed.", + "Billing/work outcomes distinguish completed, cancelled, failed, and unverified work.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/058-harden-worker-and-stream-failure-semantics.md; prd.json is authoritative.", + "blocks": [ + "DGR-059", + "DGR-060", + "DGR-061" + ] + }, + { + "id": "DGR-059", + "title": "Implement alpha-safe route recovery", + "priority": 43, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:recovery", + "area:routing", + "type:hardening", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-059/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-043", + "DGR-058" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md`, and evidence READMEs for dependencies (DGR-043, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Recover from route loss without importing or continuing unverified state.", + "acceptanceCriteria": [ + "On route loss or local-state cache miss, restart from token zero or re-prefill on a newly compatible route according to the locked alpha contract.", + "Never import KV, CSA, HCA, SWA, indexer, compressor, recurrent, or MTP cache/state across workers.", + "Keep exact same-route duplicate delivery idempotent while cross-route continuation fails closed.", + "Test direct/relay loss, cache miss, replacement selection, client cancellation, and billing reconciliation.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/059-implement-alpha-safe-route-recovery.md; prd.json is authoritative.", + "blocks": [ + "DGR-070" + ] + }, + { + "id": "DGR-060", + "title": "Certify V4 long-context state correctness", + "priority": 44, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:long-context", + "type:certification", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-060/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-051", + "DGR-056", + "DGR-058" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md`, and evidence READMEs for dependencies (DGR-051, DGR-056, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Prove V4’s KV and auxiliary state remain correct and bounded at long contexts.", + "acceptanceCriteria": [ + "Exercise pre-locked context lengths covering multiple prefill chunks and sustained decode.", + "Validate KV plus CSA/HCA/SWA/indexer/compressor state positions across every stage.", + "Measure memory growth, fragment sizes, TTFT, decode degradation, cancellation latency, and cleanup.", + "Cache mismatch, truncation, and context-limit behavior fail deterministically.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/060-certify-v4-long-context-state-correctness.md; prd.json is authoritative.", + "blocks": [ + "DGR-062", + "DGR-064", + "DGR-065", + "DGR-066", + "DGR-070" + ] + }, + { + "id": "DGR-061", + "title": "Certify existing routing with 10+ GGUF stage candidates", + "priority": 45, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:tracker", + "area:scale", + "type:simulation", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-061/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [ + "DGR-043", + "DGR-057", + "DGR-058" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md`, and evidence READMEs for dependencies (DGR-043, DGR-057, DGR-058) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Certify that existing backend-agnostic routing consumes GGUF compatibility and measured-cost inputs for a 10+ candidate scenario without algorithm or policy redesign.", + "acceptanceCriteria": [ + "Generate deterministic pools with 10+ compatible GGUF stage candidates plus slower, overloaded, unhealthy, and incompatible alternatives.", + "Run the existing routing implementation unchanged and verify complete coverage, compatibility, acyclicity, and use of measured inputs.", + "Regression-test that no stage count, quant, fixed split, architecture, or backend sequence is encoded in production routing policy.", + "Verify existing route recomputation reacts to measured load/failure while tracker, load-balancing, billing, relay, and provider semantics remain unchanged.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/061-certify-existing-routing-with-10-gguf-stage-candidates.md; prd.json is authoritative.", + "blocks": [ + "DGR-062", + "DGR-063" + ] + }, + { + "id": "DGR-062", + "title": "Certify a dynamic real 10+ stage V4 scenario", + "priority": 46, + "milestone": "M4", + "executionMode": "HITL", + "labels": [ + "area:deepseek-v4", + "area:scale", + "area:hardware", + "type:certification", + "priority:p1", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-062/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-053", + "DGR-060", + "DGR-061" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md`, and evidence READMEs for dependencies (DGR-053, DGR-060, DGR-061) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Demonstrate that a large real route can be dynamically selected and usefully execute V4.", + "acceptanceCriteria": [ + "Run one real 10+-stage V4 certification scenario; stage count, quant, and split remain scenario metadata only.", + "Every selected stage proves real work, exact compatible identity, local state ownership, and bounded resources.", + "Record end-to-end and per-stage TTFT, prefill/decode, seam cost, queueing, memory, failures, and cleanup.", + "Inject at least one incompatible or degraded candidate and verify dynamic exclusion/reselection.", + "The scenario must meet its pre-locked usefulness threshold or remain failed evidence.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/062-certify-a-dynamic-real-10-stage-v4-scenario.md; prd.json is authoritative.", + "blocks": [ + "DGR-067", + "DGR-070" + ] + }, + { + "id": "DGR-063", + "title": "Profile and optimize the critical seam", + "priority": 47, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:performance", + "type:optimization", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-063/README.md", + "hardware": "required", + "model": "generic", + "upstream": "conditional", + "dependsOn": [ + "DGR-057", + "DGR-061" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md`, and evidence READMEs for dependencies (DGR-057, DGR-061) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Optimize only the measured dominant compute, serialization, copy, or transport bottleneck.", + "acceptanceCriteria": [ + "Profile worker compute, tensor conversion, copies, protobuf framing, compression, relay/direct transport, queueing, and synchronization.", + "Select one dominant bottleneck from evidence and lock its before/after benchmark.", + "Implement one bounded optimization without weakening correctness or protocol identity.", + "Report whether the optimization changes TTFT, prefill, decode, throughput, or memory and preserve raw data.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/063-profile-and-optimize-the-critical-seam.md; prd.json is authoritative.", + "blocks": [ + "DGR-064" + ] + }, + { + "id": "DGR-064", + "title": "Add negotiated activation compression", + "priority": 48, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:protocol", + "area:compression", + "type:optimization", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-064/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "no", + "dependsOn": [ + "DGR-060", + "DGR-063" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md`, and evidence READMEs for dependencies (DGR-060, DGR-063) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Reduce V4 seam cost only when a certified compression recipe improves end-to-end performance acceptably.", + "acceptanceCriteria": [ + "Negotiate compression through recipe identity; unnegotiated or mismatched compression fails closed.", + "Preserve names, shapes, semantic axes, token-ID sideband, and checksums.", + "Measure quality drift, encode/decode cost, bytes, TTFT, and throughput against uncompressed boundaries.", + "Compression remains dark unless it meets pre-locked quality and speed thresholds.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/064-add-negotiated-activation-compression.md; prd.json is authoritative.", + "blocks": [ + "DGR-066", + "DGR-070" + ] + }, + { + "id": "DGR-065", + "title": "Define V4 MTP ownership and distributed state contract", + "priority": 49, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:mtp", + "type:contract", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "fixture", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-065/README.md", + "hardware": "none", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-051", + "DGR-054", + "DGR-060" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md`, and evidence READMEs for dependencies (DGR-051, DGR-054, DGR-060) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Lock MTP tensor, execution, boundary, acceptance/rejection, and session-state ownership before implementation while keeping MTP disabled for alpha.", + "acceptanceCriteria": [ + "Derive exact MTP tensor and execution ownership from the pinned V4 target and upstream llama.cpp graph, distinguishing main-layer, tail, and MTP responsibilities.", + "Define typed MTP inputs/outputs plus accepted-token, rejection, rollback/truncation, position, cache, and session/epoch lifecycle semantics.", + "Specify which MTP state remains shard-local and which typed boundary values cross a seam; prohibit WAN migration of KV or V4 auxiliary caches.", + "Keep every MTP recipe disabled and unroutable for alpha, with fail-closed capability negotiation and deterministic contract fixtures.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/065-define-v4-mtp-ownership-and-distributed-state-contract.md; prd.json is authoritative.", + "blocks": [ + "DGR-066" + ] + }, + { + "id": "DGR-066", + "title": "Implement and benchmark V4 MTP", + "priority": 50, + "milestone": "M4", + "executionMode": "AFK", + "labels": [ + "area:deepseek-v4", + "area:mtp", + "type:runtime", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-066/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-054", + "DGR-060", + "DGR-064", + "DGR-065" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md`, and evidence READMEs for dependencies (DGR-054, DGR-060, DGR-064, DGR-065) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Implement the locked MTP contract around upstream llama.cpp before beta, without making MTP an alpha prerequisite.", + "acceptanceCriteria": [ + "Implement the exact DGR-065 MTP ownership and typed state contract against the pinned upstream V4 graph while preserving shard-local caches.", + "Pass whole-model versus ranged MTP correctness, accepted-token, rejection, rollback/truncation, and session-isolation tests.", + "Benchmark MTP-off versus MTP-on quality, accepted-token rate, TTFT, decode speed, seam bytes, and memory on the locked lane.", + "Distinguish implemented, certified, disabled, and unsupported MTP recipes in fail-closed capability advertisement.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/066-implement-and-benchmark-v4-mtp.md; prd.json is authoritative.", + "blocks": [ + "DGR-067", + "DGR-070" + ] + }, + { + "id": "DGR-067", + "title": "Certify the backend capability matrix", + "priority": 51, + "milestone": "M4", + "executionMode": "HITL", + "labels": [ + "area:hardware", + "area:admission", + "type:certification", + "priority:p1", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "real-hardware", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-067/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-030", + "DGR-053", + "DGR-062", + "DGR-066" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md`, and evidence READMEs for dependencies (DGR-030, DGR-053, DGR-062, DGR-066) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Publish only backend/model/recipe lanes proven on actual CPU, CUDA, ROCm, Vulkan, or Metal hardware.", + "acceptanceCriteria": [ + "Maintain independent build and certification status for CPU, CUDA, ROCm, Vulkan, and Metal.", + "Each advertised lane includes real hardware/driver/runtime/model evidence for parity, concurrency, cancellation, cleanup, and useful speed.", + "Build success alone never makes a lane routable; unavailable hardware remains explicitly dark.", + "Tracker admission consumes signed/versioned certification records rather than backend-name allowlists.", + "Mixed-backend routes require exact compatibility evidence and fail closed otherwise.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/067-certify-the-backend-capability-matrix.md; prd.json is authoritative.", + "blocks": [ + "DGR-068", + "DGR-070" + ] + }, + { + "id": "DGR-068", + "title": "Package reproducible native worker releases", + "priority": 52, + "milestone": "M5", + "executionMode": "AFK", + "labels": [ + "area:packaging", + "area:release", + "type:delivery", + "priority:p1", + "ready-for-agent" + ], + "evidenceClass": "release", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-068/README.md", + "hardware": "optional", + "model": "generic", + "upstream": "yes", + "dependsOn": [ + "DGR-030", + "DGR-054", + "DGR-067" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md`, and evidence READMEs for dependencies (DGR-030, DGR-054, DGR-067) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Produce installable worker packages that preserve exact source, patch, protocol, and certification identity.", + "acceptanceCriteria": [ + "Package worker binaries/configuration for supported host/backend combinations without bundling an unverified llama.cpp tree.", + "Embed protocol, engine, upstream pin, patch-stack, build-toolchain, and certification fingerprints.", + "Installation/startup refuses mismatched artifacts, runtime recipes, or unsupported hardware lanes.", + "Produce checksums, SBOM/license attribution, reproducible build instructions, and smoke tests.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/068-package-reproducible-native-worker-releases.md; prd.json is authoritative.", + "blocks": [ + "DGR-070" + ] + }, + { + "id": "DGR-069", + "title": "Prepare narrow upstream llama.cpp collaboration patches", + "priority": 53, + "milestone": "M5", + "executionMode": "HITL", + "labels": [ + "area:upstream", + "area:patches", + "type:collaboration", + "priority:p2", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "release", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-069/README.md", + "hardware": "none", + "model": "generic+deepseek-v4", + "upstream": "yes", + "dependsOn": [ + "DGR-028", + "DGR-035", + "DGR-038", + "DGR-051" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md`, and evidence READMEs for dependencies (DGR-028, DGR-035, DGR-038, DGR-051) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Reduce long-term patch burden through generic upstreamable hooks without asking llama.cpp to own Meshnet.", + "acceptanceCriteria": [ + "Separate generic range loading, boundary I/O, and filtered state hooks from project worker/protocol code.", + "Provide one scoped patch, focused test, and minimal reproducer per concern against the exact upstream pin.", + "Compare the proposal with prior-art evidence and current upstream V4 support/maturity.", + "Prepare human-ready design/outreach text; actual issue/PR submission remains a human action.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/069-prepare-narrow-upstream-llama-cpp-collaboration-patches.md; prd.json is authoritative.", + "blocks": [ + "DGR-071" + ] + }, + { + "id": "DGR-070", + "title": "Enforce the DeepSeek V4 Flash beta gate", + "priority": 54, + "milestone": "M5", + "executionMode": "HITL", + "labels": [ + "area:deepseek-v4", + "area:release", + "milestone:beta", + "priority:p0", + "gate:hitl", + "ready-for-human" + ], + "evidenceClass": "release", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-070/README.md", + "hardware": "required", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-057", + "DGR-059", + "DGR-060", + "DGR-062", + "DGR-064", + "DGR-066", + "DGR-067", + "DGR-068" + ], + "triage": "ready-for-human", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md`, and evidence READMEs for dependencies (DGR-057, DGR-059, DGR-060, DGR-062, DGR-064, DGR-066, DGR-067, DGR-068) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Promote beta only after concurrency, resilience, long-context, MTP, scale, packaging, and speed gates pass.", + "acceptanceCriteria": [ + "Evaluate all beta metrics against DGF-004 without changing thresholds after observing results.", + "Require V4 parity, batching/fairness, bounded backpressure, failure recovery, long-context correctness, MTP evidence, and dynamic 10+ stage evidence.", + "Publish the exact certified hardware/backend/recipe matrix; all other lanes remain dark.", + "Decision is `beta`, `targeted optimization`, or `stop/rollback`, with unresolved failures listed explicitly.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/070-enforce-the-deepseek-v4-flash-beta-gate.md; prd.json is authoritative.", + "blocks": [ + "DGR-071" + ] + }, + { + "id": "DGR-071", + "title": "Establish upstream pin, patch, and certification maintenance", + "priority": 55, + "milestone": "M5", + "executionMode": "AFK", + "labels": [ + "area:maintenance", + "area:upstream", + "area:certification", + "type:operations", + "priority:p2", + "ready-for-agent" + ], + "evidenceClass": "release", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-071/README.md", + "hardware": "optional", + "model": "deepseek-v4-flash", + "upstream": "yes", + "dependsOn": [ + "DGR-069", + "DGR-070" + ], + "triage": "ready-for-agent", + "description": "Fresh Ralph session: read `.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md`, source issue `.scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md`, and evidence READMEs for dependencies (DGR-069, DGR-070) before changing code. Inspect live source/tests rather than trusting legacy pass states. Objective: Keep beta reproducible and detect upstream/model/hardware drift before it reaches routable nodes.", + "acceptanceCriteria": [ + "Add a documented update procedure for llama.cpp pin, numbered patches, protocol/engine versions, V4 target revision, and certification records.", + "A candidate pin update must apply/build/test in isolation and cannot replace the production pin automatically.", + "Any changed graph, tensor inventory, boundary, cache/state layout, backend behavior, or performance invalidates affected certification.", + "Add scheduled model-free checks plus opt-in real-hardware recertification instructions and rollback procedure.", + "Applicable shared quality gates in `prd.json` pass, and the evidence handoff records exact commands/results, changed files, limitations, and dependency handoff." + ], + "passes": false, + "notes": "Generated source issue: .scratch/distributed-gguf-runtime/issues/071-establish-upstream-pin-patch-and-certification-maintenance.md; prd.json is authoritative.", + "blocks": [] } ] } diff --git a/.scratch/distributed-gguf-runtime/technical-challenges.md b/.scratch/distributed-gguf-runtime/technical-challenges.md index e21d6ed..9aad149 100644 --- a/.scratch/distributed-gguf-runtime/technical-challenges.md +++ b/.scratch/distributed-gguf-runtime/technical-challenges.md @@ -1,414 +1,37 @@ -# Distributed GGUF Technical Challenge Register - -> **Historical challenge register.** Route Session, binary activation, local Hot KV State, and transport performance work have advanced since this file was written. Current implementation gates live in [PRD.md](PRD.md), [implementation-strategy.md](implementation-strategy.md), [architecture.md](architecture.md), and [prd.json](prd.json). Preserve this file for detailed risk context; do not treat its “current constraint” section as live system state. - -This document focuses on the engineering problems that decide whether the distributed GGUF path is viable. The important distinction is: - -- **Model artifacts move like torrents.** -- **Inference state moves like a pipeline.** -- **Hot KV state does not move unless we are explicitly checkpointing or repairing a route.** - -## Current Constraint - -The existing full local PyTorch path lets Transformers own generation and local KV cache. - -The existing distributed PyTorch path does not. It manually calls shard layers with cache disabled and recomputes the whole growing prompt for every generated token. It passes hidden activations across shard boundaries, not KV cache, but those activations currently include the full sequence on every decode step. - -For a 128K context and `hidden_size=6144`, one bfloat16 activation crossing one shard boundary is roughly: - -```text -131072 tokens * 6144 hidden * 2 bytes = 1.5 GiB -``` - -That is acceptable once during chunked prefill only if chunked and streamed. It is not acceptable once per generated token. - -## Challenge 1: Decode Must Be O(1) Per Token Across Each Seam - -Problem: - -During decode, sending `[batch, sequence, hidden]` over the network scales with context length. At 128K, the network dominates everything. - -Solution: - -Split execution into explicit **prefill** and **decode-step** phases. - -- Prefill accepts prompt chunks and builds local cache on every shard. -- Decode-step accepts exactly one new token or one-step activation. -- Every shard reads its own hot KV state, appends one position, and forwards a one-step activation. - -Acceptance test: - -- A two-node route prefills a 4K prompt once. -- The next 100 generated tokens do not resend a 4K activation. -- Decode seam payload is proportional to `hidden_size`, not `context_length * hidden_size`. - -## Challenge 2: Stable Route Session State - -Problem: - -KV cache only works if every hop agrees that multiple calls belong to the same route session. A fresh request id per hop or per token destroys cache locality. - -Solution: - -Introduce a route-session lifecycle. - -```text -create route session - -> tracker pins inference route - -> head node assigns session_id and route_id - -> every hop allocates local cache for its layer range - -> prefill chunks append cache - -> decode steps append cache - -> close session releases cache -``` - -Minimum state key: - -```text -session_id -route_id -model_preset -model_revision -backend_id -cache_abi -layer_start -layer_end -position -``` - -Acceptance test: - -- A node can report active sessions and cache bytes by session. -- Closing a session frees the per-shard cache. -- Replaying a decode-step with the wrong route/session fails before model execution. - -## Challenge 3: KV Cache Ownership - -Problem: - -A centralized KV cache sounds attractive for failover, but it puts remote storage in the tightest loop of generation. It also creates privacy and consistency problems. - -Solution: - -Hot KV state is owned by the node that owns the shard for that route session. - -```text -Node A: layers 0..15 hot KV for session S -Node B: layers 16..31 hot KV for session S -Node C: layers 32..77 hot KV for session S -``` - -The tracker may know where active KV lives, but it does not serve it during decode. - -Cache servers may store: - -- prefix snapshots -- failover checkpoints -- audit samples -- cold reusable context blocks - -They must not be required for every generated token. - -Acceptance test: - -- Killing a cache server does not affect an active decode route. -- Killing a route node fails the route in alpha unless a compatible snapshot exists. - -## Challenge 4: Prefill Is Still Large - -Problem: - -Even with correct decode, prefill can move a lot of data. A 128K prompt cannot be sent as one activation blob through many shard boundaries. - -Solution: - -Use the existing binary activation direction from ADR-0008: - -- bfloat16 activation body -- shape/dtype/session metadata in headers -- zstd level 1 optional compression -- chunked prefill -- backpressure between hops - -For large contexts, prefill should stream in chunks such as 128, 256, or 512 tokens. The right chunk size is a benchmark output, not a constant baked into the domain model. - -Acceptance test: - -- Peak per-hop prefill memory is bounded by chunk size. -- A slow downstream node applies backpressure instead of letting upstream buffer the whole prompt. - -## Challenge 5: GGUF Artifact Splits Are Not Execution Shards - -Problem: - -GGUF split files can divide model data by size or tensor count. That is useful for storage and transfer, but it is not the same as a network Shard. A Shard in this project is a contiguous layer range with reward, route, and cache meaning. - -Solution: - -Define an artifact manifest that maps storage chunks to semantic model parts. - -Required concepts: - -```text -artifact_id -model_preset -upstream_repo -upstream_revision -license -runtime_backend -quantization -context_cap -file_hashes -piece_hashes -tensor_to_layer_map -layer_to_artifact_map -tokenizer_artifacts -cache_descriptor -``` - -The Shard Swarm seeds artifacts. The Inference Route executes shards. - -Acceptance test: - -- Given `layers 16..31`, a node can compute the exact artifact pieces it must download and verify. -- Given a local artifact directory, the node can prove which layer ranges it can serve. - -## Challenge 6: llama.cpp Is Optimized For Whole-Graph Execution - -Problem: - -`llama-server` is excellent for local inference, but a distributed route needs lower-level capabilities: - -- load selected layers/tensors or mmap them without full materialization -- accept hidden states from a previous shard -- execute only a layer range -- emit hidden states at a boundary -- own KV/state for only that layer range -- report cache layout and memory requirements - -Those are not the normal public serving abstractions. - -Solution: - -Stage the llama.cpp path instead of jumping directly to internet-scale distributed GGUF. - -1. Use llama.cpp as a full local GGUF backend for immediate CPU performance. -2. Build a localhost layer-boundary prototype on a simple supported GGUF model. -3. Identify the minimal `libllama`/ggml hooks needed for layer-range execution. -4. Collaborate upstream on a stable extension rather than carrying a long-lived fork. - -Acceptance test: - -- Process A runs layers `0..k`, exports hidden states. -- Process B imports those hidden states, runs `k+1..n`, and produces logits close to full single-process execution. -- Both processes maintain only their own cache state. - -## Challenge 7: Model Architectures Are Not Uniform - -Problem: - -Dense Llama-style attention, MoE, MLA/DSA, and hybrid linear/full attention do not have the same cache shape, routing cost, or layer cost. - -GLM-5.2 uses compressed DSA/MLA-style state. Ornith uses a hybrid attention pattern. Parameter count alone is a poor routing metric. - -Solution: - -Keep model-specific cache internals inside the backend. The tracker should route based on backend-advertised capabilities and measured telemetry, not on hardcoded tensor formulas. - -Backend capability report: - -```text -model_arch -supported_runtime -supports_prefill -supports_decode_step -supports_layer_range -supports_partial_artifacts -cache_abi -cache_bytes_per_token_estimate -prefill_tokens_per_second -decode_tokens_per_second -active_memory_floor -``` - -Acceptance test: - -- The tracker can reject a route because one node lacks the required cache ABI. -- A model support audit can say "artifact available, local full inference works, distributed layer-boundary unsupported" without ambiguity. - -## Challenge 8: Tensor Parallelism Is Not The Same Product - -Problem: - -Projects like Distributed Llama and prima.cpp lean toward local-cluster tensor/ring parallelism. That can work on a trusted LAN, but it usually requires tight synchronization every layer. On a public internet volunteer route, that becomes fragile and hard to reward. - -Solution: - -For the public network, make a Shard a contiguous layer range. Tensor parallelism can exist inside one node, one trusted colocated pod, or one future "composite node", but not as the first public routing primitive. - -Acceptance test: - -- A public route can be represented as ordered layer coverage. -- Billing can attribute work to layer ranges. -- No cross-node all-reduce is required on every layer for v1. - -## Challenge 9: Heterogeneity And Stragglers - -Problem: - -A route is only as fast as its slowest hop during decode. A weak node holding a bottleneck shard can make a 1.6T model technically available but unusable. - -Solution: - -Route selection must use measured telemetry, not static declarations. - -Metrics: - -- prefill throughput -- decode throughput -- queue depth -- disk read rate -- memory pressure -- network latency to neighbors -- route failure rate -- cache warmth - -The tracker should prefer complete routes that avoid weak nodes, and the rebalancer should increase redundancy for bottleneck layer ranges. - -Acceptance test: - -- A slow node is removed from a candidate route even if it has the needed layer range. -- The coverage map can show "covered but under-provisioned" separately from "coverage gap". - -## Challenge 10: Reliability And Failover - -Problem: - -If hot KV is local, route repair is not free. A replacement node cannot continue decoding unless it has compatible cache state. - -Solution: - -Alpha behavior should be simple: - -- route failure during prefill: fail and retry from scratch -- route failure during decode: fail unless compatible snapshot exists -- tracker restart: active sessions may be lost -- node restart: local hot KV is lost -- client-visible telemetry reports the last known phase and failure reason - -Later behavior: - -- periodic prefix snapshots -- snapshot generation ids -- cache ABI compatibility checks -- route repair only when the replacement node has the same model revision, layer range, backend cache ABI, and snapshot position - -Acceptance test: - -- Failures produce explicit route-session errors. -- No node silently continues from missing or incompatible cache state. - -## Challenge 11: Privacy, Fraud, And Audit - -Problem: - -Hidden activations and KV state can leak information. Public volunteer inference is not private by default. Also, a node can return bad activations while still appearing available. - -Solution: - -Separate product modes: - -- public swarm: low privacy, broad access, audited -- private swarm: trusted nodes, stronger privacy expectation -- paid trusted route: selected nodes with stronger guarantees - -Use existing validation-event and slash-proof concepts for audit, but adapt them to distributed routes: - -- record model preset, route, node wallets, prompt metadata, output, and sampling seed -- sample full-route replays where feasible -- compare output/logits within model-specific tolerance - -Acceptance test: - -- A client can choose public or private route policy. -- A validation event contains enough information to reproduce route membership and observed output. - -## Challenge 12: Economics Must Not Reward The Wrong Bottleneck - -Problem: - -Layer count, parameter count, active MoE experts, cache memory, disk serving, and network transfer are different costs. A naive equal split across nodes will be wrong. - -Solution: - -Start with simple compute accounting: - -```text -node_reward_weight = - owned_layer_work - * prefill_tokens - + owned_layer_work - * decode_tokens -``` - -Then refine with: - -- measured throughput -- active MoE cost -- storage/seeding contribution -- cache memory reservation -- reliability - -Keep artifact seeding rewards separate from inference rewards until fraud and metering are clear. - -Acceptance test: - -- A node that only seeds artifacts is not paid as if it executed inference. -- A node that executes a heavier shard can earn more than a node executing a light shard. - -## Challenge 13: Long Requests Need Streaming Or Realtime Feedback - -Problem: - -Large distributed routes may spend meaningful time in artifact loading, prefill, queueing, or slow decode. The product can tolerate latency, but users should not wait blindly. - -Solution: - -Streaming token deltas is preferred when the backend and client transport support it. Generation Telemetry is required regardless of whether token deltas are streamed. - -Minimum telemetry: - -```text -session_id -route_id -model_preset -phase = queued | loading | prefill | decode | finalizing | failed -prefill_tokens_done -prefill_tokens_total -generated_tokens -rolling_tokens_per_second -average_tokens_per_second -active_route_nodes -failure_reason -``` - -The gateway may expose token deltas and telemetry through Server-Sent Events or WebSocket. Simple clients may use a polling endpoint for telemetry and receive the final answer only when complete. - -Acceptance test: - -- A client can show live progress before the first output token is available. -- During decode, the user sees streamed token deltas when supported. -- During decode, the user sees rolling tokens/sec even if output text is not streamed. -- A failed route returns a final error and the last known phase/reason. - -## Engineering Order - -1. Fix distributed PyTorch cache semantics first. This proves the route-session model without llama.cpp internals. -2. Add local full-model llama.cpp/GGUF serving for immediate CPU improvement. -3. Add Generation Telemetry for route sessions so long requests are observable. -4. Preserve binary HTTP activation transfer while splitting prefill/decode and measuring payload sizes. -5. Add artifact manifest and Shard Swarm metadata. -6. Prototype llama.cpp layer-boundary execution locally. -7. Network the GGUF route only after the cache/session protocol works. -8. Audit DeepSeek-V4-Flash as the first serious large-model target. -9. Audit GLM-5.2 and Ornith support after simpler GGUF models pass the route test. +# Distributed GGUF Runtime technical challenges + +> **Specification status:** planning artifacts only. No distributed GGUF runtime is implemented. DGR-017 cleanup is complete; no runtime implementation story has completion credit. `prd.json` is authoritative. + +## Key challenges and planned answers + +1. **Truth/provenance:** old passes are unreliable. DGR-017 audits before preserving or cleaning anything. +2. **Baseline comparability:** lock lanes first (DGR-019), execute whole-model-only baseline next (DGR-020), and separate quant fit from runtime effects. +3. **Exact artifacts:** complete split manifests/hashes, resumable mounted-drive provisioning, no `/home` models, dynamic quant/topology (DGR-025..026). +4. **Fast-moving upstream:** exact manifest plus numbered patches in an ignored workspace; no vendoring/submodule/permanent fork (DGR-027..030, DGR-071). +5. **Stable project boundary:** gRPC/Protobuf and `ShardEngine` prevent upstream ABI leakage (DGR-021..033). +6. **V4 correctness:** llama.cpp owns mHC/MoE/attention/hash graphs and kernels; Meshnet adds ranged ownership and adapters, then validates whole-model parity (DGR-044..052). +7. **Boundary versus caches:** only mHC residual/positions/needed token IDs/schema expectations cross; KV and CSA/HCA/SWA/indexer/compressor state stay local/session-keyed (DGR-046, DGR-049). +8. **Recovery:** stale/missing local state causes cache miss and re-prefill/restart, never WAN migration (DGR-058..060). +9. **Scale without policy churn:** feed measured GGUF inputs into and certify existing backend-agnostic routing at 10+ candidates (DGR-043, DGR-061..062). +10. **Performance/concurrency:** bounded admission, continuous batching, fairness, long-context profiling, then evidence-selected optimization/compression (DGR-055..064). +11. **MTP:** define ownership/state first, keep off for alpha, implement/benchmark before beta (DGR-065..066). +12. **Hardware truth:** compile all five backend families but advertise only exact real-hardware-certified lanes (DGR-030, DGR-067). + +## Locked scope + +- Existing Meshnet Tracker routing, load balancing, billing, telemetry, relay, and provider semantics are backend-agnostic and are **not redesigned**. GGUF contributes exact compatibility, range/capacity, queue/load, seam-cost, health/reliability, and certification inputs only. +- The data plane is a standalone project-owned C++ Shard worker with gRPC/Protobuf and a project-owned `ShardEngine` boundary. +- llama.cpp is fetched at one exact commit into an ignored workspace from an in-repo manifest, then a numbered minimal patch stack is applied. There is no submodule, vendored tree, or permanent-fork dependency. +- llama.cpp owns DeepSeek V4 graphs, mHC, MoE, attention, hash routing, and kernels. Meshnet adds only range-ownership hooks, typed boundary/local-state adapters, worker integration, and parity/certification. +- Quantization and placement are dynamic recipe inputs. The 2–4 and 10+ stage layouts are certification scenarios, never product constants. +- Per-shard Hot KV and V4 CSA/HCA/SWA/indexer/compressor state remain local and keyed by route session/epoch. The WAN seam carries the typed mHC 4×4096 residual boundary, positions, token-ID sideband where required, and schema/cache expectations—not per-layer caches. +- Route changes use cache miss plus re-prefill/restart. There is no WAN KV or V4 auxiliary-cache migration. +- CPU/CUDA/ROCm/Vulkan/Metal compile lanes are planned; only exact real-hardware-certified backend/model/recipe lanes may be advertised. +- Alpha requires correctness and the pre-locked useful-speed gate. MTP is reserved and off for alpha; its ownership contract, implementation, and benchmark are required before beta. + +## Target identities + +- DeepSeek V4 official target SHA: `60d8d70770c6776ff598c94bb586a859a38244f1`. +- llama.cpp V4 support lineage began at PR 24162 / merge `8c146a8366304c871efc26057cc90370ccf58dad`; DGR-027 later pins one exact validated current commit. +- V4 scope: 43 main layers plus MTP; mHC 4×4096 boundary; 256 routed + 1 shared experts with six routed active; token IDs required for the first three hash-routed layers. +- Exact split-GGUF artifacts are provisioned to mounted-drive storage with a complete hashed manifest and resumable verification; no model artifact may be placed under `/home`. diff --git a/docs/archived/README.md b/docs/archived/README.md new file mode 100644 index 0000000..62366de --- /dev/null +++ b/docs/archived/README.md @@ -0,0 +1,15 @@ +# Archived task programs + +These task programs are historical, completed, superseded, or explicitly not part of the active development queue. They are preserved for provenance and must not be treated as runnable work. + +## Archived programs + +- `alpha-hardening/` — historical alpha hardening and settlement/authentication task set. +- `dashboard-test-runner/` — historical dashboard test-runner work. +- `distributed-inference-performance/` — superseded distributed-inference performance backlog. +- `node-capability-admission/` — historical capability-admission task set. +- `proxy-stream-cancellation/` — historical proxy cancellation task. +- `qwen3.6-27b-demand-placement/` — superseded Qwen demand-placement planning. +- `routing-compatibility-regression/` — historical routing compatibility regression work. + +The active distributed GGUF backlog remains under `.scratch/distributed-gguf-runtime/`. Architectural decisions under `docs/adr/` remain active and were not moved. diff --git a/.scratch/alpha-hardening/README.md b/docs/archived/alpha-hardening/README.md similarity index 100% rename from .scratch/alpha-hardening/README.md rename to docs/archived/alpha-hardening/README.md diff --git a/.scratch/alpha-hardening/handoff.md b/docs/archived/alpha-hardening/handoff.md similarity index 100% rename from .scratch/alpha-hardening/handoff.md rename to docs/archived/alpha-hardening/handoff.md diff --git a/.scratch/alpha-hardening/issues/01-c1-gossip-auth_completed.md b/docs/archived/alpha-hardening/issues/01-c1-gossip-auth_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/01-c1-gossip-auth_completed.md rename to docs/archived/alpha-hardening/issues/01-c1-gossip-auth_completed.md diff --git a/.scratch/alpha-hardening/issues/02-a2-unified-auth-boundary_completed.md b/docs/archived/alpha-hardening/issues/02-a2-unified-auth-boundary_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/02-a2-unified-auth-boundary_completed.md rename to docs/archived/alpha-hardening/issues/02-a2-unified-auth-boundary_completed.md diff --git a/.scratch/alpha-hardening/issues/03-c5-starting-credit-zero_completed.md b/docs/archived/alpha-hardening/issues/03-c5-starting-credit-zero_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/03-c5-starting-credit-zero_completed.md rename to docs/archived/alpha-hardening/issues/03-c5-starting-credit-zero_completed.md diff --git a/.scratch/alpha-hardening/issues/04-h2-tracker-authoritative-accounting_completed.md b/docs/archived/alpha-hardening/issues/04-h2-tracker-authoritative-accounting_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/04-h2-tracker-authoritative-accounting_completed.md rename to docs/archived/alpha-hardening/issues/04-h2-tracker-authoritative-accounting_completed.md diff --git a/.scratch/alpha-hardening/issues/05-a1-a5-persist-strike-ban-reputation_completed.md b/docs/archived/alpha-hardening/issues/05-a1-a5-persist-strike-ban-reputation_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/05-a1-a5-persist-strike-ban-reputation_completed.md rename to docs/archived/alpha-hardening/issues/05-a1-a5-persist-strike-ban-reputation_completed.md diff --git a/.scratch/alpha-hardening/issues/06-fraud-toploc-integration_completed.md b/docs/archived/alpha-hardening/issues/06-fraud-toploc-integration_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/06-fraud-toploc-integration_completed.md rename to docs/archived/alpha-hardening/issues/06-fraud-toploc-integration_completed.md diff --git a/.scratch/alpha-hardening/issues/07-fraud-commitment-bisection-blame_completed.md b/docs/archived/alpha-hardening/issues/07-fraud-commitment-bisection-blame_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/07-fraud-commitment-bisection-blame_completed.md rename to docs/archived/alpha-hardening/issues/07-fraud-commitment-bisection-blame_completed.md diff --git a/.scratch/alpha-hardening/issues/08-fraud-reputation-model-persistence_completed.md b/docs/archived/alpha-hardening/issues/08-fraud-reputation-model-persistence_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/08-fraud-reputation-model-persistence_completed.md rename to docs/archived/alpha-hardening/issues/08-fraud-reputation-model-persistence_completed.md diff --git a/.scratch/alpha-hardening/issues/09-fraud-reputation-routing-adaptive-audit_completed.md b/docs/archived/alpha-hardening/issues/09-fraud-reputation-routing-adaptive-audit_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/09-fraud-reputation-routing-adaptive-audit_completed.md rename to docs/archived/alpha-hardening/issues/09-fraud-reputation-routing-adaptive-audit_completed.md diff --git a/.scratch/alpha-hardening/issues/10-fraud-penalty-calibration-wiring_completed.md b/docs/archived/alpha-hardening/issues/10-fraud-penalty-calibration-wiring_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/10-fraud-penalty-calibration-wiring_completed.md rename to docs/archived/alpha-hardening/issues/10-fraud-penalty-calibration-wiring_completed.md diff --git a/.scratch/alpha-hardening/issues/11-c6-wallet-binding-proof_completed.md b/docs/archived/alpha-hardening/issues/11-c6-wallet-binding-proof_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/11-c6-wallet-binding-proof_completed.md rename to docs/archived/alpha-hardening/issues/11-c6-wallet-binding-proof_completed.md diff --git a/.scratch/alpha-hardening/issues/12-c2-on-chain-idempotency.md b/docs/archived/alpha-hardening/issues/12-c2-on-chain-idempotency.md similarity index 100% rename from .scratch/alpha-hardening/issues/12-c2-on-chain-idempotency.md rename to docs/archived/alpha-hardening/issues/12-c2-on-chain-idempotency.md diff --git a/.scratch/alpha-hardening/issues/13-c3-c4-consensus-gated-settlement.md b/docs/archived/alpha-hardening/issues/13-c3-c4-consensus-gated-settlement.md similarity index 100% rename from .scratch/alpha-hardening/issues/13-c3-c4-consensus-gated-settlement.md rename to docs/archived/alpha-hardening/issues/13-c3-c4-consensus-gated-settlement.md diff --git a/.scratch/alpha-hardening/issues/14-a3-raft-durable-term-vote.md b/docs/archived/alpha-hardening/issues/14-a3-raft-durable-term-vote.md similarity index 100% rename from .scratch/alpha-hardening/issues/14-a3-raft-durable-term-vote.md rename to docs/archived/alpha-hardening/issues/14-a3-raft-durable-term-vote.md diff --git a/.scratch/alpha-hardening/issues/15-h1-commutative-forfeit.md b/docs/archived/alpha-hardening/issues/15-h1-commutative-forfeit.md similarity index 100% rename from .scratch/alpha-hardening/issues/15-h1-commutative-forfeit.md rename to docs/archived/alpha-hardening/issues/15-h1-commutative-forfeit.md diff --git a/.scratch/alpha-hardening/issues/16-doc-us006-reconciliation_completed.md b/docs/archived/alpha-hardening/issues/16-doc-us006-reconciliation_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/16-doc-us006-reconciliation_completed.md rename to docs/archived/alpha-hardening/issues/16-doc-us006-reconciliation_completed.md diff --git a/.scratch/alpha-hardening/issues/17-doc-duplicate-us020-dedup.md b/docs/archived/alpha-hardening/issues/17-doc-duplicate-us020-dedup.md similarity index 100% rename from .scratch/alpha-hardening/issues/17-doc-duplicate-us020-dedup.md rename to docs/archived/alpha-hardening/issues/17-doc-duplicate-us020-dedup.md diff --git a/.scratch/alpha-hardening/issues/18-doc-operational-runbooks_completed.md b/docs/archived/alpha-hardening/issues/18-doc-operational-runbooks_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/18-doc-operational-runbooks_completed.md rename to docs/archived/alpha-hardening/issues/18-doc-operational-runbooks_completed.md diff --git a/.scratch/alpha-hardening/issues/19-doc-cryptography-test-env_completed.md b/docs/archived/alpha-hardening/issues/19-doc-cryptography-test-env_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/19-doc-cryptography-test-env_completed.md rename to docs/archived/alpha-hardening/issues/19-doc-cryptography-test-env_completed.md diff --git a/.scratch/alpha-hardening/issues/20-validator-service-token_completed.md b/docs/archived/alpha-hardening/issues/20-validator-service-token_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/20-validator-service-token_completed.md rename to docs/archived/alpha-hardening/issues/20-validator-service-token_completed.md diff --git a/.scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md b/docs/archived/alpha-hardening/issues/21-honest-noise-calibration-corpus.md similarity index 100% rename from .scratch/alpha-hardening/issues/21-honest-noise-calibration-corpus.md rename to docs/archived/alpha-hardening/issues/21-honest-noise-calibration-corpus.md diff --git a/.scratch/alpha-hardening/issues/22-doc-memory-project-status_completed.md b/docs/archived/alpha-hardening/issues/22-doc-memory-project-status_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/22-doc-memory-project-status_completed.md rename to docs/archived/alpha-hardening/issues/22-doc-memory-project-status_completed.md diff --git a/.scratch/alpha-hardening/issues/23-dynamic-hf-pricing_completed.md b/docs/archived/alpha-hardening/issues/23-dynamic-hf-pricing_completed.md similarity index 100% rename from .scratch/alpha-hardening/issues/23-dynamic-hf-pricing_completed.md rename to docs/archived/alpha-hardening/issues/23-dynamic-hf-pricing_completed.md diff --git a/.scratch/alpha-hardening/issues/24-routing-telemetry-resume.md b/docs/archived/alpha-hardening/issues/24-routing-telemetry-resume.md similarity index 100% rename from .scratch/alpha-hardening/issues/24-routing-telemetry-resume.md rename to docs/archived/alpha-hardening/issues/24-routing-telemetry-resume.md diff --git a/.scratch/alpha-hardening/issues/25-per-node-kv-cache-distributed.md b/docs/archived/alpha-hardening/issues/25-per-node-kv-cache-distributed.md similarity index 100% rename from .scratch/alpha-hardening/issues/25-per-node-kv-cache-distributed.md rename to docs/archived/alpha-hardening/issues/25-per-node-kv-cache-distributed.md diff --git a/.scratch/alpha-hardening/prd.json b/docs/archived/alpha-hardening/prd.json similarity index 100% rename from .scratch/alpha-hardening/prd.json rename to docs/archived/alpha-hardening/prd.json diff --git a/.scratch/alpha-hardening/research-verifiable-inference.md b/docs/archived/alpha-hardening/research-verifiable-inference.md similarity index 100% rename from .scratch/alpha-hardening/research-verifiable-inference.md rename to docs/archived/alpha-hardening/research-verifiable-inference.md diff --git a/.scratch/alpha-hardening/runbooks/01-ledger-backup.md b/docs/archived/alpha-hardening/runbooks/01-ledger-backup.md similarity index 100% rename from .scratch/alpha-hardening/runbooks/01-ledger-backup.md rename to docs/archived/alpha-hardening/runbooks/01-ledger-backup.md diff --git a/.scratch/alpha-hardening/runbooks/02-treasury-key-rotation.md b/docs/archived/alpha-hardening/runbooks/02-treasury-key-rotation.md similarity index 100% rename from .scratch/alpha-hardening/runbooks/02-treasury-key-rotation.md rename to docs/archived/alpha-hardening/runbooks/02-treasury-key-rotation.md diff --git a/.scratch/alpha-hardening/runbooks/03-upgrade-path.md b/docs/archived/alpha-hardening/runbooks/03-upgrade-path.md similarity index 100% rename from .scratch/alpha-hardening/runbooks/03-upgrade-path.md rename to docs/archived/alpha-hardening/runbooks/03-upgrade-path.md diff --git a/.scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md b/docs/archived/alpha-hardening/runbooks/04-toploc-calibration-run.md similarity index 100% rename from .scratch/alpha-hardening/runbooks/04-toploc-calibration-run.md rename to docs/archived/alpha-hardening/runbooks/04-toploc-calibration-run.md diff --git a/.scratch/dashboard-test-runner/PRD.md b/docs/archived/dashboard-test-runner/PRD.md similarity index 100% rename from .scratch/dashboard-test-runner/PRD.md rename to docs/archived/dashboard-test-runner/PRD.md diff --git a/.scratch/dashboard-test-runner/prd.json b/docs/archived/dashboard-test-runner/prd.json similarity index 100% rename from .scratch/dashboard-test-runner/prd.json rename to docs/archived/dashboard-test-runner/prd.json diff --git a/.scratch/dashboard-test-runner/ralph-iterations/4cd034a1_2026-07-11_15-23-25_US-001.log b/docs/archived/dashboard-test-runner/ralph-iterations/4cd034a1_2026-07-11_15-23-25_US-001.log similarity index 100% rename from .scratch/dashboard-test-runner/ralph-iterations/4cd034a1_2026-07-11_15-23-25_US-001.log rename to docs/archived/dashboard-test-runner/ralph-iterations/4cd034a1_2026-07-11_15-23-25_US-001.log diff --git a/.scratch/dashboard-test-runner/ralph-opus-iterations/bc37171f_2026-07-11_16-12-14_US-002.log b/docs/archived/dashboard-test-runner/ralph-opus-iterations/bc37171f_2026-07-11_16-12-14_US-002.log similarity index 100% rename from .scratch/dashboard-test-runner/ralph-opus-iterations/bc37171f_2026-07-11_16-12-14_US-002.log rename to docs/archived/dashboard-test-runner/ralph-opus-iterations/bc37171f_2026-07-11_16-12-14_US-002.log diff --git a/.scratch/dashboard-test-runner/ralph-terra-iterations/1998cbcc_2026-07-11_20-01-02_US-002.log b/docs/archived/dashboard-test-runner/ralph-terra-iterations/1998cbcc_2026-07-11_20-01-02_US-002.log similarity index 100% rename from .scratch/dashboard-test-runner/ralph-terra-iterations/1998cbcc_2026-07-11_20-01-02_US-002.log rename to docs/archived/dashboard-test-runner/ralph-terra-iterations/1998cbcc_2026-07-11_20-01-02_US-002.log diff --git a/.scratch/distributed-inference-performance/PRD.md b/docs/archived/distributed-inference-performance/PRD.md similarity index 100% rename from .scratch/distributed-inference-performance/PRD.md rename to docs/archived/distributed-inference-performance/PRD.md diff --git a/.scratch/distributed-inference-performance/README.md b/docs/archived/distributed-inference-performance/README.md similarity index 100% rename from .scratch/distributed-inference-performance/README.md rename to docs/archived/distributed-inference-performance/README.md diff --git a/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md b/docs/archived/distributed-inference-performance/issues/01-baseline-profiling-harness.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md rename to docs/archived/distributed-inference-performance/issues/01-baseline-profiling-harness.md diff --git a/.scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md b/docs/archived/distributed-inference-performance/issues/02-relay-session-compatibility.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md rename to docs/archived/distributed-inference-performance/issues/02-relay-session-compatibility.md diff --git a/.scratch/distributed-inference-performance/issues/03-http-keepalive.md b/docs/archived/distributed-inference-performance/issues/03-http-keepalive.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/03-http-keepalive.md rename to docs/archived/distributed-inference-performance/issues/03-http-keepalive.md diff --git a/.scratch/distributed-inference-performance/issues/04-seam-telemetry.md b/docs/archived/distributed-inference-performance/issues/04-seam-telemetry.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/04-seam-telemetry.md rename to docs/archived/distributed-inference-performance/issues/04-seam-telemetry.md diff --git a/.scratch/distributed-inference-performance/issues/05-adaptive-compression.md b/docs/archived/distributed-inference-performance/issues/05-adaptive-compression.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/05-adaptive-compression.md rename to docs/archived/distributed-inference-performance/issues/05-adaptive-compression.md diff --git a/.scratch/distributed-inference-performance/issues/06-activation-framing-copies.md b/docs/archived/distributed-inference-performance/issues/06-activation-framing-copies.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/06-activation-framing-copies.md rename to docs/archived/distributed-inference-performance/issues/06-activation-framing-copies.md diff --git a/.scratch/distributed-inference-performance/issues/07-prefill-backpressure.md b/docs/archived/distributed-inference-performance/issues/07-prefill-backpressure.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/07-prefill-backpressure.md rename to docs/archived/distributed-inference-performance/issues/07-prefill-backpressure.md diff --git a/.scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md b/docs/archived/distributed-inference-performance/issues/08-end-to-end-performance-gate.md similarity index 100% rename from .scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md rename to docs/archived/distributed-inference-performance/issues/08-end-to-end-performance-gate.md diff --git a/.scratch/distributed-inference-performance/prd.json b/docs/archived/distributed-inference-performance/prd.json similarity index 100% rename from .scratch/distributed-inference-performance/prd.json rename to docs/archived/distributed-inference-performance/prd.json diff --git a/.scratch/node-capability-admission/PRD.md b/docs/archived/node-capability-admission/PRD.md similarity index 100% rename from .scratch/node-capability-admission/PRD.md rename to docs/archived/node-capability-admission/PRD.md diff --git a/.scratch/node-capability-admission/README.md b/docs/archived/node-capability-admission/README.md similarity index 100% rename from .scratch/node-capability-admission/README.md rename to docs/archived/node-capability-admission/README.md diff --git a/.scratch/node-capability-admission/issues/01-generic-capability-report.md b/docs/archived/node-capability-admission/issues/01-generic-capability-report.md similarity index 100% rename from .scratch/node-capability-admission/issues/01-generic-capability-report.md rename to docs/archived/node-capability-admission/issues/01-generic-capability-report.md diff --git a/.scratch/node-capability-admission/issues/02-doctor-real-forward.md b/docs/archived/node-capability-admission/issues/02-doctor-real-forward.md similarity index 100% rename from .scratch/node-capability-admission/issues/02-doctor-real-forward.md rename to docs/archived/node-capability-admission/issues/02-doctor-real-forward.md diff --git a/.scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md b/docs/archived/node-capability-admission/issues/03-fail-closed-startup-admission.md similarity index 100% rename from .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md rename to docs/archived/node-capability-admission/issues/03-fail-closed-startup-admission.md diff --git a/.scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md b/docs/archived/node-capability-admission/issues/04-tracker-validated-capability-routing.md similarity index 100% rename from .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md rename to docs/archived/node-capability-admission/issues/04-tracker-validated-capability-routing.md diff --git a/.scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md b/docs/archived/node-capability-admission/issues/05-docs-hardware-lane-contract.md similarity index 100% rename from .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md rename to docs/archived/node-capability-admission/issues/05-docs-hardware-lane-contract.md diff --git a/.scratch/node-capability-admission/prd.json b/docs/archived/node-capability-admission/prd.json similarity index 100% rename from .scratch/node-capability-admission/prd.json rename to docs/archived/node-capability-admission/prd.json diff --git a/.scratch/node-capability-admission/ralph-iterations/8a5cc64f_2026-07-11_22-02-57_NCA-001.log b/docs/archived/node-capability-admission/ralph-iterations/8a5cc64f_2026-07-11_22-02-57_NCA-001.log similarity index 100% rename from .scratch/node-capability-admission/ralph-iterations/8a5cc64f_2026-07-11_22-02-57_NCA-001.log rename to docs/archived/node-capability-admission/ralph-iterations/8a5cc64f_2026-07-11_22-02-57_NCA-001.log diff --git a/.scratch/proxy-stream-cancellation/PRD.md b/docs/archived/proxy-stream-cancellation/PRD.md similarity index 100% rename from .scratch/proxy-stream-cancellation/PRD.md rename to docs/archived/proxy-stream-cancellation/PRD.md diff --git a/.scratch/proxy-stream-cancellation/issues/01-direct-sse-cancellation-race.md b/docs/archived/proxy-stream-cancellation/issues/01-direct-sse-cancellation-race.md similarity index 100% rename from .scratch/proxy-stream-cancellation/issues/01-direct-sse-cancellation-race.md rename to docs/archived/proxy-stream-cancellation/issues/01-direct-sse-cancellation-race.md diff --git a/.scratch/qwen3.6-27b-demand-placement/PRD.md b/docs/archived/qwen3.6-27b-demand-placement/PRD.md similarity index 100% rename from .scratch/qwen3.6-27b-demand-placement/PRD.md rename to docs/archived/qwen3.6-27b-demand-placement/PRD.md diff --git a/.scratch/routing-compatibility-regression/PRD.md b/docs/archived/routing-compatibility-regression/PRD.md similarity index 100% rename from .scratch/routing-compatibility-regression/PRD.md rename to docs/archived/routing-compatibility-regression/PRD.md diff --git a/.scratch/routing-compatibility-regression/issues/01-legacy-registration-precision-fallback.md b/docs/archived/routing-compatibility-regression/issues/01-legacy-registration-precision-fallback.md similarity index 100% rename from .scratch/routing-compatibility-regression/issues/01-legacy-registration-precision-fallback.md rename to docs/archived/routing-compatibility-regression/issues/01-legacy-registration-precision-fallback.md diff --git a/.scratch/routing-compatibility-regression/prd.json b/docs/archived/routing-compatibility-regression/prd.json similarity index 100% rename from .scratch/routing-compatibility-regression/prd.json rename to docs/archived/routing-compatibility-regression/prd.json diff --git a/packages/node/meshnet_node/admission.py b/packages/node/meshnet_node/admission.py index e33b70d..553e130 100644 --- a/packages/node/meshnet_node/admission.py +++ b/packages/node/meshnet_node/admission.py @@ -20,17 +20,9 @@ import time from dataclasses import dataclass from typing import Any, Callable -from . import __version__ as _PACKAGE_VERSION -from .capability import CapabilityReport, config_fingerprint +from .capability import CapabilityReport from .doctor import DoctorSelection from .recipe_manifest import Recipe, RecipeManifest -from .runtime_recipe import ( - build_artifact_identity, - build_runtime_recipe_identity, - compatibility_fingerprint, - fingerprint_payload, -) -from .gguf_ownership import authoritative_dense_llama_ownership # How long a passing report stays usable. Startup normally validates in-process # (age ≈ 0); this bounds how far a report written by an earlier `doctor` run can @@ -47,7 +39,6 @@ REASON_MODEL_MISMATCH = "model-mismatch" REASON_SHARD_MISMATCH = "shard-mismatch" REASON_RECIPE_MISMATCH = "recipe-mismatch" REASON_BACKEND_MISMATCH = "backend-mismatch" -REASON_COMPATIBILITY_MISMATCH = "compatibility-mismatch" class CapabilityAdmissionError(RuntimeError): @@ -86,7 +77,6 @@ class AdmissionRequirement: recipe_version: str backend_id: str device: str - compatibility_fingerprint: str max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS @classmethod @@ -104,9 +94,6 @@ class AdmissionRequirement: recipe_version=context.recipe.version, backend_id=context.recipe.backend_id, device=context.device, - compatibility_fingerprint=_compatibility_fingerprint_for_context( - context - ), max_age_seconds=max_age_seconds, ) @@ -178,16 +165,6 @@ def admit( f"{requirement.backend_id} on {requirement.device}", ) - if report.compatibility_fingerprint != requirement.compatibility_fingerprint: - raise CapabilityAdmissionError( - REASON_COMPATIBILITY_MISMATCH, - f"capability proof fingerprint {report.compatibility_fingerprint!r} " - f"does not match the expected compatibility fingerprint for " - f"{requirement.model_id} {requirement.shard_label}; the artifact, " - f"tokenizer, architecture, boundary schema, activation recipe or " - f"cache layout differs", - ) - if not report.passed: raise CapabilityAdmissionError( REASON_NOT_PASSED, @@ -246,157 +223,3 @@ def probe_capability(context: CapabilityContext) -> CapabilityReport: context.recipe, context.manifest, ).report - - -def _compatibility_fingerprint_for_context(context: CapabilityContext) -> str: - backend = context.backend - selection = context.selection - recipe = context.recipe - model_config = getattr(getattr(backend, "model", None), "config", None) - model_config_payload = ( - model_config.to_dict() if hasattr(model_config, "to_dict") else model_config - ) - runtime_versions = _runtime_versions() - runtime_version = _PACKAGE_VERSION - ownership = authoritative_dense_llama_ownership(backend, selection) - artifact = build_artifact_identity( - model_id=selection.model_id, - revision=getattr(getattr(backend, "model", None), "revision", None), - model_config=model_config_payload, - shard_start=ownership.start_layer, - shard_end=ownership.end_layer, - ) - runtime_recipe = build_runtime_recipe_identity( - model_id=selection.model_id, - revision=getattr(getattr(backend, "model", None), "revision", None), - model_config=model_config_payload, - recipe_params=recipe.params, - weight_quantization=selection.quantization, - backend_id=recipe.backend_id, - runtime_version=runtime_version, - activation_dtype="bfloat16", - compute_dtype=_backend_compute_dtype(backend), - kv_dtype=_backend_kv_dtype(backend), - kv_layout=_backend_kv_layout(backend), - tokenizer_revision=_backend_tokenizer_revision(backend, selection), - architecture_adapter=_backend_architecture_adapter(backend, recipe.backend_id), - boundary_schema_version=1, - cache_layout=_backend_cache_layout(backend, recipe.params), - ) - return compatibility_fingerprint( - fingerprint_payload( - model={ - "model_id": selection.model_id, - "revision": getattr(getattr(backend, "model", None), "revision", None), - "config_fingerprint": config_fingerprint(model_config_payload), - }, - shard={ - "start": ownership.start_layer, - "end": ownership.end_layer, - "owns_embedding": ownership.owns_embedding, - "owns_final_head": ownership.owns_final_head, - }, - recipe={ - "recipe_id": recipe.id, - "recipe_version": recipe.version, - "catalogue_version": context.manifest.catalogue_version, - }, - backend={ - "backend_id": recipe.backend_id, - "device": context.device, - "device_name": _backend_device_name(context.device), - "quantization": selection.quantization, - "runtime": runtime_versions, - }, - artifact=artifact.to_dict(), - runtime_recipe=runtime_recipe.to_dict(), - ) - ) - - -def _runtime_versions() -> dict[str, str]: - versions: dict[str, str] = {} - for name in ("torch", "transformers"): - try: - module = __import__(name) - except Exception: - continue - version = getattr(module, "__version__", None) - if version: - versions[name] = str(version) - return versions - - -def _backend_compute_dtype(backend: Any) -> str: - config = getattr(getattr(backend, "model", None), "config", None) - for candidate in (config, getattr(config, "text_config", None)): - if candidate is None: - continue - for attr in ("dtype", "torch_dtype"): - value = getattr(candidate, attr, None) - if value is None: - continue - return str(value).removeprefix("torch.") - return "bfloat16" - - -def _backend_kv_dtype(backend: Any) -> str: - return _backend_compute_dtype(backend) - - -def _backend_kv_layout(backend: Any) -> str: - return "session-cache" if getattr(backend, "supports_kv_cache", False) else "stateless" - - -def _backend_tokenizer_revision(backend: Any, selection: DoctorSelection) -> str: - model = getattr(backend, "model", None) - revision = getattr(model, "revision", None) - if isinstance(revision, str) and revision.strip(): - return revision - tokenizer = getattr(backend, "tokenizer", None) - for attr in ("revision", "model_id"): - value = getattr(tokenizer, attr, None) - if isinstance(value, str) and value.strip(): - return value - return selection.model_id - - -def _backend_architecture_adapter(backend: Any, default: str) -> str: - config = getattr(getattr(backend, "model", None), "config", None) - for candidate in (config, getattr(config, "text_config", None)): - if candidate is None: - continue - for attr in ("architecture_adapter", "model_type"): - value = getattr(candidate, attr, None) - if isinstance(value, str) and value.strip(): - return value - architectures = getattr(candidate, "architectures", None) - if isinstance(architectures, (list, tuple)) and architectures: - first = architectures[0] - if isinstance(first, str) and first.strip(): - return first - return default - - -def _backend_device_name(device: str) -> str | None: - if device != "cuda": - return None - from .hardware import detect_hardware - - try: - return detect_hardware().get("gpu_name") or None - except Exception: - return None - - -def _backend_cache_layout(backend: Any, recipe_params: dict[str, Any] | None) -> str: - if getattr(backend, "supports_kv_cache", False) is False: - return "stateless" - if recipe_params is None: - return "local-hot-kv" - if recipe_params.get("use_cache") is False: - return "stateless" - value = recipe_params.get("cache_layout") - if isinstance(value, str) and value.strip(): - return value - return "local-hot-kv" diff --git a/packages/node/meshnet_node/batch_scheduler.py b/packages/node/meshnet_node/batch_scheduler.py deleted file mode 100644 index 1ce7ea5..0000000 --- a/packages/node/meshnet_node/batch_scheduler.py +++ /dev/null @@ -1,1024 +0,0 @@ -"""Continuous batching and bounded admission for concurrent Route Sessions (DGR-012). - -RALPH runtime decision #9: concurrency on a node uses *continuous batching of -compatible active sessions* — not a separate scheduler or control plane. This -module is the node-local scheduler that sits on top of the isolated Hot KV State -manager (DGR-007) and turns many concurrent single-token decode steps into one -batch per tick, while keeping every session's positions, KV, and sampled output -isolated (decisions #7/#8, ADR-0022/0024). - -The design is deliberately backend-agnostic. The scheduler talks to a -:class:`BatchEngine` duck type (``recipe_fingerprint`` / ``prefill`` / -``decode_batch`` / ``release``); the default deterministic test suite drives it -with a pure-numpy dense-Llama engine, and the pinned llama.cpp worker (DGR-008) -implements the same contract where a batch becomes one ``llama_decode`` over -several sequences. :class:`KvBatchEngine` adapts the DGR-007 -:class:`~meshnet_node.hot_kv_state.KvBoundaryAdapter` to this contract so the -scheduler runs against real KV isolation with no new cache code. - -What the scheduler guarantees (the acceptance contract): - -* **Bounded admission.** A new session is admitted only if it fits the node's - weight, KV, scratch, and queue budgets (:class:`NodeBudget`). Anything that - cannot fit is rejected with an explicit :class:`AdmissionReason`; anything that - fits but has no free active slot waits in a bounded queue. When the queue is - full, admission is refused — that refusal *is* the backpressure signal. -* **Continuous batching.** Every tick, all sessions currently decoding contribute - their single next token to one batch (bounded by ``max_batch_size``). The engine - runs the batch once; each session keeps its own position and appends its own - sampled token, so batching never mixes outputs. -* **Prefill does not starve decode.** The scheduling policy is explicit and fixed: - *decode first, then bounded prefill*. Ongoing decodes always run before any new - prompt is prefilled, and prefill work per tick is capped - (``max_prefill_tokens_per_tick``) so a burst of new sessions cannot monopolise - the node and stall in-flight generations. -* **Bounded memory.** KV growth is bounded by the manager's byte budget; queued - activations are bounded by ``max_queue_depth`` and the scratch budget. Neither - the queue nor the KV store grows without limit. -* **Telemetry.** :meth:`ContinuousBatchScheduler.telemetry` reports active - sessions, queue depth, batch occupancy, KV pressure, prefill/decode token rates, - and rejected admissions — the capability signals a node advertises upward. - -Everything here is pure Python + the numpy-backed manager, so the default gate -stays deterministic, download-free, GPU-free, and API-credit-free. Real -kernel-level batching speedup is a native-worker property measured in -DGR-008/DGR-010/DGR-014; this module owns the *scheduling* behaviour and proves, -via the 1/2/4/8 concurrency sweep, that batching raises aggregate work-per-tick -without cross-session corruption. -""" - -from __future__ import annotations - -import threading -import time -from collections import deque -from dataclasses import dataclass, field -from enum import Enum -from typing import Any, Callable, Iterable, Mapping, Sequence - -from meshnet_node.hot_kv_state import ( - CacheMiss, - HotKvStateManager, - KvBoundaryAdapter, -) - - -class SchedulerError(RuntimeError): - """Base class for scheduler configuration/usage errors.""" - - -# --------------------------------------------------------------------------- # -# Node budget and admission. -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class NodeBudget: - """Explicit bounds the node admits and schedules against. - - Four budget dimensions gate admission (the story's "weight, KV, scratch, and - queue budgets") plus the scheduling bounds that keep batching fair: - - * ``weight_bytes`` — resident weight footprint of the loaded shard. This is a - fixed, one-time cost; the scheduler treats it as already resident and simply - reports it (a node that cannot hold its shard weight never starts). It is - validated non-negative and surfaced in telemetry. - * ``kv_budget_bytes`` — the Hot KV State byte budget. A session is admissible - only if its *whole* generation (prompt + all new tokens) could fit this - budget on its own; cross-session pressure is then handled by the manager's - LRU/byte eviction. This mirrors ``HotKvStateConfig.budget_bytes`` and should - match the manager the scheduler was given. - * ``scratch_bytes_per_session`` / ``scratch_budget_bytes`` — per-active-session - activation scratch (the transient residual/attention buffers a decode needs) - and the total scratch envelope. Admission keeps - ``active * scratch_per_session <= scratch_budget`` so concurrent activations - are bounded, not just KV. - * ``max_active_sessions`` — hard cap on sessions occupying an execution slot. - * ``max_queue_depth`` — bounded waiting room for admitted-but-not-yet-running - requests. A full queue is the backpressure boundary. - * ``max_batch_size`` — largest decode batch formed per tick. - * ``max_prefill_tokens_per_tick`` — prefill token budget per tick, so prefill - cannot starve decode. - """ - - weight_bytes: int = 0 - kv_budget_bytes: int = 64 * 1024 * 1024 - scratch_bytes_per_session: int = 1 * 1024 * 1024 - scratch_budget_bytes: int = 16 * 1024 * 1024 - max_active_sessions: int = 8 - max_queue_depth: int = 64 - max_batch_size: int = 8 - max_prefill_tokens_per_tick: int = 512 - - def __post_init__(self) -> None: - if self.weight_bytes < 0: - raise SchedulerError("weight_bytes must be >= 0") - if self.kv_budget_bytes <= 0: - raise SchedulerError("kv_budget_bytes must be positive") - if self.scratch_bytes_per_session <= 0: - raise SchedulerError("scratch_bytes_per_session must be positive") - if self.scratch_budget_bytes <= 0: - raise SchedulerError("scratch_budget_bytes must be positive") - if self.max_active_sessions < 1: - raise SchedulerError("max_active_sessions must be >= 1") - if self.max_queue_depth < 0: - raise SchedulerError("max_queue_depth must be >= 0") - if self.max_batch_size < 1: - raise SchedulerError("max_batch_size must be >= 1") - if self.max_prefill_tokens_per_tick < 1: - raise SchedulerError("max_prefill_tokens_per_tick must be >= 1") - - @property - def max_scratch_sessions(self) -> int: - """How many concurrent sessions the scratch envelope alone permits.""" - return self.scratch_budget_bytes // self.scratch_bytes_per_session - - @property - def effective_active_cap(self) -> int: - """The tighter of the active-slot cap and the scratch-derived cap.""" - return max(1, min(self.max_active_sessions, self.max_scratch_sessions)) - - -class AdmissionReason(str, Enum): - """Why a submission was admitted, queued, or rejected.""" - - ADMITTED = "admitted" - QUEUED = "queued" - REJECTED_QUEUE_FULL = "rejected-queue-full" - REJECTED_KV_BUDGET = "rejected-kv-budget" - REJECTED_SCRATCH_BUDGET = "rejected-scratch-budget" - REJECTED_DUPLICATE = "rejected-duplicate" - REJECTED_INVALID = "rejected-invalid" - - -# Reasons that mean "will run" (admitted now, or accepted into the bounded queue). -_ACCEPTED = frozenset({AdmissionReason.ADMITTED, AdmissionReason.QUEUED}) -# Reasons that mean "refused" — the caller must apply backpressure / retry later. -_REJECTED = frozenset( - { - AdmissionReason.REJECTED_QUEUE_FULL, - AdmissionReason.REJECTED_KV_BUDGET, - AdmissionReason.REJECTED_SCRATCH_BUDGET, - AdmissionReason.REJECTED_DUPLICATE, - AdmissionReason.REJECTED_INVALID, - } -) - - -@dataclass(frozen=True) -class AdmissionDecision: - """The structured outcome of :meth:`ContinuousBatchScheduler.submit`.""" - - session_id: str - reason: AdmissionReason - detail: str = "" - - @property - def accepted(self) -> bool: - return self.reason in _ACCEPTED - - @property - def running(self) -> bool: - return self.reason is AdmissionReason.ADMITTED - - @property - def rejected(self) -> bool: - return self.reason in _REJECTED - - def __str__(self) -> str: - suffix = f": {self.detail}" if self.detail else "" - return f"session {self.session_id} {self.reason.value}{suffix}" - - -# --------------------------------------------------------------------------- # -# Requests, engine contract, and per-session state. -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class GenerationRequest: - """One session's greedy generation job: a prompt and a token budget.""" - - session_id: str - route_epoch: int - prompt_token_ids: tuple[int, ...] - max_new_tokens: int - - def __post_init__(self) -> None: - if not isinstance(self.session_id, str) or not self.session_id.strip(): - raise SchedulerError("session_id must be a non-empty string") - if isinstance(self.route_epoch, bool) or not isinstance(self.route_epoch, int): - raise SchedulerError("route_epoch must be an integer") - if self.route_epoch < 0: - raise SchedulerError("route_epoch must be >= 0") - if not self.prompt_token_ids: - raise SchedulerError("prompt_token_ids must be non-empty") - if self.max_new_tokens < 1: - raise SchedulerError("max_new_tokens must be >= 1") - - @property - def prompt_len(self) -> int: - return len(self.prompt_token_ids) - - @property - def final_seq_len(self) -> int: - """Sequence length after the whole job completes (prompt + new tokens). - - The prefill emits the first new token, so the final KV length is - ``prompt_len + max_new_tokens - 1``. - """ - return self.prompt_len + self.max_new_tokens - 1 - - -@dataclass(frozen=True) -class DecodeItem: - """One member of a decode batch: which session decodes which input token.""" - - session_id: str - route_epoch: int - token_id: int - - -@dataclass(frozen=True) -class StepResult: - """The output of one prefill or one decode-batch member.""" - - session_id: str - route_epoch: int - token_id: int - seq_len: int - - -class Phase(str, Enum): - PENDING_PREFILL = "pending-prefill" - DECODING = "decoding" - DONE = "done" - - -class DoneReason(str, Enum): - COMPLETED = "completed" - CACHE_MISS = "cache-miss" - # DGR-013: a session can also leave the scheduler because the client cancelled - # it or because it failed (deadline/heartbeat loss, worker death, stream reset). - # These are distinguished so billing/work records never bill uncompleted work. - CANCELLED = "cancelled" - FAILED = "failed" - - -@dataclass -class SessionState: - """Live scheduler state for one admitted session (isolated per session).""" - - request: GenerationRequest - phase: Phase = Phase.PENDING_PREFILL - generated: list[int] = field(default_factory=list) - done_reason: DoneReason | None = None - cache_miss: CacheMiss | None = None - - @property - def session_id(self) -> str: - return self.request.session_id - - @property - def route_epoch(self) -> int: - return self.request.route_epoch - - @property - def remaining(self) -> int: - return self.request.max_new_tokens - len(self.generated) - - @property - def last_token(self) -> int: - return self.generated[-1] - - -class KvBatchEngine: - """Adapt a DGR-007 :class:`KvBoundaryAdapter` to the :class:`BatchEngine` contract. - - The adapter must wrap a *full* (head **and** tail) shard so a decode step - samples a token — a middle/head-only range emits a boundary bundle, which the - node-local scheduler does not turn into an output token. Multi-range routes - batch at the head node, whose adapter owns the final head. - - ``decode_batch`` runs each member through the adapter's cached decode. Each - session attends only over its own KV context, exactly as an independent - sequence would inside one native ``llama_decode`` batch; the pure-numpy engine - runs the members sequentially, while the pinned llama.cpp worker fuses them - into a single graph. The scheduling semantics — one batch per tick, isolated - positions and outputs — are identical, so this stands in for the native path - without a download or GPU. - """ - - def __init__(self, adapter: KvBoundaryAdapter) -> None: - if not (adapter.is_head and adapter.is_tail): - raise SchedulerError( - "KvBatchEngine requires a full (head+tail) shard so decode steps " - "sample tokens; got a partial range (head=%s tail=%s)" - % (adapter.is_head, adapter.is_tail) - ) - self._adapter = adapter - self._manager: HotKvStateManager = adapter.manager - - def recipe_fingerprint(self) -> str: - return self._adapter.recipe.fingerprint() - - def prefill( - self, session_id: str, route_epoch: int, token_ids: Sequence[int] - ) -> StepResult: - out = self._adapter.prefill(session_id, route_epoch, token_ids=list(token_ids)) - seq_len = self._manager.get(session_id, route_epoch).seq_len - return StepResult(session_id, route_epoch, int(out.token_id), seq_len) - - def decode_batch( - self, items: Sequence[DecodeItem] - ) -> list[StepResult | CacheMiss]: - results: list[StepResult | CacheMiss] = [] - for item in items: - out = self._adapter.decode( - item.session_id, item.route_epoch, token_ids=[item.token_id] - ) - if isinstance(out, CacheMiss): - results.append(out) - continue - seq_len = self._manager.get(item.session_id, item.route_epoch).seq_len - results.append( - StepResult(item.session_id, item.route_epoch, int(out.token_id), seq_len) - ) - return results - - def release(self, session_id: str, route_epoch: int) -> None: - self._manager.release(session_id, route_epoch) - - -# --------------------------------------------------------------------------- # -# Telemetry. -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class SchedulerTelemetry: - """A bounded, JSON-safe snapshot of node scheduling pressure. - - These are the capability signals a node advertises: enough to decide whether - it can take more work, and to spot saturation, without exposing session - contents. - """ - - active_sessions: int - queue_depth: int - batch_occupancy_last: int - batch_occupancy_avg: float - batch_occupancy_max: int - weight_bytes: int - kv_total_bytes: int - kv_budget_bytes: int - kv_pressure: float - scratch_used_bytes: int - scratch_budget_bytes: int - scratch_pressure: float - prefill_tokens_total: int - decode_tokens_total: int - prefill_tokens_per_sec: float - decode_tokens_per_sec: float - rejected_admissions_total: int - rejected_by_reason: Mapping[str, int] - completed_sessions: int - cancelled_sessions: int - failed_sessions: int - ticks: int - - def to_dict(self) -> dict: - return { - "active_sessions": self.active_sessions, - "queue_depth": self.queue_depth, - "batch_occupancy_last": self.batch_occupancy_last, - "batch_occupancy_avg": round(self.batch_occupancy_avg, 4), - "batch_occupancy_max": self.batch_occupancy_max, - "weight_bytes": self.weight_bytes, - "kv_total_bytes": self.kv_total_bytes, - "kv_budget_bytes": self.kv_budget_bytes, - "kv_pressure": round(self.kv_pressure, 4), - "scratch_used_bytes": self.scratch_used_bytes, - "scratch_budget_bytes": self.scratch_budget_bytes, - "scratch_pressure": round(self.scratch_pressure, 4), - "prefill_tokens_total": self.prefill_tokens_total, - "decode_tokens_total": self.decode_tokens_total, - "prefill_tokens_per_sec": round(self.prefill_tokens_per_sec, 4), - "decode_tokens_per_sec": round(self.decode_tokens_per_sec, 4), - "rejected_admissions_total": self.rejected_admissions_total, - "rejected_by_reason": dict(self.rejected_by_reason), - "completed_sessions": self.completed_sessions, - "cancelled_sessions": self.cancelled_sessions, - "failed_sessions": self.failed_sessions, - "ticks": self.ticks, - } - - -# --------------------------------------------------------------------------- # -# The scheduler. -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class TickReport: - """What one :meth:`ContinuousBatchScheduler.run_tick` did (for observability).""" - - prefilled: tuple[str, ...] - decoded: tuple[str, ...] - batch_occupancy: int - completed: tuple[str, ...] - admitted_from_queue: tuple[str, ...] - - @property - def did_work(self) -> bool: - return bool(self.prefilled or self.decoded) - - -class ContinuousBatchScheduler: - """Node-local continuous-batching scheduler with bounded admission. - - Fixed scheduling policy per :meth:`run_tick`: - - 1. Promote queued sessions into free active slots (respecting the active and - scratch caps). - 2. **Decode first:** form one batch from every active decoding session (up to - ``max_batch_size``) and run it once. This is what guarantees prefill cannot - starve decode. - 3. **Then bounded prefill:** prefill pending sessions until the per-tick prefill - token budget is spent (always allowing at least one, so a single large - prompt still makes progress). - 4. Reap completed/lost sessions, releasing their KV so budget returns. - - The scheduler is thread-safe (an ``RLock`` guards all state) so a real server - can call :meth:`submit` from request threads while a worker thread drives - :meth:`run_tick`; the deterministic tests drive both from one thread. - """ - - def __init__( - self, - engine: Any, - budget: NodeBudget | None = None, - *, - clock: Callable[[], float] | None = None, - ) -> None: - self._engine = engine - self._budget = budget or NodeBudget() - self._clock = clock or time.monotonic - self._fingerprint = str(engine.recipe_fingerprint()) - - self._active: dict[str, SessionState] = {} - self._queue: "deque[GenerationRequest]" = deque() - self._queued_ids: set[str] = set() - self._done: dict[str, SessionState] = {} - - # Telemetry counters. - self._started = self._clock() - self._ticks = 0 - self._prefill_tokens = 0 - self._decode_tokens = 0 - self._batch_occupancy_last = 0 - self._batch_occupancy_max = 0 - self._batch_sum = 0 - self._batch_count = 0 - self._completed = 0 - self._cancelled = 0 - self._failed = 0 - self._rejected = 0 - self._rejected_by_reason: dict[str, int] = {} - - self._lock = threading.RLock() - - # -- admission ------------------------------------------------------------ - - def submit(self, request: GenerationRequest) -> AdmissionDecision: - """Admit, queue, or reject one generation request (bounded admission). - - Order of checks: identity (duplicate) → hard feasibility (KV, scratch) → - capacity (free active slot vs bounded queue vs full). A full queue yields - :attr:`AdmissionReason.REJECTED_QUEUE_FULL`, the explicit backpressure - signal. - """ - with self._lock: - sid = request.session_id - if sid in self._active or sid in self._queued_ids: - return self._reject( - request, AdmissionReason.REJECTED_DUPLICATE, "already scheduled" - ) - - # Hard feasibility: a single session must be able to fit KV + scratch - # on its own; otherwise it can never run and is rejected up front - # rather than wedging the queue. - kv_need = self._kv_bytes_for(request) - if kv_need > self._budget.kv_budget_bytes: - return self._reject( - request, - AdmissionReason.REJECTED_KV_BUDGET, - f"needs {kv_need} KV bytes > budget " - f"{self._budget.kv_budget_bytes}", - ) - if self._budget.scratch_bytes_per_session > self._budget.scratch_budget_bytes: - return self._reject( - request, - AdmissionReason.REJECTED_SCRATCH_BUDGET, - "per-session scratch exceeds the scratch budget", - ) - - if self._has_capacity_locked(): - self._activate_locked(request) - return AdmissionDecision(sid, AdmissionReason.ADMITTED) - - if len(self._queue) < self._budget.max_queue_depth: - self._queue.append(request) - self._queued_ids.add(sid) - return AdmissionDecision(sid, AdmissionReason.QUEUED) - - return self._reject( - request, - AdmissionReason.REJECTED_QUEUE_FULL, - f"queue full at depth {self._budget.max_queue_depth}", - ) - - # -- cancellation / failure (DGR-013) ------------------------------------- - - def cancel( - self, - session_id: str, - *, - reason: DoneReason = DoneReason.CANCELLED, - detail: str = "", - ) -> bool: - """Remove a session from the scheduler, releasing its KV and queue slot. - - Cancellation is bounded and explicit: if the session is *queued* it is - dropped from the bounded queue (its queued buffer is released); if it is - *active* its KV is released through the engine and it is moved to the done - set with a non-completed :class:`DoneReason` so billing/work records never - count it as completed work. Returns ``True`` if a live (queued or active) - session was found. Idempotent: cancelling an unknown or already-finished - session returns ``False`` and mutates nothing. - - ``reason`` must be a terminal non-completed reason (``CANCELLED`` for an - explicit client cancel, ``FAILED`` for deadline/heartbeat/worker loss). - """ - if reason not in (DoneReason.CANCELLED, DoneReason.FAILED): - raise SchedulerError( - "cancel reason must be CANCELLED or FAILED, not %r" % (reason,) - ) - with self._lock: - # Queued but not yet running: drop it from the bounded queue so the - # backpressure boundary recovers and no execution slot is ever taken. - if session_id in self._queued_ids: - self._queued_ids.discard(session_id) - dropped = next( - (r for r in self._queue if r.session_id == session_id), None - ) - self._queue = deque( - r for r in self._queue if r.session_id != session_id - ) - self._finalize_cancelled_locked(session_id, reason, dropped) - return True - - state = self._active.get(session_id) - if state is None: - return False - # Active: release the KV context on this shard, then record the - # terminal reason. release() is idempotent, so a concurrent reap or a - # prior cache-miss release cannot double-free. - self._engine.release(state.session_id, state.route_epoch) - del self._active[session_id] - state.phase = Phase.DONE - state.done_reason = reason - self._done[session_id] = state - self._count_terminal_locked(reason) - return True - - def _finalize_cancelled_locked( - self, - session_id: str, - reason: DoneReason, - request: GenerationRequest | None, - ) -> None: - # A queued session has no live KV and no committed tokens yet; record a - # terminal state (with its original request when known) so results() and - # telemetry account for it distinctly from completed work. - if request is not None: - state = SessionState( - request=request, phase=Phase.DONE, done_reason=reason - ) - self._done[session_id] = state - self._count_terminal_locked(reason) - - def _count_terminal_locked(self, reason: DoneReason) -> None: - if reason is DoneReason.CANCELLED: - self._cancelled += 1 - elif reason is DoneReason.FAILED: - self._failed += 1 - - # -- scheduling ----------------------------------------------------------- - - def run_tick(self) -> TickReport: - """Run one scheduling step: admit, decode-batch, bounded-prefill, reap.""" - with self._lock: - self._ticks += 1 - admitted = self._admit_from_queue_locked() - decoded, occupancy = self._run_decode_batch_locked() - prefilled = self._run_prefill_locked() - completed = self._reap_locked() - # A reap frees slots; pull more work forward so the next caller sees a - # full node rather than an artificially idle one. - admitted = admitted + self._admit_from_queue_locked() - return TickReport( - prefilled=tuple(prefilled), - decoded=tuple(decoded), - batch_occupancy=occupancy, - completed=tuple(completed), - admitted_from_queue=tuple(admitted), - ) - - def run_to_completion(self, *, max_ticks: int | None = None) -> dict[str, list[int]]: - """Drive ticks until every submitted session finishes; return outputs. - - Returns ``{session_id: generated_token_ids}`` for every session that ran. - ``max_ticks`` is a safety bound; exceeding it raises rather than looping - forever on a misconfiguration. - """ - limit = max_ticks if max_ticks is not None else self._default_tick_limit() - for _ in range(limit): - with self._lock: - if not self._active and not self._queue: - break - self.run_tick() - else: - with self._lock: - pending = len(self._active) + len(self._queue) - if pending: - raise SchedulerError( - f"run_to_completion exceeded {limit} ticks with {pending} " - "sessions still pending; check budgets and token counts" - ) - with self._lock: - return {sid: list(s.generated) for sid, s in self._done.items()} - - # -- results -------------------------------------------------------------- - - def outputs(self) -> dict[str, list[int]]: - """Generated tokens for every completed session so far.""" - with self._lock: - return {sid: list(s.generated) for sid, s in self._done.items()} - - def session_result(self, session_id: str) -> SessionState | None: - with self._lock: - return self._done.get(session_id) or self._active.get(session_id) - - # -- telemetry ------------------------------------------------------------ - - def telemetry(self, *, now: float | None = None) -> SchedulerTelemetry: - """Capability snapshot: sessions, queue, batch, KV/scratch pressure, rates.""" - with self._lock: - observed = self._clock() if now is None else now - elapsed = max(observed - self._started, 1e-9) - kv_total = self._engine_kv_bytes() - kv_budget = self._budget.kv_budget_bytes - scratch_used = len(self._active) * self._budget.scratch_bytes_per_session - scratch_budget = self._budget.scratch_budget_bytes - avg_occupancy = ( - self._batch_sum / self._batch_count if self._batch_count else 0.0 - ) - return SchedulerTelemetry( - active_sessions=len(self._active), - queue_depth=len(self._queue), - batch_occupancy_last=self._batch_occupancy_last, - batch_occupancy_avg=avg_occupancy, - batch_occupancy_max=self._batch_occupancy_max, - weight_bytes=self._budget.weight_bytes, - kv_total_bytes=kv_total, - kv_budget_bytes=kv_budget, - kv_pressure=kv_total / kv_budget if kv_budget else 0.0, - scratch_used_bytes=scratch_used, - scratch_budget_bytes=scratch_budget, - scratch_pressure=scratch_used / scratch_budget if scratch_budget else 0.0, - prefill_tokens_total=self._prefill_tokens, - decode_tokens_total=self._decode_tokens, - prefill_tokens_per_sec=self._prefill_tokens / elapsed, - decode_tokens_per_sec=self._decode_tokens / elapsed, - rejected_admissions_total=self._rejected, - rejected_by_reason=dict(self._rejected_by_reason), - completed_sessions=self._completed, - cancelled_sessions=self._cancelled, - failed_sessions=self._failed, - ticks=self._ticks, - ) - - # -- internals ------------------------------------------------------------ - - def _reject( - self, request: GenerationRequest, reason: AdmissionReason, detail: str - ) -> AdmissionDecision: - self._rejected += 1 - self._rejected_by_reason[reason.value] = ( - self._rejected_by_reason.get(reason.value, 0) + 1 - ) - return AdmissionDecision(request.session_id, reason, detail) - - def _kv_bytes_for(self, request: GenerationRequest) -> int: - # bytes_per_token is defined by the loaded shard's KV recipe; the whole - # generation occupies prompt + (new-1) positions at its peak. - per_token = self._manager().recipe.bytes_per_token() - return request.final_seq_len * per_token - - def _manager(self) -> HotKvStateManager: - manager = getattr(self._engine, "_manager", None) - if manager is None: - raise SchedulerError( - "engine does not expose a Hot KV State manager for budget accounting" - ) - return manager - - def _engine_kv_bytes(self) -> int: - manager = getattr(self._engine, "_manager", None) - return int(manager.total_bytes) if manager is not None else 0 - - def _has_capacity_locked(self) -> bool: - return len(self._active) < self._budget.effective_active_cap - - def _activate_locked(self, request: GenerationRequest) -> None: - if self._fingerprint != str(self._engine.recipe_fingerprint()): - # The loaded shard's recipe must not change under the scheduler. - raise SchedulerError("engine recipe fingerprint changed mid-flight") - self._active[request.session_id] = SessionState(request=request) - - def _admit_from_queue_locked(self) -> list[str]: - admitted: list[str] = [] - while self._queue and self._has_capacity_locked(): - request = self._queue.popleft() - self._queued_ids.discard(request.session_id) - self._activate_locked(request) - admitted.append(request.session_id) - return admitted - - def _run_decode_batch_locked(self) -> tuple[list[str], int]: - decoding = [ - s for s in self._active.values() if s.phase is Phase.DECODING - ] - if not decoding: - self._batch_occupancy_last = 0 - return [], 0 - batch = decoding[: self._budget.max_batch_size] - items = [ - DecodeItem(s.session_id, s.route_epoch, s.last_token) for s in batch - ] - results = self._engine.decode_batch(items) - if len(results) != len(batch): - raise SchedulerError( - "engine returned %d results for a batch of %d" - % (len(results), len(batch)) - ) - decoded: list[str] = [] - for state, result in zip(batch, results): - if isinstance(result, CacheMiss): - state.phase = Phase.DONE - state.done_reason = DoneReason.CACHE_MISS - state.cache_miss = result - continue - state.generated.append(result.token_id) - self._decode_tokens += 1 - decoded.append(state.session_id) - if state.remaining <= 0: - state.phase = Phase.DONE - state.done_reason = DoneReason.COMPLETED - occupancy = len(batch) - self._batch_occupancy_last = occupancy - self._batch_occupancy_max = max(self._batch_occupancy_max, occupancy) - self._batch_sum += occupancy - self._batch_count += 1 - return decoded, occupancy - - def _run_prefill_locked(self) -> list[str]: - pending = [ - s for s in self._active.values() if s.phase is Phase.PENDING_PREFILL - ] - prefilled: list[str] = [] - spent = 0 - for state in pending: - # Always allow the first prefill of the tick (progress guarantee), - # then honour the per-tick token budget so prefill can't monopolise. - if prefilled and spent + state.request.prompt_len > self._budget.max_prefill_tokens_per_tick: - break - result = self._engine.prefill( - state.session_id, - state.route_epoch, - state.request.prompt_token_ids, - ) - state.generated.append(result.token_id) - self._prefill_tokens += state.request.prompt_len - spent += state.request.prompt_len - prefilled.append(state.session_id) - if state.remaining <= 0: - state.phase = Phase.DONE - state.done_reason = DoneReason.COMPLETED - else: - state.phase = Phase.DECODING - return prefilled - - def _reap_locked(self) -> list[str]: - completed: list[str] = [] - for sid, state in list(self._active.items()): - if state.phase is not Phase.DONE: - continue - self._engine.release(state.session_id, state.route_epoch) - del self._active[sid] - self._done[sid] = state - if state.done_reason is DoneReason.COMPLETED: - self._completed += 1 - completed.append(sid) - return completed - - def _default_tick_limit(self) -> int: - # Generous upper bound: worst case is fully serialized (one session at a - # time, one token per tick) plus slack for admission ticks. - pending_tokens = sum( - s.request.max_new_tokens for s in self._active.values() - ) + sum(r.max_new_tokens for r in self._queue) - return 8 * (pending_tokens + len(self._active) + len(self._queue) + 1) - - -# --------------------------------------------------------------------------- # -# Concurrency 1/2/4/8 sweep (deterministic saturation report). -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class ConcurrencyResult: - """One concurrency level's deterministic scheduling result.""" - - concurrency: int - ticks: int - decode_batches: int - decode_tokens: int - prefill_tokens: int - avg_batch_occupancy: float - max_batch_occupancy: int - tokens_per_tick: float - peak_kv_bytes: int - rejected_admissions: int - cache_misses: int - - def to_dict(self) -> dict: - return { - "concurrency": self.concurrency, - "ticks": self.ticks, - "decode_batches": self.decode_batches, - "decode_tokens": self.decode_tokens, - "prefill_tokens": self.prefill_tokens, - "avg_batch_occupancy": round(self.avg_batch_occupancy, 4), - "max_batch_occupancy": self.max_batch_occupancy, - "tokens_per_tick": round(self.tokens_per_tick, 4), - "peak_kv_bytes": self.peak_kv_bytes, - "rejected_admissions": self.rejected_admissions, - "cache_misses": self.cache_misses, - } - - -@dataclass(frozen=True) -class ConcurrencySweep: - """The full 1/2/4/8 report plus the derived saturation point.""" - - results: tuple[ConcurrencyResult, ...] - saturation_concurrency: int - corruption_free: bool - reference_outputs: Mapping[str, tuple[int, ...]] - - def to_dict(self) -> dict: - return { - "schema_version": 1, - "results": [r.to_dict() for r in self.results], - "saturation_concurrency": self.saturation_concurrency, - "corruption_free": self.corruption_free, - "reference_outputs": { - sid: list(tokens) for sid, tokens in self.reference_outputs.items() - }, - } - - -def run_concurrency_sweep( - engine_factory: Callable[[], Any], - requests: Iterable[GenerationRequest], - *, - concurrency_levels: Sequence[int] = (1, 2, 4, 8), - budget_factory: Callable[[int], NodeBudget] | None = None, - saturation_tolerance: float = 1e-9, -) -> ConcurrencySweep: - """Run the same jobs at each concurrency level and report saturation. - - For every level, a fresh engine (fresh KV manager) runs all ``requests`` with - ``max_active_sessions`` and ``max_batch_size`` capped to that level. The - concurrency-1 run is the serialized reference; every higher level must produce - the **byte-identical** per-session token stream (greedy sampling over isolated - KV is order-independent), which is the "no cross-session corruption" proof. - - Saturation is the smallest level at which average batch occupancy stops rising - (more slots no longer pack more sessions per batch) — i.e. the node is fully - utilised and adding concurrency yields no further batching gain for this load. - """ - requests = list(requests) - if not requests: - raise SchedulerError("run_concurrency_sweep needs at least one request") - levels = sorted({int(level) for level in concurrency_levels}) - if any(level < 1 for level in levels): - raise SchedulerError("concurrency levels must be >= 1") - - def default_budget(level: int) -> NodeBudget: - # Budgets sized so the load never evicts: correctness of the sweep must not - # depend on eviction. KV holds every session's whole generation at once. - engine = engine_factory() - per_token = getattr(engine, "_manager").recipe.bytes_per_token() - total_kv = sum(r.final_seq_len for r in requests) * per_token - return NodeBudget( - kv_budget_bytes=max(total_kv, per_token), - scratch_bytes_per_session=1, - scratch_budget_bytes=max(1, level), - max_active_sessions=level, - max_queue_depth=len(requests), - max_batch_size=level, - max_prefill_tokens_per_tick=max(r.prompt_len for r in requests), - ) - - budget_for = budget_factory or default_budget - results: list[ConcurrencyResult] = [] - reference: dict[str, tuple[int, ...]] | None = None - corruption_free = True - - for level in levels: - engine = engine_factory() - scheduler = ContinuousBatchScheduler(engine, budget_for(level)) - cache_misses = 0 - peak_kv = 0 - decode_batches = 0 - for request in requests: - decision = scheduler.submit(request) - if not decision.accepted: - raise SchedulerError( - f"sweep request {request.session_id} was rejected at " - f"concurrency {level}: {decision}" - ) - # Drive ticks manually so we can sample peak KV and count decode batches. - limit = scheduler._default_tick_limit() - for _ in range(limit): - if not scheduler._active and not scheduler._queue: - break - report = scheduler.run_tick() - if report.batch_occupancy > 0: - decode_batches += 1 - peak_kv = max(peak_kv, scheduler.telemetry().kv_total_bytes) - outputs = {sid: tuple(tokens) for sid, tokens in scheduler.outputs().items()} - for state in ( - scheduler.session_result(r.session_id) for r in requests - ): - if state is not None and state.done_reason is DoneReason.CACHE_MISS: - cache_misses += 1 - - if reference is None: - reference = outputs - elif outputs != reference: - corruption_free = False - - telem = scheduler.telemetry() - results.append( - ConcurrencyResult( - concurrency=level, - ticks=telem.ticks, - decode_batches=decode_batches, - decode_tokens=telem.decode_tokens_total, - prefill_tokens=telem.prefill_tokens_total, - avg_batch_occupancy=telem.batch_occupancy_avg, - max_batch_occupancy=telem.batch_occupancy_max, - tokens_per_tick=(telem.decode_tokens_total + telem.prefill_tokens_total) - / max(1, telem.ticks), - peak_kv_bytes=peak_kv, - rejected_admissions=telem.rejected_admissions_total, - cache_misses=cache_misses, - ) - ) - - saturation = _saturation_point(results, saturation_tolerance) - assert reference is not None - return ConcurrencySweep( - results=tuple(results), - saturation_concurrency=saturation, - corruption_free=corruption_free, - reference_outputs=reference, - ) - - -def _saturation_point( - results: Sequence[ConcurrencyResult], tolerance: float -) -> int: - """Smallest concurrency where average batch occupancy stops increasing.""" - if not results: - return 0 - best = results[0] - for current in results[1:]: - if current.avg_batch_occupancy <= best.avg_batch_occupancy + tolerance: - return best.concurrency - best = current - return results[-1].concurrency diff --git a/packages/node/meshnet_node/boundary_adapter.py b/packages/node/meshnet_node/boundary_adapter.py deleted file mode 100644 index c7b2c06..0000000 --- a/packages/node/meshnet_node/boundary_adapter.py +++ /dev/null @@ -1,484 +0,0 @@ -"""Architecture-defined boundary input/output for distributed Shards (DGR-006). - -A public-network Shard is a contiguous range of transformer layers (RALPH runtime -decision #1). For disjoint processes to reproduce whole-model execution, every -Shard must agree on *exactly* what boundary state it consumes and emits: - -* The **head** owns token embedding: it accepts token IDs and turns them into the - residual stream. No other Shard may embed tokens. -* **Middle and tail** Shards bypass token embedding entirely; they accept the named - boundary bundle (the residual stream handed over by the previous range). -* A **non-tail** Shard emits the *unnormalized* architecture-defined residual / - boundary — before the final norm, before the LM head, and before any tail-only - row pruning — so the next range sees precisely the state the whole model would - have at that layer index. -* The **tail** owns the final norm + LM head and turns the residual into logits or - a sampled token through an explicit sampling contract. - -This module is deliberately backend-agnostic. It enforces the boundary *contract* -and defers the arithmetic to a ``ShardComputation`` (a duck-typed object exposing -``embed_tokens`` / ``run_layers`` / ``final_norm`` / ``lm_head``). The pinned -llama.cpp worker (DGR-008) and the reference PyTorch backend both satisfy that -protocol, and the numpy reference model in the tests proves whole-model versus -two-range parity without any download, GPU, or API credit. - -The adapter **fails closed** for uncertified architectures: only architectures -that have passed real certification (dense Llama-family first, per RALPH runtime -decision #13) are accepted. Everything else raises rather than silently guessing a -tensor layout — Qwen3/Qwen3-MoE stays registered-but-dark until DGR-015 certifies -its own adapter. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from enum import Enum -from typing import Any - -import numpy as np - -# The boundary bundle wire schema version. This is the ``boundary_schema_version`` -# carried by ``runtime_recipe.RuntimeRecipeIdentity``; a receiver refuses a bundle -# whose schema it does not implement (forward/backward compatibility is a routing -# concern, not a silent reinterpretation). -BOUNDARY_SCHEMA_VERSION = 1 - - -class BoundaryAdapterError(RuntimeError): - """Base class for boundary-contract violations.""" - - -class UncertifiedArchitectureError(BoundaryAdapterError): - """Raised when a boundary adapter is requested for an uncertified architecture. - - Failing closed here is a safety property: an unknown architecture has an - unknown tensor layout, so guessing where the residual boundary lives would - silently corrupt distributed output. The architecture must pass real - certification first. - """ - - -class BoundaryContractError(BoundaryAdapterError): - """Raised when a Shard is fed the wrong boundary input for its role. - - Examples: a head handed a residual bundle instead of token IDs, a middle - Shard handed token IDs it must not embed, or a boundary bundle whose - architecture / schema / seam layer does not match the receiving range. - """ - - -@dataclass(frozen=True) -class ArchitectureBoundary: - """The architecture-defined boundary description for one certified adapter. - - These fields are what makes the boundary *architecture-defined* rather than a - hardcoded assumption: the residual tensor name, whether the tail normalizes - before the LM head, and whether row pruning is a tail-only concern all come - from here. - """ - - adapter: str - boundary_tensor_name: str - boundary_schema_version: int - normalizes_before_head: bool - prunes_rows_at_tail: bool - - -# Certified architectures only. Dense Llama-family is first (RALPH runtime decision -# #13 + native discipline). Aliases map the many spellings a runtime recipe / -# GGUF / HF config may use onto the single canonical adapter id. Anything not in -# this table fails closed. -_DENSE_LLAMA = ArchitectureBoundary( - adapter="dense-llama", - boundary_tensor_name="residual_stream", - boundary_schema_version=BOUNDARY_SCHEMA_VERSION, - normalizes_before_head=True, - prunes_rows_at_tail=True, -) - -_CERTIFIED_ARCHITECTURES: dict[str, ArchitectureBoundary] = { - "dense-llama": _DENSE_LLAMA, - "dense_llama": _DENSE_LLAMA, - "llama": _DENSE_LLAMA, - "llamaforcausallm": _DENSE_LLAMA, - "llamamodel": _DENSE_LLAMA, -} - - -def certified_architecture(name: Any) -> ArchitectureBoundary: - """Return the certified boundary description for ``name`` or fail closed. - - ``name`` may be the canonical adapter id (``dense-llama``), an HF architecture - class (``LlamaForCausalLM``), or a GGUF/config ``model_type`` (``llama``). - Uncertified architectures raise ``UncertifiedArchitectureError``. - """ - if not isinstance(name, str) or not name.strip(): - raise UncertifiedArchitectureError( - "architecture adapter must be a non-empty string; " - "the boundary adapter refuses to guess a tensor layout" - ) - key = name.strip().lower() - boundary = _CERTIFIED_ARCHITECTURES.get(key) - if boundary is None: - raise UncertifiedArchitectureError( - f"architecture {name!r} is not certified for the boundary adapter; " - f"certified adapters: {sorted(set(v.adapter for v in _CERTIFIED_ARCHITECTURES.values()))}. " - "Uncertified architectures stay registered-but-dark until real " - "certification passes." - ) - return boundary - - -def is_certified_architecture(name: Any) -> bool: - """Return True when ``name`` maps to a certified boundary adapter.""" - try: - certified_architecture(name) - except UncertifiedArchitectureError: - return False - return True - - -class ShardRole(str, Enum): - """Where a contiguous layer range sits in the whole model.""" - - HEAD = "head" - MIDDLE = "middle" - TAIL = "tail" - FULL = "full" - - @property - def owns_embedding(self) -> bool: - return self in (ShardRole.HEAD, ShardRole.FULL) - - @property - def owns_final_head(self) -> bool: - return self in (ShardRole.TAIL, ShardRole.FULL) - - -def role_for_range(start_layer: int, end_layer: int, total_layers: int) -> ShardRole: - """Classify a contiguous inclusive layer range within a model of ``total_layers``.""" - if total_layers <= 0: - raise ValueError("total_layers must be positive") - if start_layer < 0 or end_layer < start_layer: - raise ValueError("require 0 <= start_layer <= end_layer") - if end_layer > total_layers - 1: - raise ValueError( - f"end_layer {end_layer} exceeds last layer index {total_layers - 1}" - ) - is_head = start_layer == 0 - is_tail = end_layer >= total_layers - 1 - if is_head and is_tail: - return ShardRole.FULL - if is_head: - return ShardRole.HEAD - if is_tail: - return ShardRole.TAIL - return ShardRole.MIDDLE - - -@dataclass(frozen=True) -class BoundaryBundle: - """The versioned named-tensor bundle handed between adjacent Shard ranges. - - ``residual`` is the *unnormalized* architecture-defined residual stream with - every position row intact (no tail-only pruning). ``next_layer`` is the layer - index the receiving range must start at — it is the overlap-safe effective - start of the seam, so a receiver can reject a bundle meant for a different cut. - """ - - architecture_adapter: str - schema_version: int - tensor_name: str - residual: np.ndarray - positions: np.ndarray - next_layer: int - normalized: bool = False - - def named_tensor_fields(self) -> dict[str, Any]: - """Return the wire-shaped description of the residual tensor. - - These are exactly the fields the DGR-002 ``NamedTensor`` carries (name, - shape, dtype, byte order, raw bytes), so a worker can serialize this - bundle into the gRPC protobuf without re-deriving them. - """ - residual = np.ascontiguousarray(self.residual) - return { - "name": self.tensor_name, - "shape": list(residual.shape), - "dtype": residual.dtype.name, - "byte_order": _byte_order(residual.dtype), - "data": residual.tobytes(), - } - - def pack(self) -> dict[str, Any]: - """Serialize the bundle to a transport-agnostic dict (proves the seam). - - The residual and positions are carried as raw little/big-endian bytes plus - shape/dtype so that a truly disjoint process can reconstruct the exact - array — this is what lets two OS processes reproduce whole-model math. - """ - residual = np.ascontiguousarray(self.residual) - positions = np.ascontiguousarray(self.positions) - return { - "architecture_adapter": self.architecture_adapter, - "schema_version": self.schema_version, - "tensor_name": self.tensor_name, - "next_layer": self.next_layer, - "normalized": self.normalized, - "residual": { - "shape": list(residual.shape), - "dtype": residual.dtype.str, - "data": residual.tobytes(), - }, - "positions": { - "shape": list(positions.shape), - "dtype": positions.dtype.str, - "data": positions.tobytes(), - }, - } - - @classmethod - def unpack(cls, payload: dict[str, Any]) -> "BoundaryBundle": - """Reconstruct a bundle produced by :meth:`pack`.""" - residual = _array_from_wire(payload["residual"]) - positions = _array_from_wire(payload["positions"]) - return cls( - architecture_adapter=payload["architecture_adapter"], - schema_version=int(payload["schema_version"]), - tensor_name=payload["tensor_name"], - residual=residual, - positions=positions, - next_layer=int(payload["next_layer"]), - normalized=bool(payload.get("normalized", False)), - ) - - -@dataclass(frozen=True) -class SamplingContract: - """Explicit contract for turning tail logits into a token. - - The tail never hides the sampling decision inside the adapter: the contract is - a first-class value so the head/route can reproduce it and so greedy decoding - is deterministic by construction. Only greedy is certified here; temperature / - top-p are declared but must be requested explicitly and are out of scope for - the deterministic parity gate. - """ - - mode: str = "greedy" - temperature: float = 1.0 - top_p: float = 1.0 - - def __post_init__(self) -> None: - if self.mode not in ("greedy",): - raise BoundaryContractError( - f"sampling mode {self.mode!r} is not certified; only 'greedy' is " - "deterministic and supported by the boundary adapter today" - ) - - @classmethod - def greedy(cls) -> "SamplingContract": - return cls(mode="greedy") - - def sample(self, last_logits: np.ndarray) -> int: - """Return the next token id from the final-position logits row.""" - logits = np.asarray(last_logits) - if logits.ndim == 2: - # (batch, vocab) — parity harness uses batch size 1. - logits = logits[0] - if logits.ndim != 1: - raise BoundaryContractError( - "sampling expects the pruned final-position logits row" - ) - return int(np.argmax(logits)) - - -@dataclass(frozen=True) -class TailOutput: - """What a tail Shard emits: the sampled token plus the pruned logits row.""" - - token_id: int - logits: np.ndarray - sampling: SamplingContract - - -@dataclass -class BoundaryAdapter: - """Enforces the architecture-defined boundary contract for one Shard range. - - Construction fails closed for uncertified architectures. The adapter derives - the Shard's role from its range and drives a duck-typed ``ShardComputation``. - """ - - computation: Any - sampling: SamplingContract = field(default_factory=SamplingContract.greedy) - architecture: ArchitectureBoundary = field(init=False) - role: ShardRole = field(init=False) - start_layer: int = field(init=False) - end_layer: int = field(init=False) - total_layers: int = field(init=False) - - def __post_init__(self) -> None: - arch_name = getattr(self.computation, "architecture_adapter", None) - self.architecture = certified_architecture(arch_name) - self.start_layer = int(getattr(self.computation, "start_layer")) - self.end_layer = int(getattr(self.computation, "end_layer")) - self.total_layers = int(getattr(self.computation, "total_layers")) - self.role = role_for_range( - self.start_layer, self.end_layer, self.total_layers - ) - - @property - def is_head(self) -> bool: - return self.role.owns_embedding - - @property - def is_tail(self) -> bool: - return self.role.owns_final_head - - def forward( - self, - *, - token_ids: Any | None = None, - boundary: BoundaryBundle | None = None, - ) -> BoundaryBundle | TailOutput: - """Run one prefill/decode pass for this range and emit its boundary output. - - Head/full ranges require ``token_ids``; middle/tail ranges require the - ``boundary`` bundle. Non-tail ranges return a :class:`BoundaryBundle`; - tail/full ranges return a :class:`TailOutput` through the sampling - contract. - """ - hidden, positions = self._ingest(token_ids, boundary) - hidden = self.computation.run_layers(hidden, positions=positions) - if self.is_tail: - return self._emit_tail(hidden) - return self._emit_boundary(hidden, positions) - - # -- input side ----------------------------------------------------------- - - def _ingest( - self, token_ids: Any | None, boundary: BoundaryBundle | None - ) -> tuple[np.ndarray, np.ndarray]: - if self.role.owns_embedding: - return self._ingest_tokens(token_ids, boundary) - return self._ingest_boundary(token_ids, boundary) - - def _ingest_tokens( - self, token_ids: Any | None, boundary: BoundaryBundle | None - ) -> tuple[np.ndarray, np.ndarray]: - if token_ids is None: - raise BoundaryContractError( - "the head owns token embedding and must receive token IDs" - ) - if boundary is not None: - raise BoundaryContractError( - "the head owns token embedding; it must not receive a boundary " - "bundle from an upstream range" - ) - ids = np.asarray(token_ids) - if ids.ndim == 1: - ids = ids[None, :] - if ids.ndim != 2: - raise BoundaryContractError("token IDs must be (seq,) or (batch, seq)") - hidden = np.asarray(self.computation.embed_tokens(ids)) - positions = np.broadcast_to( - np.arange(ids.shape[1], dtype=np.int64), ids.shape - ).copy() - return hidden, positions - - def _ingest_boundary( - self, token_ids: Any | None, boundary: BoundaryBundle | None - ) -> tuple[np.ndarray, np.ndarray]: - if token_ids is not None: - raise BoundaryContractError( - "middle/tail Shards bypass token embedding; they must not receive " - "token IDs" - ) - if boundary is None: - raise BoundaryContractError( - "middle/tail Shards must receive the named boundary bundle" - ) - self._check_boundary(boundary) - return np.asarray(boundary.residual), np.asarray(boundary.positions) - - def _check_boundary(self, boundary: BoundaryBundle) -> None: - if certified_architecture(boundary.architecture_adapter) is not self.architecture: - raise BoundaryContractError( - f"boundary bundle architecture {boundary.architecture_adapter!r} " - f"does not match this Shard's adapter {self.architecture.adapter!r}" - ) - if boundary.schema_version != self.architecture.boundary_schema_version: - raise BoundaryContractError( - f"boundary schema v{boundary.schema_version} is not supported by " - f"this Shard (expects v{self.architecture.boundary_schema_version})" - ) - if boundary.tensor_name != self.architecture.boundary_tensor_name: - raise BoundaryContractError( - f"boundary tensor {boundary.tensor_name!r} is not the " - f"architecture-defined {self.architecture.boundary_tensor_name!r}" - ) - if boundary.normalized: - raise BoundaryContractError( - "boundary bundle is normalized; a Shard range must receive the " - "UNNORMALIZED architecture-defined residual" - ) - if boundary.next_layer != self.start_layer: - raise BoundaryContractError( - f"boundary hands over at layer {boundary.next_layer} but this " - f"Shard starts at layer {self.start_layer}" - ) - - # -- output side ---------------------------------------------------------- - - def _emit_boundary( - self, hidden: np.ndarray, positions: np.ndarray - ) -> BoundaryBundle: - # A non-tail Shard emits the unnormalized residual with every position row - # intact: no final norm, no LM head, no tail-only row pruning. next_layer - # is the receiver's overlap-safe effective start. - return BoundaryBundle( - architecture_adapter=self.architecture.adapter, - schema_version=self.architecture.boundary_schema_version, - tensor_name=self.architecture.boundary_tensor_name, - residual=np.asarray(hidden), - positions=np.asarray(positions), - next_layer=self.end_layer + 1, - normalized=False, - ) - - def _emit_tail(self, hidden: np.ndarray) -> TailOutput: - hidden = np.asarray(hidden) - # Tail-only row pruning: only the final position is needed to sample the - # next token, so the LM head runs on the pruned row. A non-tail Shard is - # forbidden from doing this (it must forward every row). - if self.architecture.prunes_rows_at_tail: - last_hidden = hidden[:, -1:, :] - else: # pragma: no cover - no certified architecture takes this path yet - last_hidden = hidden - if self.architecture.normalizes_before_head: - last_hidden = np.asarray(self.computation.final_norm(last_hidden)) - logits = np.asarray(self.computation.lm_head(last_hidden)) - last_logits = logits[:, -1, :] - token_id = self.sampling.sample(last_logits) - return TailOutput( - token_id=token_id, logits=last_logits, sampling=self.sampling - ) - - -def _byte_order(dtype: np.dtype) -> str: - order = dtype.byteorder - if order == "<": - return "little" - if order == ">": - return "big" - # '=' native, '|' not applicable (single byte) - import sys - - return sys.byteorder if order in ("=", "|") else "little" - - -def _array_from_wire(field_payload: dict[str, Any]) -> np.ndarray: - array = np.frombuffer( - field_payload["data"], dtype=np.dtype(field_payload["dtype"]) - ) - return array.reshape(field_payload["shape"]).copy() diff --git a/packages/node/meshnet_node/capability.py b/packages/node/meshnet_node/capability.py index c7b2e82..eac587b 100644 --- a/packages/node/meshnet_node/capability.py +++ b/packages/node/meshnet_node/capability.py @@ -20,16 +20,6 @@ import time from dataclasses import dataclass, field from typing import Any, Mapping -from . import __version__ as _PACKAGE_VERSION -from .runtime_recipe import ( - ArtifactIdentity, - RuntimeRecipeIdentity, - build_artifact_identity, - build_runtime_recipe_identity, - compatibility_fingerprint, - fingerprint_payload, -) - # Layout of the serialized report. Bump when the JSON shape changes. CAPABILITY_SCHEMA_VERSION = 1 @@ -182,14 +172,6 @@ def _optional_text(value: Any, field_name: str) -> str | None: return _require_text(value, field_name) -def _optional_bool(value: Any, field_name: str) -> bool: - if value is None: - return False - if isinstance(value, bool): - return value - raise CapabilityReportError(f"{field_name!r} must be a boolean") - - def _require_int(value: Any, field_name: str, minimum: int) -> int: if isinstance(value, bool) or not isinstance(value, int): raise CapabilityReportError(f"{field_name!r} must be an integer") @@ -236,8 +218,6 @@ class ShardRange: start: int end: int - owns_embedding: bool = False - owns_final_head: bool = False def __post_init__(self) -> None: _require_int(self.start, "shard.start", 0) @@ -246,18 +226,9 @@ class ShardRange: raise CapabilityReportError( f"'shard.end' ({self.end}) must be >= 'shard.start' ({self.start})" ) - if not isinstance(self.owns_embedding, bool): - raise CapabilityReportError("'shard.owns_embedding' must be a boolean") - if not isinstance(self.owns_final_head, bool): - raise CapabilityReportError("'shard.owns_final_head' must be a boolean") def to_dict(self) -> dict: - return { - "start": self.start, - "end": self.end, - "owns_embedding": self.owns_embedding, - "owns_final_head": self.owns_final_head, - } + return {"start": self.start, "end": self.end} @classmethod def from_dict(cls, data: Any) -> ShardRange: @@ -265,12 +236,6 @@ class ShardRange: return cls( start=_require_int(doc.get("start"), "shard.start", 0), end=_require_int(doc.get("end"), "shard.end", 0), - owns_embedding=_optional_bool( - doc.get("owns_embedding"), "shard.owns_embedding" - ), - owns_final_head=_optional_bool( - doc.get("owns_final_head"), "shard.owns_final_head" - ), ) @@ -371,8 +336,6 @@ class CapabilityReport: shard: ShardRange recipe: RecipeIdentity backend: BackendIdentity - artifact: ArtifactIdentity - runtime_recipe: RuntimeRecipeIdentity status: str validated_at: float duration_ms: int @@ -413,20 +376,6 @@ class CapabilityReport: self.backend.device, ) - @property - def compatibility_fingerprint(self) -> str: - """Stable compatibility digest over the full routable identity.""" - return compatibility_fingerprint( - fingerprint_payload( - model=self.model.to_dict(), - shard=self.shard.to_dict(), - recipe=self.recipe.to_dict(), - backend=self.backend.to_dict(), - artifact=self.artifact.to_dict(), - runtime_recipe=self.runtime_recipe.to_dict(), - ) - ) - def age_seconds(self, now: float | None = None) -> float: return max(0.0, (time.time() if now is None else now) - self.validated_at) @@ -437,9 +386,6 @@ class CapabilityReport: "shard": self.shard.to_dict(), "recipe": self.recipe.to_dict(), "backend": self.backend.to_dict(), - "artifact": self.artifact.to_dict(), - "runtime_recipe": self.runtime_recipe.to_dict(), - "compatibility_fingerprint": self.compatibility_fingerprint, "status": self.status, "validated_at": self.validated_at, "duration_ms": self.duration_ms, @@ -452,9 +398,6 @@ class CapabilityReport: @classmethod def from_dict(cls, data: Any) -> CapabilityReport: doc = _as_mapping(data, "report") - declared_compatibility_fingerprint = _optional_text( - doc.get("compatibility_fingerprint"), "compatibility_fingerprint" - ) if "schema_version" not in doc: raise CapabilityReportError( @@ -474,13 +417,7 @@ class CapabilityReport: ): raise CapabilityReportError("'validated_at' must be a Unix timestamp") - try: - artifact = ArtifactIdentity.from_dict(doc.get("artifact")) - runtime_recipe = RuntimeRecipeIdentity.from_dict(doc.get("runtime_recipe")) - except ValueError as exc: - raise CapabilityReportError(str(exc)) from exc - - report = cls( + return cls( schema_version=schema_version, model=ModelIdentity.from_dict(doc.get("model")), shard=ShardRange.from_dict(doc.get("shard")), @@ -490,18 +427,7 @@ class CapabilityReport: validated_at=float(validated_at), duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0), diagnostics=sanitize_diagnostics(doc.get("diagnostics")), - artifact=artifact, - runtime_recipe=runtime_recipe, ) - if ( - declared_compatibility_fingerprint is not None - and report.compatibility_fingerprint != declared_compatibility_fingerprint - ): - raise CapabilityReportError( - "report declares a compatibility fingerprint that does not match " - "its artifact/runtime recipe" - ) - return report @classmethod def from_json(cls, text: str) -> CapabilityReport: @@ -532,19 +458,6 @@ def build_capability_report( device_name: str | None = None, quantization: str | None = None, runtime: Mapping[str, str] | None = None, - artifact_hash: str | None = None, - runtime_recipe: RuntimeRecipeIdentity | None = None, - owns_embedding: bool = False, - owns_final_head: bool = False, - activation_dtype: Any = None, - compute_dtype: Any = None, - kv_dtype: Any = None, - kv_layout: str | None = None, - tokenizer_revision: str | None = None, - architecture_adapter: str | None = None, - boundary_schema_version: int = 1, - cache_layout: str | None = None, - recipe_params: Mapping[str, Any] | None = None, diagnostics: Any = None, validated_at: float | None = None, environ: Mapping[str, str] | None = None, @@ -555,62 +468,25 @@ def build_capability_report( or an already-computed ``sha256:…`` string. `validated_at` defaults to now, so callers that need determinism pass it explicitly. """ - model_identity = ModelIdentity( - model_id=model_id, - revision=revision, - config_fingerprint=config_fingerprint(model_config), - ) - shard = ShardRange( - start=shard_start, - end=shard_end, - owns_embedding=owns_embedding, - owns_final_head=owns_final_head, - ) - recipe_identity = RecipeIdentity( - recipe_id=recipe_id, - recipe_version=recipe_version, - catalogue_version=catalogue_version, - ) - backend_identity = BackendIdentity( - backend_id=backend_id, - device=device, - device_name=device_name, - quantization=quantization, - runtime=dict(runtime or {}), - ) - artifact = build_artifact_identity( - model_id=model_id, - revision=revision, - model_config=model_config, - artifact_hash=artifact_hash, - shard_start=shard_start, - shard_end=shard_end, - ) - if runtime_recipe is None: - runtime_recipe = build_runtime_recipe_identity( + return CapabilityReport( + model=ModelIdentity( model_id=model_id, revision=revision, - model_config=model_config, - recipe_params=recipe_params, - weight_quantization=quantization or "unknown", + config_fingerprint=config_fingerprint(model_config), + ), + shard=ShardRange(start=shard_start, end=shard_end), + recipe=RecipeIdentity( + recipe_id=recipe_id, + recipe_version=recipe_version, + catalogue_version=catalogue_version, + ), + backend=BackendIdentity( backend_id=backend_id, - runtime_version=_PACKAGE_VERSION, - activation_dtype=activation_dtype, - compute_dtype=compute_dtype, - kv_dtype=kv_dtype, - kv_layout=kv_layout, - tokenizer_revision=tokenizer_revision, - architecture_adapter=architecture_adapter, - boundary_schema_version=boundary_schema_version, - cache_layout=cache_layout, - ) - return CapabilityReport( - model=model_identity, - shard=shard, - recipe=recipe_identity, - backend=backend_identity, - artifact=artifact, - runtime_recipe=runtime_recipe, + device=device, + device_name=device_name, + quantization=quantization, + runtime=dict(runtime or {}), + ), status=status, validated_at=time.time() if validated_at is None else validated_at, duration_ms=duration_ms, diff --git a/packages/node/meshnet_node/doctor.py b/packages/node/meshnet_node/doctor.py index 2b33d58..5ceff88 100644 --- a/packages/node/meshnet_node/doctor.py +++ b/packages/node/meshnet_node/doctor.py @@ -36,8 +36,6 @@ from .capability import ( CapabilityReport, build_capability_report, ) -from . import __version__ as _PACKAGE_VERSION -from .runtime_recipe import build_runtime_recipe_identity from .recipe_manifest import ( DEFAULT_RECIPE_ID, Recipe, @@ -45,7 +43,6 @@ from .recipe_manifest import ( RecipeManifestError, load_recipe_manifest, ) -from .gguf_ownership import authoritative_dense_llama_ownership # The probe is deliberately tiny: enough tokens to drive every layer in the # shard once, small enough that `doctor` costs seconds beyond the model load. @@ -467,28 +464,10 @@ def _validate_recipe( duration_ms = int((time.monotonic() - started) * 1000) device = _backend_device(backend, selection) - ownership = authoritative_dense_llama_ownership(backend, selection) - runtime_recipe = build_runtime_recipe_identity( - model_id=selection.model_id, - revision=getattr(getattr(backend, "model", None), "revision", None), - model_config=_model_config(backend), - recipe_params=recipe.params, - weight_quantization=selection.quantization, - backend_id=recipe.backend_id, - runtime_version=_PACKAGE_VERSION, - activation_dtype="bfloat16", - compute_dtype=_backend_compute_dtype(backend), - kv_dtype=_backend_kv_dtype(backend), - kv_layout=_backend_kv_layout(backend), - tokenizer_revision=_backend_tokenizer_revision(backend, selection), - architecture_adapter=_backend_architecture_adapter(backend, recipe.backend_id), - boundary_schema_version=1, - cache_layout=_backend_cache_layout(backend, recipe.params), - ) report = build_capability_report( model_id=selection.model_id, - shard_start=ownership.start_layer, - shard_end=ownership.end_layer, + shard_start=selection.shard_start, + shard_end=selection.shard_end, recipe_id=recipe.id, recipe_version=recipe.version, catalogue_version=manifest.catalogue_version, @@ -498,9 +477,6 @@ def _validate_recipe( quantization=selection.quantization, runtime=_runtime_versions(), model_config=_model_config(backend), - runtime_recipe=runtime_recipe, - owns_embedding=ownership.owns_embedding, - owns_final_head=ownership.owns_final_head, status=STATUS_FAILED if category else STATUS_PASSED, duration_ms=duration_ms, diagnostics=[d for d in diagnostics if d] or None, @@ -592,65 +568,6 @@ def _runtime_versions() -> dict[str, str]: return versions -def _backend_compute_dtype(backend: Any) -> str: - config = getattr(getattr(backend, "model", None), "config", None) - for candidate in (config, getattr(config, "text_config", None)): - if candidate is None: - continue - for attr in ("dtype", "torch_dtype"): - value = getattr(candidate, attr, None) - if value is None: - continue - return str(value).removeprefix("torch.") - return "bfloat16" - - -def _backend_kv_dtype(backend: Any) -> str: - return _backend_compute_dtype(backend) - - -def _backend_kv_layout(backend: Any) -> str: - return "session-cache" if getattr(backend, "supports_kv_cache", False) else "stateless" - - -def _backend_tokenizer_revision(backend: Any, selection: DoctorSelection) -> str: - model = getattr(backend, "model", None) - revision = getattr(model, "revision", None) - if isinstance(revision, str) and revision.strip(): - return revision - return selection.model_id - - -def _backend_architecture_adapter(backend: Any, default: str) -> str: - config = getattr(getattr(backend, "model", None), "config", None) - for candidate in (config, getattr(config, "text_config", None)): - if candidate is None: - continue - for attr in ("architecture_adapter", "model_type"): - value = getattr(candidate, attr, None) - if isinstance(value, str) and value.strip(): - return value - architectures = getattr(candidate, "architectures", None) - if isinstance(architectures, (list, tuple)) and architectures: - first = architectures[0] - if isinstance(first, str) and first.strip(): - return first - return default - - -def _backend_cache_layout(backend: Any, recipe_params: Mapping[str, Any] | None) -> str: - if getattr(backend, "supports_kv_cache", False) is False: - return "stateless" - if recipe_params is None: - return "local-hot-kv" - if recipe_params.get("use_cache") is False: - return "stateless" - value = recipe_params.get("cache_layout") - if isinstance(value, str) and value.strip(): - return value - return "local-hot-kv" - - # --- output ----------------------------------------------------------------- DEFAULT_REPORT_FILENAME = "capability.json" diff --git a/packages/node/meshnet_node/failure_semantics.py b/packages/node/meshnet_node/failure_semantics.py deleted file mode 100644 index 8419441..0000000 --- a/packages/node/meshnet_node/failure_semantics.py +++ /dev/null @@ -1,893 +0,0 @@ -"""Bounded failure, cancellation, and restart semantics for Shard streams (DGR-013). - -Distributed speed must not come with hanging or corrupted generations. This module -hardens the per-Route-Session decode stream that runs over the DGR-007 Hot KV State -manager (isolated ``(session, epoch)`` KV) and the DGR-012 continuous-batch -scheduler. It is deliberately backend-agnostic and pure-Python: it drives the same -``KvBoundaryAdapter`` the default deterministic gate uses, so the whole matrix stays -download-free, GPU-free, and API-credit-free while exercising the *real* KV -isolation path (the pinned llama.cpp worker, DGR-008, implements the identical -adapter contract). - -The guarantees, mapped to the story's acceptance criteria: - -* **Deadlines and heartbeat/health loss terminate blocked stream operations.** - :class:`DeadlineGuard` bounds every step against an absolute deadline and a - heartbeat-timeout; when either is breached it raises :class:`StreamTerminated` - so a blocked stream never hangs. -* **Cancellation propagates across every Shard and releases local KV and queued - buffers.** :class:`ShardCancellationGroup` fans a single cancel across every - node-local KV manager serving a Route Session and releases queued activation - buffers; the DGR-012 scheduler's :meth:`~meshnet_node.batch_scheduler. - ContinuousBatchScheduler.cancel` drops queued/active work on this node. -* **Duplicate steps are idempotent; uncertain mutations are never replayed - silently.** :class:`IdempotencyLedger` records each committed - ``(session, epoch, step)`` and returns the recorded token for a duplicate - delivery instead of re-running it. A step whose outcome is *uncertain* (the - worker died mid-mutation) is marked uncertain and can never be silently - replayed — a replay attempt raises :class:`UncertainMutationError`, forcing an - explicit verify-or-restart. -* **Alpha failover restarts from token zero on a newly compatible route rather - than importing unverified KV.** :class:`RestartController` opens a *new* route - epoch, releases every shard's prior-epoch KV, and the restart re-prefills the - whole prompt from token zero. The old epoch becomes stale (rejected by the KV - manager); unverified KV is never migrated (RALPH runtime decision #14). -* **Billing/work records distinguish completed, cancelled, failed, and unverified - work.** :class:`WorkLedger` records a typed :class:`WorkRecord` per attempt; - only :attr:`WorkStatus.COMPLETED` records are billable, so cancelled, failed, - and uncertain (unverified) work is accounted but never charged. - -:class:`HardenedSessionRunner` composes these into one drivable stream: it runs a -single session's prefill+decode through the adapter under a deadline/heartbeat -guard and a cancellation token, records the typed work outcome, and — via -:meth:`HardenedSessionRunner.run_with_failover` — restarts a transient failure -from token zero on a fresh epoch. -""" - -from __future__ import annotations - -import threading -import time -from dataclasses import dataclass, field, replace -from enum import Enum -from typing import Any, Callable, Mapping, Sequence - -from meshnet_node.batch_scheduler import DoneReason, GenerationRequest -from meshnet_node.boundary_adapter import BoundaryContractError, TailOutput -from meshnet_node.hot_kv_state import ( - CacheMiss, - HotKvStateManager, - IncompatibleCacheRecipeError, - KvBoundaryAdapter, - KvCacheMissError, - StaleRouteEpochError, -) - - -class FailureSemanticsError(RuntimeError): - """Base class for failure/cancellation/restart errors.""" - - -# --------------------------------------------------------------------------- # -# Typed outcomes: failure kinds and billing/work statuses. -# --------------------------------------------------------------------------- # - - -class FailureKind(str, Enum): - """Why a stream step failed. Stable strings for the protocol's structured status.""" - - # Bounded termination of a blocked op. - DEADLINE_EXCEEDED = "deadline-exceeded" - HEARTBEAT_LOST = "heartbeat-lost" - # Transport / worker loss (transient — a restart from token zero may succeed). - WORKER_DEATH = "worker-death" - STREAM_RESET = "stream-reset" - # Protocol violations (deterministic — a restart would fail identically). - MALFORMED_BUNDLE = "malformed-bundle" - STALE_EPOCH = "stale-epoch" - INCOMPATIBLE_RECIPE = "incompatible-recipe" - # KV state expected by the caller is gone; re-prefill from token zero. - CACHE_MISS = "cache-miss" - # Explicit client cancellation. - CANCELLED = "cancelled" - - -# Failure kinds that a from-token-zero restart on a fresh route may recover from. -# A protocol violation or an explicit bound (deadline/cancel) is NOT restartable — -# retrying it would hang or fail identically, so we surface it instead. -_RESTARTABLE = frozenset( - { - FailureKind.WORKER_DEATH, - FailureKind.STREAM_RESET, - FailureKind.CACHE_MISS, - } -) - -# Failure kinds whose mutation outcome is *uncertain* — the KV may or may not have -# advanced, so the confirmed work is billed as UNVERIFIED and never replayed -# silently. Only an *unexpected* error raised while a step was executing is -# uncertain (mapped to WORKER_DEATH). A stream reset, deadline, or cache miss -# detected at a step boundary is certain: nothing committed for that step. -_UNCERTAIN = frozenset({FailureKind.WORKER_DEATH}) - - -class WorkStatus(str, Enum): - """The billing-relevant outcome class of a unit of work (AC: billing records). - - Only :attr:`COMPLETED` work is billable. Cancelled, failed, and unverified - work is recorded distinctly so a client is never charged for a generation that - hung, was cancelled, or whose mutations could not be verified. - """ - - COMPLETED = "completed" - CANCELLED = "cancelled" - FAILED = "failed" - UNVERIFIED = "unverified" - - -def work_status_for(kind: FailureKind) -> WorkStatus: - """Map a terminal failure kind to its billing/work status.""" - if kind is FailureKind.CANCELLED: - return WorkStatus.CANCELLED - if kind in _UNCERTAIN: - return WorkStatus.UNVERIFIED - return WorkStatus.FAILED - - -def classify_exception(exc: BaseException) -> FailureKind: - """Classify a raised error into a :class:`FailureKind`. - - Protocol violations map to their specific kind; a :class:`StreamTerminated` - carries its own kind; any *unexpected* error is treated as worker death - (an uncertain, transient loss), never silently ignored. - """ - if isinstance(exc, StreamTerminated): - return exc.kind - if isinstance(exc, OperationCancelled): - return FailureKind.CANCELLED - if isinstance(exc, StaleRouteEpochError): - return FailureKind.STALE_EPOCH - if isinstance(exc, IncompatibleCacheRecipeError): - return FailureKind.INCOMPATIBLE_RECIPE - if isinstance(exc, BoundaryContractError): - return FailureKind.MALFORMED_BUNDLE - if isinstance(exc, KvCacheMissError): - return FailureKind.CACHE_MISS - return FailureKind.WORKER_DEATH - - -# --------------------------------------------------------------------------- # -# Deadlines and heartbeat/health loss. -# --------------------------------------------------------------------------- # - - -class StreamTerminated(FailureSemanticsError): - """A blocked stream op was terminated by a deadline or heartbeat/health loss.""" - - def __init__(self, kind: FailureKind, detail: str = "") -> None: - self.kind = kind - self.detail = detail - suffix = f": {detail}" if detail else "" - super().__init__(f"stream terminated ({kind.value}){suffix}") - - -class OperationCancelled(FailureSemanticsError): - """Raised when a step observes its :class:`CancellationToken` is cancelled.""" - - def __init__(self, reason: str = "client-cancel") -> None: - self.reason = reason - super().__init__(f"operation cancelled: {reason}") - - -@dataclass -class DeadlineGuard: - """Bounds a blocked stream op against an absolute deadline and heartbeat loss. - - ``deadline`` is an absolute time on ``clock``'s scale (``None`` disables it). - ``heartbeat_timeout`` is the maximum tolerated gap since the last observed - heartbeat; when the peer stops sending heartbeats (its health is lost) the gap - grows past the timeout and :meth:`check` raises rather than blocking forever. - Both bounds are checked with an injected ``clock`` so the matrix is - deterministic. - """ - - deadline: float | None = None - heartbeat_timeout: float | None = None - clock: Callable[[], float] = time.monotonic - _last_heartbeat: float = field(default=0.0, init=False) - _started: bool = field(default=False, init=False) - - def __post_init__(self) -> None: - if self.heartbeat_timeout is not None and self.heartbeat_timeout <= 0: - raise FailureSemanticsError("heartbeat_timeout must be positive") - - def start(self) -> None: - self._last_heartbeat = self.clock() - self._started = True - - def heartbeat(self) -> None: - """Record that the peer is alive (resets the heartbeat gap).""" - self._last_heartbeat = self.clock() - - def check(self) -> None: - """Raise :class:`StreamTerminated` if the deadline or heartbeat lapsed.""" - if not self._started: - self.start() - now = self.clock() - if self.deadline is not None and now >= self.deadline: - raise StreamTerminated( - FailureKind.DEADLINE_EXCEEDED, - f"deadline {self.deadline} reached at {now}", - ) - if self.heartbeat_timeout is not None: - gap = now - self._last_heartbeat - if gap > self.heartbeat_timeout: - raise StreamTerminated( - FailureKind.HEARTBEAT_LOST, - f"no heartbeat for {gap} > {self.heartbeat_timeout}", - ) - - def remaining(self) -> float | None: - if self.deadline is None: - return None - return self.deadline - self.clock() - - -# --------------------------------------------------------------------------- # -# Cancellation that propagates across shards and releases KV + queued buffers. -# --------------------------------------------------------------------------- # - - -class CancellationToken: - """A thread-safe one-shot cancellation flag shared by a Route Session's steps.""" - - def __init__(self) -> None: - self._cancelled = False - self._reason = "" - self._lock = threading.Lock() - - def cancel(self, reason: str = "client-cancel") -> None: - with self._lock: - if not self._cancelled: - self._cancelled = True - self._reason = reason - - @property - def cancelled(self) -> bool: - with self._lock: - return self._cancelled - - @property - def reason(self) -> str: - with self._lock: - return self._reason - - def raise_if_cancelled(self) -> None: - with self._lock: - if self._cancelled: - raise OperationCancelled(self._reason) - - -@dataclass(frozen=True) -class CancellationOutcome: - """What a :meth:`ShardCancellationGroup.cancel` released (for observability).""" - - session_id: str - route_epoch: int - shards_released: int - buffers_released: int - - def to_dict(self) -> dict: - return { - "session_id": self.session_id, - "route_epoch": self.route_epoch, - "shards_released": self.shards_released, - "buffers_released": self.buffers_released, - } - - -class ShardCancellationGroup: - """Fan one cancellation across every node-local Shard of a Route Session. - - A Route Session spans a chain of Shards, each with its own local Hot KV State - manager (KV is never migrated between nodes). Cancelling the session must free - *all* of that state: this group releases the ``(session, epoch)`` KV on every - registered manager and invokes every registered queued-buffer release callback - (the pending activation bundles a node holds for the session). Release is - idempotent, so cancelling twice is safe. - """ - - def __init__(self, session_id: str, route_epoch: int) -> None: - if not isinstance(session_id, str) or not session_id.strip(): - raise FailureSemanticsError("session_id must be a non-empty string") - self.session_id = session_id - self.route_epoch = int(route_epoch) - self._managers: list[HotKvStateManager] = [] - self._buffers: list[Callable[[], None]] = [] - self._lock = threading.Lock() - self._cancelled = False - - def add_shard(self, manager: HotKvStateManager) -> "ShardCancellationGroup": - with self._lock: - self._managers.append(manager) - return self - - def add_queued_buffer( - self, release: Callable[[], None] - ) -> "ShardCancellationGroup": - """Register a queued activation buffer's release callback.""" - with self._lock: - self._buffers.append(release) - return self - - @property - def cancelled(self) -> bool: - with self._lock: - return self._cancelled - - def cancel(self) -> CancellationOutcome: - """Release every shard's KV and every queued buffer for this session.""" - with self._lock: - managers = list(self._managers) - buffers = list(self._buffers) - self._buffers.clear() - self._cancelled = True - shards_released = 0 - for manager in managers: - if manager.release(self.session_id, self.route_epoch): - shards_released += 1 - buffers_released = 0 - for release in buffers: - release() - buffers_released += 1 - return CancellationOutcome( - session_id=self.session_id, - route_epoch=self.route_epoch, - shards_released=shards_released, - buffers_released=buffers_released, - ) - - -# --------------------------------------------------------------------------- # -# Idempotency: duplicate steps are no-ops; uncertain mutations never replay. -# --------------------------------------------------------------------------- # - - -class StepPhase(str, Enum): - IN_FLIGHT = "in-flight" - COMMITTED = "committed" - UNCERTAIN = "uncertain" - - -class UncertainMutationError(FailureSemanticsError): - """Raised when a caller tries to replay a step whose outcome is uncertain. - - A step is uncertain when its mutation may or may not have been applied (worker - death / stream reset mid-append). Replaying it silently could double-apply KV - or bill unverified work, so the ledger refuses: the caller must verify against - the actual KV length or restart from token zero on a fresh epoch instead. - """ - - -@dataclass(frozen=True) -class StepKey: - """Identity of one idempotent stream step within a route epoch.""" - - session_id: str - route_epoch: int - step_index: int - - -@dataclass(frozen=True) -class StepDisposition: - """What :meth:`IdempotencyLedger.begin` decided for a step.""" - - fresh: bool - token: int | None = None - - @property - def duplicate(self) -> bool: - return not self.fresh - - -class IdempotencyLedger: - """Records committed/uncertain stream steps so duplicates never re-mutate. - - Keyed by ``(session, epoch, step_index)`` — the protocol's idempotency step. - - * :meth:`begin` on a *fresh* key marks it in-flight and returns "execute". - * :meth:`begin` on a *committed* key returns the recorded token so a duplicate - delivery is a no-op (idempotent replay). - * :meth:`begin` on an *in-flight* or *uncertain* key raises - :class:`UncertainMutationError` — a concurrent duplicate or a replay of an - unverified mutation is never silently applied. - """ - - def __init__(self) -> None: - self._phase: dict[StepKey, StepPhase] = {} - self._token: dict[StepKey, int] = {} - self._lock = threading.Lock() - - def begin(self, key: StepKey) -> StepDisposition: - with self._lock: - phase = self._phase.get(key) - if phase is None: - self._phase[key] = StepPhase.IN_FLIGHT - return StepDisposition(fresh=True) - if phase is StepPhase.COMMITTED: - return StepDisposition(fresh=False, token=self._token[key]) - # IN_FLIGHT (concurrent duplicate) or UNCERTAIN (post-crash replay): - # both are unverified and must not be silently re-applied. - raise UncertainMutationError( - f"step {key.step_index} for session {key.session_id[:8]} epoch " - f"{key.route_epoch} is {phase.value}; refusing silent replay" - ) - - def commit(self, key: StepKey, token: int) -> None: - with self._lock: - self._phase[key] = StepPhase.COMMITTED - self._token[key] = int(token) - - def mark_uncertain(self, key: StepKey, detail: str = "") -> None: - with self._lock: - # A committed step is verified; never downgrade it. - if self._phase.get(key) is StepPhase.COMMITTED: - return - self._phase[key] = StepPhase.UNCERTAIN - - def phase_of(self, key: StepKey) -> StepPhase | None: - with self._lock: - return self._phase.get(key) - - def committed_token(self, key: StepKey) -> int | None: - with self._lock: - return self._token.get(key) - - def has_uncertain(self) -> bool: - with self._lock: - return any(p is StepPhase.UNCERTAIN for p in self._phase.values()) - - -# --------------------------------------------------------------------------- # -# Restart / alpha failover: from token zero on a fresh compatible route. -# --------------------------------------------------------------------------- # - - -class RestartController: - """Alpha failover that restarts from token zero, never importing prior KV. - - RALPH runtime decision #14: when the alpha (the head owning embedding + final - head) fails, the route retries from token zero; unverified KV is never - migrated. :meth:`failover` opens the *next* route epoch and releases every - node-local shard's prior-epoch KV, so the restart begins with empty caches. The - KV manager then treats the failed epoch as stale (a later reference to it is - rejected), which is what keeps a half-computed cache from being reused. - """ - - def __init__(self, managers: Sequence[HotKvStateManager]) -> None: - self._managers = list(managers) - - def failover(self, session_id: str, failed_epoch: int) -> int: - """Advance to a fresh epoch and drop the failed epoch's KV on every shard.""" - new_epoch = int(failed_epoch) + 1 - for manager in self._managers: - manager.release(session_id, failed_epoch) - return new_epoch - - def assert_fresh_start(self, session_id: str, new_epoch: int) -> None: - """Verify no shard carries KV for the new epoch (a true token-zero restart). - - Any residual KV under the new epoch would be unverified imported state; - this fails closed so a restart can never silently attend over it. - """ - for manager in self._managers: - result = manager.resolve(session_id, new_epoch) - if not isinstance(result, CacheMiss): - raise FailureSemanticsError( - f"restart epoch {new_epoch} for session {session_id[:8]} is not " - "empty; refusing to import unverified KV" - ) - - -# --------------------------------------------------------------------------- # -# Billing / work records. -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class WorkRecord: - """A typed unit of served work, distinguishing what may be billed. - - ``tokens`` counts only *committed* generated tokens. Only a - :attr:`WorkStatus.COMPLETED` record is billable; cancelled/failed/unverified - records carry their confirmed token count for observability but are excluded - from billing so uncompleted or unverified work is never charged. - """ - - session_id: str - route_epoch: int - status: WorkStatus - tokens: int - failure_kind: FailureKind | None = None - detail: str = "" - - @property - def billable(self) -> bool: - return self.status is WorkStatus.COMPLETED - - def to_dict(self) -> dict: - return { - "session_id": self.session_id, - "route_epoch": self.route_epoch, - "status": self.status.value, - "tokens": self.tokens, - "failure_kind": self.failure_kind.value if self.failure_kind else None, - "detail": self.detail, - "billable": self.billable, - } - - -class WorkLedger: - """Append-only ledger of :class:`WorkRecord`, split by billing status.""" - - def __init__(self) -> None: - self._records: list[WorkRecord] = [] - self._lock = threading.Lock() - - def record(self, record: WorkRecord) -> WorkRecord: - with self._lock: - self._records.append(record) - return record - - def records(self) -> list[WorkRecord]: - with self._lock: - return list(self._records) - - def records_for(self, session_id: str) -> list[WorkRecord]: - with self._lock: - return [r for r in self._records if r.session_id == session_id] - - def billable_records(self) -> list[WorkRecord]: - with self._lock: - return [r for r in self._records if r.billable] - - def billable_tokens(self) -> int: - """Total tokens that may be charged (completed work only).""" - with self._lock: - return sum(r.tokens for r in self._records if r.billable) - - def counts_by_status(self) -> dict[str, int]: - counts: dict[str, int] = {s.value: 0 for s in WorkStatus} - with self._lock: - for record in self._records: - counts[record.status.value] += 1 - return counts - - def to_dict(self) -> dict: - with self._lock: - records = [r.to_dict() for r in self._records] - counts: dict[str, int] = {s.value: 0 for s in WorkStatus} - for record in records: - counts[record["status"]] += 1 - return { - "schema_version": 1, - "records": records, - "counts_by_status": counts, - "billable_tokens": sum(r["tokens"] for r in records if r["billable"]), - } - - -# --------------------------------------------------------------------------- # -# The hardened single-session stream runner. -# --------------------------------------------------------------------------- # - - -@dataclass(frozen=True) -class RunOutcome: - """The typed result of one hardened generation attempt.""" - - session_id: str - route_epoch: int - status: WorkStatus - tokens: tuple[int, ...] - failure_kind: FailureKind | None - detail: str - - @property - def completed(self) -> bool: - return self.status is WorkStatus.COMPLETED - - @property - def token_count(self) -> int: - return len(self.tokens) - - @property - def restartable(self) -> bool: - return self.failure_kind in _RESTARTABLE - - def work_record(self) -> WorkRecord: - return WorkRecord( - session_id=self.session_id, - route_epoch=self.route_epoch, - status=self.status, - tokens=len(self.tokens), - failure_kind=self.failure_kind, - detail=self.detail, - ) - - -@dataclass(frozen=True) -class FailoverResult: - """The result of a run that may have restarted from token zero after a failure.""" - - outcome: RunOutcome - attempts: tuple[RunOutcome, ...] - restarts: int - - @property - def completed(self) -> bool: - return self.outcome.completed - - def to_dict(self) -> dict: - return { - "final_status": self.outcome.status.value, - "final_epoch": self.outcome.route_epoch, - "restarts": self.restarts, - "attempts": [ - { - "route_epoch": a.route_epoch, - "status": a.status.value, - "failure_kind": a.failure_kind.value if a.failure_kind else None, - "tokens": a.token_count, - } - for a in self.attempts - ], - } - - -class HardenedSessionRunner: - """Drive one Route Session's decode stream with bounded failure semantics. - - The runner owns a single full-shard :class:`KvBoundaryAdapter` (head **and** - tail, so a step samples a token) and threads every DGR-013 guarantee through a - step loop: - - * every step is bounded by a :class:`DeadlineGuard` and can observe a - :class:`CancellationToken`; - * every step is idempotent through an :class:`IdempotencyLedger` (a duplicate - returns the recorded token; an uncertain mutation is never replayed); - * any failure releases this session's KV (cancellation) and is recorded as a - typed :class:`WorkRecord` in the :class:`WorkLedger`; - * :meth:`run_with_failover` restarts a transient failure from token zero on a - fresh epoch via a :class:`RestartController`. - """ - - def __init__( - self, - adapter: KvBoundaryAdapter, - *, - clock: Callable[[], float] | None = None, - work_ledger: WorkLedger | None = None, - idempotency: IdempotencyLedger | None = None, - ) -> None: - if not (adapter.is_head and adapter.is_tail): - raise FailureSemanticsError( - "HardenedSessionRunner needs a full (head+tail) shard so decode " - "steps sample tokens; got a partial range " - f"(head={adapter.is_head} tail={adapter.is_tail})" - ) - self._adapter = adapter - self._manager: HotKvStateManager = adapter.manager - self._clock = clock or time.monotonic - self.work_ledger = work_ledger or WorkLedger() - self.idempotency = idempotency or IdempotencyLedger() - - # -- single attempt ------------------------------------------------------- - - def run( - self, - request: GenerationRequest, - *, - deadline: float | None = None, - heartbeat_timeout: float | None = None, - cancel_token: CancellationToken | None = None, - heartbeat: Callable[[int], bool] | None = None, - before_step: Callable[[int], None] | None = None, - ) -> RunOutcome: - """Run one attempt of ``request``; record and return a typed outcome. - - ``deadline`` (absolute, on the injected clock) and ``heartbeat_timeout`` - bound blocked steps. ``cancel_token`` lets a client cancel mid-stream. - ``heartbeat(step)`` returns ``True`` when a heartbeat was heard before that - step (resetting the health timer); ``before_step(step)`` is a fault- - injection / clock-advance hook run before each step and may raise - :class:`StreamTerminated` (e.g. a stream reset) or - :class:`OperationCancelled`. - """ - sid = request.session_id - epoch = request.route_epoch - guard = DeadlineGuard( - deadline=deadline, - heartbeat_timeout=heartbeat_timeout, - clock=self._clock, - ) - guard.start() - tokens: list[int] = [] - current_key: StepKey | None = None - try: - # step 0 is the prefill (emits the first token); steps 1..N are decodes. - for step_index in range(request.max_new_tokens): - # before_step is the fault-injection / clock-advance hook and may - # itself terminate the step (stream reset, cancel); run it first so - # a fault it raises takes effect on this step, then re-check the - # bounds it may have advanced (deadline / heartbeat / cancel). - if before_step is not None: - before_step(step_index) - if cancel_token is not None: - cancel_token.raise_if_cancelled() - if heartbeat is not None and heartbeat(step_index): - guard.heartbeat() - guard.check() - - current_key = StepKey(sid, epoch, step_index) - disposition = self.idempotency.begin(current_key) - if disposition.duplicate: - # Idempotent replay: reuse the recorded token, do not re-mutate. - assert disposition.token is not None - tokens.append(disposition.token) - continue - - token = self._execute_step(request, step_index, tokens) - if isinstance(token, CacheMiss): - # The expected KV was gone; the append never started, so this is - # a certain (not uncertain) miss — restartable from token zero. - return self._finish_failure( - request, - tokens, - FailureKind.CACHE_MISS, - str(token), - cancel_token, - ) - self.idempotency.commit(current_key, token) - tokens.append(token) - except (StreamTerminated, OperationCancelled) as exc: - return self._finish_failure( - request, tokens, classify_exception(exc), str(exc), cancel_token - ) - except ( - BoundaryContractError, - StaleRouteEpochError, - IncompatibleCacheRecipeError, - KvCacheMissError, - ) as exc: - # Deterministic protocol/state errors, all validated before any KV - # append committed — certain, not uncertain. - return self._finish_failure( - request, tokens, classify_exception(exc), str(exc), cancel_token - ) - except UncertainMutationError as exc: - # A replay of an unverified step reached the ledger — never silent. - return self._finish_failure( - request, tokens, FailureKind.WORKER_DEATH, str(exc), cancel_token - ) - except Exception as exc: # noqa: BLE001 - unexpected == worker death - # An unexpected error mid-step may have left the KV half-mutated; mark - # the step uncertain so it can never be silently replayed, then fail - # closed as unverified work. - if current_key is not None: - self.idempotency.mark_uncertain(current_key, str(exc)) - return self._finish_failure( - request, tokens, FailureKind.WORKER_DEATH, str(exc), cancel_token - ) - - return self._finish_completed(request, tokens) - - def _execute_step( - self, request: GenerationRequest, step_index: int, tokens: list[int] - ) -> int | CacheMiss: - sid = request.session_id - epoch = request.route_epoch - if step_index == 0: - out = self._adapter.prefill( - sid, epoch, token_ids=list(request.prompt_token_ids) - ) - else: - # expected_seq_len defends the KV layer against a desynchronised decode: - # prompt positions plus the tokens already committed this run. - expected = request.prompt_len + (step_index - 1) - out = self._adapter.decode( - sid, - epoch, - token_ids=[tokens[-1]], - expected_seq_len=expected, - ) - if isinstance(out, CacheMiss): - return out - if not isinstance(out, TailOutput): - raise FailureSemanticsError( - "full-shard step did not yield a sampled token; got " - f"{type(out).__name__}" - ) - return int(out.token_id) - - # -- failover across restarts -------------------------------------------- - - def run_with_failover( - self, - request: GenerationRequest, - controller: RestartController, - *, - max_restarts: int = 3, - **run_kwargs: Any, - ) -> FailoverResult: - """Run ``request``, restarting a transient failure from token zero. - - On a restartable failure (worker death, stream reset, cache miss) the - controller advances to a fresh epoch and drops the failed epoch's KV; the - next attempt re-prefills the whole prompt from token zero. A deterministic - failure (deadline, cancel, malformed bundle, stale epoch) is returned as-is - — retrying it would hang or fail identically. Per-attempt fault-injection - hooks (``before_step`` / ``heartbeat``) are only applied to the *first* - attempt so a restart runs clean. - """ - if max_restarts < 0: - raise FailureSemanticsError("max_restarts must be >= 0") - epoch = request.route_epoch - attempts: list[RunOutcome] = [] - first_kwargs = run_kwargs - for attempt in range(max_restarts + 1): - attempt_request = replace(request, route_epoch=epoch) - kwargs = first_kwargs if attempt == 0 else {} - outcome = self.run(attempt_request, **kwargs) - attempts.append(outcome) - if outcome.completed or not outcome.restartable or attempt == max_restarts: - return FailoverResult( - outcome=outcome, attempts=tuple(attempts), restarts=attempt - ) - # Alpha failover: fresh epoch, drop prior-epoch KV on every shard, and - # verify the new epoch starts empty (no unverified KV import). - epoch = controller.failover(request.session_id, epoch) - controller.assert_fresh_start(request.session_id, epoch) - # Unreachable: the loop always returns, but keep the type-checker happy. - raise FailureSemanticsError("run_with_failover exhausted without returning") - - # -- outcome bookkeeping -------------------------------------------------- - - def _finish_completed( - self, request: GenerationRequest, tokens: list[int] - ) -> RunOutcome: - outcome = RunOutcome( - session_id=request.session_id, - route_epoch=request.route_epoch, - status=WorkStatus.COMPLETED, - tokens=tuple(tokens), - failure_kind=None, - detail="", - ) - self.work_ledger.record(outcome.work_record()) - return outcome - - def _finish_failure( - self, - request: GenerationRequest, - tokens: list[int], - kind: FailureKind, - detail: str, - cancel_token: CancellationToken | None, - ) -> RunOutcome: - # Cancellation semantics: release this session's local KV so a failed or - # cancelled stream never leaks its cache. release() is idempotent. - self._manager.release(request.session_id, request.route_epoch) - if cancel_token is not None and kind is not FailureKind.CANCELLED: - # Ensure downstream shards sharing the token also stop. - cancel_token.cancel(kind.value) - outcome = RunOutcome( - session_id=request.session_id, - route_epoch=request.route_epoch, - status=work_status_for(kind), - tokens=tuple(tokens), - failure_kind=kind, - detail=detail, - ) - self.work_ledger.record(outcome.work_record()) - return outcome diff --git a/packages/node/meshnet_node/gguf_backend.py b/packages/node/meshnet_node/gguf_backend.py deleted file mode 100644 index 382375e..0000000 --- a/packages/node/meshnet_node/gguf_backend.py +++ /dev/null @@ -1,423 +0,0 @@ -"""Native llama.cpp/GGUF backend adapter for Meshnet node startup. - -This module keeps the node-side GGUF seam separate from the Torch-backed -reference path. The public object intentionally looks like the existing -``TorchModelShard`` surface so ``TorchNodeServer`` can serve it without changing -the HTTP/control-plane code that already correlates request ids, telemetry and -billing. - -The transport layer is intentionally explicit: - -* direct worker calls are expected to use the versioned gRPC Shard protocol - from :mod:`meshnet_node.native_protocol`; -* the backend itself stays transport-agnostic and delegates to a worker - transport object with the same method surface as the existing node backend. - -The default factory is strict: if no worker endpoint is configured, it fails -closed rather than silently pretending the native worker exists. -""" - -from __future__ import annotations - -import os -from dataclasses import dataclass, field -from types import SimpleNamespace -from typing import Any, Protocol, runtime_checkable - -from .model_backend import ( - MissingModelDependencyError, - ModelBackendError, - TailTokenResult, - TensorPayload, -) - -_BACKEND_ID = "llama.cpp" - - -@runtime_checkable -class NativeWorkerTransport(Protocol): - """Backend-shaped transport for the supervised native worker.""" - - def encode_prompt( - self, - prompt: str, - session_id: str | None = None, - ) -> TensorPayload | TailTokenResult | str: ... - - def encode_next_token( - self, - token_id: int, - session_id: str, - ) -> TensorPayload | TailTokenResult | str: ... - - def forward_bytes( - self, - body: bytes, - shape: list[int], - attention_mask_header: str | None, - position_ids_header: str | None, - *, - start_layer: int | None = None, - session_id: str | None = None, - cache_mode: str | None = None, - past_len: int | None = None, - ) -> TensorPayload | TailTokenResult | str: ... - - def decode_tail_token(self, hidden_states: Any) -> TailTokenResult: ... - - def generate_text( - self, - messages: list[dict], - max_new_tokens: int = 5120, - temperature: float = 1.0, - top_p: float = 1.0, - ) -> str: ... - - def generate_text_streaming( - self, - messages: list[dict], - max_new_tokens: int = 5120, - temperature: float = 1.0, - top_p: float = 1.0, - ): ... - - def count_prompt_tokens(self, messages: list[dict]) -> int: ... - - def count_text_tokens(self, text: str) -> int: ... - - def eos_token_ids(self) -> list[int]: ... - - def release_session(self, session_id: str) -> None: ... - - -@dataclass(frozen=True) -class _NativeModelConfig: - """Enough model metadata for admission and capability reporting.""" - - model_type: str = "llama" - architecture_adapter: str = "dense-llama" - num_hidden_layers: int = 1 - torch_dtype: str = "bfloat16" - - def to_dict(self) -> dict[str, Any]: - return { - "model_type": self.model_type, - "architecture_adapter": self.architecture_adapter, - "num_hidden_layers": self.num_hidden_layers, - "torch_dtype": self.torch_dtype, - } - - -@dataclass -class GgufNodeBackend: - """GGUF shard backend shaped like ``TorchModelShard``. - - The adapter keeps the Meshnet-facing surface stable while the actual model - execution is delegated to a worker transport. The backend carries the exact - model, shard and runtime metadata required for admission and registration. - """ - - model_id: str - shard_start: int - shard_end: int - quantization: str = "bfloat16" - transport: NativeWorkerTransport | None = None - total_layers: int | None = None - model_revision: str | None = None - loaded_tensor_names: tuple[str, ...] = () - device_type: str = "cpu" - supports_kv_cache: bool = True - worker_url: str | None = None - architecture_adapter: str = "dense-llama" - tokenizer_revision: str | None = None - runtime_recipe_fingerprint: str | None = None - _model: SimpleNamespace = field(init=False, repr=False) - _tokenizer: SimpleNamespace = field(init=False, repr=False) - is_head: bool = field(init=False) - is_tail: bool = field(init=False) - loaded_shard_start: int = field(init=False) - loaded_shard_end: int = field(init=False) - owns_embedding: bool = field(init=False) - owns_final_head: bool = field(init=False) - - backend_id = _BACKEND_ID - - def __post_init__(self) -> None: - if self.shard_start < 0 or self.shard_end < self.shard_start: - raise ValueError("shard_start must be <= shard_end and non-negative") - total_layers = self.total_layers or (self.shard_end + 1) - object.__setattr__( - self, - "total_layers", - int(total_layers), - ) - object.__setattr__( - self, - "_model", - SimpleNamespace( - revision=self.model_revision or self.model_id, - config=_NativeModelConfig( - num_hidden_layers=int(total_layers), - torch_dtype=self.quantization, - ), - ), - ) - object.__setattr__( - self, - "_tokenizer", - SimpleNamespace( - model_id=self.model_id, - revision=self.tokenizer_revision or self.model_revision or self.model_id, - eos_token="", - eos_token_id=[], - ), - ) - object.__setattr__(self, "is_head", self.shard_start == 0) - object.__setattr__(self, "is_tail", self.shard_end >= int(total_layers) - 1) - object.__setattr__(self, "loaded_shard_start", self.shard_start) - object.__setattr__(self, "loaded_shard_end", self.shard_end) - object.__setattr__(self, "owns_embedding", self.is_head) - object.__setattr__(self, "owns_final_head", self.is_tail) - if not self.loaded_tensor_names: - object.__setattr__( - self, - "loaded_tensor_names", - self._default_tensor_inventory(), - ) - - @property - def model(self) -> Any: - return self._model - - @property - def tokenizer(self) -> Any: - return self._tokenizer - - @property - def device(self) -> SimpleNamespace: - return SimpleNamespace(type=self.device_type) - - @property - def shard_range(self) -> tuple[int, int]: - return self.shard_start, self.shard_end - - def encode_prompt(self, prompt: str, session_id: str | None = None) -> TensorPayload | TailTokenResult | str: - return self._transport().encode_prompt(prompt, session_id=session_id) - - def encode_next_token(self, token_id: int, session_id: str) -> TensorPayload | TailTokenResult | str: - return self._transport().encode_next_token(token_id, session_id) - - def forward_bytes( - self, - body: bytes, - shape: list[int], - attention_mask_header: str | None, - position_ids_header: str | None, - start_layer: int | None = None, - session_id: str | None = None, - cache_mode: str | None = None, - past_len: int | None = None, - ) -> TensorPayload | TailTokenResult | str: - return self._transport().forward_bytes( - body, - shape, - attention_mask_header, - position_ids_header, - start_layer=start_layer, - session_id=session_id, - cache_mode=cache_mode, - past_len=past_len, - ) - - def decode_tail(self, hidden_states: Any) -> str: - return self.decode_tail_token(hidden_states).text - - def decode_tail_token(self, hidden_states: Any) -> TailTokenResult: - return self._transport().decode_tail_token(hidden_states) - - def generate_text( - self, - messages: list[dict], - max_new_tokens: int = 5120, - temperature: float = 1.0, - top_p: float = 1.0, - ) -> str: - return self._transport().generate_text(messages, max_new_tokens, temperature, top_p) - - def generate_text_streaming( - self, - messages: list[dict], - max_new_tokens: int = 5120, - temperature: float = 1.0, - top_p: float = 1.0, - ): - yield from self._transport().generate_text_streaming(messages, max_new_tokens, temperature, top_p) - - def count_prompt_tokens(self, messages: list[dict]) -> int: - return self._transport().count_prompt_tokens(messages) - - def count_text_tokens(self, text: str) -> int: - return self._transport().count_text_tokens(text) - - def eos_token_ids(self) -> list[int]: - return self._transport().eos_token_ids() - - def release_session(self, session_id: str) -> None: - self._transport().release_session(session_id) - - def _transport(self) -> NativeWorkerTransport: - if self.transport is None: - raise MissingModelDependencyError( - "native GGUF backend needs a worker transport; set MESHNET_NATIVE_WORKER_URL " - "or inject a test transport" - ) - return self.transport - - def _default_tensor_inventory(self) -> tuple[str, ...]: - tensor_names = [f"blk.{layer}.weight" for layer in range(self.shard_start, self.shard_end + 1)] - if self.is_head: - tensor_names.append("token_embd.weight") - if self.is_tail: - tensor_names.extend(["output_norm.weight", "output.weight"]) - return tuple(tensor_names) - - -class GrpcNativeWorkerTransport: - """Transport that speaks the versioned gRPC worker protocol. - - The transport is intentionally conservative: it provides the unary service - hooks and carries the protocol metadata, but it does not guess at worker - behavior beyond what the compiled protobuf schema already describes. - """ - - def __init__(self, worker_url: str, *, timeout: float = 30.0) -> None: - self.worker_url = worker_url - self.timeout = timeout - self._grpc = None - self._channel = None - self._stub = None - - def _ensure_stub(self) -> Any: - if self._stub is not None: - return self._stub - try: - import grpc # type: ignore[import] - except ImportError as exc: # pragma: no cover - environment dependent - raise MissingModelDependencyError( - "grpc is required for the native GGUF worker transport" - ) from exc - from . import native_protocol - - grpc_mod = native_protocol.load_grpc() - self._grpc = grpc - self._channel = grpc.insecure_channel(self.worker_url) - self._stub = grpc_mod.ShardRuntimeStub(self._channel) - return self._stub - - def encode_prompt(self, prompt: str, session_id: str | None = None) -> TensorPayload | TailTokenResult | str: - raise ModelBackendError( - "gRPC transport is present, but prompt-to-activation translation is provided " - "by the backend wrapper so it can keep worker framing and tokenizer state aligned" - ) - - def encode_next_token(self, token_id: int, session_id: str) -> TensorPayload | TailTokenResult | str: - raise ModelBackendError( - "gRPC transport is present, but decode translation is provided by the backend wrapper" - ) - - def forward_bytes( - self, - body: bytes, - shape: list[int], - attention_mask_header: str | None, - position_ids_header: str | None, - *, - start_layer: int | None = None, - session_id: str | None = None, - cache_mode: str | None = None, - past_len: int | None = None, - ) -> TensorPayload | TailTokenResult | str: - raise ModelBackendError( - "gRPC transport is present, but activation streaming is handled by the backend wrapper" - ) - - def decode_tail_token(self, hidden_states: Any) -> TailTokenResult: - raise ModelBackendError("tail decoding is handled by the backend wrapper") - - def generate_text( - self, - messages: list[dict], - max_new_tokens: int = 5120, - temperature: float = 1.0, - top_p: float = 1.0, - ) -> str: - raise ModelBackendError("text generation is handled by the backend wrapper") - - def generate_text_streaming( - self, - messages: list[dict], - max_new_tokens: int = 5120, - temperature: float = 1.0, - top_p: float = 1.0, - ): - raise ModelBackendError("streaming generation is handled by the backend wrapper") - - def count_prompt_tokens(self, messages: list[dict]) -> int: - return sum(1 for message in messages if isinstance(message, dict)) - - def count_text_tokens(self, text: str) -> int: - return len(text.split()) or (1 if text else 0) - - def eos_token_ids(self) -> list[int]: - return [] - - def release_session(self, session_id: str) -> None: - stub = self._ensure_stub() - from . import native_protocol - - pb2 = native_protocol.load() - stub.Release(pb2.ReleaseRequest(reason="release from adapter")) - - -def build_gguf_backend( - *, - model_id: str, - shard_start: int, - shard_end: int, - quantization: str = "bfloat16", - transport: NativeWorkerTransport | None = None, - worker_url: str | None = None, - total_layers: int | None = None, - model_revision: str | None = None, - loaded_tensor_names: tuple[str, ...] = (), - device_type: str = "cpu", - architecture_adapter: str = "dense-llama", - tokenizer_revision: str | None = None, - runtime_recipe_fingerprint: str | None = None, - supports_kv_cache: bool = True, -) -> GgufNodeBackend: - """Construct a native-worker-backed GGUF node backend.""" - if transport is None: - worker_url = worker_url or os.environ.get("MESHNET_NATIVE_WORKER_URL") - if not worker_url: - raise MissingModelDependencyError( - "set MESHNET_NATIVE_WORKER_URL to the local gRPC worker endpoint " - "or inject a fake transport in tests" - ) - transport = GrpcNativeWorkerTransport(worker_url) - return GgufNodeBackend( - model_id=model_id, - shard_start=shard_start, - shard_end=shard_end, - quantization=quantization, - transport=transport, - total_layers=total_layers, - model_revision=model_revision, - loaded_tensor_names=loaded_tensor_names, - device_type=device_type, - supports_kv_cache=supports_kv_cache, - worker_url=worker_url, - architecture_adapter=architecture_adapter, - tokenizer_revision=tokenizer_revision, - runtime_recipe_fingerprint=runtime_recipe_fingerprint, - ) diff --git a/packages/node/meshnet_node/gguf_ownership.py b/packages/node/meshnet_node/gguf_ownership.py deleted file mode 100644 index c7c783e..0000000 --- a/packages/node/meshnet_node/gguf_ownership.py +++ /dev/null @@ -1,287 +0,0 @@ -"""Dense-Llama GGUF ownership helpers. - -This module keeps two related concerns together: - -* selecting the tensors a dense-Llama GGUF shard is allowed to own; and -* inferring the authoritative loaded range / endpoint ownership from the - tensors the model actually exposes. - -The first is used by the range-aware loader seam. The second is used by the -doctor/admission/reporting path so the tracker sees what the model loaded, not -what a CLI flag claimed. -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass -from typing import Any, Iterable, Mapping - -_BLOCK_RE = re.compile(r"^blk\.(\d+)\.") - -_HEAD_TENSOR_NAMES = { - "token_embd.weight", - "token_embd.bias", - "tok_embeddings.weight", - "tok_embeddings.bias", - "embed_tokens.weight", - "embed_tokens.bias", -} - -_TAIL_TENSOR_NAMES = { - "output_norm.weight", - "output_norm.bias", - "output.weight", - "output.bias", - "lm_head.weight", - "lm_head.bias", -} - - -@dataclass(frozen=True) -class DenseLlamaShardOwnership: - """Authoritative ownership for one dense-Llama shard.""" - - start_layer: int - end_layer: int - owns_embedding: bool - owns_final_head: bool - tensor_names: tuple[str, ...] = () - source_artifact_hash: str | None = None - slice_artifact_hash: str | None = None - derivative_slice: bool = False - final_artifact_semantics: bool = True - - def __post_init__(self) -> None: - if self.start_layer < 0: - raise ValueError("start_layer must be non-negative") - if self.end_layer < self.start_layer: - raise ValueError("end_layer must be >= start_layer") - if self.derivative_slice: - if not self.source_artifact_hash or not self.slice_artifact_hash: - raise ValueError( - "temporary derivative sub-GGUFs must carry source and slice hashes" - ) - if self.final_artifact_semantics: - raise ValueError( - "temporary derivative sub-GGUFs must not be claimed as final artifacts" - ) - - @property - def range(self) -> tuple[int, int]: - return self.start_layer, self.end_layer - - def to_dict(self) -> dict[str, Any]: - return { - "start_layer": self.start_layer, - "end_layer": self.end_layer, - "owns_embedding": self.owns_embedding, - "owns_final_head": self.owns_final_head, - "tensor_names": list(self.tensor_names), - "source_artifact_hash": self.source_artifact_hash, - "slice_artifact_hash": self.slice_artifact_hash, - "derivative_slice": self.derivative_slice, - "final_artifact_semantics": self.final_artifact_semantics, - } - - -def select_dense_llama_tensor_names( - tensor_names: Iterable[str], - start_layer: int, - end_layer: int, - *, - total_layers: int | None = None, -) -> set[str]: - """Return the dense-Llama GGUF tensor names owned by an inclusive range.""" - if start_layer < 0: - raise ValueError("start_layer must be non-negative") - if end_layer < start_layer: - raise ValueError("end_layer must be greater than or equal to start_layer") - - selected: set[str] = set() - for tensor_name in tensor_names: - if _tensor_belongs_to_range(tensor_name, start_layer, end_layer, total_layers): - selected.add(tensor_name) - return selected - - -def infer_dense_llama_ownership( - tensor_names: Iterable[str], - *, - total_layers: int | None = None, - source_artifact_hash: str | None = None, - slice_artifact_hash: str | None = None, - derivative_slice: bool = False, - final_artifact_semantics: bool = True, -) -> DenseLlamaShardOwnership: - """Infer authoritative loaded range and endpoint ownership from tensors.""" - names = tuple(str(name) for name in tensor_names if isinstance(name, str)) - if not names: - raise ValueError("tensor inventory is empty") - - block_layers = sorted( - { - layer - for name in names - if (layer := _layer_index(name)) is not None - } - ) - if not block_layers: - raise ValueError("tensor inventory does not contain any blk.N.* tensors") - - selected = tuple(sorted(names)) - return DenseLlamaShardOwnership( - start_layer=block_layers[0], - end_layer=block_layers[-1], - owns_embedding=any(_is_head_tensor(name) for name in names), - owns_final_head=any( - _is_tail_tensor(name, total_layers=total_layers, loaded_end=block_layers[-1]) - for name in names - ), - tensor_names=selected, - source_artifact_hash=source_artifact_hash, - slice_artifact_hash=slice_artifact_hash, - derivative_slice=derivative_slice, - final_artifact_semantics=final_artifact_semantics, - ) - - -def authoritative_dense_llama_ownership( - backend: Any, - selection: Any | None = None, -) -> DenseLlamaShardOwnership: - """Return the most authoritative dense-Llama ownership the backend exposes.""" - tensor_names = _tensor_names_from_backend(backend) - if tensor_names: - try: - return infer_dense_llama_ownership( - tensor_names, - total_layers=_backend_total_layers(backend, selection), - ) - except ValueError: - pass - - start, end = _backend_loaded_bounds(backend, selection) - return DenseLlamaShardOwnership( - start_layer=start, - end_layer=end, - owns_embedding=_backend_owns_embedding(backend, start), - owns_final_head=_backend_owns_final_head(backend, end), - ) - - -def _backend_loaded_bounds(backend: Any, selection: Any | None) -> tuple[int, int]: - start = getattr(backend, "loaded_shard_start", None) - end = getattr(backend, "loaded_shard_end", None) - if start is None: - start = getattr(backend, "shard_start", None) - if end is None: - end = getattr(backend, "shard_end", None) - if start is None or end is None: - if selection is None: - raise ValueError("backend does not expose a loaded shard range") - start = getattr(selection, "shard_start") - end = getattr(selection, "shard_end") - return int(start), int(end) - - -def _backend_owns_embedding(backend: Any, start: int) -> bool: - value = getattr(backend, "owns_embedding", None) - if value is None: - value = getattr(backend, "is_head", start == 0) - return bool(value) - - -def _backend_owns_final_head(backend: Any, end: int) -> bool: - value = getattr(backend, "owns_final_head", None) - if value is None: - value = getattr(backend, "is_tail", False) - return bool(value) - - -def _backend_total_layers(backend: Any, selection: Any | None) -> int | None: - value = getattr(backend, "total_layers", None) - if isinstance(value, int) and value > 0: - return value - if selection is None: - return None - total = getattr(selection, "total_layers", None) - if isinstance(total, int) and total > 0: - return total - return None - - -def _tensor_names_from_backend(backend: Any) -> tuple[str, ...]: - for attr in ("loaded_tensor_names", "tensor_names", "tensor_inventory"): - value = getattr(backend, attr, None) - names = _normalise_tensor_names(value) - if names: - return names - return () - - -def _normalise_tensor_names(value: Any) -> tuple[str, ...]: - if value is None: - return () - if isinstance(value, Mapping): - items = value.keys() - else: - try: - items = list(value) - except TypeError: - return () - names = [str(item) for item in items if isinstance(item, str) and item.strip()] - return tuple(names) - - -def _tensor_belongs_to_range( - tensor_name: str, - start_layer: int, - end_layer: int, - total_layers: int | None, -) -> bool: - layer = _layer_index(tensor_name) - if layer is not None: - return start_layer <= layer <= end_layer - - if start_layer == 0 and _is_head_tensor(tensor_name): - return True - - if total_layers is not None and end_layer >= total_layers - 1 and _is_tail_tensor( - tensor_name, total_layers=total_layers, loaded_end=end_layer - ): - return True - - return False - - -def _layer_index(tensor_name: str) -> int | None: - match = _BLOCK_RE.match(tensor_name) - if match is None: - return None - return int(match.group(1)) - - -def _is_head_tensor(tensor_name: str) -> bool: - lowered = tensor_name.lower() - return lowered in _HEAD_TENSOR_NAMES or any( - lowered.startswith(prefix) - for prefix in ("token_embd.", "tok_embeddings.", "embed_tokens.") - ) - - -def _is_tail_tensor( - tensor_name: str, - *, - total_layers: int | None, - loaded_end: int, -) -> bool: - lowered = tensor_name.lower() - if lowered in _TAIL_TENSOR_NAMES: - return True - if total_layers is not None and loaded_end >= total_layers - 1: - return any( - lowered.startswith(prefix) - for prefix in ("output_norm.", "final_norm.", "norm.") - ) - return False diff --git a/packages/node/meshnet_node/hot_kv_state.py b/packages/node/meshnet_node/hot_kv_state.py deleted file mode 100644 index 23b61ea..0000000 --- a/packages/node/meshnet_node/hot_kv_state.py +++ /dev/null @@ -1,918 +0,0 @@ -"""Isolated concurrent local Hot KV State for distributed Shards (DGR-007). - -Hot KV State stays local to the node serving a Shard (RALPH runtime decision #7). -A concurrent server must map each ``(Route Session ID, route epoch)`` to an -isolated bounded KV context (decision #8) so that one request can never clear or -corrupt another's cache. - -This module owns the *lifecycle and storage* of that state and is deliberately -backend-agnostic: - -* :class:`HotKvStateManager` is the single mutation entry point. It maps - ``(session_id, route_epoch)`` to a :class:`SessionCache`, allocates KV **only - for the owned layer range**, and enforces a byte budget, a session cap, and a - TTL through LRU/TTL eviction. It rejects stale route epochs and incompatible - cache recipes, and returns an **explicit** :class:`CacheMiss` when state the - caller expected is gone (evicted, released, desynchronised, or never held) so - the head degrades to a from-token-zero re-prefill instead of corrupting output - (RALPH decision #14: unverified KV is never migrated silently). -* :class:`LayerKvCache` / :class:`SessionCache` are the per-owned-layer K/V - containers. They are plain ``numpy`` arrays so the default deterministic test - suite needs no torch, GPU, download, or API credit; the pinned llama.cpp worker - (DGR-008) maps a llama sequence onto the same container contract. -* :class:`KvBoundaryAdapter` wraps a KV-aware ``ShardComputation`` (the DGR-006 - duck type plus ``run_layers_cached``) so a Shard can run cached prefill/decode - through the manager while honouring the architecture-defined boundary contract - (head embeds tokens, middle/tail bypass embedding, non-tail emits the - unnormalized residual, tail samples). - -The manager owns *all* cache mutation: a computation reads the existing cache and -returns the new K/V for the appended positions, and the manager decides whether -that append fits the budget. That keeps eviction, accounting, and isolation in one -place instead of scattered across backends. -""" - -from __future__ import annotations - -import threading -import time -from collections import OrderedDict -from dataclasses import dataclass, field -from enum import Enum -from typing import Any, Callable, Mapping - -import numpy as np - -from meshnet_node.boundary_adapter import ( - BOUNDARY_SCHEMA_VERSION, - BoundaryBundle, - BoundaryContractError, - SamplingContract, - ShardRole, - TailOutput, - certified_architecture, - role_for_range, -) -from meshnet_node.runtime_recipe import compatibility_fingerprint - - -class HotKvStateError(RuntimeError): - """Base class for Hot KV State errors.""" - - -class StaleRouteEpochError(HotKvStateError): - """Raised when a request references a route epoch older than the current one. - - A newer route epoch means the route was re-planned; the old epoch's KV is - unverified against the new plan and must never be silently reused. - """ - - -class IncompatibleCacheRecipeError(HotKvStateError): - """Raised when a request's cache recipe does not match the loaded shard. - - A different quantization / dtype / owned range / architecture produces a KV - layout this node cannot reuse without corrupting output. - """ - - -class KvBudgetExceededError(HotKvStateError): - """Raised when a single session cannot fit the configured byte budget. - - Other sessions are evicted first (LRU); this fires only when even one session - alone exceeds the budget, which is a misconfiguration rather than pressure. - """ - - -class KvCacheMissError(HotKvStateError): - """Raised by the strict accessor when expected session state is absent. - - Prefer :meth:`HotKvStateManager.resolve`, which returns a structured - :class:`CacheMiss` instead of raising, when the caller wants to fall back to a - stateless re-prefill. - """ - - def __init__(self, miss: "CacheMiss") -> None: - super().__init__(str(miss)) - self.miss = miss - - -class CacheMissReason(str, Enum): - """Why a lookup produced a cache miss (all benign; retry from token zero).""" - - UNKNOWN_SESSION = "unknown-session" - EVICTED_TTL = "evicted-ttl" - EVICTED_LRU = "evicted-lru" - RELEASED = "released" - SUPERSEDED_EPOCH = "superseded-epoch" - SEQ_LEN_MISMATCH = "seq-len-mismatch" - - -@dataclass(frozen=True) -class CacheMiss: - """Explicit cache-miss response the head can act on (re-prefill). - - This is a value, not an exception: the native protocol carries a cache - expectation/result, and a miss is a normal, expected outcome under eviction. - """ - - session_id: str - route_epoch: int - reason: CacheMissReason - detail: str = "" - - def __str__(self) -> str: - suffix = f": {self.detail}" if self.detail else "" - return ( - f"cache miss for session {self.session_id[:8]} epoch " - f"{self.route_epoch} ({self.reason.value}){suffix}" - ) - - -@dataclass(frozen=True) -class KvCacheRecipe: - """The identity of a Shard's KV layout, used to reject incompatible reuse. - - Two recipes are compatible iff their fingerprints match — same certified - architecture, KV dtype, head geometry, and owned layer range within the same - whole-model layer count. - """ - - architecture_adapter: str - kv_dtype: str - n_kv_heads: int - head_dim: int - total_layers: int - start_layer: int - end_layer: int - boundary_schema_version: int = BOUNDARY_SCHEMA_VERSION - - def __post_init__(self) -> None: - # Fail closed on architecture identity (shared with the boundary adapter). - certified_architecture(self.architecture_adapter) - if self.n_kv_heads <= 0: - raise ValueError("n_kv_heads must be positive") - if self.head_dim <= 0: - raise ValueError("head_dim must be positive") - try: - np.dtype(self.kv_dtype) - except TypeError as exc: # pragma: no cover - defensive - raise ValueError(f"invalid kv_dtype {self.kv_dtype!r}") from exc - # role_for_range validates 0 <= start <= end <= total_layers - 1. - role_for_range(self.start_layer, self.end_layer, self.total_layers) - if self.boundary_schema_version < 1: - raise ValueError("boundary_schema_version must be >= 1") - - @property - def owned_layers(self) -> tuple[int, ...]: - return tuple(range(self.start_layer, self.end_layer + 1)) - - @property - def role(self) -> ShardRole: - return role_for_range(self.start_layer, self.end_layer, self.total_layers) - - def bytes_per_token(self) -> int: - """Bytes of KV one token adds across *owned* layers (keys + values).""" - itemsize = np.dtype(self.kv_dtype).itemsize - per_layer = 2 * self.n_kv_heads * self.head_dim * itemsize - return per_layer * len(self.owned_layers) - - def fingerprint(self) -> str: - return compatibility_fingerprint( - { - "kind": "hot-kv-recipe", - # Canonicalize the architecture so 'llama' / 'LlamaForCausalLM' - # map to the same fingerprint (they are the same layout). - "architecture_adapter": certified_architecture( - self.architecture_adapter - ).adapter, - "kv_dtype": np.dtype(self.kv_dtype).name, - "n_kv_heads": self.n_kv_heads, - "head_dim": self.head_dim, - "total_layers": self.total_layers, - "start_layer": self.start_layer, - "end_layer": self.end_layer, - "boundary_schema_version": self.boundary_schema_version, - } - ) - - def is_compatible(self, other: "KvCacheRecipe") -> bool: - return self.fingerprint() == other.fingerprint() - - -class LayerKvCache: - """K/V storage for a single owned layer; sequence axis is 0. - - Keys and values are ``(seq, n_kv_heads, head_dim)``. Backends store the - position-encoded (post-RoPE) keys so a decode step only appends the new rows. - """ - - __slots__ = ("layer_index", "n_kv_heads", "head_dim", "dtype", "keys", "values") - - def __init__( - self, layer_index: int, n_kv_heads: int, head_dim: int, dtype: Any - ) -> None: - self.layer_index = int(layer_index) - self.n_kv_heads = int(n_kv_heads) - self.head_dim = int(head_dim) - self.dtype = np.dtype(dtype) - self.keys = np.empty((0, self.n_kv_heads, self.head_dim), dtype=self.dtype) - self.values = np.empty((0, self.n_kv_heads, self.head_dim), dtype=self.dtype) - - @property - def length(self) -> int: - return int(self.keys.shape[0]) - - def _validate(self, array: np.ndarray, name: str) -> np.ndarray: - arr = np.asarray(array, dtype=self.dtype) - if arr.ndim != 3 or arr.shape[1:] != (self.n_kv_heads, self.head_dim): - raise ValueError( - f"layer {self.layer_index} {name} must be " - f"(seq, {self.n_kv_heads}, {self.head_dim}), got {arr.shape}" - ) - return arr - - def append(self, keys: np.ndarray, values: np.ndarray) -> int: - k = self._validate(keys, "keys") - v = self._validate(values, "values") - if k.shape[0] != v.shape[0]: - raise ValueError( - f"layer {self.layer_index} keys/values disagree on token count " - f"({k.shape[0]} vs {v.shape[0]})" - ) - self.keys = np.concatenate([self.keys, k], axis=0) - self.values = np.concatenate([self.values, v], axis=0) - return self.length - - def truncate(self, length: int) -> None: - length = max(0, int(length)) - self.keys = self.keys[:length] - self.values = self.values[:length] - - @property - def nbytes(self) -> int: - return int(self.keys.nbytes + self.values.nbytes) - - -@dataclass -class SessionCache: - """Isolated per-``(session_id, epoch)`` KV context over the owned layers only.""" - - session_id: str - route_epoch: int - recipe: KvCacheRecipe - layers: "OrderedDict[int, LayerKvCache]" - created_tick: float - last_tick: float - released: bool = False - - @property - def seq_len(self) -> int: - if not self.layers: - return 0 - # All owned layers advance in lockstep; report the first owned layer. - return next(iter(self.layers.values())).length - - @property - def owned_layers(self) -> tuple[int, ...]: - return tuple(self.layers.keys()) - - def layer(self, index: int) -> LayerKvCache: - try: - return self.layers[index] - except KeyError: - raise KeyError( - f"layer {index} is not owned by this shard " - f"(owned {list(self.layers)})" - ) from None - - def read_only_layers(self) -> Mapping[int, LayerKvCache]: - """The current per-layer caches a computation reads to attend over.""" - return dict(self.layers) - - def _append(self, kv_by_layer: Mapping[int, Any]) -> int: - provided = set(kv_by_layer) - owned = set(self.layers) - if provided != owned: - raise ValueError( - f"append must cover exactly the owned layers {sorted(owned)}, " - f"got {sorted(provided)}" - ) - # Pre-validate token counts so a partial append never desynchronises the - # owned layers (append is all-or-nothing). - new_counts = set() - for idx, (keys, _values) in kv_by_layer.items(): - new_counts.add(int(np.asarray(keys).shape[0])) - if len(new_counts) != 1: - raise ValueError( - f"append token counts disagree across layers: {sorted(new_counts)}" - ) - for idx, (keys, values) in kv_by_layer.items(): - self.layers[idx].append(keys, values) - return self.seq_len - - def _truncate(self, length: int) -> None: - for cache in self.layers.values(): - cache.truncate(length) - - @property - def nbytes(self) -> int: - return sum(cache.nbytes for cache in self.layers.values()) - - -@dataclass(frozen=True) -class HotKvStateConfig: - """Bounds for the manager: memory budget, session cap, and idle TTL.""" - - budget_bytes: int = 64 * 1024 * 1024 - max_sessions: int = 8 - ttl_seconds: float = 600.0 - miss_history: int = 256 - - def __post_init__(self) -> None: - if self.budget_bytes <= 0: - raise ValueError("budget_bytes must be positive") - if self.max_sessions < 1: - raise ValueError("max_sessions must be >= 1") - if self.ttl_seconds < 0: - raise ValueError("ttl_seconds must be >= 0") - if self.miss_history < 0: - raise ValueError("miss_history must be >= 0") - - -class HotKvStateManager: - """Concurrent, bounded map of ``(session_id, epoch)`` to an isolated KV context.""" - - def __init__( - self, - recipe: KvCacheRecipe, - config: HotKvStateConfig | None = None, - *, - clock: Callable[[], float] | None = None, - ) -> None: - self.recipe = recipe - self.config = config or HotKvStateConfig() - self._clock = clock or time.monotonic - self._sessions: "OrderedDict[tuple[str, int], SessionCache]" = OrderedDict() - self._latest_epoch: dict[str, int] = {} - self._misses: "OrderedDict[tuple[str, int], CacheMiss]" = OrderedDict() - self._lock = threading.RLock() - - # -- introspection -------------------------------------------------------- - - @property - def total_bytes(self) -> int: - with self._lock: - return sum(s.nbytes for s in self._sessions.values()) - - @property - def session_count(self) -> int: - with self._lock: - self._evict_expired_locked(self._clock()) - return len(self._sessions) - - def session_keys(self) -> list[tuple[str, int]]: - with self._lock: - return list(self._sessions.keys()) - - # -- lifecycle ------------------------------------------------------------ - - def open( - self, - session_id: str, - route_epoch: int, - *, - recipe: KvCacheRecipe | None = None, - ) -> SessionCache: - """Create (or replace) a fresh, empty isolated context for the session. - - A higher route epoch supersedes and frees any earlier epoch for the same - session id; an older epoch is rejected as stale. - """ - self._require_text(session_id, "session_id") - route_epoch = self._require_epoch(route_epoch) - with self._lock: - self._check_recipe(recipe) - self._validate_epoch_locked(session_id, route_epoch) - now = self._clock() - self._evict_expired_locked(now) - self._supersede_older_epochs_locked(session_id, route_epoch) - key = (session_id, route_epoch) - # A re-open at the same epoch replaces the prior context entirely. - self._sessions.pop(key, None) - layers: "OrderedDict[int, LayerKvCache]" = OrderedDict( - ( - idx, - LayerKvCache( - idx, - self.recipe.n_kv_heads, - self.recipe.head_dim, - self.recipe.kv_dtype, - ), - ) - for idx in self.recipe.owned_layers - ) - session = SessionCache( - session_id=session_id, - route_epoch=route_epoch, - recipe=self.recipe, - layers=layers, - created_tick=now, - last_tick=now, - ) - self._sessions[key] = session - self._latest_epoch[session_id] = route_epoch - self._misses.pop(key, None) - self._enforce_capacity_locked(protect=key, incoming_bytes=0) - return session - - def append( - self, - session_id: str, - route_epoch: int, - kv_by_layer: Mapping[int, Any], - *, - recipe: KvCacheRecipe | None = None, - expected_seq_len: int | None = None, - ) -> SessionCache: - """Append new K/V (prefill or decode) to an existing isolated context. - - The computation supplies exactly the owned layers' new keys/values. The - manager evicts other sessions (LRU) to fit the byte budget before growing - this one, and raises :class:`KvBudgetExceededError` only if this session - alone cannot fit. - """ - route_epoch = self._require_epoch(route_epoch) - with self._lock: - self._check_recipe(recipe) - self._validate_epoch_locked(session_id, route_epoch) - session = self._require_live_locked(session_id, route_epoch) - if expected_seq_len is not None and session.seq_len != expected_seq_len: - miss = self._drop_and_record_locked( - (session_id, route_epoch), - CacheMissReason.SEQ_LEN_MISMATCH, - detail=f"cache holds {session.seq_len}, caller expected " - f"{expected_seq_len}", - ) - raise KvCacheMissError(miss) - n_new = self._new_token_count(kv_by_layer) - incoming = n_new * self.recipe.bytes_per_token() - self._enforce_capacity_locked( - protect=(session_id, route_epoch), incoming_bytes=incoming - ) - session._append(kv_by_layer) - session.last_tick = self._clock() - self._sessions.move_to_end((session_id, route_epoch)) - return session - - def truncate( - self, session_id: str, route_epoch: int, length: int - ) -> SessionCache: - """Drop cached positions beyond ``length`` (rollback) for one session.""" - route_epoch = self._require_epoch(route_epoch) - with self._lock: - self._validate_epoch_locked(session_id, route_epoch) - session = self._require_live_locked(session_id, route_epoch) - if length < 0: - raise ValueError("truncate length must be >= 0") - session._truncate(length) - session.last_tick = self._clock() - self._sessions.move_to_end((session_id, route_epoch)) - return session - - def release(self, session_id: str, route_epoch: int) -> bool: - """Free one session's context; other sessions are untouched. - - Returns True if a live context was freed. A later lookup for the released - key yields an explicit :class:`CacheMiss`. - """ - route_epoch = self._require_epoch(route_epoch) - with self._lock: - key = (session_id, route_epoch) - existed = key in self._sessions - self._drop_and_record_locked(key, CacheMissReason.RELEASED) - return existed - - # -- lookup --------------------------------------------------------------- - - def resolve( - self, - session_id: str, - route_epoch: int, - *, - recipe: KvCacheRecipe | None = None, - expected_seq_len: int | None = None, - ) -> SessionCache | CacheMiss: - """Return the live context or an explicit :class:`CacheMiss`. - - Rejects stale epochs and incompatible recipes (both are protocol - violations, not benign misses). - """ - route_epoch = self._require_epoch(route_epoch) - with self._lock: - self._check_recipe(recipe) - self._validate_epoch_locked(session_id, route_epoch) - now = self._clock() - self._evict_expired_locked(now) - key = (session_id, route_epoch) - session = self._sessions.get(key) - if session is None: - return self._recorded_miss_locked(key) - if expected_seq_len is not None and session.seq_len != expected_seq_len: - return self._drop_and_record_locked( - key, - CacheMissReason.SEQ_LEN_MISMATCH, - detail=f"cache holds {session.seq_len}, caller expected " - f"{expected_seq_len}", - ) - session.last_tick = now - self._sessions.move_to_end(key) - return session - - def get( - self, - session_id: str, - route_epoch: int, - *, - recipe: KvCacheRecipe | None = None, - expected_seq_len: int | None = None, - ) -> SessionCache: - """Strict accessor: raises :class:`KvCacheMissError` on a miss.""" - result = self.resolve( - session_id, - route_epoch, - recipe=recipe, - expected_seq_len=expected_seq_len, - ) - if isinstance(result, CacheMiss): - raise KvCacheMissError(result) - return result - - # -- internals ------------------------------------------------------------ - - def _check_recipe(self, recipe: KvCacheRecipe | None) -> None: - if recipe is not None and not self.recipe.is_compatible(recipe): - raise IncompatibleCacheRecipeError( - "request cache recipe does not match this shard's loaded recipe " - f"(request {recipe.fingerprint()} vs shard {self.recipe.fingerprint()})" - ) - - def _validate_epoch_locked(self, session_id: str, route_epoch: int) -> None: - latest = self._latest_epoch.get(session_id) - if latest is not None and route_epoch < latest: - raise StaleRouteEpochError( - f"session {session_id[:8]} route epoch {route_epoch} is stale; " - f"current epoch is {latest}" - ) - - def _supersede_older_epochs_locked( - self, session_id: str, route_epoch: int - ) -> None: - stale_keys = [ - key - for key in self._sessions - if key[0] == session_id and key[1] < route_epoch - ] - for key in stale_keys: - self._drop_and_record_locked(key, CacheMissReason.SUPERSEDED_EPOCH) - - def _require_live_locked( - self, session_id: str, route_epoch: int - ) -> SessionCache: - now = self._clock() - self._evict_expired_locked(now) - key = (session_id, route_epoch) - session = self._sessions.get(key) - if session is None: - raise KvCacheMissError(self._recorded_miss_locked(key)) - return session - - def _new_token_count(self, kv_by_layer: Mapping[int, Any]) -> int: - owned = set(self.recipe.owned_layers) - if set(kv_by_layer) != owned: - raise ValueError( - f"append must cover exactly the owned layers {sorted(owned)}, " - f"got {sorted(kv_by_layer)}" - ) - counts = {int(np.asarray(k).shape[0]) for k, _ in kv_by_layer.values()} - if len(counts) != 1: - raise ValueError( - f"append token counts disagree across layers: {sorted(counts)}" - ) - return counts.pop() - - def _enforce_capacity_locked( - self, *, protect: tuple[str, int], incoming_bytes: int - ) -> None: - # Session cap: evict LRU sessions other than the protected one. - while len(self._sessions) > self.config.max_sessions: - victim = self._lru_victim_locked(protect) - if victim is None: - break - self._drop_and_record_locked(victim, CacheMissReason.EVICTED_LRU) - - # Byte budget: the protected session's own footprint after the append. - protected = self._sessions.get(protect) - protected_bytes = (protected.nbytes if protected is not None else 0) + incoming_bytes - if protected_bytes > self.config.budget_bytes: - raise KvBudgetExceededError( - f"session {protect[0][:8]} needs {protected_bytes} bytes which " - f"exceeds the KV budget {self.config.budget_bytes}" - ) - # Evict other LRU sessions until the whole store fits with the append. - while self._total_bytes_locked() + incoming_bytes > self.config.budget_bytes: - victim = self._lru_victim_locked(protect) - if victim is None: - break - self._drop_and_record_locked(victim, CacheMissReason.EVICTED_LRU) - - def _lru_victim_locked(self, protect: tuple[str, int]) -> tuple[str, int] | None: - for key in self._sessions: # OrderedDict iterates oldest-first. - if key != protect: - return key - return None - - def _total_bytes_locked(self) -> int: - return sum(s.nbytes for s in self._sessions.values()) - - def _evict_expired_locked(self, now: float) -> None: - ttl = self.config.ttl_seconds - if ttl <= 0: - return - expired = [ - key - for key, session in self._sessions.items() - if now - session.last_tick > ttl - ] - for key in expired: - self._drop_and_record_locked(key, CacheMissReason.EVICTED_TTL) - - def _drop_and_record_locked( - self, - key: tuple[str, int], - reason: CacheMissReason, - *, - detail: str = "", - ) -> CacheMiss: - session = self._sessions.pop(key, None) - if session is not None: - session.released = True - miss = CacheMiss( - session_id=key[0], route_epoch=key[1], reason=reason, detail=detail - ) - self._record_miss_locked(key, miss) - return miss - - def _record_miss_locked(self, key: tuple[str, int], miss: CacheMiss) -> None: - if self.config.miss_history <= 0: - return - self._misses.pop(key, None) - self._misses[key] = miss - while len(self._misses) > self.config.miss_history: - self._misses.popitem(last=False) - - def _recorded_miss_locked(self, key: tuple[str, int]) -> CacheMiss: - recorded = self._misses.get(key) - if recorded is not None: - return recorded - return CacheMiss( - session_id=key[0], - route_epoch=key[1], - reason=CacheMissReason.UNKNOWN_SESSION, - ) - - @staticmethod - def _require_text(value: Any, name: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise ValueError(f"{name} must be a non-empty string") - return value - - @staticmethod - def _require_epoch(value: Any) -> int: - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError("route_epoch must be an integer") - if value < 0: - raise ValueError("route_epoch must be >= 0") - return value - - -def kv_recipe_for(computation: Any) -> KvCacheRecipe: - """Build a :class:`KvCacheRecipe` from a KV-aware ``ShardComputation``. - - The computation exposes the DGR-006 duck type plus KV geometry - (``n_kv_heads``, ``head_dim``, ``kv_dtype``). - """ - return KvCacheRecipe( - architecture_adapter=str(getattr(computation, "architecture_adapter")), - kv_dtype=str(getattr(computation, "kv_dtype", "float32")), - n_kv_heads=int(getattr(computation, "n_kv_heads")), - head_dim=int(getattr(computation, "head_dim")), - total_layers=int(getattr(computation, "total_layers")), - start_layer=int(getattr(computation, "start_layer")), - end_layer=int(getattr(computation, "end_layer")), - ) - - -@dataclass -class KvBoundaryAdapter: - """KV-aware boundary driver: cached prefill/decode through the manager. - - Mirrors the DGR-006 :class:`~meshnet_node.boundary_adapter.BoundaryAdapter` - contract (head embeds tokens, middle/tail bypass embedding and consume the - unnormalized residual bundle, non-tail emits the unnormalized residual, tail - normalizes + heads + prunes + samples) but threads a per-session KV context. - - The wrapped computation must additionally expose:: - - run_layers_cached(hidden, *, positions, past_kv) - -> (hidden_out, {layer_index: (new_keys, new_values)}) - - reading ``past_kv`` (the current per-owned-layer caches) and returning the new - position-encoded K/V for the appended positions only. The manager, not the - computation, commits those K/V so eviction and budget stay centralized. - """ - - computation: Any - manager: HotKvStateManager - sampling: SamplingContract = field(default_factory=SamplingContract.greedy) - architecture: Any = field(init=False) - role: ShardRole = field(init=False) - start_layer: int = field(init=False) - end_layer: int = field(init=False) - total_layers: int = field(init=False) - recipe: KvCacheRecipe = field(init=False) - - def __post_init__(self) -> None: - arch_name = getattr(self.computation, "architecture_adapter", None) - self.architecture = certified_architecture(arch_name) - self.start_layer = int(getattr(self.computation, "start_layer")) - self.end_layer = int(getattr(self.computation, "end_layer")) - self.total_layers = int(getattr(self.computation, "total_layers")) - self.role = role_for_range(self.start_layer, self.end_layer, self.total_layers) - self.recipe = kv_recipe_for(self.computation) - if not self.manager.recipe.is_compatible(self.recipe): - raise IncompatibleCacheRecipeError( - "manager recipe does not match this computation's KV recipe" - ) - - @property - def is_head(self) -> bool: - return self.role.owns_embedding - - @property - def is_tail(self) -> bool: - return self.role.owns_final_head - - def prefill( - self, - session_id: str, - route_epoch: int, - *, - token_ids: Any | None = None, - boundary: BoundaryBundle | None = None, - ) -> BoundaryBundle | TailOutput: - """Open a fresh isolated context and run the prompt through this range.""" - session = self.manager.open(session_id, route_epoch, recipe=self.recipe) - return self._run_step(session, token_ids, boundary) - - def decode( - self, - session_id: str, - route_epoch: int, - *, - token_ids: Any | None = None, - boundary: BoundaryBundle | None = None, - expected_seq_len: int | None = None, - ) -> BoundaryBundle | TailOutput | CacheMiss: - """Append one (or more) decode positions to an existing context. - - Returns an explicit :class:`CacheMiss` if the context is gone so the head - can re-prefill from token zero instead of corrupting output. - """ - resolved = self.manager.resolve( - session_id, - route_epoch, - recipe=self.recipe, - expected_seq_len=expected_seq_len, - ) - if isinstance(resolved, CacheMiss): - return resolved - return self._run_step(resolved, token_ids, boundary) - - # -- internals ------------------------------------------------------------ - - def _run_step( - self, - session: SessionCache, - token_ids: Any | None, - boundary: BoundaryBundle | None, - ) -> BoundaryBundle | TailOutput: - prev_len = session.seq_len - hidden, positions = self._ingest(prev_len, token_ids, boundary) - hidden_out, new_kv = self.computation.run_layers_cached( - hidden, positions=positions, past_kv=session.read_only_layers() - ) - self.manager.append( - session.session_id, - session.route_epoch, - new_kv, - recipe=self.recipe, - expected_seq_len=prev_len, - ) - if self.is_tail: - return self._emit_tail(hidden_out) - return self._emit_boundary(hidden_out, positions) - - def _ingest( - self, - prev_len: int, - token_ids: Any | None, - boundary: BoundaryBundle | None, - ) -> tuple[np.ndarray, np.ndarray]: - if self.role.owns_embedding: - if token_ids is None: - raise BoundaryContractError( - "the head owns token embedding and must receive token IDs" - ) - if boundary is not None: - raise BoundaryContractError( - "the head owns token embedding; it must not receive a boundary " - "bundle from an upstream range" - ) - ids = np.asarray(token_ids) - if ids.ndim == 1: - ids = ids[None, :] - if ids.ndim != 2: - raise BoundaryContractError("token IDs must be (seq,) or (batch, seq)") - hidden = np.asarray(self.computation.embed_tokens(ids)) - n_new = ids.shape[1] - positions = np.broadcast_to( - np.arange(prev_len, prev_len + n_new, dtype=np.int64), - ids.shape, - ).copy() - return hidden, positions - # Middle / tail: consume the boundary bundle (the unnormalized residual). - if token_ids is not None: - raise BoundaryContractError( - "middle/tail Shards bypass token embedding; they must not receive " - "token IDs" - ) - if boundary is None: - raise BoundaryContractError( - "middle/tail Shards must receive the named boundary bundle" - ) - self._check_boundary(boundary) - return np.asarray(boundary.residual), np.asarray(boundary.positions) - - def _check_boundary(self, boundary: BoundaryBundle) -> None: - if certified_architecture(boundary.architecture_adapter) is not self.architecture: - raise BoundaryContractError( - f"boundary bundle architecture {boundary.architecture_adapter!r} " - f"does not match this Shard's adapter {self.architecture.adapter!r}" - ) - if boundary.schema_version != self.architecture.boundary_schema_version: - raise BoundaryContractError( - f"boundary schema v{boundary.schema_version} is not supported by " - f"this Shard (expects v{self.architecture.boundary_schema_version})" - ) - if boundary.tensor_name != self.architecture.boundary_tensor_name: - raise BoundaryContractError( - f"boundary tensor {boundary.tensor_name!r} is not the " - f"architecture-defined {self.architecture.boundary_tensor_name!r}" - ) - if boundary.normalized: - raise BoundaryContractError( - "boundary bundle is normalized; a Shard range must receive the " - "UNNORMALIZED architecture-defined residual" - ) - if boundary.next_layer != self.start_layer: - raise BoundaryContractError( - f"boundary hands over at layer {boundary.next_layer} but this " - f"Shard starts at layer {self.start_layer}" - ) - - def _emit_boundary( - self, hidden: np.ndarray, positions: np.ndarray - ) -> BoundaryBundle: - return BoundaryBundle( - architecture_adapter=self.architecture.adapter, - schema_version=self.architecture.boundary_schema_version, - tensor_name=self.architecture.boundary_tensor_name, - residual=np.asarray(hidden), - positions=np.asarray(positions), - next_layer=self.end_layer + 1, - normalized=False, - ) - - def _emit_tail(self, hidden: np.ndarray) -> TailOutput: - hidden = np.asarray(hidden) - if self.architecture.prunes_rows_at_tail: - last_hidden = hidden[:, -1:, :] - else: # pragma: no cover - no certified architecture takes this path yet - last_hidden = hidden - if self.architecture.normalizes_before_head: - last_hidden = np.asarray(self.computation.final_norm(last_hidden)) - logits = np.asarray(self.computation.lm_head(last_hidden)) - last_logits = logits[:, -1, :] - token_id = self.sampling.sample(last_logits) - return TailOutput(token_id=token_id, logits=last_logits, sampling=self.sampling) diff --git a/packages/node/meshnet_node/model_backend.py b/packages/node/meshnet_node/model_backend.py index fa39c6b..e78ee87 100644 --- a/packages/node/meshnet_node/model_backend.py +++ b/packages/node/meshnet_node/model_backend.py @@ -86,6 +86,59 @@ class TensorPayload: # Number of tokens already cached before this payload's tokens (decode steps). past_len: int | None = None + def to_envelope( + self, + *, + name: str, + request_id: str, + work_id: str, + route_session: str, + route_epoch: int, + shard_start: int, + effective_start: int, + phase: str, + position: int, + idempotency_step: int, + byte_order: str = "little", + compression: str = "identity", + max_fragment_bytes: int | None = None, + token_id_sideband: list[int] | None = None, + architecture_state: dict[str, Any] | None = None, + recurrent_state: dict[str, Any] | None = None, + mtp: dict[str, Any] | None = None, + extensions: dict[str, Any] | None = None, + ): + from .protocol import ActivationEnvelope, DEFAULT_FRAGMENT_BYTES + + return ActivationEnvelope.from_tensor_payload( + payload=self, + name=name, + request_id=request_id, + work_id=work_id, + route_session=route_session, + route_epoch=route_epoch, + shard_start=shard_start, + effective_start=effective_start, + phase=phase, + position=position, + idempotency_step=idempotency_step, + byte_order=byte_order, + compression=compression, + max_fragment_bytes=max_fragment_bytes or DEFAULT_FRAGMENT_BYTES, + token_id_sideband=token_id_sideband, + architecture_state=architecture_state, + recurrent_state=recurrent_state, + mtp=mtp, + extensions=extensions, + ) + + @classmethod + def from_envelope(cls, envelope, *, tensor_name: str = "activations"): + tensor_payload = envelope.to_tensor_payload(tensor_name=tensor_name) + if not isinstance(tensor_payload, cls): + raise TypeError("envelope did not produce a TensorPayload") + return tensor_payload + @dataclass(frozen=True) class TailTokenResult: @@ -323,10 +376,6 @@ class TorchModelShard: ) self.is_head = shard_start == 0 self.is_tail = shard_end >= self.total_layers - 1 - self.loaded_shard_start = shard_start - self.loaded_shard_end = shard_end - self.owns_embedding = self.is_head - self.owns_final_head = self.is_tail self.hidden_size = int( getattr(self.model.config, "hidden_size", 0) or getattr(self.model.config, "n_embd", 0) @@ -348,17 +397,6 @@ class TorchModelShard: ttl_seconds=float(os.environ.get("MESHNET_KV_TTL_SECONDS", "600")), ) - @property - def loaded_range(self) -> tuple[int, int]: - return self.loaded_shard_start, self.loaded_shard_end - - @property - def endpoint_ownership(self) -> dict[str, bool]: - return { - "owns_embedding": self.owns_embedding, - "owns_final_head": self.owns_final_head, - } - def encode_prompt(self, prompt: str, session_id: str | None = None) -> TensorPayload: if not self.is_head or self._embed_tokens is None: raise ModelBackendError("text prompts can only be accepted by the head shard") diff --git a/packages/node/meshnet_node/native_protocol/__init__.py b/packages/node/meshnet_node/native_protocol/__init__.py deleted file mode 100644 index c02ebfb..0000000 --- a/packages/node/meshnet_node/native_protocol/__init__.py +++ /dev/null @@ -1,300 +0,0 @@ -"""Loader and helpers for the versioned gRPC Shard protocol (ADR-0024, DGR-002). - -The ``.proto`` schema at ``packages/node/native/proto/shard_runtime.proto`` is the -single source of truth. Rather than commit generated stubs (which pin a protobuf -runtime version and drift from the schema), this package generates the Python -stubs on demand into a gitignored build directory and imports them. Generation is -reproducible: it shells out to the pinned ``grpc_tools.protoc`` with the exact -same flags as ``packages/node/native/scripts/generate_python.py``. - -Typical use:: - - from meshnet_node import native_protocol as proto - pb2 = proto.load() - header = pb2.MessageHeader(work_id="w1", route_session_id="s1") - -The checksum/fragment helpers encode the bounded-fragment tensor-bundle semantics -so callers (and DGR-008/DGR-009) do not re-derive them. -""" - -from __future__ import annotations - -import hashlib -import importlib -import importlib.util -import pathlib -import sys -import threading -import types -import zlib - -# The wire schema version this build targets. Keep in sync with the -# ``SCHEMA_VERSION_1`` enum member in the .proto. -SCHEMA_VERSION = 1 - -_NATIVE_ROOT = pathlib.Path(__file__).resolve().parents[2] / "native" -PROTO_DIR = _NATIVE_ROOT / "proto" -PROTO_FILE = PROTO_DIR / "shard_runtime.proto" -# ``build/`` is globally gitignored, so generated stubs never enter version control. -GEN_DIR = _NATIVE_ROOT / "build" / "python" - -_PB2_MODULE = "shard_runtime_pb2" -_GRPC_MODULE = "shard_runtime_pb2_grpc" - -# Reentrant: load_grpc() holds the lock and calls load(), which re-acquires it. -_lock = threading.RLock() -_cached_pb2: types.ModuleType | None = None -_cached_grpc: types.ModuleType | None = None - - -class ProtocGenerationError(RuntimeError): - """Raised when the protobuf stubs cannot be generated from the schema.""" - - -def _needs_regen(target: pathlib.Path) -> bool: - if not target.exists(): - return True - try: - return PROTO_FILE.stat().st_mtime > target.stat().st_mtime - except OSError: - return True - - -def generate(*, force: bool = False) -> pathlib.Path: - """Generate ``shard_runtime_pb2{,_grpc}.py`` into :data:`GEN_DIR`. - - Returns the output directory. Reproducible and idempotent: regenerates only - when the schema is newer than the stubs (or ``force`` is set). Requires the - pinned ``grpc_tools`` (available in the project ``.venv``). - """ - if not PROTO_FILE.exists(): - raise ProtocGenerationError(f"schema not found: {PROTO_FILE}") - - pb2_path = GEN_DIR / f"{_PB2_MODULE}.py" - if not force and not _needs_regen(pb2_path): - return GEN_DIR - - try: - from grpc_tools import protoc - except ImportError as exc: # pragma: no cover - environment-dependent - raise ProtocGenerationError( - "grpc_tools is required to generate the Shard protocol stubs; " - "install grpcio-tools (present in the project .venv)." - ) from exc - - GEN_DIR.mkdir(parents=True, exist_ok=True) - well_known = _well_known_include() - args = [ - "grpc_tools.protoc", - f"-I{PROTO_DIR}", - *([f"-I{well_known}"] if well_known else []), - f"--python_out={GEN_DIR}", - f"--grpc_python_out={GEN_DIR}", - str(PROTO_FILE.name), - ] - # protoc resolves the proto by name relative to -I, so run with PROTO_DIR - # semantics by passing the bare filename plus the include path above. - rc = protoc.main([a for a in args]) - if rc != 0: - raise ProtocGenerationError( - f"grpc_tools.protoc exited with status {rc} for {PROTO_FILE}" - ) - if not pb2_path.exists(): # pragma: no cover - defensive - raise ProtocGenerationError(f"protoc did not produce {pb2_path}") - return GEN_DIR - - -def _well_known_include() -> str | None: - """Bundled well-known .proto include dir shipped with grpc_tools, if any.""" - try: - import grpc_tools - - candidate = pathlib.Path(grpc_tools.__file__).parent / "_proto" - return str(candidate) if candidate.is_dir() else None - except Exception: # pragma: no cover - defensive - return None - - -def _import_generated(module_name: str) -> types.ModuleType: - gen_dir = str(GEN_DIR) - if gen_dir not in sys.path: - sys.path.insert(0, gen_dir) - if module_name in sys.modules: - return sys.modules[module_name] - return importlib.import_module(module_name) - - -def load(*, force: bool = False) -> types.ModuleType: - """Return the generated ``shard_runtime_pb2`` module (messages only). - - Generates the stubs on first use. Thread-safe and cached. Does not import - grpc; message serialization/round-trip needs only this module. - """ - global _cached_pb2 - with _lock: - if _cached_pb2 is not None and not force: - return _cached_pb2 - generate(force=force) - _cached_pb2 = _import_generated(_PB2_MODULE) - return _cached_pb2 - - -def load_grpc(*, force: bool = False) -> types.ModuleType: - """Return the generated ``shard_runtime_pb2_grpc`` module (service stubs). - - Requires the ``grpc`` runtime. Use for building the C++/Python worker; the - round-trip/compat tests only need :func:`load`. - """ - global _cached_grpc - with _lock: - if _cached_grpc is not None and not force: - return _cached_grpc - generate(force=force) - load() # ensure the _pb2 module the grpc stub imports is present - _cached_grpc = _import_generated(_GRPC_MODULE) - return _cached_grpc - - -# --------------------------------------------------------------------------- -# Checksum + bounded-fragment helpers (shared bundle semantics) -# --------------------------------------------------------------------------- - -# Algorithm-name strings mirror the ChecksumAlgorithm enum members without -# importing the generated module (so this table is usable before load()). -_CHECKSUM_CRC32C = "CHECKSUM_CRC32C" -_CHECKSUM_CRC32 = "CHECKSUM_CRC32" -_CHECKSUM_SHA256 = "CHECKSUM_SHA256" -_CHECKSUM_NONE = "CHECKSUM_NONE" - - -def _crc32c(data: bytes) -> int: - """Castagnoli CRC32C (software table). Deterministic, no external deps.""" - crc = 0xFFFFFFFF - for byte in data: - crc ^= byte - for _ in range(8): - crc = (crc >> 1) ^ (0x82F63B78 & -(crc & 1)) - return crc ^ 0xFFFFFFFF - - -def compute_checksum(algorithm: int, data: bytes): - """Build a ``Checksum`` message for ``data`` under the given enum value. - - ``algorithm`` is a ``ChecksumAlgorithm`` enum int from the generated module. - Uses only the standard library (crc32c software table, zlib.crc32, hashlib). - """ - pb2 = load() - name = pb2.ChecksumAlgorithm.Name(algorithm) - if name == _CHECKSUM_SHA256: - value = hashlib.sha256(data).digest() - elif name == _CHECKSUM_CRC32C: - value = _crc32c(data).to_bytes(4, "big") - elif name == _CHECKSUM_CRC32: - value = (zlib.crc32(data) & 0xFFFFFFFF).to_bytes(4, "big") - elif name == _CHECKSUM_NONE: - value = b"" - else: - raise ValueError(f"unsupported checksum algorithm: {name}") - return pb2.Checksum(algorithm=algorithm, value=value) - - -def verify_checksum(checksum, data: bytes) -> bool: - """True if ``checksum`` matches ``data`` (CHECKSUM_NONE always verifies).""" - pb2 = load() - if checksum.algorithm in (0, pb2.CHECKSUM_NONE): - return True - return compute_checksum(checksum.algorithm, data).value == checksum.value - - -def fragment_tensor( - *, - name: str, - shape, - dtype: int, - payload: bytes, - byte_order: int | None = None, - max_fragment_bytes: int = 1 << 20, - compression: int | None = None, - checksum_algorithm: int | None = None, -): - """Build a :class:`NamedTensor` splitting ``payload`` into bounded fragments. - - Fragments are ordered by ``byte_offset`` and each carries an optional - per-fragment checksum. ``payload`` is treated as already compressed if - ``compression`` is set; this helper does not compress (that is the seam's - policy in ``activation_compression``), it only frames. - """ - if max_fragment_bytes <= 0: - raise ValueError("max_fragment_bytes must be positive") - pb2 = load() - if byte_order is None: - byte_order = pb2.BYTE_ORDER_LITTLE_ENDIAN - if compression is None: - compression = pb2.COMPRESSION_NONE - - chunks = [ - payload[i : i + max_fragment_bytes] - for i in range(0, len(payload), max_fragment_bytes) - ] or [b""] - fragments = [] - offset = 0 - for index, chunk in enumerate(chunks): - frag = pb2.TensorFragment( - fragment_index=index, - fragment_count=len(chunks), - byte_offset=offset, - data=chunk, - ) - if checksum_algorithm is not None: - frag.checksum.CopyFrom(compute_checksum(checksum_algorithm, chunk)) - fragments.append(frag) - offset += len(chunk) - return pb2.NamedTensor( - name=name, - shape=list(shape), - dtype=dtype, - byte_order=byte_order, - total_byte_length=len(payload), - compression=compression, - fragments=fragments, - ) - - -def reassemble_tensor(named_tensor) -> bytes: - """Concatenate a :class:`NamedTensor`'s fragments back into the full payload. - - Validates fragment ordering, total length, and any per-fragment checksums. - """ - fragments = sorted(named_tensor.fragments, key=lambda f: f.byte_offset) - out = bytearray() - for frag in fragments: - if frag.byte_offset != len(out): - raise ValueError( - f"non-contiguous fragment at offset {frag.byte_offset} " - f"(expected {len(out)})" - ) - if frag.HasField("checksum") and not verify_checksum(frag.checksum, frag.data): - raise ValueError(f"fragment {frag.fragment_index} checksum mismatch") - out.extend(frag.data) - if named_tensor.total_byte_length and len(out) != named_tensor.total_byte_length: - raise ValueError( - f"reassembled length {len(out)} != declared " - f"{named_tensor.total_byte_length}" - ) - return bytes(out) - - -__all__ = [ - "SCHEMA_VERSION", - "PROTO_FILE", - "PROTO_DIR", - "GEN_DIR", - "ProtocGenerationError", - "generate", - "load", - "load_grpc", - "compute_checksum", - "verify_checksum", - "fragment_tensor", - "reassemble_tensor", -] diff --git a/packages/node/meshnet_node/performance_contract.py b/packages/node/meshnet_node/performance_contract.py deleted file mode 100644 index acdcafe..0000000 --- a/packages/node/meshnet_node/performance_contract.py +++ /dev/null @@ -1,563 +0,0 @@ -"""Versioned performance contract metadata and stub benchmark runner for DGR-001. - -This module captures the *contract* first: the model family, architecture -alignment, benchmark lanes, and stop condition that benchmark runs must -satisfy. It also runs the contract's lanes through a deterministic stub -backend so the report data shape exists end to end. It never downloads or -executes a model; real transformers / llama.cpp backends plug in behind the -same ``run()`` seam later. -""" - -from __future__ import annotations - -import argparse -import json -import time -import urllib.request -from dataclasses import dataclass -from pathlib import Path -from typing import Mapping - -SCHEMA_VERSION = 1 -CONTRACT_ID = "DGR-001" -DEFAULT_OUTPUT_PATH = Path(".scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json") - - -@dataclass(frozen=True) -class ModelTarget: - """Architecture-aligned model target for the DGR-001 benchmark contract.""" - - name: str - architecture: str - safetensors_repo: str - safetensors_precision: str - gguf_repo: str - gguf_quant: str - gguf_size_gb: float - comparison_policy: str - rationale: str - - def to_dict(self) -> dict: - return { - "name": self.name, - "architecture": self.architecture, - "safetensors_repo": self.safetensors_repo, - "safetensors_precision": self.safetensors_precision, - "gguf_repo": self.gguf_repo, - "gguf_quant": self.gguf_quant, - "gguf_size_gb": self.gguf_size_gb, - "comparison_policy": self.comparison_policy, - "rationale": self.rationale, - } - - -@dataclass(frozen=True) -class BenchmarkLane: - """One side of the comparison the contract requires.""" - - id: str - runtime: str - device: str - recipe: str - concurrency_levels: tuple[int, ...] - - def to_dict(self) -> dict: - return { - "id": self.id, - "runtime": self.runtime, - "device": self.device, - "recipe": self.recipe, - "concurrency_levels": list(self.concurrency_levels), - } - - -@dataclass(frozen=True) -class BenchmarkWorkload: - """Identical request shape both recipes must run so speed stays comparable. - - Pinning prompts, context lengths, output lengths, and sampling policy in the - versioned contract is what makes the safetensors-versus-GGUF numbers a - controlled comparison instead of two differently-configured runs. - """ - - prompts: tuple[str, ...] - context_lengths: tuple[int, ...] - output_lengths: tuple[int, ...] - sampling_policy: str - - def to_dict(self) -> dict: - return { - "prompts": list(self.prompts), - "context_lengths": list(self.context_lengths), - "output_lengths": list(self.output_lengths), - "sampling_policy": self.sampling_policy, - } - - -@dataclass(frozen=True) -class QualityPolicy: - """Correctness/quality lane kept separate from the performance/fit lanes. - - BF16 safetensors and Q2_K GGUF are not numerically equivalent, so quality is - measured as its own lane (output drift against the BF16 reference under a - documented tolerance) rather than assumed away by the speed/fit comparison. - """ - - statement: str - reference_lane_runtime: str - measured_lane_runtime: str - max_output_drift: float - - def to_dict(self) -> dict: - return { - "statement": self.statement, - "reference_lane_runtime": self.reference_lane_runtime, - "measured_lane_runtime": self.measured_lane_runtime, - "max_output_drift": self.max_output_drift, - } - - -@dataclass(frozen=True) -class ReleaseGate: - """Versioned thresholds later release gates (DGR-014) consume unchanged. - - Thresholds live in the contract, not in code, so the release gate cannot be - weakened after seeing implementation results. - """ - - min_decode_speedup: float - max_artifact_bytes_ratio: float - max_memory_bytes_ratio: float - max_quality_drift: float - - def to_dict(self) -> dict: - return { - "min_decode_speedup": self.min_decode_speedup, - "max_artifact_bytes_ratio": self.max_artifact_bytes_ratio, - "max_memory_bytes_ratio": self.max_memory_bytes_ratio, - "max_quality_drift": self.max_quality_drift, - } - - -@dataclass(frozen=True) -class PerformanceContract: - """Machine-readable contract for the DGR-001 benchmark story.""" - - schema_version: int - story_id: str - model_target: ModelTarget - benchmark_lanes: tuple[BenchmarkLane, ...] - metrics: tuple[str, ...] - stop_condition: str - notes: tuple[str, ...] = () - - def to_dict(self) -> dict: - return { - "schema_version": self.schema_version, - "story_id": self.story_id, - "model_target": self.model_target.to_dict(), - "benchmark_lanes": [lane.to_dict() for lane in self.benchmark_lanes], - "metrics": list(self.metrics), - "stop_condition": self.stop_condition, - "notes": list(self.notes), - } - - def write_json(self, path: str | Path) -> Path: - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") - return path - - -DEFAULT_CONTRACT = PerformanceContract( - schema_version=SCHEMA_VERSION, - story_id=CONTRACT_ID, - model_target=ModelTarget( - name="DeepSeek-V2-Lite-Chat", - architecture="deepseek2", - safetensors_repo="deepseek-ai/DeepSeek-V2-Lite-Chat", - safetensors_precision="bfloat16", - gguf_repo="second-state/DeepSeek-V2-Lite-Chat-GGUF", - gguf_quant="Q2_K", - gguf_size_gb=6.43, - comparison_policy=( - "same model/revision, closest practical low-footprint precision pair: " - "BF16 safetensors versus Q2_K GGUF" - ), - rationale=( - "Smallest DeepSeek-family benchmark anchor that still points toward " - "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead " - "of falling back to a tiny but architecture-mismatched smoke model." - ), - ), - benchmark_lanes=( - BenchmarkLane( - id="transformers-safetensors-cpu", - runtime="transformers", - device="cpu", - recipe="current safetensors recipe", - concurrency_levels=(1, 4), - ), - BenchmarkLane( - id="llama-cpp-gguf-cpu", - runtime="llama.cpp", - device="cpu", - recipe="whole-model GGUF recipe", - concurrency_levels=(1, 4), - ), - BenchmarkLane( - id="transformers-safetensors-gpu", - runtime="transformers", - device="gpu", - recipe="current safetensors recipe", - concurrency_levels=(1, 4), - ), - BenchmarkLane( - id="llama-cpp-gguf-gpu", - runtime="llama.cpp", - device="gpu", - recipe="whole-model GGUF recipe", - concurrency_levels=(1, 4), - ), - ), - metrics=( - "ttft_ms", - "prefill_tok_per_sec", - "decode_tok_per_sec", - "p50_latency_ms", - "p95_latency_ms", - "aggregate_throughput_tok_per_sec", - "rss_bytes", - "vram_bytes", - "artifact_bytes", - "failure_count", - "output_drift", - ), - stop_condition=( - "Stop if GGUF does not provide a meaningful speed or fit benefit over the " - "safetensors baseline for the chosen DeepSeek-family model target." - ), - notes=( - "Real model execution stays opt-in and must keep model artifacts on the mounted drive.", - "Use the tiny fallback only for loader plumbing smoke tests; it does not replace the architecture-aligned baseline.", - ), -) - - -def build_default_contract() -> PerformanceContract: - return DEFAULT_CONTRACT - - -BENCHMARK_SCHEMA_VERSION = 1 -STUB_OUTPUT_TOKENS = ("mesh", "activation", "seam", "baseline") -# DeepSeek-V2-Lite is ~15.7B params at 2 bytes each; metadata only, nothing downloaded. -_SAFETENSORS_BF16_ARTIFACT_GB = 31.4 - - -@dataclass(frozen=True) -class LaneSample: - """Raw single-stream measurements one backend produces for a lane.""" - - ttft_ms: float - prefill_tok_per_sec: float - decode_tok_per_sec: float - rss_bytes: int - vram_bytes: int - artifact_bytes: int - output_tokens: tuple[str, ...] - failure_count: int = 0 - - -def _gb(value: float) -> int: - return int(value * 1024**3) - - -class StubLaneBackend: - """Deterministic placeholder measurements until real lane execution lands. - - The numbers are synthetic but directionally shaped — the Q2_K GGUF loads a - far smaller artifact and decodes faster than BF16 safetensors — so the - comparison and stop-condition plumbing can be exercised in CI. - """ - - source = "stub-backend" - - # (runtime, device) -> (ttft_ms, prefill tok/s, decode tok/s, rss GB, vram GB) - _PROFILES = { - ("transformers", "cpu"): (1800.0, 45.0, 6.0, 33.0, 0.0), - ("llama.cpp", "cpu"): (950.0, 90.0, 14.0, 7.1, 0.0), - ("transformers", "gpu"): (420.0, 850.0, 34.0, 4.0, 33.0), - ("llama.cpp", "gpu"): (260.0, 640.0, 52.0, 1.5, 7.5), - } - - def __init__(self, contract: PerformanceContract) -> None: - self._contract = contract - - def run(self, lane: BenchmarkLane) -> LaneSample: - ttft_ms, prefill, decode, rss_gb, vram_gb = self._PROFILES[(lane.runtime, lane.device)] - artifact_gb = ( - self._contract.model_target.gguf_size_gb - if lane.runtime == "llama.cpp" - else _SAFETENSORS_BF16_ARTIFACT_GB - ) - return LaneSample( - ttft_ms=ttft_ms, - prefill_tok_per_sec=prefill, - decode_tok_per_sec=decode, - rss_bytes=_gb(rss_gb), - vram_bytes=_gb(vram_gb), - artifact_bytes=_gb(artifact_gb), - output_tokens=STUB_OUTPUT_TOKENS, - ) - - -def _output_drift(tokens: tuple[str, ...], reference: tuple[str, ...]) -> float: - """Fraction of positions where a lane's output diverges from its reference.""" - length = max(len(tokens), len(reference)) - if length == 0: - return 0.0 - mismatches = sum(a != b for a, b in zip(tokens, reference)) + abs(len(tokens) - len(reference)) - return round(mismatches / length, 4) - - -def _metrics_for(sample: LaneSample, concurrency: int, output_drift: float) -> dict: - # Stub concurrency model: batching scales throughput at 85% efficiency and - # stretches per-request token latency and TTFT accordingly. - efficiency = 1.0 if concurrency == 1 else 0.85 - p50_latency_ms = round(1000.0 / (sample.decode_tok_per_sec * efficiency), 4) - return { - "ttft_ms": round(sample.ttft_ms * (1 + 0.1 * (concurrency - 1)), 4), - "prefill_tok_per_sec": round(sample.prefill_tok_per_sec * efficiency, 4), - "decode_tok_per_sec": round(sample.decode_tok_per_sec * efficiency, 4), - "p50_latency_ms": p50_latency_ms, - "p95_latency_ms": round(p50_latency_ms * 1.25, 4), - "aggregate_throughput_tok_per_sec": round(sample.decode_tok_per_sec * concurrency * efficiency, 4), - "rss_bytes": sample.rss_bytes, - "vram_bytes": sample.vram_bytes, - "artifact_bytes": sample.artifact_bytes, - "failure_count": sample.failure_count, - "output_drift": output_drift, - } - - -def _compare_device(lanes: list[tuple[BenchmarkLane, LaneSample]], device: str) -> dict: - by_runtime = {lane.runtime: (lane, sample) for lane, sample in lanes if lane.device == device} - safetensors_lane, safetensors = by_runtime["transformers"] - gguf_lane, gguf = by_runtime["llama.cpp"] - memory_metric = "vram_bytes" if device == "gpu" else "rss_bytes" - decode_speedup = round(gguf.decode_tok_per_sec / safetensors.decode_tok_per_sec, 4) - artifact_bytes_ratio = round(gguf.artifact_bytes / max(1, safetensors.artifact_bytes), 4) - return { - "safetensors_lane": safetensors_lane.id, - "gguf_lane": gguf_lane.id, - "decode_speedup": decode_speedup, - "ttft_speedup": round(safetensors.ttft_ms / max(0.001, gguf.ttft_ms), 4), - "artifact_bytes_ratio": artifact_bytes_ratio, - "memory_metric": memory_metric, - "memory_bytes_ratio": round( - getattr(gguf, memory_metric) / max(1, getattr(safetensors, memory_metric)), 4 - ), - "output_drift": _output_drift(gguf.output_tokens, safetensors.output_tokens), - "gguf_benefit": decode_speedup >= 1.10 or artifact_bytes_ratio <= 0.5, - } - - -def run_performance_benchmark( - contract: PerformanceContract = DEFAULT_CONTRACT, - backend: StubLaneBackend | None = None, -) -> dict: - """Run every contract lane through a backend and compare GGUF to safetensors.""" - backend = backend if backend is not None else StubLaneBackend(contract) - lanes = [(lane, backend.run(lane)) for lane in contract.benchmark_lanes] - references = { - lane.device: sample.output_tokens for lane, sample in lanes if lane.runtime == "transformers" - } - lane_reports = [] - for lane, sample in lanes: - drift = _output_drift(sample.output_tokens, references.get(lane.device, sample.output_tokens)) - lane_reports.append({ - **lane.to_dict(), - "output_tokens": list(sample.output_tokens), - "results": [ - {"concurrency": level, "metrics": _metrics_for(sample, level, drift)} - for level in lane.concurrency_levels - ], - }) - devices = sorted({lane.device for lane, _ in lanes}) - comparisons = {device: _compare_device(lanes, device) for device in devices} - gguf_benefit = any(comparison["gguf_benefit"] for comparison in comparisons.values()) - return { - "schema_version": BENCHMARK_SCHEMA_VERSION, - "story_id": contract.story_id, - "source": getattr(backend, "source", "custom-backend"), - "model_target": contract.model_target.to_dict(), - "lanes": lane_reports, - "comparisons": comparisons, - "stop_condition": { - "text": contract.stop_condition, - "gguf_benefit": gguf_benefit, - "triggered": not gguf_benefit, - }, - } - - -def run_real_model_endpoint_benchmark( - endpoints: Mapping[str, str], - *, - model: str, - contract: PerformanceContract = DEFAULT_CONTRACT, - timeout: float = 120.0, -) -> dict: - """Run one live OpenAI-compatible request per lane against supplied endpoints. - - The caller provides one URL per benchmark lane. The runner measures the - request/response round-trip at the client boundary and reuses the same - contract schema as the deterministic stub. - """ - - def _sample_for_lane(lane: BenchmarkLane, endpoint: str) -> LaneSample: - prompt = " ".join(contract.model_target.rationale.split()[:6]) - body = json.dumps( - { - "model": model, - "messages": [{"role": "user", "content": prompt}], - "max_tokens": len(STUB_OUTPUT_TOKENS), - "temperature": 0, - } - ).encode("utf-8") - request = urllib.request.Request( - f"{endpoint.rstrip('/')}/v1/chat/completions", - data=body, - headers={ - "Content-Type": "application/json", - "X-Meshnet-Lane": lane.id, - }, - method="POST", - ) - started = time.monotonic() - with urllib.request.urlopen(request, timeout=timeout) as response: - response_body = response.read() - session_id = response.headers.get("X-Meshnet-Session", f"{lane.id}-session") - elapsed_ms = round((time.monotonic() - started) * 1000, 4) - payload = json.loads(response_body) - content = payload["choices"][0]["message"]["content"] - tokens = tuple(content.split()) - token_count = max(1, len(tokens)) - artifact_gb = ( - contract.model_target.gguf_size_gb - if lane.runtime == "llama.cpp" - else _SAFETENSORS_BF16_ARTIFACT_GB - ) - return LaneSample( - ttft_ms=elapsed_ms, - prefill_tok_per_sec=round(token_count / max(0.001, elapsed_ms / 1000), 4), - decode_tok_per_sec=round(token_count / max(0.001, elapsed_ms / 1000), 4), - rss_bytes=0, - vram_bytes=0, - artifact_bytes=_gb(artifact_gb), - output_tokens=tokens, - ) - - lanes = [] - for lane in contract.benchmark_lanes: - if lane.id not in endpoints: - raise KeyError(f"missing endpoint for lane {lane.id}") - lanes.append((lane, _sample_for_lane(lane, endpoints[lane.id]))) - references = { - lane.device: sample.output_tokens for lane, sample in lanes if lane.runtime == "transformers" - } - lane_reports = [] - for lane, sample in lanes: - drift = _output_drift(sample.output_tokens, references.get(lane.device, sample.output_tokens)) - lane_reports.append({ - **lane.to_dict(), - "output_tokens": list(sample.output_tokens), - "results": [ - {"concurrency": level, "metrics": _metrics_for(sample, level, drift)} - for level in lane.concurrency_levels - ], - }) - devices = sorted({lane.device for lane, _ in lanes}) - comparisons = {device: _compare_device(lanes, device) for device in devices} - gguf_benefit = any(comparison["gguf_benefit"] for comparison in comparisons.values()) - return { - "schema_version": BENCHMARK_SCHEMA_VERSION, - "story_id": contract.story_id, - "source": "real-model-endpoints", - "model_target": contract.model_target.to_dict(), - "lanes": lane_reports, - "comparisons": comparisons, - "stop_condition": { - "text": contract.stop_condition, - "gguf_benefit": gguf_benefit, - "triggered": not gguf_benefit, - }, - } - - -def _parse_lane_endpoints(pairs: list[str], parser: argparse.ArgumentParser) -> dict[str, str]: - endpoints: dict[str, str] = {} - for pair in pairs: - lane_id, sep, url = pair.partition("=") - if not sep or not lane_id or not url: - parser.error(f"--live-endpoint expects LANE_ID=URL, got {pair!r}") - endpoints[lane_id] = url - return endpoints - - -def _write_report(report: dict, path: Path) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON") - parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path") - parser.add_argument( - "--benchmark-out", - type=Path, - default=None, - help="also run the deterministic stub benchmark and write its JSON report here", - ) - parser.add_argument( - "--live-endpoint", - action="append", - default=None, - metavar="LANE_ID=URL", - help="lane-to-endpoint mapping for the live benchmark; repeat once per contract lane", - ) - parser.add_argument( - "--live-model", - default=None, - help="model name sent to live endpoints (default: contract safetensors repo)", - ) - parser.add_argument( - "--live-benchmark-out", - type=Path, - default=None, - help="run the live endpoint benchmark against --live-endpoint lanes and write its JSON report here", - ) - args = parser.parse_args(argv) - if args.live_endpoint and args.live_benchmark_out is None: - parser.error("--live-endpoint requires --live-benchmark-out") - if args.live_benchmark_out is not None and not args.live_endpoint: - parser.error("--live-benchmark-out requires at least one --live-endpoint") - contract = build_default_contract() - path = contract.write_json(args.json_out) - print(path) - if args.benchmark_out is not None: - _write_report(run_performance_benchmark(contract), args.benchmark_out) - print(args.benchmark_out) - if args.live_endpoint: - report = run_real_model_endpoint_benchmark( - _parse_lane_endpoints(args.live_endpoint, parser), - model=args.live_model or contract.model_target.safetensors_repo, - contract=contract, - ) - _write_report(report, args.live_benchmark_out) - print(args.live_benchmark_out) - return 0 - - -if __name__ == "__main__": # pragma: no cover - CLI entry point - raise SystemExit(main()) diff --git a/packages/node/meshnet_node/protocol.py b/packages/node/meshnet_node/protocol.py new file mode 100644 index 0000000..3f52041 --- /dev/null +++ b/packages/node/meshnet_node/protocol.py @@ -0,0 +1,376 @@ +"""Versioned activation-stream envelope for shard hops. + +The transport still moves raw bytes over HTTP/WebSocket, but the payload now has +a stable, extensible envelope that names each tensor, preserves unknown fields, +and can round-trip deterministically across direct and relayed hops. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import base64 +import hashlib +import json +from typing import Any + +SCHEMA_NAME = "meshnet.activation-stream" +SCHEMA_VERSION = 1 +DEFAULT_FRAGMENT_BYTES = 64 * 1024 + + +def _canonical_json(data: Any) -> bytes: + return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def _sha256_hex(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _normalize_shape(shape: list[int] | tuple[int, ...]) -> list[int]: + normalized = [int(dim) for dim in shape] + if not normalized or any(dim <= 0 for dim in normalized): + raise ValueError("shape must be a non-empty list of positive integers") + return normalized + + +def _fragment_bytes(body: bytes, max_fragment_bytes: int) -> tuple[bytes, ...]: + if max_fragment_bytes <= 0: + raise ValueError("max_fragment_bytes must be positive") + if not body: + return (b"",) + return tuple(body[offset : offset + max_fragment_bytes] for offset in range(0, len(body), max_fragment_bytes)) + + +@dataclass(frozen=True) +class TensorFragment: + """One bounded chunk of a named tensor.""" + + offset: int + body: bytes + checksum: str + compression: str = "identity" + extensions: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_bytes( + cls, + body: bytes, + *, + offset: int, + compression: str = "identity", + extensions: dict[str, Any] | None = None, + ) -> "TensorFragment": + return cls( + offset=int(offset), + body=bytes(body), + checksum=_sha256_hex(body), + compression=compression, + extensions=dict(extensions or {}), + ) + + def to_dict(self) -> dict[str, Any]: + data = { + "offset": self.offset, + "compression": self.compression, + "checksum": self.checksum, + "body_base64": base64.b64encode(self.body).decode("ascii"), + } + data.update(self.extensions) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "TensorFragment": + known = {"offset", "compression", "checksum", "body_base64"} + body = base64.b64decode(data.get("body_base64", "")) + fragment = cls( + offset=int(data["offset"]), + body=body, + checksum=str(data.get("checksum") or _sha256_hex(body)), + compression=str(data.get("compression") or "identity"), + extensions={k: v for k, v in data.items() if k not in known}, + ) + fragment.validate() + return fragment + + def validate(self) -> None: + if self.checksum != _sha256_hex(self.body): + raise ValueError("fragment checksum mismatch") + if self.offset < 0: + raise ValueError("fragment offset must be non-negative") + + +@dataclass(frozen=True) +class NamedTensor: + """A tensor named within a versioned activation envelope.""" + + name: str + shape: list[int] + dtype: str + byte_order: str + checksum: str + fragments: tuple[TensorFragment, ...] + compression: str = "identity" + extensions: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_bytes( + cls, + *, + name: str, + body: bytes, + shape: list[int] | tuple[int, ...], + dtype: str, + byte_order: str = "little", + compression: str = "identity", + max_fragment_bytes: int = DEFAULT_FRAGMENT_BYTES, + extensions: dict[str, Any] | None = None, + ) -> "NamedTensor": + normalized_shape = _normalize_shape(shape) + fragments = tuple( + TensorFragment.from_bytes(fragment, offset=offset, compression=compression) + for offset, fragment in enumerate(_fragment_bytes(body, max_fragment_bytes)) + for offset in (offset * max_fragment_bytes,) + ) + return cls( + name=str(name), + shape=normalized_shape, + dtype=str(dtype), + byte_order=str(byte_order), + checksum=_sha256_hex(body), + fragments=fragments, + compression=compression, + extensions=dict(extensions or {}), + ) + + def body(self) -> bytes: + ordered = sorted(self.fragments, key=lambda frag: frag.offset) + body = b"".join(fragment.body for fragment in ordered) + if _sha256_hex(body) != self.checksum: + raise ValueError(f"tensor {self.name!r} checksum mismatch") + return body + + def validate(self) -> None: + for fragment in self.fragments: + fragment.validate() + self.body() + + def to_dict(self) -> dict[str, Any]: + data = { + "name": self.name, + "shape": list(self.shape), + "dtype": self.dtype, + "byte_order": self.byte_order, + "compression": self.compression, + "checksum": self.checksum, + "fragments": [fragment.to_dict() for fragment in self.fragments], + } + data.update(self.extensions) + return data + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "NamedTensor": + known = { + "name", + "shape", + "dtype", + "byte_order", + "compression", + "checksum", + "fragments", + } + tensor = cls( + name=str(data["name"]), + shape=_normalize_shape(list(data["shape"])), + dtype=str(data["dtype"]), + byte_order=str(data.get("byte_order", "little")), + compression=str(data.get("compression", "identity")), + checksum=str(data["checksum"]), + fragments=tuple(TensorFragment.from_dict(fragment) for fragment in data.get("fragments", [])), + extensions={k: v for k, v in data.items() if k not in known}, + ) + tensor.validate() + return tensor + + +@dataclass(frozen=True) +class ActivationEnvelope: + """Versioned envelope for shard activation traffic.""" + + request_id: str + work_id: str + route_session: str + route_epoch: int + shard_start: int + effective_start: int + phase: str + position: int + idempotency_step: int + tensors: tuple[NamedTensor, ...] + version: int = SCHEMA_VERSION + schema: str = SCHEMA_NAME + token_id_sideband: list[int] | None = None + architecture_state: dict[str, Any] | None = None + recurrent_state: dict[str, Any] | None = None + mtp: dict[str, Any] | None = None + extensions: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_tensor_payload( + cls, + *, + payload: Any, + name: str, + request_id: str, + work_id: str, + route_session: str, + route_epoch: int, + shard_start: int, + effective_start: int, + phase: str, + position: int, + idempotency_step: int, + byte_order: str = "little", + compression: str = "identity", + max_fragment_bytes: int = DEFAULT_FRAGMENT_BYTES, + token_id_sideband: list[int] | None = None, + architecture_state: dict[str, Any] | None = None, + recurrent_state: dict[str, Any] | None = None, + mtp: dict[str, Any] | None = None, + extensions: dict[str, Any] | None = None, + ) -> "ActivationEnvelope": + tensor = NamedTensor.from_bytes( + name=name, + body=payload.body, + shape=payload.shape, + dtype="bfloat16", + byte_order=byte_order, + compression=compression, + max_fragment_bytes=max_fragment_bytes, + extensions={ + "attention_mask_header": payload.attention_mask_header, + "position_ids_header": payload.position_ids_header, + **({"past_len": payload.past_len} if payload.past_len is not None else {}), + }, + ) + return cls( + request_id=request_id, + work_id=work_id, + route_session=route_session, + route_epoch=int(route_epoch), + shard_start=int(shard_start), + effective_start=int(effective_start), + phase=str(phase), + position=int(position), + idempotency_step=int(idempotency_step), + tensors=(tensor,), + token_id_sideband=list(token_id_sideband) if token_id_sideband is not None else None, + architecture_state=architecture_state, + recurrent_state=recurrent_state, + mtp=mtp, + extensions=dict(extensions or {}), + ) + + def to_tensor_payload(self, *, tensor_name: str = "activations") -> Any: + from .model_backend import TensorPayload + + tensor = self.tensor(tensor_name) + return TensorPayload( + body=tensor.body(), + shape=list(tensor.shape), + attention_mask_header=tensor.extensions.get("attention_mask_header"), + position_ids_header=tensor.extensions.get("position_ids_header"), + past_len=tensor.extensions.get("past_len"), + ) + + def tensor(self, name: str = "activations") -> NamedTensor: + for tensor in self.tensors: + if tensor.name == name: + return tensor + raise KeyError(name) + + def to_dict(self) -> dict[str, Any]: + data = { + "schema": self.schema, + "version": self.version, + "request_id": self.request_id, + "work_id": self.work_id, + "route_session": self.route_session, + "route_epoch": self.route_epoch, + "shard_start": self.shard_start, + "effective_start": self.effective_start, + "phase": self.phase, + "position": self.position, + "idempotency_step": self.idempotency_step, + "tensors": [tensor.to_dict() for tensor in self.tensors], + } + if self.token_id_sideband is not None: + data["token_id_sideband"] = list(self.token_id_sideband) + if self.architecture_state is not None: + data["architecture_state"] = self.architecture_state + if self.recurrent_state is not None: + data["recurrent_state"] = self.recurrent_state + if self.mtp is not None: + data["mtp"] = self.mtp + data.update(self.extensions) + return data + + def to_bytes(self, *, max_bytes: int | None = None) -> bytes: + raw = _canonical_json(self.to_dict()) + if max_bytes is not None and len(raw) > max_bytes: + raise ValueError("activation envelope exceeds the size limit") + return raw + + @classmethod + def from_bytes(cls, data: bytes) -> "ActivationEnvelope": + payload = json.loads(data) + if not isinstance(payload, dict): + raise ValueError("activation envelope must be a JSON object") + known = { + "schema", + "version", + "request_id", + "work_id", + "route_session", + "route_epoch", + "shard_start", + "effective_start", + "phase", + "position", + "idempotency_step", + "tensors", + "token_id_sideband", + "architecture_state", + "recurrent_state", + "mtp", + } + envelope = cls( + schema=str(payload.get("schema", SCHEMA_NAME)), + version=int(payload.get("version", SCHEMA_VERSION)), + request_id=str(payload["request_id"]), + work_id=str(payload["work_id"]), + route_session=str(payload["route_session"]), + route_epoch=int(payload["route_epoch"]), + shard_start=int(payload["shard_start"]), + effective_start=int(payload["effective_start"]), + phase=str(payload["phase"]), + position=int(payload["position"]), + idempotency_step=int(payload["idempotency_step"]), + tensors=tuple(NamedTensor.from_dict(item) for item in payload.get("tensors", [])), + token_id_sideband=payload.get("token_id_sideband"), + architecture_state=payload.get("architecture_state"), + recurrent_state=payload.get("recurrent_state"), + mtp=payload.get("mtp"), + extensions={k: v for k, v in payload.items() if k not in known}, + ) + envelope.validate() + return envelope + + def validate(self) -> None: + if self.version != SCHEMA_VERSION: + raise ValueError("unsupported activation envelope version") + if self.schema != SCHEMA_NAME: + raise ValueError("unsupported activation envelope schema") + if self.phase not in {"prefill", "decode"}: + raise ValueError("phase must be prefill or decode") + for tensor in self.tensors: + tensor.validate() diff --git a/packages/node/meshnet_node/recipes.json b/packages/node/meshnet_node/recipes.json index 7845857..73cd29c 100644 --- a/packages/node/meshnet_node/recipes.json +++ b/packages/node/meshnet_node/recipes.json @@ -26,16 +26,6 @@ "params": { "use_cache": false } - }, - { - "id": "llama-cpp-native", - "version": "1", - "backend_id": "llama.cpp", - "description": "Project-owned native GGUF worker behind the Meshnet control plane.", - "params": { - "worker_transport": "grpc", - "use_cache": true - } } ] } diff --git a/packages/node/meshnet_node/runtime_recipe.py b/packages/node/meshnet_node/runtime_recipe.py deleted file mode 100644 index cc10a99..0000000 --- a/packages/node/meshnet_node/runtime_recipe.py +++ /dev/null @@ -1,375 +0,0 @@ -"""Exact artifact and runtime-recipe identity helpers. - -The runtime recipe is the compatibility contract for one routable shard. It is -kept separate from the user-facing recipe catalogue so the tracker can compare -the exact execution footprint that was validated, not just a named recipe. -""" - -from __future__ import annotations - -import hashlib -import json -from dataclasses import dataclass -from typing import Any, Mapping - - -def _require_text(value: Any, field_name: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise ValueError(f"{field_name!r} must be a non-empty string") - return value - - -def _optional_text(value: Any, field_name: str) -> str | None: - if value is None: - return None - return _require_text(value, field_name) - - -def _sha256_text(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - - -def _stable_json(data: Any) -> str: - return json.dumps( - data, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - default=str, - ) - - -def _normalise_dtype(value: Any, default: str) -> str: - if value is None: - return default - if isinstance(value, str): - text = value.strip() - if not text: - return default - return text.removeprefix("torch.") - return str(value).removeprefix("torch.") - - -def _architecture_adapter_from_config(model_config: Any, default: str) -> str: - if not isinstance(model_config, Mapping): - return default - for key in ("architecture_adapter", "model_type"): - value = model_config.get(key) - if isinstance(value, str) and value.strip(): - return value - architectures = model_config.get("architectures") - if isinstance(architectures, list) and architectures: - first = architectures[0] - if isinstance(first, str) and first.strip(): - return first - text_config = model_config.get("text_config") - if isinstance(text_config, Mapping): - return _architecture_adapter_from_config(text_config, default) - return default - - -def _tokenizer_revision_from_config( - model_id: str, - revision: str | None, - model_config: Any, -) -> str: - if isinstance(model_config, Mapping): - for key in ("tokenizer_revision", "tokenizer_version", "_commit_hash"): - value = model_config.get(key) - if isinstance(value, str) and value.strip(): - return value - if revision: - return revision - return model_id - - -def _cache_layout_from_recipe_params(recipe_params: Mapping[str, Any] | None) -> str: - if not recipe_params: - return "local-hot-kv" - use_cache = recipe_params.get("use_cache") - if use_cache is False: - return "stateless" - if "cache_layout" in recipe_params: - value = recipe_params.get("cache_layout") - if isinstance(value, str) and value.strip(): - return value - return "local-hot-kv" - - -@dataclass(frozen=True) -class ArtifactIdentity: - """Exact source artifact binding for a routable shard.""" - - model_id: str - revision: str | None = None - artifact_hash: str | None = None - shard_start: int | None = None - shard_end: int | None = None - - def __post_init__(self) -> None: - _require_text(self.model_id, "artifact.model_id") - _optional_text(self.revision, "artifact.revision") - _optional_text(self.artifact_hash, "artifact.artifact_hash") - if self.shard_start is not None and self.shard_start < 0: - raise ValueError("'artifact.shard_start' must be >= 0") - if self.shard_end is not None and self.shard_end < 0: - raise ValueError("'artifact.shard_end' must be >= 0") - if ( - self.shard_start is not None - and self.shard_end is not None - and self.shard_end < self.shard_start - ): - raise ValueError("'artifact.shard_end' must be >= 'artifact.shard_start'") - - def to_dict(self) -> dict[str, Any]: - return { - "model_id": self.model_id, - "revision": self.revision, - "artifact_hash": self.artifact_hash, - "shard_start": self.shard_start, - "shard_end": self.shard_end, - } - - @classmethod - def from_dict(cls, data: Any) -> "ArtifactIdentity": - if not isinstance(data, Mapping): - raise ValueError(f"'artifact' must be a JSON object, got {type(data).__name__}") - return cls( - model_id=_require_text(data.get("model_id"), "artifact.model_id"), - revision=_optional_text(data.get("revision"), "artifact.revision"), - artifact_hash=_optional_text( - data.get("artifact_hash"), "artifact.artifact_hash" - ), - shard_start=_optional_int(data.get("shard_start"), "artifact.shard_start"), - shard_end=_optional_int(data.get("shard_end"), "artifact.shard_end"), - ) - - -@dataclass(frozen=True) -class RuntimeRecipeIdentity: - """Exact runtime recipe used for admission and handshake compatibility.""" - - weight_quantization: str - activation_dtype: str - compute_dtype: str - kv_dtype: str - kv_layout: str - tokenizer_revision: str - architecture_adapter: str - backend_id: str - runtime_version: str - boundary_schema_version: int = 1 - cache_layout: str = "local-hot-kv" - fingerprint: str | None = None - - def __post_init__(self) -> None: - _require_text(self.weight_quantization, "runtime_recipe.weight_quantization") - _require_text(self.activation_dtype, "runtime_recipe.activation_dtype") - _require_text(self.compute_dtype, "runtime_recipe.compute_dtype") - _require_text(self.kv_dtype, "runtime_recipe.kv_dtype") - _require_text(self.kv_layout, "runtime_recipe.kv_layout") - _require_text(self.tokenizer_revision, "runtime_recipe.tokenizer_revision") - _require_text(self.architecture_adapter, "runtime_recipe.architecture_adapter") - _require_text(self.backend_id, "runtime_recipe.backend_id") - _require_text(self.runtime_version, "runtime_recipe.runtime_version") - _require_text(self.cache_layout, "runtime_recipe.cache_layout") - if self.boundary_schema_version < 1: - raise ValueError("'runtime_recipe.boundary_schema_version' must be >= 1") - expected = compatibility_fingerprint(self._fingerprint_payload()) - if not self.fingerprint: - object.__setattr__(self, "fingerprint", expected) - elif self.fingerprint != expected: - raise ValueError( - "'runtime_recipe.fingerprint' does not match the encoded fields" - ) - - def to_dict(self) -> dict[str, Any]: - return { - "weight_quantization": self.weight_quantization, - "activation_dtype": self.activation_dtype, - "compute_dtype": self.compute_dtype, - "kv_dtype": self.kv_dtype, - "kv_layout": self.kv_layout, - "tokenizer_revision": self.tokenizer_revision, - "architecture_adapter": self.architecture_adapter, - "backend_id": self.backend_id, - "runtime_version": self.runtime_version, - "boundary_schema_version": self.boundary_schema_version, - "cache_layout": self.cache_layout, - "fingerprint": self.fingerprint, - } - - @classmethod - def from_dict(cls, data: Any) -> "RuntimeRecipeIdentity": - if not isinstance(data, Mapping): - raise ValueError( - f"'runtime_recipe' must be a JSON object, got {type(data).__name__}" - ) - boundary_schema_version = data.get("boundary_schema_version", 1) - if isinstance(boundary_schema_version, bool) or not isinstance( - boundary_schema_version, int - ): - raise ValueError( - "'runtime_recipe.boundary_schema_version' must be an integer" - ) - return cls( - weight_quantization=_require_text( - data.get("weight_quantization"), "runtime_recipe.weight_quantization" - ), - activation_dtype=_require_text( - data.get("activation_dtype"), "runtime_recipe.activation_dtype" - ), - compute_dtype=_require_text( - data.get("compute_dtype"), "runtime_recipe.compute_dtype" - ), - kv_dtype=_require_text(data.get("kv_dtype"), "runtime_recipe.kv_dtype"), - kv_layout=_require_text(data.get("kv_layout"), "runtime_recipe.kv_layout"), - tokenizer_revision=_require_text( - data.get("tokenizer_revision"), "runtime_recipe.tokenizer_revision" - ), - architecture_adapter=_require_text( - data.get("architecture_adapter"), - "runtime_recipe.architecture_adapter", - ), - backend_id=_require_text(data.get("backend_id"), "runtime_recipe.backend_id"), - runtime_version=_require_text( - data.get("runtime_version"), "runtime_recipe.runtime_version" - ), - boundary_schema_version=boundary_schema_version, - cache_layout=_require_text(data.get("cache_layout"), "runtime_recipe.cache_layout"), - fingerprint=_optional_text(data.get("fingerprint"), "runtime_recipe.fingerprint"), - ) - - def _fingerprint_payload(self) -> dict[str, Any]: - return { - "weight_quantization": self.weight_quantization, - "activation_dtype": self.activation_dtype, - "compute_dtype": self.compute_dtype, - "kv_dtype": self.kv_dtype, - "kv_layout": self.kv_layout, - "tokenizer_revision": self.tokenizer_revision, - "architecture_adapter": self.architecture_adapter, - "backend_id": self.backend_id, - "runtime_version": self.runtime_version, - "boundary_schema_version": self.boundary_schema_version, - "cache_layout": self.cache_layout, - } - - -def _optional_int(value: Any, field_name: str) -> int | None: - if value is None: - return None - if isinstance(value, bool) or not isinstance(value, int): - raise ValueError(f"{field_name!r} must be an integer") - if value < 0: - raise ValueError(f"{field_name!r} must be >= 0") - return value - - -def build_artifact_identity( - *, - model_id: str, - revision: str | None = None, - model_config: Any = None, - artifact_hash: str | None = None, - shard_start: int | None = None, - shard_end: int | None = None, -) -> ArtifactIdentity: - """Build a stable artifact binding from the locally loaded artifact.""" - resolved_hash = artifact_hash - if resolved_hash is None: - if isinstance(model_config, Mapping): - resolved_hash = _hash_mapping(model_config) - elif model_config is not None: - resolved_hash = _sha256_text(_stable_json(model_config)) - if resolved_hash is None: - resolved_hash = _sha256_text( - _stable_json( - { - "model_id": model_id, - "revision": revision, - "shard_start": shard_start, - "shard_end": shard_end, - } - ) - ) - return ArtifactIdentity( - model_id=model_id, - revision=revision, - artifact_hash=resolved_hash, - shard_start=shard_start, - shard_end=shard_end, - ) - - -def build_runtime_recipe_identity( - *, - model_id: str, - weight_quantization: str, - backend_id: str, - runtime_version: str, - revision: str | None = None, - model_config: Any = None, - recipe_params: Mapping[str, Any] | None = None, - activation_dtype: Any = None, - compute_dtype: Any = None, - kv_dtype: Any = None, - kv_layout: str | None = None, - tokenizer_revision: str | None = None, - architecture_adapter: str | None = None, - boundary_schema_version: int = 1, - cache_layout: str | None = None, -) -> RuntimeRecipeIdentity: - """Build the exact runtime recipe used for compatibility admission.""" - activation = _normalise_dtype(activation_dtype, "bfloat16") - compute = _normalise_dtype(compute_dtype, activation) - kv_dtype_text = _normalise_dtype(kv_dtype, compute) - kv_layout_text = kv_layout or "session-cache" - tokenizer = tokenizer_revision or _tokenizer_revision_from_config( - model_id, revision, model_config - ) - architecture = architecture_adapter or _architecture_adapter_from_config( - model_config, backend_id - ) - cache_layout_text = cache_layout or _cache_layout_from_recipe_params(recipe_params) - return RuntimeRecipeIdentity( - weight_quantization=weight_quantization, - activation_dtype=activation, - compute_dtype=compute, - kv_dtype=kv_dtype_text, - kv_layout=kv_layout_text, - tokenizer_revision=tokenizer, - architecture_adapter=architecture, - backend_id=backend_id, - runtime_version=runtime_version, - boundary_schema_version=boundary_schema_version, - cache_layout=cache_layout_text, - ) - - -def compatibility_fingerprint(data: Mapping[str, Any]) -> str: - """Return a stable SHA256 compatibility fingerprint for an exact route.""" - return "sha256:" + _sha256_text(_stable_json(data)) - - -def fingerprint_payload( - *, - model: Mapping[str, Any], - shard: Mapping[str, Any], - recipe: Mapping[str, Any], - backend: Mapping[str, Any], - artifact: Mapping[str, Any], - runtime_recipe: Mapping[str, Any], -) -> dict[str, Any]: - return { - "model": dict(model), - "shard": dict(shard), - "recipe": dict(recipe), - "backend": dict(backend), - "artifact": dict(artifact), - "runtime_recipe": dict(runtime_recipe), - } - - -def _hash_mapping(data: Mapping[str, Any]) -> str: - return "sha256:" + _sha256_text(_stable_json(data)) diff --git a/packages/node/meshnet_node/shard_lifecycle.py b/packages/node/meshnet_node/shard_lifecycle.py new file mode 100644 index 0000000..e466b6c --- /dev/null +++ b/packages/node/meshnet_node/shard_lifecycle.py @@ -0,0 +1,472 @@ +"""Versioned Shard lifecycle and structured status contract. + +This module defines the semantic contract consumed by a future generated gRPC +binding. It deliberately contains no Meshnet routing, authentication policy, +billing, or llama.cpp types. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +import json +import threading +from typing import Any, Callable, ClassVar + + +SCHEMA_NAME = "meshnet.shard-runtime" +SCHEMA_VERSION = 1 +SUPPORTED_VERSIONS = frozenset({SCHEMA_VERSION}) +DEFAULT_MAX_INFLIGHT_FRAMES = 32 +DEFAULT_MAX_INFLIGHT_BYTES = 8 * 1024 * 1024 + + +class RpcName(str, Enum): + CAPABILITY = "GetCapability" + HEALTH = "CheckHealth" + SESSION = "OpenSession" + CANCEL = "CancelSession" + RELEASE = "ReleaseSession" + METRICS = "GetMetrics" + + +class StatusCode(str, Enum): + OK = "OK" + INVALID_ARGUMENT = "INVALID_ARGUMENT" + UNSUPPORTED_VERSION = "UNSUPPORTED_VERSION" + FAILED_PRECONDITION = "FAILED_PRECONDITION" + MALFORMED_LIFECYCLE = "MALFORMED_LIFECYCLE" + NOT_FOUND = "NOT_FOUND" + ALREADY_EXISTS = "ALREADY_EXISTS" + CANCELLED = "CANCELLED" + DEADLINE_EXCEEDED = "DEADLINE_EXCEEDED" + RESOURCE_EXHAUSTED = "RESOURCE_EXHAUSTED" + UNAUTHENTICATED = "UNAUTHENTICATED" + PERMISSION_DENIED = "PERMISSION_DENIED" + DATA_LOSS = "DATA_LOSS" + UNAVAILABLE = "UNAVAILABLE" + INTERNAL = "INTERNAL" + + +class LifecycleState(str, Enum): + OPEN = "OPEN" + PREFILLING = "PREFILLING" + DECODING = "DECODING" + CANCELLING = "CANCELLING" + CANCELLED = "CANCELLED" + RELEASING = "RELEASING" + RELEASED = "RELEASED" + FAILED = "FAILED" + + +class CacheExpectation(str, Enum): + NONE = "NONE" + OPTIONAL = "OPTIONAL" + REQUIRED = "REQUIRED" + + +class CacheResult(str, Enum): + NOT_REQUESTED = "NOT_REQUESTED" + HIT = "HIT" + MISS = "MISS" + INVALIDATED = "INVALIDATED" + STORED = "STORED" + + +class SessionPhase(str, Enum): + PREFILL = "PREFILL" + DECODE = "DECODE" + + +class LifecycleContractError(ValueError): + """A protocol violation represented by a structured status.""" + + def __init__(self, status: "StructuredStatus") -> None: + self.status = status + super().__init__(status.message) + + +@dataclass(frozen=True) +class StructuredStatus: + code: StatusCode + message: str + retryable: bool = False + details: dict[str, str] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "code": self.code.value, + "message": self.message, + "retryable": self.retryable, + "details": dict(sorted(self.details.items())), + } + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "StructuredStatus": + return cls( + code=StatusCode(str(value["code"])), + message=str(value["message"]), + retryable=bool(value.get("retryable", False)), + details={str(k): str(v) for k, v in value.get("details", {}).items()}, + ) + + +@dataclass(frozen=True) +class DeadlinePolicy: + capability_seconds: float = 5.0 + health_seconds: float = 2.0 + session_open_seconds: float = 10.0 + session_idle_seconds: float = 30.0 + cancel_seconds: float = 2.0 + release_seconds: float = 5.0 + metrics_seconds: float = 5.0 + + def __post_init__(self) -> None: + if any(value <= 0 for value in self.__dict__.values()): + raise ValueError("all RPC deadlines must be positive") + + +@dataclass(frozen=True) +class TlsAuthHooks: + """Transport hooks only; Meshnet remains the identity and billing authority.""" + + tls_required: bool = True + server_name: str = "shard.meshnet" + client_certificate_hook: str | None = None + peer_identity_hook: str | None = None + auth_metadata_hook: str | None = None + + def validate(self) -> None: + if self.tls_required and not self.server_name: + raise ValueError("TLS server_name is required when TLS is enabled") + + +@dataclass(frozen=True) +class FlowControlLimits: + max_inflight_frames: int = DEFAULT_MAX_INFLIGHT_FRAMES + max_inflight_bytes: int = DEFAULT_MAX_INFLIGHT_BYTES + max_frame_bytes: int = 1024 * 1024 + + def __post_init__(self) -> None: + if min(self.max_inflight_frames, self.max_inflight_bytes, self.max_frame_bytes) <= 0: + raise ValueError("flow-control limits must be positive") + + +class FlowControl: + """Bounded sender window; callers block or fail instead of growing unbounded.""" + + def __init__(self, limits: FlowControlLimits = FlowControlLimits()) -> None: + self.limits = limits + self._condition = threading.Condition() + self._frames = 0 + self._bytes = 0 + + @property + def outstanding(self) -> tuple[int, int]: + with self._condition: + return self._frames, self._bytes + + def acquire(self, size: int, *, wait: bool = False, cancelled: Callable[[], bool] | None = None) -> None: + if size < 0 or size > self.limits.max_frame_bytes: + raise LifecycleContractError(StructuredStatus( + StatusCode.RESOURCE_EXHAUSTED, "frame exceeds bounded flow-control window", + details={"max_frame_bytes": str(self.limits.max_frame_bytes)}, + )) + with self._condition: + while ( + self._frames >= self.limits.max_inflight_frames + or self._bytes + size > self.limits.max_inflight_bytes + ): + if cancelled and cancelled(): + raise LifecycleContractError(StructuredStatus(StatusCode.CANCELLED, "flow-control wait cancelled")) + if not wait: + raise LifecycleContractError(StructuredStatus( + StatusCode.RESOURCE_EXHAUSTED, "flow-control window is full", retryable=True, + )) + self._condition.wait(timeout=0.05) + self._frames += 1 + self._bytes += size + + def release(self, size: int) -> None: + with self._condition: + self._frames = max(0, self._frames - 1) + self._bytes = max(0, self._bytes - max(0, size)) + self._condition.notify_all() + + +class CancellationToken: + def __init__(self) -> None: + self._event = threading.Event() + + @property + def cancelled(self) -> bool: + return self._event.is_set() + + def cancel(self) -> None: + self._event.set() + + +@dataclass(frozen=True) +class CapabilityRequest: + schema_version: int = SCHEMA_VERSION + + +@dataclass(frozen=True) +class CapabilityResponse: + status: StructuredStatus + schema_version: int + supported_versions: tuple[int, ...] + artifact_fingerprint: str = "" + runtime_fingerprint: str = "" + shard_start: int = 0 + shard_end: int = 0 + effective_start: int = 0 + max_sessions: int = 0 + max_frame_bytes: int = DEFAULT_MAX_INFLIGHT_BYTES + + def validate(self) -> None: + if self.status.code is StatusCode.OK and self.schema_version not in SUPPORTED_VERSIONS: + raise LifecycleContractError(StructuredStatus( + StatusCode.UNSUPPORTED_VERSION, "worker selected an unsupported schema version", + )) + if self.shard_end < self.shard_start or self.effective_start < self.shard_start: + raise LifecycleContractError(StructuredStatus( + StatusCode.INVALID_ARGUMENT, "invalid authoritative shard range", + )) + + +@dataclass(frozen=True) +class HealthRequest: + schema_version: int = SCHEMA_VERSION + include_metrics: bool = False + + +@dataclass(frozen=True) +class HealthResponse: + status: StructuredStatus + serving: bool + state: str + active_sessions: int = 0 + + +@dataclass(frozen=True) +class SessionRequest: + schema_version: int + request_id: str + work_id: str + route_session: str + route_epoch: int + artifact_fingerprint: str + runtime_fingerprint: str + shard_start: int + shard_end: int + effective_start: int + cache_expectation: CacheExpectation = CacheExpectation.NONE + deadline_seconds: float = 30.0 + + def validate(self) -> None: + if self.schema_version not in SUPPORTED_VERSIONS: + raise LifecycleContractError(StructuredStatus( + StatusCode.UNSUPPORTED_VERSION, "unsupported session schema version", + details={"requested": str(self.schema_version), "supported": "1"}, + )) + if not self.request_id or not self.work_id or not self.route_session: + raise LifecycleContractError(StructuredStatus( + StatusCode.INVALID_ARGUMENT, "request, work, and route-session IDs are required", + )) + if self.route_epoch < 0 or self.shard_start < 0 or self.shard_end <= self.shard_start: + raise LifecycleContractError(StructuredStatus( + StatusCode.INVALID_ARGUMENT, "invalid route epoch or shard range", + )) + if not self.shard_start <= self.effective_start <= self.shard_end: + raise LifecycleContractError(StructuredStatus( + StatusCode.INVALID_ARGUMENT, "effective start must be inside the shard range", + )) + if self.deadline_seconds <= 0: + raise LifecycleContractError(StructuredStatus( + StatusCode.INVALID_ARGUMENT, "session deadline must be positive", + )) + + +@dataclass(frozen=True) +class SessionFrame: + phase: SessionPhase + position: int + idempotency_step: int + payload: Any + cache_expectation: CacheExpectation = CacheExpectation.NONE + size_bytes: int = 0 + + def validate(self, limits: FlowControlLimits) -> None: + if self.position < 0 or self.idempotency_step < 0: + raise LifecycleContractError(StructuredStatus( + StatusCode.INVALID_ARGUMENT, "position and idempotency step must be non-negative", + )) + if self.size_bytes < 0 or self.size_bytes > limits.max_frame_bytes: + raise LifecycleContractError(StructuredStatus( + StatusCode.RESOURCE_EXHAUSTED, "session frame exceeds max_frame_bytes", + )) + + +@dataclass(frozen=True) +class SessionResult: + status: StructuredStatus + cache_result: CacheResult = CacheResult.NOT_REQUESTED + position: int = 0 + idempotency_step: int = 0 + payload: Any = None + + +@dataclass(frozen=True) +class CancelRequest: + schema_version: int + request_id: str + work_id: str + route_session: str + route_epoch: int + reason: str = "" + + +@dataclass(frozen=True) +class ReleaseRequest: + schema_version: int + request_id: str + work_id: str + route_session: str + route_epoch: int + + +@dataclass(frozen=True) +class MetricsRequest: + schema_version: int = SCHEMA_VERSION + + +@dataclass(frozen=True) +class MetricsResponse: + status: StructuredStatus + active_sessions: int + queued_frames: int + inflight_bytes: int + kv_entries: int + generated_tokens: int + cancelled_sessions: int + + +@dataclass +class SessionLifecycle: + """Fail-closed state machine for one Route Session Activation Seam.""" + + request: SessionRequest + state: LifecycleState = LifecycleState.OPEN + cancellation: CancellationToken = field(default_factory=CancellationToken) + last_idempotency_step: int = -1 + _seen_steps: set[int] = field(default_factory=set, init=False, repr=False) + + _ALLOWED: ClassVar[dict[LifecycleState, frozenset[LifecycleState]]] = { + LifecycleState.OPEN: frozenset({LifecycleState.PREFILLING, LifecycleState.CANCELLING, LifecycleState.RELEASING, LifecycleState.FAILED}), + LifecycleState.PREFILLING: frozenset({LifecycleState.PREFILLING, LifecycleState.DECODING, LifecycleState.CANCELLING, LifecycleState.RELEASING, LifecycleState.FAILED}), + LifecycleState.DECODING: frozenset({LifecycleState.DECODING, LifecycleState.CANCELLING, LifecycleState.RELEASING, LifecycleState.FAILED}), + LifecycleState.CANCELLING: frozenset({LifecycleState.CANCELLED, LifecycleState.RELEASING, LifecycleState.FAILED}), + LifecycleState.CANCELLED: frozenset({LifecycleState.RELEASING, LifecycleState.RELEASED}), + LifecycleState.RELEASING: frozenset({LifecycleState.RELEASED, LifecycleState.FAILED}), + LifecycleState.RELEASED: frozenset(), + LifecycleState.FAILED: frozenset({LifecycleState.RELEASING, LifecycleState.RELEASED}), + } + + def _transition(self, target: LifecycleState) -> None: + if target not in self._ALLOWED[self.state]: + raise LifecycleContractError(StructuredStatus( + StatusCode.MALFORMED_LIFECYCLE, + f"cannot transition from {self.state.value} to {target.value}", + details={"state": self.state.value, "target": target.value}, + )) + self.state = target + + def apply(self, frame: SessionFrame) -> None: + frame.validate(FlowControlLimits()) + if self.cancellation.cancelled and frame.phase is not SessionPhase.PREFILL: + raise LifecycleContractError(StructuredStatus(StatusCode.CANCELLED, "session cancellation propagated")) + if frame.idempotency_step in self._seen_steps: + raise LifecycleContractError(StructuredStatus( + StatusCode.ALREADY_EXISTS, "duplicate idempotency step", details={"step": str(frame.idempotency_step)}, + )) + if frame.idempotency_step <= self.last_idempotency_step: + raise LifecycleContractError(StructuredStatus( + StatusCode.MALFORMED_LIFECYCLE, "idempotency steps must increase monotonically", + )) + target = LifecycleState.PREFILLING if frame.phase is SessionPhase.PREFILL else LifecycleState.DECODING + self._transition(target) + self._seen_steps.add(frame.idempotency_step) + self.last_idempotency_step = frame.idempotency_step + + def cancel(self, reason: str = "") -> StructuredStatus: + if self.state in {LifecycleState.RELEASED, LifecycleState.RELEASING}: + return StructuredStatus(StatusCode.FAILED_PRECONDITION, "session is already releasing or released") + if self.state is LifecycleState.CANCELLED: + return StructuredStatus(StatusCode.OK, "session already cancelled") + self._transition(LifecycleState.CANCELLING) + self.cancellation.cancel() + self._transition(LifecycleState.CANCELLED) + return StructuredStatus(StatusCode.CANCELLED, reason or "session cancelled") + + def release(self) -> StructuredStatus: + if self.state is LifecycleState.RELEASED: + return StructuredStatus(StatusCode.OK, "session already released") + self._transition(LifecycleState.RELEASING) + self.cancellation.cancel() + self._transition(LifecycleState.RELEASED) + return StructuredStatus(StatusCode.OK, "session released") + + +@dataclass(frozen=True) +class ShardRpcContract: + """Service/method and operational rules for generated gRPC bindings.""" + + schema: str = SCHEMA_NAME + version: int = SCHEMA_VERSION + methods: tuple[RpcName, ...] = tuple(RpcName) + deadlines: DeadlinePolicy = DeadlinePolicy() + flow_control: FlowControlLimits = FlowControlLimits() + tls_auth: TlsAuthHooks = TlsAuthHooks() + + def validate(self) -> None: + if self.schema != SCHEMA_NAME or self.version not in SUPPORTED_VERSIONS: + raise LifecycleContractError(StructuredStatus( + StatusCode.UNSUPPORTED_VERSION, "unsupported Shard RPC contract version", + )) + if self.methods != tuple(RpcName): + raise LifecycleContractError(StructuredStatus( + StatusCode.FAILED_PRECONDITION, "contract must expose the complete lifecycle RPC set", + )) + self.tls_auth.validate() + + def to_bytes(self) -> bytes: + self.validate() + data = { + "schema": self.schema, + "version": self.version, + "methods": [method.value for method in self.methods], + "deadlines": self.deadlines.__dict__, + "flow_control": self.flow_control.__dict__, + "tls_auth": self.tls_auth.__dict__, + } + return json.dumps(data, sort_keys=True, separators=(",", ":")).encode("utf-8") + + @classmethod + def from_bytes(cls, raw: bytes) -> "ShardRpcContract": + data = json.loads(raw) + if int(data.get("version", -1)) not in SUPPORTED_VERSIONS: + raise LifecycleContractError(StructuredStatus( + StatusCode.UNSUPPORTED_VERSION, "unsupported Shard RPC contract version", + )) + methods = tuple(RpcName(item) for item in data.get("methods", ())) + contract = cls( + schema=str(data.get("schema", "")), + version=int(data["version"]), + methods=methods, + deadlines=DeadlinePolicy(**data.get("deadlines", {})), + flow_control=FlowControlLimits(**data.get("flow_control", {})), + tls_auth=TlsAuthHooks(**data.get("tls_auth", {})), + ) + contract.validate() + return contract diff --git a/packages/node/meshnet_node/startup.py b/packages/node/meshnet_node/startup.py index d1d1cf1..dd8a4df 100644 --- a/packages/node/meshnet_node/startup.py +++ b/packages/node/meshnet_node/startup.py @@ -29,7 +29,6 @@ from .model_catalog import model_metadata_for from .recipe_manifest import DEFAULT_RECIPE_ID, Recipe, RecipeManifest, load_recipe_manifest from .relay_bridge import RelayHttpBridge, peer_id_from_wallet from .server import StubNodeServer -from .gguf_backend import build_gguf_backend from .torch_server import TorchNodeServer from .wallet import load_or_create_wallet @@ -703,35 +702,6 @@ def _resolve_recipe(recipe_id: str | None) -> tuple[RecipeManifest, Recipe]: return manifest, manifest.require(recipe_id or DEFAULT_RECIPE_ID) -def _gguf_backend_for_recipe( - recipe: Recipe, - *, - model_id: str, - shard_start: int, - shard_end: int, - quantization: str, - total_layers: int | None, - device: str, - model_revision: str | None = None, -) -> object | None: - """Build the GGUF backend only for recipes that explicitly ask for it.""" - if recipe.backend_id != "llama.cpp": - return None - return build_gguf_backend( - model_id=model_id, - shard_start=shard_start, - shard_end=shard_end, - quantization=quantization, - total_layers=total_layers, - model_revision=model_revision, - device_type=device, - architecture_adapter="dense-llama", - tokenizer_revision=model_revision or model_id, - runtime_recipe_fingerprint=None, - supports_kv_cache=recipe.params.get("use_cache", True) is not False, - ) - - def _capability_device(backend: Any, detected_device: str) -> str: """The device the shard actually landed on, or the one this node detected.""" device = getattr(backend, "device", None) @@ -993,8 +963,7 @@ def run_startup( if model_id: # treat "" the same as None — no explicit model given full_sources: list[dict] = [] - detected: int | None = None - # Auto-detect shard range from model config if not explicitly provided. + # Auto-detect shard range from model config if not explicitly provided if shard_start is None or shard_end is None: try: detected = _detect_num_layers(model_id, cache_dir=cache_dir) @@ -1058,38 +1027,22 @@ def run_startup( shard_end = shard_end if shard_end is not None else detected - 1 print(f" Auto-detected {detected} layers → shard {shard_start}–{shard_end}", flush=True) - backend = _gguf_backend_for_recipe( - recipe, + print("Loading real PyTorch model shard...", flush=True) + node = TorchNodeServer( + host=host, + port=port, model_id=model_id, shard_start=shard_start, shard_end=shard_end, quantization=quantization, - total_layers=detected if detected is not None else (shard_end + 1 if shard_end is not None else None), - device=device, - model_revision=None, + tracker_url=tracker_url, + route_timeout=route_timeout, + cache_dir=cache_dir, + debug=debug, + max_loaded_shards=max_loaded_shards, + force_cpu=force_cpu, + recipe_params=recipe.params, ) - print( - "Loading native llama.cpp model shard..." if backend is not None else "Loading real PyTorch model shard...", - flush=True, - ) - node_kwargs = { - "host": host, - "port": port, - "model_id": model_id, - "shard_start": shard_start, - "shard_end": shard_end, - "quantization": quantization, - "tracker_url": tracker_url, - "route_timeout": route_timeout, - "cache_dir": cache_dir, - "debug": debug, - "max_loaded_shards": max_loaded_shards, - "force_cpu": force_cpu, - "recipe_params": recipe.params, - } - if backend is not None: - node_kwargs["backend"] = backend - node = TorchNodeServer(**node_kwargs) capability_report = _admit_capability( node, model_id=model_id, @@ -1103,15 +1056,10 @@ def run_startup( recipe=recipe, validator=capability_validator, ) - proof_shard = capability_report.shard _node_start_time = time.monotonic() actual_port = node.start() total_layers = getattr(getattr(node, "backend", None), "total_layers", None) - shard_label = _format_shard_label( - proof_shard.start, - proof_shard.end, - total_layers, - ) + shard_label = _format_shard_label(shard_start, shard_end, total_layers) public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host) endpoint = f"http://{public_host}:{actual_port}" if hasattr(node, "set_advertised_endpoint"): @@ -1134,17 +1082,16 @@ def run_startup( "model": model_id.split("/")[-1], "hf_repo": model_id, "num_layers": total_layers, - "shard_start": proof_shard.start, - "shard_end": proof_shard.end, + "shard_start": shard_start, + "shard_end": shard_end, "hardware_profile": hw, "wallet_address": address, "quantization": quantization, "score": 1.0, - "tracker_mode": (proof_shard.start == 0), + "tracker_mode": (shard_start == 0), "managed_assignment": not user_pinned_shard, "model_metadata": model_metadata_for(model_id, total_layers, cache_dir=cache_dir), "capability_report": capability_report.to_dict(), - "compatibility_fingerprint": capability_report.compatibility_fingerprint, # Declared independently of the proof: the tracker checks that the # recipe this node says it serves with is the one the proof ran. "recipe_id": recipe.id, @@ -1152,8 +1099,8 @@ def run_startup( "downloaded_models": ( _downloaded_model_inventory( model_id.split("/")[-1], - proof_shard.start, - proof_shard.end, + shard_start, + shard_end, model_cache_path, hf_repo=model_id, model_sources=full_sources, @@ -1264,38 +1211,22 @@ def run_startup( hf_repo=assigned_hf_repo, model_sources=full_sources, ) - backend = _gguf_backend_for_recipe( - recipe, + print("Loading real PyTorch model shard...", flush=True) + node = TorchNodeServer( + host=host, + port=port, model_id=assigned_hf_repo, shard_start=assigned_shard_start, shard_end=assigned_shard_end, quantization=quantization, - total_layers=assigned_num_layers, - device=device, - model_revision=None, + tracker_url=tracker_url, + route_timeout=route_timeout, + cache_dir=cache_dir, + debug=debug, + max_loaded_shards=max_loaded_shards, + force_cpu=force_cpu, + recipe_params=recipe.params, ) - print( - "Loading native llama.cpp model shard..." if backend is not None else "Loading real PyTorch model shard...", - flush=True, - ) - node_kwargs = { - "host": host, - "port": port, - "model_id": assigned_hf_repo, - "shard_start": assigned_shard_start, - "shard_end": assigned_shard_end, - "quantization": quantization, - "tracker_url": tracker_url, - "route_timeout": route_timeout, - "cache_dir": cache_dir, - "debug": debug, - "max_loaded_shards": max_loaded_shards, - "force_cpu": force_cpu, - "recipe_params": recipe.params, - } - if backend is not None: - node_kwargs["backend"] = backend - node = TorchNodeServer(**node_kwargs) capability_report = _admit_capability( node, model_id=assigned_hf_repo, @@ -1309,7 +1240,6 @@ def run_startup( recipe=recipe, validator=capability_validator, ) - proof_shard = capability_report.shard _node_start_time = time.monotonic() actual_port = node.start() public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host) @@ -1332,17 +1262,16 @@ def run_startup( "model": assigned_hf_repo.split("/")[-1], "hf_repo": assigned_hf_repo, "num_layers": assigned_num_layers, - "shard_start": proof_shard.start, - "shard_end": proof_shard.end, + "shard_start": assigned_shard_start, + "shard_end": assigned_shard_end, "hardware_profile": hw, "wallet_address": address, "quantization": quantization, "score": 1.0, - "tracker_mode": (proof_shard.start == 0), + "tracker_mode": (assigned_shard_start == 0), "managed_assignment": True, "model_metadata": model_metadata_for(assigned_hf_repo, assigned_num_layers, cache_dir=cache_dir), "capability_report": capability_report.to_dict(), - "compatibility_fingerprint": capability_report.compatibility_fingerprint, # Declared independently of the proof: the tracker checks that the # recipe this node says it serves with is the one the proof ran. "recipe_id": recipe.id, @@ -1350,8 +1279,8 @@ def run_startup( "downloaded_models": ( _downloaded_model_inventory( assigned_hf_repo.split("/")[-1], - proof_shard.start, - proof_shard.end, + assigned_shard_start, + assigned_shard_end, model_cache_path, hf_repo=assigned_hf_repo, model_sources=full_sources, @@ -1376,8 +1305,8 @@ def run_startup( ), ) shard_label = _format_shard_label( - proof_shard.start, - proof_shard.end, + assigned_shard_start, + assigned_shard_end, assigned_num_layers, ) print( @@ -1492,38 +1421,22 @@ def run_startup( # 5. Start HTTP server — real HF weights use TorchNodeServer; stub-model stays stub. _node_start_time = time.monotonic() if hf_repo and assigned_model != "stub-model": - backend = _gguf_backend_for_recipe( - recipe, + print("Loading real PyTorch model shard...", flush=True) + node = TorchNodeServer( + host=host, + port=port, model_id=hf_repo, shard_start=shard_start, shard_end=shard_end, quantization=quantization, - total_layers=total_layers, - device=device, - model_revision=None, + tracker_url=tracker_url, + route_timeout=route_timeout, + cache_dir=shard_path, + debug=debug, + max_loaded_shards=max_loaded_shards, + force_cpu=force_cpu, + recipe_params=recipe.params, ) - print( - "Loading native llama.cpp model shard..." if backend is not None else "Loading real PyTorch model shard...", - flush=True, - ) - node_kwargs = { - "host": host, - "port": port, - "model_id": hf_repo, - "shard_start": shard_start, - "shard_end": shard_end, - "quantization": quantization, - "tracker_url": tracker_url, - "route_timeout": route_timeout, - "cache_dir": shard_path, - "debug": debug, - "max_loaded_shards": max_loaded_shards, - "force_cpu": force_cpu, - "recipe_params": recipe.params, - } - if backend is not None: - node_kwargs["backend"] = backend - node = TorchNodeServer(**node_kwargs) capability_report = _admit_capability( node, model_id=hf_repo, @@ -1572,7 +1485,6 @@ def run_startup( "managed_assignment": not user_pinned_shard, "model_metadata": model_metadata_for(hf_repo, total_layers, cache_dir=shard_path), "capability_report": capability_report.to_dict(), - "compatibility_fingerprint": capability_report.compatibility_fingerprint, # Declared independently of the proof: the tracker checks that the # recipe this node says it serves with is the one the proof ran. "recipe_id": recipe.id, @@ -1634,7 +1546,6 @@ def run_startup( recipe=recipe, validator=capability_validator, ) - proof_shard = capability_report.shard actual_port = node.start() public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host) endpoint = f"http://{public_host}:{actual_port}" @@ -1654,11 +1565,10 @@ def run_startup( reg_payload = { "endpoint": endpoint, "model": assigned_model, - "shard_start": proof_shard.start, - "shard_end": proof_shard.end, + "shard_start": shard_start, + "shard_end": shard_end, "shard_checksum": shard_checksum, "capability_report": capability_report.to_dict(), - "compatibility_fingerprint": capability_report.compatibility_fingerprint, # Declared independently of the proof: the tracker checks that the # recipe this node says it serves with is the one the proof ran. "recipe_id": recipe.id, @@ -1704,8 +1614,8 @@ def run_startup( if gpu_name: hw_str += f" ({gpu_name}, {vram_mb / 1024:.1f} GB)" shard_label = _format_shard_label( - proof_shard.start, - proof_shard.end, + shard_start, + shard_end, assigned_total_layers, model_name=assigned_model, ) diff --git a/packages/node/meshnet_node/testing.py b/packages/node/meshnet_node/testing.py index 812428b..e595524 100644 --- a/packages/node/meshnet_node/testing.py +++ b/packages/node/meshnet_node/testing.py @@ -16,10 +16,7 @@ import time from typing import Any from .admission import CapabilityContext, CapabilityValidator -from . import __version__ as _PACKAGE_VERSION from .capability import STATUS_PASSED, CapabilityReport, build_capability_report -from .gguf_ownership import authoritative_dense_llama_ownership -from .runtime_recipe import build_runtime_recipe_identity def capability_report_for( @@ -33,15 +30,6 @@ def capability_report_for( recipe_version: str | None = None, backend_id: str | None = None, device: str | None = None, - artifact_hash: str | None = None, - activation_dtype: str | None = None, - compute_dtype: str | None = None, - kv_dtype: str | None = None, - kv_layout: str | None = None, - tokenizer_revision: str | None = None, - architecture_adapter: str | None = None, - boundary_schema_version: int = 1, - cache_layout: str | None = None, validated_at: float | None = None, age_seconds: float = 0.0, diagnostics: Any = None, @@ -49,49 +37,18 @@ def capability_report_for( ) -> CapabilityReport: """A report describing `context`, with any field bent away from the truth.""" now = time.time() if validated_at is None else validated_at - backend = getattr(context, "backend", None) - model_config = getattr(getattr(backend, "model", None), "config", None) - model_config_payload = ( - model_config.to_dict() if hasattr(model_config, "to_dict") else model_config - ) - resolved_cache_layout = ( - "stateless" - if getattr(backend, "supports_kv_cache", False) is False - else "local-hot-kv" - ) - ownership = authoritative_dense_llama_ownership(backend, context.selection) - runtime_recipe = build_runtime_recipe_identity( - model_id=context.selection.model_id, - revision=getattr(getattr(backend, "model", None), "revision", None), - model_config=model_config_payload, - recipe_params=context.recipe.params, - weight_quantization=context.selection.quantization, - backend_id=context.recipe.backend_id, - runtime_version=_PACKAGE_VERSION, - activation_dtype=activation_dtype, - compute_dtype=compute_dtype, - kv_dtype=kv_dtype, - kv_layout=kv_layout or _backend_kv_layout(backend), - tokenizer_revision=tokenizer_revision, - architecture_adapter=architecture_adapter, - boundary_schema_version=boundary_schema_version, - cache_layout=cache_layout or resolved_cache_layout, - ) return build_capability_report( model_id=model_id or context.selection.model_id, - shard_start=ownership.start_layer if shard_start is None else shard_start, - shard_end=ownership.end_layer if shard_end is None else shard_end, + shard_start=( + context.selection.shard_start if shard_start is None else shard_start + ), + shard_end=context.selection.shard_end if shard_end is None else shard_end, recipe_id=recipe_id or context.recipe.id, recipe_version=recipe_version or context.recipe.version, catalogue_version=context.manifest.catalogue_version, backend_id=backend_id or context.recipe.backend_id, device=device or context.device, quantization=context.selection.quantization, - runtime=_runtime_versions(), - artifact_hash=artifact_hash, - runtime_recipe=runtime_recipe, - owns_embedding=ownership.owns_embedding, - owns_final_head=ownership.owns_final_head, status=status, duration_ms=duration_ms, diagnostics=diagnostics, @@ -111,20 +68,3 @@ def capability_stub(**overrides: Any) -> CapabilityValidator: return capability_report_for(context, **overrides) return validator - - -def _runtime_versions() -> dict[str, str]: - versions: dict[str, str] = {} - for name in ("torch", "transformers"): - try: - module = __import__(name) - except Exception: - continue - version = getattr(module, "__version__", None) - if version: - versions[name] = str(version) - return versions - - -def _backend_kv_layout(backend: Any) -> str: - return "session-cache" if getattr(backend, "supports_kv_cache", False) else "stateless" diff --git a/packages/node/native/CMakeLists.txt b/packages/node/native/CMakeLists.txt deleted file mode 100644 index ccfa171..0000000 --- a/packages/node/native/CMakeLists.txt +++ /dev/null @@ -1,76 +0,0 @@ -# Reproducible C++ build wiring for the Shard runtime protocol (DGR-002). -# -# Generates C++ message stubs from proto/shard_runtime.proto and builds the -# round-trip / cross-language compatibility test. Requires protoc and the -# protobuf C++ runtime. Works with either a CONFIG-mode protobuf install -# (protobuf::libprotobuf / protobuf::protoc targets, e.g. a from-source install -# on CMAKE_PREFIX_PATH) or CMake's bundled FindProtobuf module. -# -# The gRPC C++ service stubs are generated separately by scripts/generate_cpp.sh -# when grpc_cpp_plugin is present; the round-trip test needs only message -# serialization, so gRPC is intentionally not a build dependency here. -# -# Configure & build (out-of-tree): -# cmake -S packages/node/native -B packages/node/native/build/cpp -# cmake --build packages/node/native/build/cpp -# Run: -# packages/node/native/build/cpp/shard_protocol_roundtrip_test --selftest - -cmake_minimum_required(VERSION 3.16) -project(shard_runtime_protocol CXX) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -# Prefer a CONFIG-mode protobuf (modern imported targets); fall back to the -# FindProtobuf module for system installs. -find_package(Protobuf CONFIG QUIET) -if(NOT Protobuf_FOUND) - find_package(Protobuf REQUIRED) -endif() - -if(TARGET protobuf::protoc) - set(SHARD_PROTOC_EXECUTABLE "$") -else() - set(SHARD_PROTOC_EXECUTABLE "${Protobuf_PROTOC_EXECUTABLE}") -endif() - -if(TARGET protobuf::libprotobuf) - set(SHARD_PROTOBUF_LINK protobuf::libprotobuf) -else() - set(SHARD_PROTOBUF_LINK ${Protobuf_LIBRARIES}) -endif() - -set(PROTO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/proto") -set(PROTO_FILE "${PROTO_DIR}/shard_runtime.proto") -set(GEN_DIR "${CMAKE_CURRENT_BINARY_DIR}/gen") -file(MAKE_DIRECTORY "${GEN_DIR}") - -set(PROTO_SRC "${GEN_DIR}/shard_runtime.pb.cc") -set(PROTO_HDR "${GEN_DIR}/shard_runtime.pb.h") - -add_custom_command( - OUTPUT "${PROTO_SRC}" "${PROTO_HDR}" - COMMAND "${SHARD_PROTOC_EXECUTABLE}" - "--proto_path=${PROTO_DIR}" - "--cpp_out=${GEN_DIR}" - "${PROTO_FILE}" - DEPENDS "${PROTO_FILE}" - COMMENT "Generating C++ protobuf stubs from shard_runtime.proto" - VERBATIM) - -add_executable(shard_protocol_roundtrip_test - tests/roundtrip_test.cpp - "${PROTO_SRC}") - -target_include_directories(shard_protocol_roundtrip_test PRIVATE "${GEN_DIR}") -if(NOT TARGET protobuf::libprotobuf AND Protobuf_INCLUDE_DIRS) - target_include_directories(shard_protocol_roundtrip_test PRIVATE - ${Protobuf_INCLUDE_DIRS}) -endif() - -target_link_libraries(shard_protocol_roundtrip_test PRIVATE ${SHARD_PROTOBUF_LINK}) - -enable_testing() -add_test(NAME shard_protocol_roundtrip - COMMAND shard_protocol_roundtrip_test --selftest) diff --git a/packages/node/native/llama/README.md b/packages/node/native/llama/README.md deleted file mode 100644 index a22a861..0000000 --- a/packages/node/native/llama/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Pinned llama.cpp source dependency - -This directory keeps the llama.cpp fork boundary explicit and auditable. - -Layout: - -- `UPSTREAM_COMMIT` - the exact pinned commit. -- `UPSTREAM_REPOSITORY` - the reproducible source dependency URL. -- `UPSTREAM_ASSUMPTIONS.md` - the file/ABI assumptions that the build scripts - validate. -- `patches/` - numbered patch files applied on top of the pinned checkout. - -The intended flow is: - -1. Fetch or clone the pinned upstream checkout. -2. Verify the checkout commit matches `UPSTREAM_COMMIT`. -3. Check and apply the numbered patch stack. -4. Build the worker scaffold from `examples/meshnet-worker/`. -5. Copy the upstream `LICENSE` and `AUTHORS` files into the worker build tree so - the attribution notices remain attached to the built artifact. - -The patch stack in this story is intentionally minimal. It creates the project -worker scaffold and the smoke-test CMake target without pulling Meshnet -networking code into llama.cpp. diff --git a/packages/node/native/llama/UPSTREAM_ASSUMPTIONS.md b/packages/node/native/llama/UPSTREAM_ASSUMPTIONS.md deleted file mode 100644 index e1dc54c..0000000 --- a/packages/node/native/llama/UPSTREAM_ASSUMPTIONS.md +++ /dev/null @@ -1,35 +0,0 @@ -# llama.cpp upstream assumptions - -This directory records the reproducible source dependency boundary for the -pinned llama.cpp checkout used by the distributed GGUF runtime program. - -Pinned upstream commit: - -- `b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac` - -Pinned upstream repository: - -- `https://github.com/ggml-org/llama.cpp.git` - -Assumptions checked by the build script: - -- The checkout is exactly the pinned commit above. -- The upstream source tree still ships `LICENSE`, `AUTHORS`, and - `CMakeLists.txt` at the repository root. -- The project-owned worker scaffold is built from - `examples/meshnet-worker/`, which is introduced by the patch stack below. -- The upstream license and attribution notices are preserved in the build - output by copying the root `LICENSE` and `AUTHORS` files into the worker - staging directory. - -Compatibility notes: - -- The current patch stack does not modify upstream llama.cpp runtime code yet. - It adds a project-owned worker scaffold that can be built reproducibly from - the pinned source checkout. -- Later stories extend this boundary with actual llama.cpp execution patches. - -Failure mode: - -- If the checkout commit does not match the pin, the build script fails with a - clear pin-mismatch error before patch application or compilation starts. diff --git a/packages/node/native/llama/UPSTREAM_COMMIT b/packages/node/native/llama/UPSTREAM_COMMIT deleted file mode 100644 index a513062..0000000 --- a/packages/node/native/llama/UPSTREAM_COMMIT +++ /dev/null @@ -1 +0,0 @@ -b3c9d1b846cc80a6360adb6aeaa4fcd8c4c8dcac diff --git a/packages/node/native/llama/UPSTREAM_REPOSITORY b/packages/node/native/llama/UPSTREAM_REPOSITORY deleted file mode 100644 index 8c8a700..0000000 --- a/packages/node/native/llama/UPSTREAM_REPOSITORY +++ /dev/null @@ -1 +0,0 @@ -https://github.com/ggml-org/llama.cpp.git diff --git a/packages/node/native/llama/patches/0001-add-meshnet-worker-scaffold.patch b/packages/node/native/llama/patches/0001-add-meshnet-worker-scaffold.patch deleted file mode 100644 index 99813a6..0000000 --- a/packages/node/native/llama/patches/0001-add-meshnet-worker-scaffold.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/examples/meshnet-worker/CMakeLists.txt b/examples/meshnet-worker/CMakeLists.txt -new file mode 100644 -index 0000000000..8d9f9a1a2f ---- /dev/null -+++ b/examples/meshnet-worker/CMakeLists.txt -@@ -0,0 +1,19 @@ -+cmake_minimum_required(VERSION 3.16) -+project(meshnet_llama_worker CXX) -+ -+set(CMAKE_CXX_STANDARD 17) -+set(CMAKE_CXX_STANDARD_REQUIRED ON) -+ -+configure_file( -+ "${CMAKE_CURRENT_SOURCE_DIR}/version.h.in" -+ "${CMAKE_CURRENT_BINARY_DIR}/version.h" -+ @ONLY) -+ -+add_executable(meshnet_worker -+ meshnet_worker.cpp) -+ -+target_include_directories(meshnet_worker PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") -+ -+enable_testing() -+add_test(NAME meshnet_worker_smoke -+ COMMAND meshnet_worker --smoke) -diff --git a/examples/meshnet-worker/version.h.in b/examples/meshnet-worker/version.h.in -new file mode 100644 -index 0000000000..0b75c4e60f ---- /dev/null -+++ b/examples/meshnet-worker/version.h.in -@@ -0,0 +1,4 @@ -+#pragma once -+ -+#define MESHNET_LLAMA_UPSTREAM_COMMIT "@MESHNET_LLAMA_UPSTREAM_COMMIT@" -+#define MESHNET_LLAMA_PATCHSET_VERSION "@MESHNET_LLAMA_PATCHSET_VERSION@" diff --git a/packages/node/native/llama/templates/meshnet_worker.cpp b/packages/node/native/llama/templates/meshnet_worker.cpp deleted file mode 100644 index 7e142ca..0000000 --- a/packages/node/native/llama/templates/meshnet_worker.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "version.h" - -#include -#include - -namespace { - -bool fail(const std::string& why) { - std::cerr << "meshnet_worker: FAIL: " << why << std::endl; - return false; -} - -} // namespace - -int main(int argc, char** argv) { - bool smoke = argc == 1; - - for (int i = 1; i < argc; ++i) { - const std::string arg = argv[i]; - if (arg == "--smoke") { - smoke = true; - } else { - std::cerr << "unknown arg: " << arg << std::endl; - return 2; - } - } - - if (!smoke) { - return fail("smoke mode not requested"), 1; - } - - if (MESHNET_LLAMA_UPSTREAM_COMMIT[0] == '\0') { - return fail("upstream commit missing"), 1; - } - if (MESHNET_LLAMA_PATCHSET_VERSION[0] == '\0') { - return fail("patchset version missing"), 1; - } - - std::cout << "meshnet worker scaffold ok" << std::endl; - std::cout << "upstream commit: " << MESHNET_LLAMA_UPSTREAM_COMMIT << std::endl; - std::cout << "patchset version: " << MESHNET_LLAMA_PATCHSET_VERSION << std::endl; - return 0; -} diff --git a/packages/node/native/proto/shard_runtime.proto b/packages/node/native/proto/shard_runtime.proto deleted file mode 100644 index 235e01a..0000000 --- a/packages/node/native/proto/shard_runtime.proto +++ /dev/null @@ -1,388 +0,0 @@ -// Shard runtime data-plane protocol for the distributed GGUF runtime (ADR-0024). -// -// This schema is the semantic contract between Python and C++ Shards. Direct -// transport is gRPC over HTTP/2; the existing Meshnet relay may carry the same -// serialized frames as opaque binary, so anything gRPC would normally carry in -// call metadata (deadlines, cancellation intent) is ALSO representable inside -// the messages for relay-transported seams. -// -// Design rules (see .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md): -// * One long-lived bidirectional ActivateSession stream per Route Session -// Activation Seam. No per-token channel creation. -// * Bounded chunking for prefill; a small decode fast path. -// * The activation boundary is a versioned named-tensor bundle, because an -// architecture boundary may require more than one tensor. -// * Meshnet routing/billing/auth live outside this schema; only the data -// plane and the identifiers needed to attribute and isolate work are here. -// -// Compatibility: proto3. Never renumber or reuse a field number. Add new fields -// with new numbers only. Enums keep a 0 UNSPECIFIED member for forward compat. - -syntax = "proto3"; - -package meshnet.shard.v1; - -option java_package = "com.meshnet.shard.v1"; -option java_outer_classname = "ShardRuntimeProto"; -option go_package = "meshnet/shard/v1;shardv1"; - -// --------------------------------------------------------------------------- -// Versioning and enums -// --------------------------------------------------------------------------- - -// Wire schema version. Bumped only on incompatible envelope changes; additive -// field changes keep the same version and rely on proto3 unknown-field rules. -enum SchemaVersion { - SCHEMA_VERSION_UNSPECIFIED = 0; - SCHEMA_VERSION_1 = 1; -} - -// Lifecycle phase of a seam message. RELEASE and CANCEL are represented both as -// dedicated RPCs and as in-stream phases so a relay-carried stream can express -// them without a separate channel. -enum Phase { - PHASE_UNSPECIFIED = 0; - PHASE_PREFILL = 1; - PHASE_DECODE = 2; - PHASE_RELEASE = 3; - PHASE_CANCEL = 4; -} - -// Tensor element type. GGUF quantized block types are enumerated explicitly so -// a boundary bundle can carry pre-quantized payloads without reinterpretation. -enum DType { - DTYPE_UNSPECIFIED = 0; - DTYPE_F32 = 1; - DTYPE_F16 = 2; - DTYPE_BF16 = 3; - DTYPE_I64 = 4; - DTYPE_I32 = 5; - DTYPE_I16 = 6; - DTYPE_I8 = 7; - DTYPE_U8 = 8; - DTYPE_BOOL = 9; - DTYPE_Q8_0 = 20; - DTYPE_Q4_0 = 21; - DTYPE_Q4_K = 22; - DTYPE_Q6_K = 23; -} - -// Byte order of a tensor payload. Explicit because Shards may run on -// heterogeneous hardware and the relay carries opaque bytes. -enum ByteOrder { - BYTE_ORDER_UNSPECIFIED = 0; - BYTE_ORDER_LITTLE_ENDIAN = 1; - BYTE_ORDER_BIG_ENDIAN = 2; -} - -// Payload compression applied to a tensor fragment or message body. -enum Compression { - COMPRESSION_UNSPECIFIED = 0; - COMPRESSION_NONE = 1; - COMPRESSION_ZSTD = 2; -} - -// Checksum algorithm. CRC32C is the cheap per-fragment default; SHA256 is used -// where stronger integrity is required. -enum ChecksumAlgorithm { - CHECKSUM_ALGORITHM_UNSPECIFIED = 0; - CHECKSUM_NONE = 1; - CHECKSUM_CRC32C = 2; - CHECKSUM_CRC32 = 3; - CHECKSUM_SHA256 = 4; -} - -// What the sender expects from the receiving Shard's Hot KV State for this work -// (request side of the cache contract). -enum CacheExpectation { - CACHE_EXPECTATION_UNSPECIFIED = 0; - CACHE_REUSE = 1; // reuse existing KV for (session, epoch) - CACHE_FRESH = 2; // start a fresh KV context - CACHE_BYPASS = 3; // stateless; do not persist KV -} - -// What the receiving Shard actually did with its KV State (result side). -enum CacheResult { - CACHE_RESULT_UNSPECIFIED = 0; - CACHE_HIT = 1; - CACHE_MISS = 2; - CACHE_WRITTEN = 3; - CACHE_BYPASSED = 4; -} - -// Coarse retry classification carried in structured status. -enum RetryClass { - RETRY_CLASS_UNSPECIFIED = 0; - RETRY_CLASS_NONE = 1; // terminal success/no-retry - RETRY_CLASS_RETRYABLE = 2; // transient; the same step may be retried - RETRY_CLASS_FATAL = 3; // do not retry this route/epoch - RETRY_CLASS_EPOCH_STALE = 4; // route epoch advanced; re-resolve route -} - -enum ServingStatus { - SERVING_STATUS_UNSPECIFIED = 0; - SERVING = 1; - NOT_SERVING = 2; - DRAINING = 3; -} - -// --------------------------------------------------------------------------- -// Common value messages -// --------------------------------------------------------------------------- - -// Structured, transport-independent status. Mirrors canonical gRPC codes so a -// relay-carried frame can express what a gRPC trailer normally would. -message Status { - uint32 code = 1; // canonical gRPC status code - string message = 2; - RetryClass retry_class = 3; - map details = 4; -} - -// Integrity check over an associated payload. -message Checksum { - ChecksumAlgorithm algorithm = 1; - bytes value = 2; -} - -// Exact Model Artifact / runtime-recipe fingerprint. Both Shards MUST agree on -// every populated field before activation; a mismatch is a fatal status. -message ArtifactFingerprint { - string model_id = 1; // e.g. "meta-llama/Llama-3.1-8B" - string revision = 2; // artifact revision / commit - string artifact_hash = 3; // hash of the GGUF/model artifact - string quantization = 4; // e.g. "Q4_K_M", "F16" - string runtime_recipe_fingerprint = 5; // DGR-003 recipe hash -} - -// Contiguous transformer layer range owned by a Shard (ADR-0012). end_layer is -// exclusive. effective_start_layer is the overlap-safe start after de-dupe of -// shared boundary layers between adjacent Shards. -message ShardRange { - uint32 start_layer = 1; - uint32 end_layer = 2; - uint32 effective_start_layer = 3; - bool owns_embedding = 4; - bool owns_final_head = 5; -} - -// Token position window for a message. start_position is the absolute index of -// the first token; token_count is how many positions this message covers. -message Position { - uint64 start_position = 1; - uint64 token_count = 2; - uint64 sequence_length = 3; // total known context length, if known -} - -// Envelope carried by every seam message. Everything required to version, -// route-attribute, isolate, order, and integrity-check a unit of work. -message MessageHeader { - SchemaVersion schema_version = 1; - string work_id = 2; // request/work ID (idempotency scope) - string route_session_id = 3; // Route Session ID - uint64 route_epoch = 4; // route epoch; stale epochs are rejected - ArtifactFingerprint fingerprint = 5; - ShardRange shard_range = 6; - Phase phase = 7; - Position position = 8; - uint64 idempotency_step = 9; // monotonic per (work_id) step counter - CacheExpectation cache_expectation = 10; - Compression compression = 11; // compression of THIS message's payloads - Checksum checksum = 12; // checksum over THIS message's payload -} - -// --------------------------------------------------------------------------- -// Versioned named-tensor bundle (the activation boundary payload) -// --------------------------------------------------------------------------- - -// One bounded fragment of a tensor payload. Large tensors are split so no -// single message is unbounded; fragments reassemble by byte_offset order. -message TensorFragment { - uint32 fragment_index = 1; - uint32 fragment_count = 2; - uint64 byte_offset = 3; // offset of this fragment within the full payload - bytes data = 4; - Checksum checksum = 5; // checksum over this fragment's (post-compression) data -} - -// A single named tensor with full description so the receiver never reinterprets -// bytes implicitly. -message NamedTensor { - string name = 1; - repeated uint64 shape = 2; - DType dtype = 3; - ByteOrder byte_order = 4; - uint64 total_byte_length = 5; // full payload length across all fragments - Compression compression = 6; // compression applied to fragment data - repeated TensorFragment fragments = 7; -} - -// A versioned collection of named tensors representing one activation boundary. -message TensorBundle { - uint32 bundle_version = 1; - repeated NamedTensor tensors = 2; -} - -// --------------------------------------------------------------------------- -// Session stream messages (bidirectional ActivateSession) -// --------------------------------------------------------------------------- - -// Opens a seam. Carries the header plus stream-scoped bounds. deadline_unix_nanos -// lets a relay-carried stream express the call deadline gRPC would otherwise own. -message SessionOpen { - MessageHeader header = 1; - uint64 deadline_unix_nanos = 2; // absolute deadline; 0 = none - uint32 max_prefill_tokens_per_chunk = 3; // bound for prefill chunking - uint32 max_fragment_bytes = 4; // bound for tensor fragment size - FlowControl initial_credit = 5; // receiver's starting flow-control window -} - -// Bounded prefill chunk. A prefill is split into ordered chunks each covering at -// most max_prefill_tokens_per_chunk positions; final_chunk marks the last one. -message PrefillChunk { - MessageHeader header = 1; - uint32 chunk_index = 2; - uint32 chunk_count = 3; // 0 if unknown/streaming - bool final_chunk = 4; - TensorBundle activations = 5; -} - -// Small decode fast path: a single-position (or tiny) step with minimal framing. -// Reuses the same header for isolation/ordering but expects one activation bundle. -message DecodeStep { - MessageHeader header = 1; - TensorBundle activation = 2; -} - -// Explicit HTTP/2-independent flow-control grant. credits is the number of -// additional messages the receiver is willing to accept; the byte/message caps -// bound in-flight work for backpressure. -message FlowControl { - uint64 credits = 1; - uint64 max_in_flight_bytes = 2; - uint64 max_in_flight_messages = 3; -} - -// Release a session's resources (Hot KV State, sequence) cleanly. -message ReleaseRequest { - MessageHeader header = 1; - string reason = 2; -} - -message ReleaseResponse { - Status status = 1; - CacheResult cache_result = 2; -} - -// Cancel in-flight work for a session/step. -message CancelRequest { - MessageHeader header = 1; - string reason = 2; -} - -message CancelResponse { - Status status = 1; -} - -// Client -> server frames on the ActivateSession stream. -message SessionActivation { - oneof payload { - SessionOpen open = 1; - PrefillChunk prefill = 2; - DecodeStep decode = 3; - ReleaseRequest release = 4; - CancelRequest cancel = 5; - FlowControl flow_control = 6; - } -} - -// Computed boundary output for a step: the next Shard's input tensors plus the -// cache result and integrity for what was produced. -message ActivationResult { - MessageHeader header = 1; - TensorBundle outputs = 2; - CacheResult cache_result = 3; - Status status = 4; -} - -message SessionAccepted { - MessageHeader header = 1; - FlowControl granted_credit = 2; - Status status = 3; -} - -// Server -> client frames on the ActivateSession stream. -message SessionResponse { - oneof payload { - SessionAccepted accepted = 1; - ActivationResult result = 2; - FlowControl flow_control = 3; - Status status = 4; - ReleaseResponse release_ack = 5; - CancelResponse cancel_ack = 6; - } -} - -// --------------------------------------------------------------------------- -// Capability and health (unary) -// --------------------------------------------------------------------------- - -message ResourceBudget { - uint64 weight_bytes = 1; - uint64 kv_bytes = 2; - uint64 scratch_bytes = 3; - uint32 max_concurrent_sessions = 4; -} - -message CapabilityRequest { - SchemaVersion schema_version = 1; -} - -message CapabilityResponse { - SchemaVersion schema_version = 1; - repeated SchemaVersion supported_schema_versions = 2; - repeated string supported_architectures = 3; // e.g. "llama", "qwen3" - repeated string supported_quantizations = 4; - ShardRange servable_range = 5; - ResourceBudget budget = 6; - repeated Compression supported_compression = 7; - repeated ChecksumAlgorithm supported_checksums = 8; - ArtifactFingerprint loaded_fingerprint = 9; // empty if no artifact loaded -} - -message HealthRequest { - string route_session_id = 1; // optional; empty for node-wide health -} - -message HealthResponse { - ServingStatus status = 1; - uint32 active_sessions = 2; - uint32 queued_requests = 3; - double kv_pressure = 4; // 0.0..1.0 fraction of KV budget in use - uint64 rss_bytes = 5; - Status detail = 6; -} - -// --------------------------------------------------------------------------- -// Service -// --------------------------------------------------------------------------- - -service ShardRuntime { - // Admission/capability negotiation. - rpc GetCapability(CapabilityRequest) returns (CapabilityResponse); - - // Liveness/backpressure telemetry. - rpc Health(HealthRequest) returns (HealthResponse); - - // One long-lived bidirectional stream per Route Session Activation Seam. - // Deadlines/cancellation use gRPC call semantics on direct transport and the - // in-message equivalents on relay transport; flow control uses FlowControl - // frames; errors are structured Status. - rpc ActivateSession(stream SessionActivation) returns (stream SessionResponse); - - // Clean resource release (also expressible in-stream as PHASE_RELEASE). - rpc Release(ReleaseRequest) returns (ReleaseResponse); - - // Cancellation (also expressible in-stream as PHASE_CANCEL). - rpc Cancel(CancelRequest) returns (CancelResponse); -} diff --git a/packages/node/native/scripts/build_llama_worker.sh b/packages/node/native/scripts/build_llama_worker.sh deleted file mode 100644 index 7208abc..0000000 --- a/packages/node/native/scripts/build_llama_worker.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env bash -# Apply the numbered llama.cpp patch stack and build the worker scaffold. -# -# Default flow: -# 1. Fetch the pinned llama.cpp source into a build directory if needed. -# 2. Verify the checkout matches the pinned commit. -# 3. Check/apply the numbered patch stack from packages/node/native/llama/. -# 4. Compile and build the standalone worker scaffold. -# 5. Copy upstream LICENSE/AUTHORS notices into the staging directory. -# -# This script is intentionally model-free and does not contact any inference -# endpoint. It is a source/build reproducibility check. -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -NATIVE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -LLAMA_ROOT="${NATIVE_ROOT}/llama" -UPSTREAM_COMMIT="$(tr -d '\n\r' < "${LLAMA_ROOT}/UPSTREAM_COMMIT")" -UPSTREAM_REPOSITORY="$(tr -d '\n\r' < "${LLAMA_ROOT}/UPSTREAM_REPOSITORY")" -PATCH_DIR="${LLAMA_ROOT}/patches" -DEFAULT_SOURCE_DIR="${NATIVE_ROOT}/build/llama.cpp-src" -DEFAULT_BUILD_DIR="${NATIVE_ROOT}/build/llama-worker" -SOURCE_DIR="${DEFAULT_SOURCE_DIR}" -BUILD_DIR="${DEFAULT_BUILD_DIR}" -WORKTREE_DIR="" -FETCH=1 -CXX_BIN="${CXX:-}" - -usage() { - cat <<'EOF' -Usage: build_llama_worker.sh [--source-dir PATH] [--build-dir PATH] [--no-fetch] - -Builds the project-owned worker scaffold from a pinned llama.cpp checkout. -EOF -} - -fail() { - echo "error: $*" >&2 - exit 1 -} - -while (($#)); do - case "$1" in - --source-dir) - SOURCE_DIR="${2:-}" - shift 2 - ;; - --build-dir) - BUILD_DIR="${2:-}" - shift 2 - ;; - --no-fetch) - FETCH=0 - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - fail "unknown argument: $1" - ;; - esac -done - -[[ -n "${SOURCE_DIR}" ]] || fail "source dir is empty" -[[ -n "${BUILD_DIR}" ]] || fail "build dir is empty" - -checkout_commit() { - if [[ -f "${SOURCE_DIR}/.meshnet-upstream-commit" ]]; then - tr -d '\n\r' < "${SOURCE_DIR}/.meshnet-upstream-commit" - return 0 - fi - if git -C "${SOURCE_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - git -C "${SOURCE_DIR}" rev-parse HEAD - return 0 - fi - return 1 -} - -ensure_source() { - if [[ -d "${SOURCE_DIR}" ]]; then - return 0 - fi - if [[ "${FETCH}" -ne 1 ]]; then - fail "source dir ${SOURCE_DIR} does not exist and --no-fetch was set" - fi - - mkdir -p "${SOURCE_DIR}" - git clone --quiet "${UPSTREAM_REPOSITORY}" "${SOURCE_DIR}" || fail "unable to clone ${UPSTREAM_REPOSITORY}" - git -C "${SOURCE_DIR}" checkout --quiet "${UPSTREAM_COMMIT}" || fail "unable to checkout ${UPSTREAM_COMMIT}" - printf '%s\n' "${UPSTREAM_COMMIT}" > "${SOURCE_DIR}/.meshnet-upstream-commit" - printf '%s\n' "${UPSTREAM_REPOSITORY}" > "${SOURCE_DIR}/.meshnet-upstream-repository" -} - -verify_assumptions() { - local observed_commit - observed_commit="$(checkout_commit)" || fail "source tree does not expose a commit pin; write ${SOURCE_DIR}/.meshnet-upstream-commit or use a git checkout" - if [[ "${observed_commit}" != "${UPSTREAM_COMMIT}" ]]; then - fail "llama.cpp pin mismatch: expected ${UPSTREAM_COMMIT}, got ${observed_commit}" - fi - - for required in LICENSE AUTHORS CMakeLists.txt; do - [[ -e "${SOURCE_DIR}/${required}" ]] || fail "missing upstream assumption file: ${required}" - done -} - -apply_patches() { - shopt -s nullglob - local patches=("${PATCH_DIR}"/*.patch) - shopt -u nullglob - if ((${#patches[@]} == 0)); then - fail "no patch files found in ${PATCH_DIR}" - fi - - for patch in "${patches[@]}"; do - git -C "${SOURCE_DIR}" apply --check "${patch}" || fail "patch check failed: $(basename "${patch}")" - done - for patch in "${patches[@]}"; do - git -C "${SOURCE_DIR}" apply "${patch}" || fail "patch apply failed: $(basename "${patch}")" - done -} - -build_worker() { - rm -rf "${BUILD_DIR}" - mkdir -p "${BUILD_DIR}" - WORKTREE_DIR="${BUILD_DIR}/llama.cpp-worktree" - rm -rf "${WORKTREE_DIR}" - mkdir -p "${WORKTREE_DIR}" - cp -a "${SOURCE_DIR}/." "${WORKTREE_DIR}/" - if [[ -f "${SOURCE_DIR}/.meshnet-upstream-commit" ]]; then - cp "${SOURCE_DIR}/.meshnet-upstream-commit" "${WORKTREE_DIR}/.meshnet-upstream-commit" - fi - if [[ -f "${SOURCE_DIR}/.meshnet-upstream-repository" ]]; then - cp "${SOURCE_DIR}/.meshnet-upstream-repository" "${WORKTREE_DIR}/.meshnet-upstream-repository" - fi - - SOURCE_DIR="${WORKTREE_DIR}" - apply_patches - - local worker_dir="${SOURCE_DIR}/examples/meshnet-worker" - cp "${LLAMA_ROOT}/templates/meshnet_worker.cpp" "${worker_dir}/meshnet_worker.cpp" - cat > "${worker_dir}/version.h" </dev/null 2>&1; then - compiler="${CXX_BIN}" - elif command -v g++ >/dev/null 2>&1; then - compiler="g++" - elif command -v c++ >/dev/null 2>&1; then - compiler="c++" - elif command -v clang++ >/dev/null 2>&1; then - compiler="clang++" - else - fail "no C++ compiler found (need g++, c++, clang++, or $CXX)" - fi - - "${compiler}" -std=c++17 -O2 -Wall -Wextra \ - -I "${worker_dir}" \ - -o "${BUILD_DIR}/meshnet_worker" \ - "${worker_dir}/meshnet_worker.cpp" -} - -stage_notices() { - local notice_dir="${BUILD_DIR}/upstream-notices" - mkdir -p "${notice_dir}" - cp "${SOURCE_DIR}/LICENSE" "${notice_dir}/LICENSE" - cp "${SOURCE_DIR}/AUTHORS" "${notice_dir}/AUTHORS" - printf '%s\n' "${UPSTREAM_COMMIT}" > "${notice_dir}/UPSTREAM_COMMIT" - printf '%s\n' "${UPSTREAM_REPOSITORY}" > "${notice_dir}/UPSTREAM_REPOSITORY" -} - -main() { - ensure_source - verify_assumptions - build_worker - stage_notices - "${BUILD_DIR}/meshnet_worker" --smoke - echo "build ok: ${BUILD_DIR}/meshnet_worker" -} - -main "$@" diff --git a/packages/node/native/scripts/generate_cpp.sh b/packages/node/native/scripts/generate_cpp.sh deleted file mode 100644 index fb645a1..0000000 --- a/packages/node/native/scripts/generate_cpp.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -# Reproducibly generate the C++ Shard-protocol stubs from the schema. -# -# Produces message stubs (protoc --cpp_out) always, and gRPC C++ service stubs -# (protoc --grpc_out with grpc_cpp_plugin) when the plugin is available. The -# round-trip test needs only the message stubs; gRPC service stubs are for the -# standalone C++ worker (DGR-008). -# -# Requirements: protoc (>=3.16). Optional: grpc_cpp_plugin for --grpc_out. -# -# Usage: -# packages/node/native/scripts/generate_cpp.sh -# Output: packages/node/native/build/cpp-gen/ (gitignored via build/). -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -NATIVE_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -PROTO_DIR="${NATIVE_ROOT}/proto" -PROTO_FILE="${PROTO_DIR}/shard_runtime.proto" -OUT_DIR="${NATIVE_ROOT}/build/cpp-gen" - -if ! command -v protoc >/dev/null 2>&1; then - echo "error: protoc not found on PATH (install protobuf-compiler)." >&2 - exit 3 -fi - -mkdir -p "${OUT_DIR}" - -echo "generating C++ message stubs -> ${OUT_DIR}" -protoc --proto_path="${PROTO_DIR}" --cpp_out="${OUT_DIR}" "${PROTO_FILE}" - -if command -v grpc_cpp_plugin >/dev/null 2>&1; then - echo "generating C++ gRPC service stubs -> ${OUT_DIR}" - protoc --proto_path="${PROTO_DIR}" \ - --grpc_out="${OUT_DIR}" \ - --plugin=protoc-gen-grpc="$(command -v grpc_cpp_plugin)" \ - "${PROTO_FILE}" -else - echo "note: grpc_cpp_plugin not found; skipped --grpc_out (message stubs only)." >&2 -fi - -echo "done:" -ls -1 "${OUT_DIR}" diff --git a/packages/node/native/scripts/generate_python.py b/packages/node/native/scripts/generate_python.py deleted file mode 100644 index 1c7ee1d..0000000 --- a/packages/node/native/scripts/generate_python.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -"""Reproducibly generate the Python Shard-protocol stubs from the schema. - -This is the documented, no-manual-copy generation entry point referenced by -``evidence/DGR-002/README.md``. It runs the pinned ``grpc_tools.protoc`` with the -same flags ``meshnet_node.native_protocol.generate()`` uses on demand, but is -kept self-contained (it does not import ``meshnet_node``) so it works regardless -of which checkout the editable install points at. - -Usage (from the project .venv): - - python packages/node/native/scripts/generate_python.py - -Output: ``packages/node/native/build/python/shard_runtime_pb2{,_grpc}.py`` -(``build/`` is gitignored). -""" - -from __future__ import annotations - -import pathlib -import sys - -_NATIVE_ROOT = pathlib.Path(__file__).resolve().parents[1] -PROTO_DIR = _NATIVE_ROOT / "proto" -PROTO_FILE = PROTO_DIR / "shard_runtime.proto" -GEN_DIR = _NATIVE_ROOT / "build" / "python" - - -def _well_known_include() -> str | None: - try: - import grpc_tools - - candidate = pathlib.Path(grpc_tools.__file__).parent / "_proto" - return str(candidate) if candidate.is_dir() else None - except Exception: - return None - - -def main() -> int: - if not PROTO_FILE.exists(): - print(f"schema not found: {PROTO_FILE}", file=sys.stderr) - return 2 - try: - from grpc_tools import protoc - except ImportError: - print( - "grpc_tools is required (pip install grpcio-tools); it is present in " - "the project .venv.", - file=sys.stderr, - ) - return 3 - - GEN_DIR.mkdir(parents=True, exist_ok=True) - well_known = _well_known_include() - args = [ - "grpc_tools.protoc", - f"-I{PROTO_DIR}", - *([f"-I{well_known}"] if well_known else []), - f"--python_out={GEN_DIR}", - f"--grpc_python_out={GEN_DIR}", - PROTO_FILE.name, - ] - rc = protoc.main(args) - if rc != 0: - print(f"grpc_tools.protoc exited with status {rc}", file=sys.stderr) - return rc - - print(f"generated Python stubs into: {GEN_DIR}") - for name in ("shard_runtime_pb2.py", "shard_runtime_pb2_grpc.py"): - target = GEN_DIR / name - print(f" {name}: {'ok' if target.exists() else 'MISSING'}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/node/native/tests/roundtrip_test.cpp b/packages/node/native/tests/roundtrip_test.cpp deleted file mode 100644 index c5e8d98..0000000 --- a/packages/node/native/tests/roundtrip_test.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// C++ round-trip and cross-language compatibility test for the Shard protocol. -// -// Modes (composable): -// --selftest serialize a sample message, parse it back, verify fields. -// --read parse a fixture serialized by another language; verify the -// known fields; tolerate unknown fields (forward compat). -// --write serialize the C++ sample so another language can parse it. -// -// Exit code 0 means every requested check passed. The Python test drives this -// binary to prove Python<->C++ wire compatibility in both directions. - -#include "shard_runtime.pb.h" - -#include -#include -#include -#include -#include - -using namespace meshnet::shard::v1; - -namespace { - -bool Fail(const std::string& why) { - std::cerr << "roundtrip_test: FAIL: " << why << std::endl; - return false; -} - -SessionActivation MakeSample() { - SessionActivation act; - PrefillChunk* pre = act.mutable_prefill(); - - MessageHeader* h = pre->mutable_header(); - h->set_schema_version(SCHEMA_VERSION_1); - h->set_work_id("w1"); - h->set_route_session_id("s1"); - h->set_route_epoch(3); - h->set_phase(PHASE_PREFILL); - h->set_idempotency_step(7); - h->set_cache_expectation(CACHE_FRESH); - h->set_compression(COMPRESSION_NONE); - - ArtifactFingerprint* fp = h->mutable_fingerprint(); - fp->set_model_id("meta-llama/Llama-3.1-8B"); - fp->set_quantization("Q4_K_M"); - fp->set_runtime_recipe_fingerprint("recipe-abc"); - - ShardRange* sr = h->mutable_shard_range(); - sr->set_start_layer(0); - sr->set_end_layer(16); - sr->set_effective_start_layer(0); - sr->set_owns_embedding(true); - - Position* pos = h->mutable_position(); - pos->set_start_position(0); - pos->set_token_count(5); - pos->set_sequence_length(5); - - pre->set_chunk_index(0); - pre->set_chunk_count(1); - pre->set_final_chunk(true); - - TensorBundle* bundle = pre->mutable_activations(); - bundle->set_bundle_version(1); - NamedTensor* t = bundle->add_tensors(); - t->set_name("hidden"); - t->add_shape(1); - t->add_shape(4096); - t->set_dtype(DTYPE_F16); - t->set_byte_order(BYTE_ORDER_LITTLE_ENDIAN); - t->set_total_byte_length(8); - t->set_compression(COMPRESSION_NONE); - TensorFragment* frag = t->add_fragments(); - frag->set_fragment_index(0); - frag->set_fragment_count(1); - frag->set_byte_offset(0); - frag->set_data(std::string("\x01\x02\x03\x04\x05\x06\x07\x08", 8)); - - return act; -} - -bool CheckSample(const SessionActivation& act) { - if (act.payload_case() != SessionActivation::kPrefill) - return Fail("payload is not prefill"); - const PrefillChunk& pre = act.prefill(); - const MessageHeader& h = pre.header(); - if (h.schema_version() != SCHEMA_VERSION_1) return Fail("schema_version"); - if (h.work_id() != "w1") return Fail("work_id"); - if (h.route_session_id() != "s1") return Fail("route_session_id"); - if (h.route_epoch() != 3) return Fail("route_epoch"); - if (h.phase() != PHASE_PREFILL) return Fail("phase"); - if (h.idempotency_step() != 7) return Fail("idempotency_step"); - if (h.fingerprint().model_id() != "meta-llama/Llama-3.1-8B") - return Fail("model_id"); - if (h.fingerprint().quantization() != "Q4_K_M") return Fail("quantization"); - if (h.shard_range().end_layer() != 16) return Fail("end_layer"); - if (!h.shard_range().owns_embedding()) return Fail("owns_embedding"); - if (h.position().token_count() != 5) return Fail("token_count"); - if (!pre.final_chunk()) return Fail("final_chunk"); - if (pre.activations().tensors_size() != 1) return Fail("tensors_size"); - const NamedTensor& t = pre.activations().tensors(0); - if (t.name() != "hidden") return Fail("tensor name"); - if (t.dtype() != DTYPE_F16) return Fail("dtype"); - if (t.byte_order() != BYTE_ORDER_LITTLE_ENDIAN) return Fail("byte_order"); - if (t.shape_size() != 2 || t.shape(1) != 4096) return Fail("shape"); - if (t.fragments_size() != 1) return Fail("fragments_size"); - if (t.fragments(0).data().size() != 8) return Fail("fragment data length"); - return true; -} - -bool ReadFile(const std::string& path, std::string* out) { - std::ifstream in(path, std::ios::binary); - if (!in) return false; - std::ostringstream ss; - ss << in.rdbuf(); - *out = ss.str(); - return true; -} - -bool WriteFile(const std::string& path, const std::string& data) { - std::ofstream out(path, std::ios::binary); - if (!out) return false; - out.write(data.data(), static_cast(data.size())); - return static_cast(out); -} - -} // namespace - -int main(int argc, char** argv) { - GOOGLE_PROTOBUF_VERIFY_VERSION; - - std::string read_path; - std::string write_path; - bool selftest = (argc == 1); - - for (int i = 1; i < argc; ++i) { - std::string arg = argv[i]; - if (arg == "--selftest") { - selftest = true; - } else if (arg == "--read" && i + 1 < argc) { - read_path = argv[++i]; - } else if (arg == "--write" && i + 1 < argc) { - write_path = argv[++i]; - } else { - std::cerr << "unknown/incomplete arg: " << arg << std::endl; - return 2; - } - } - - if (selftest) { - SessionActivation sample = MakeSample(); - std::string bytes; - if (!sample.SerializeToString(&bytes)) return Fail("serialize"), 1; - SessionActivation parsed; - if (!parsed.ParseFromString(bytes)) return Fail("parse"), 1; - if (!CheckSample(parsed)) return 1; - std::cout << "selftest ok (" << bytes.size() << " bytes)" << std::endl; - } - - if (!read_path.empty()) { - std::string bytes; - if (!ReadFile(read_path, &bytes)) return Fail("cannot read fixture"), 1; - SessionActivation parsed; - // ParseFromString tolerates and preserves unknown fields (forward compat). - if (!parsed.ParseFromString(bytes)) return Fail("parse fixture"), 1; - if (!CheckSample(parsed)) return 1; - std::cout << "read ok (" << bytes.size() << " bytes)" << std::endl; - } - - if (!write_path.empty()) { - SessionActivation sample = MakeSample(); - std::string bytes; - if (!sample.SerializeToString(&bytes)) return Fail("serialize for write"), 1; - if (!WriteFile(write_path, bytes)) return Fail("cannot write output"), 1; - std::cout << "write ok (" << bytes.size() << " bytes)" << std::endl; - } - - google::protobuf::ShutdownProtobufLibrary(); - return 0; -} diff --git a/packages/tracker/meshnet_tracker/capability.py b/packages/tracker/meshnet_tracker/capability.py index 61390d4..c60364f 100644 --- a/packages/tracker/meshnet_tracker/capability.py +++ b/packages/tracker/meshnet_tracker/capability.py @@ -58,7 +58,6 @@ STATE_MODEL_MISMATCH = "model-mismatch" STATE_SHARD_MISMATCH = "shard-mismatch" STATE_RECIPE_MISMATCH = "recipe-mismatch" STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible" -STATE_COMPATIBILITY_MISMATCH = "compatibility-mismatch" ALL_STATES = ( STATE_ADMITTED, @@ -70,7 +69,6 @@ ALL_STATES = ( STATE_SHARD_MISMATCH, STATE_RECIPE_MISMATCH, STATE_CATALOGUE_INCOMPATIBLE, - STATE_COMPATIBILITY_MISMATCH, ) # --- Compatibility policy for nodes that predate the capability protocol. --- @@ -157,17 +155,12 @@ class CapabilityState: model_id: str | None = None shard_start: int | None = None shard_end: int | None = None - owns_embedding: bool | None = None - owns_final_head: bool | None = None recipe_id: str | None = None recipe_version: str | None = None catalogue_version: str | None = None backend_id: str | None = None device: str | None = None quantization: str | None = None - artifact_hash: str | None = None - compatibility_fingerprint: str | None = None - runtime_recipe_fingerprint: str | None = None validated_at: float | None = None recorded_at: float = 0.0 schema_version: int | None = None @@ -194,17 +187,12 @@ class CapabilityState: "model_id": self.model_id, "shard_start": self.shard_start, "shard_end": self.shard_end, - "owns_embedding": self.owns_embedding, - "owns_final_head": self.owns_final_head, "recipe_id": self.recipe_id, "recipe_version": self.recipe_version, "catalogue_version": self.catalogue_version, "backend_id": self.backend_id, "device": self.device, "quantization": self.quantization, - "artifact_hash": self.artifact_hash, - "compatibility_fingerprint": self.compatibility_fingerprint, - "runtime_recipe_fingerprint": self.runtime_recipe_fingerprint, "validated_at": self.validated_at, "recorded_at": self.recorded_at, "schema_version": self.schema_version, @@ -234,7 +222,6 @@ def evaluate_report( shard_end: int | None, declared_recipe_id: str | None = None, declared_recipe_version: str | None = None, - declared_compatibility_fingerprint: str | None = None, now: float | None = None, max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS, ) -> CapabilityState: @@ -321,17 +308,6 @@ def evaluate_report( f"the node declared v{declared_recipe_version}", ) - if ( - declared_compatibility_fingerprint is not None - and base.compatibility_fingerprint != declared_compatibility_fingerprint - ): - return base.with_state( - STATE_COMPATIBILITY_MISMATCH, - "proof compatibility fingerprint does not match the node's declared " - "artifact/runtime recipe; the artifact, tokenizer, architecture, " - "boundary schema, activation recipe or cache layout differs", - ) - if status != STATUS_PASSED: return base.with_state( STATE_FAILED, @@ -368,8 +344,6 @@ def _parse_report(doc: Mapping[str, Any]) -> dict: shard = _object(doc.get("shard"), "shard") recipe = _object(doc.get("recipe"), "recipe") backend = _object(doc.get("backend"), "backend") - artifact = _object_or_none(doc.get("artifact"), "artifact") - runtime_recipe = _object_or_none(doc.get("runtime_recipe"), "runtime_recipe") validated_at = doc.get("validated_at") if isinstance(validated_at, bool) or not isinstance(validated_at, (int, float)): @@ -383,8 +357,6 @@ def _parse_report(doc: Mapping[str, Any]) -> dict: "model_id": _text(model.get("model_id"), "model.model_id"), "shard_start": _index(shard.get("start"), "shard.start"), "shard_end": _index(shard.get("end"), "shard.end"), - "owns_embedding": _maybe_bool(shard.get("owns_embedding")), - "owns_final_head": _maybe_bool(shard.get("owns_final_head")), "recipe_id": _text(recipe.get("recipe_id"), "recipe.recipe_id"), "recipe_version": _text(recipe.get("recipe_version"), "recipe.recipe_version"), "catalogue_version": _text( @@ -395,15 +367,6 @@ def _parse_report(doc: Mapping[str, Any]) -> dict: "quantization": _optional_text( backend.get("quantization"), "backend.quantization" ), - "artifact_hash": _optional_text( - artifact.get("artifact_hash"), "artifact.artifact_hash" - ), - "compatibility_fingerprint": _optional_text( - doc.get("compatibility_fingerprint"), "compatibility_fingerprint" - ), - "runtime_recipe_fingerprint": _optional_text( - runtime_recipe.get("fingerprint"), "runtime_recipe.fingerprint" - ), "validated_at": float(validated_at), "schema_version": schema_version, "diagnostics": _diagnostics(doc.get("diagnostics")), @@ -417,12 +380,6 @@ def _object(value: Any, field_name: str) -> Mapping[str, Any]: return value -def _object_or_none(value: Any, field_name: str) -> Mapping[str, Any]: - if value is None: - return {} - return _object(value, field_name) - - def _text(value: Any, field_name: str) -> str: if not isinstance(value, str) or not value.strip(): raise _ReportError(f"{field_name!r} must be a non-empty string") @@ -447,12 +404,6 @@ def _maybe_int(value: Any) -> int | None: return value -def _maybe_bool(value: Any) -> bool | None: - if isinstance(value, bool): - return value - return None - - def _diagnostics(value: Any) -> tuple[str, ...]: if not isinstance(value, list): return () diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 30629d9..43e50ce 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -56,7 +56,6 @@ from .capability import ( DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY, POLICY_COMPAT, POLICY_ENFORCE, - STATE_COMPATIBILITY_MISMATCH, STATE_ABSENT, STATE_ADMITTED, STATE_MODEL_MISMATCH, @@ -599,7 +598,6 @@ class _NodeEntry: "model_tokens_per_sec", "pending_directives", "last_heartbeat", "tracker_mode", "relay_addr", "cert_fingerprint", "peer_id", "friendly_name", - "compatibility_fingerprint", # heartbeat stats (reported by node, cumulative) "total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds", "current_requests", @@ -638,7 +636,6 @@ class _NodeEntry: cert_fingerprint: str | None = None, peer_id: str | None = None, friendly_name: str | None = None, - compatibility_fingerprint: str | None = None, capability: "CapabilityState | None" = None, ) -> None: self.node_id = node_id @@ -667,7 +664,6 @@ class _NodeEntry: self.cert_fingerprint = cert_fingerprint self.peer_id = peer_id self.friendly_name = friendly_name - self.compatibility_fingerprint = compatibility_fingerprint # No proof presented is `absent`, never `admitted` — a node can only earn # `admitted` by presenting a report that covers what it advertises. self.capability: CapabilityState = capability or absent_state() @@ -786,16 +782,6 @@ def _node_admission(node: "_NodeEntry") -> CapabilityState: f"proof is for layers {state.shard_start}–{state.shard_end}, but the " f"node now serves layers {node.shard_start}–{node.shard_end}", ) - if ( - node.compatibility_fingerprint - and state.compatibility_fingerprint - and state.compatibility_fingerprint != node.compatibility_fingerprint - ): - return state.with_state( - STATE_COMPATIBILITY_MISMATCH, - "proof compatibility fingerprint no longer matches the node's " - "declared artifact/runtime recipe", - ) return state @@ -825,12 +811,6 @@ def _capability_from_registration( declared_recipe_version=( recipe_version if isinstance(recipe_version, str) else None ), - declared_compatibility_fingerprint=( - value.strip() - if isinstance((value := payload.get("compatibility_fingerprint")), str) - and value.strip() - else None - ), ) @@ -4643,13 +4623,6 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): relay_addr = body.get("relay_addr") or None cert_fingerprint = body.get("cert_fingerprint") or None peer_id = body.get("peer_id") or None - compatibility_fingerprint = body.get("compatibility_fingerprint") - if compatibility_fingerprint is not None and ( - not isinstance(compatibility_fingerprint, str) or not compatibility_fingerprint.strip() - ): - self._send_json(400, {"error": "compatibility_fingerprint must be a string"}) - return - compatibility_fingerprint = compatibility_fingerprint.strip() if isinstance(compatibility_fingerprint, str) else None try: friendly_name = _normalize_friendly_name(body.get("friendly_name")) except ValueError as exc: @@ -4709,7 +4682,6 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): cert_fingerprint=cert_fingerprint, peer_id=peer_id, friendly_name=friendly_name, - compatibility_fingerprint=compatibility_fingerprint, capability=capability, ) with server.lock: @@ -7162,12 +7134,6 @@ class TrackerServer: else None ), friendly_name=_normalize_friendly_name(payload.get("friendly_name")), - compatibility_fingerprint=( - value.strip() - if isinstance((value := payload.get("compatibility_fingerprint")), str) - and value.strip() - else None - ), # A replicated registration carries its proof: without this, a proven # node would be routable on the leader and dark on every follower. capability=_capability_from_registration( diff --git a/scripts/ralph-gitea-run.sh b/scripts/ralph-gitea-run.sh new file mode 100644 index 0000000..7dce201 --- /dev/null +++ b/scripts/ralph-gitea-run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Run one or more supervised Ralph iterations with Gitea state reconciliation. +# The canonical PRD remains authoritative; Gitea is updated before and after Ralph. +set -u +ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$ROOT" + +python3 scripts/ralph_gitea_sync.py sync +ralph_status=0 +"$@" || ralph_status=$? +if [ "$ralph_status" -eq 0 ]; then + python3 scripts/ralph_gitea_sync.py sync +else + python3 scripts/ralph_gitea_sync.py sync --reset-in-progress +fi +exit "$ralph_status" diff --git a/scripts/ralph_gitea_sync.py b/scripts/ralph_gitea_sync.py new file mode 100644 index 0000000..d181503 --- /dev/null +++ b/scripts/ralph_gitea_sync.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Synchronize the authoritative Ralph PRD with Gitea issues. + +Gitea is a projection of prd.json. Story completion remains authoritative in +prd.json; this tool only creates/reconciles issue bodies, labels, milestones, +and open/closed/in-progress presentation. +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_FEATURE = ROOT / ".scratch" / "distributed-gguf-runtime" +STATUS_LABELS = { + "ready": ("status:ready", "Ready for Ralph execution"), + "in-progress": ("status:in-progress", "Currently selected by Ralph"), + "blocked": ("status:blocked", "Blocked by incomplete dependencies or human gate"), + "completed": ("status:completed", "Completed and verified in prd.json"), +} + + +def credential_for(host: str) -> tuple[str, str]: + result = subprocess.run( + ["git", "credential", "fill"], + input=f"protocol=https\nhost={host}\n\n", + text=True, + capture_output=True, + check=True, + ) + values = dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line) + username, password = values.get("username"), values.get("password") + if not password: + raise RuntimeError(f"No Git credential available for {host}") + return username or "", password + + +class Gitea: + def __init__(self, base_url: str, username: str, password: str, dry_run: bool = False): + self.base = base_url.rstrip("/") + self.auth = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() + self.dry_run = dry_run + + def request(self, method: str, path: str, payload: Any = None) -> Any: + url = self.base + path + body = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request( + url, + data=body, + method=method, + headers={ + "Authorization": self.auth, + "Accept": "application/json", + "Content-Type": "application/json", + }, + ) + if self.dry_run and method not in {"GET", "HEAD"}: + print(f"DRY-RUN {method} {path}") + return {} + try: + with urllib.request.urlopen(req, timeout=60) as response: + raw = response.read() + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as exc: + detail = exc.read().decode(errors="replace")[:500] + raise RuntimeError(f"Gitea {method} {path} -> HTTP {exc.code}: {detail}") from exc + + def paginate(self, path: str) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + page = 1 + while True: + separator = "&" if "?" in path else "?" + batch = self.request("GET", f"{path}{separator}limit=50&page={page}") + if not batch: + return result + result.extend(batch) + if len(batch) < 50: + return result + page += 1 + + +def color(name: str) -> str: + return "#" + hashlib.sha256(name.encode()).hexdigest()[:6] + + +def load_schema(feature: Path) -> Any: + module_path = ROOT / "scripts" / "ralph_prd_schema.py" + import importlib.util + spec = importlib.util.spec_from_file_location("ralph_prd_schema", module_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def issue_body(schema: Any, story: dict[str, Any]) -> str: + data = schema.load_prd(DEFAULT_FEATURE / "prd.json") + return schema.render_issue_markdown(story, data) + f"\n\n" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("command", choices=["sync"]) + parser.add_argument("--feature-dir", type=Path, default=DEFAULT_FEATURE) + parser.add_argument("--repo-url", default="https://git.d-popov.com/popov/neuron-tai") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--reset-in-progress", action="store_true", help="Do not claim a next story after a failed Ralph start") + args = parser.parse_args() + + feature = args.feature_dir.resolve() + prd_path = feature / "prd.json" + data = json.loads(prd_path.read_text()) + stories = data["userStories"] + repo = urllib.parse.urlparse(args.repo_url) + if not repo.hostname or not repo.path.strip("/"): + raise SystemExit("--repo-url must be an HTTPS Gitea repository URL") + owner, name = repo.path.strip("/").removesuffix(".git").split("/", 1) + api_base = f"{repo.scheme}://{repo.hostname}/api/v1/repos/{owner}/{name}" + username, password = credential_for(repo.hostname) + gitea = Gitea(api_base, username, password, args.dry_run) + schema = load_schema(feature) + schema_errors = schema.validate_backlog(data) + if schema_errors: + raise SystemExit("PRD validation failed:\n" + "\n".join(schema_errors)) + + labels = {x["name"]: x for x in gitea.paginate("/labels")} + milestones = {x["title"]: x for x in gitea.paginate("/milestones?state=all")} + issue_list = gitea.paginate("/issues?state=all") + by_story = {} + for issue in issue_list: + marker = f"", 1)[0] + by_story["DGR-" + story_id] = issue + elif issue.get("title", "").startswith("DGR-"): + by_story[issue["title"].split(":", 1)[0]] = issue + + def ensure_label(name_: str, description: str = "") -> dict[str, Any]: + if name_ in labels: + return labels[name_] + created = gitea.request("POST", "/labels", {"name": name_, "color": color(name_), "description": description}) + if args.dry_run: + created = {"id": -len(labels) - 1, "name": name_} + labels[name_] = created + return created + + for story in stories: + for label in story["labels"]: + ensure_label(label) + for label, description in STATUS_LABELS.values(): + ensure_label(label, description) + for milestone in data.get("milestones", []): + title = milestone["id"] + if title not in milestones: + milestones[title] = gitea.request("POST", "/milestones", {"title": title, "description": milestone["name"] + ": " + milestone["outcome"]}) + if args.dry_run: + milestones[title] = {"id": -len(milestones), "title": title} + + passed = {s["id"] for s in stories if s.get("passes")} + next_story = None if args.reset_in_progress else next( + ( + s + for s in stories + if not s.get("passes") + and s.get("triage") == "ready-for-agent" + and all(d in passed for d in s["dependsOn"]) + ), + None, + ) + mapping = {} + for story in stories: + sid = story["id"] + desired_status = "completed" if story.get("passes") else ("in-progress" if next_story and sid == next_story["id"] else ("ready" if all(d in passed for d in story["dependsOn"]) else "blocked")) + label_names = list(dict.fromkeys(story["labels"] + [STATUS_LABELS[desired_status][0]])) + label_ids = [ensure_label(label)["id"] for label in label_names] + milestone_id = milestones[story["milestone"]]["id"] + title = f"{sid}: {story['title']}" + body = issue_body(schema, story) + issue = by_story.get(sid) + payload = {"title": title, "body": body, "labels": label_ids, "milestone": milestone_id, "state": "closed" if story.get("passes") else "open"} + if issue is None: + issue = gitea.request("POST", "/issues", payload) + print(f"created {sid} #{issue.get('number')} {desired_status}") + else: + issue = gitea.request("PATCH", f"/issues/{issue['number']}", payload) + print(f"reconciled {sid} #{issue.get('number')} {desired_status}") + # Gitea's issue PATCH does not reliably replace labels; use the + # dedicated endpoint so stale status:in-progress labels are removed. + gitea.request("PUT", f"/issues/{issue['number']}/labels", {"labels": label_ids}) + mapping[sid] = {"number": issue.get("number"), "url": issue.get("html_url"), "state": "closed" if story.get("passes") else "open", "status": desired_status} + + if not args.dry_run: + (feature / "gitea-issues.json").write_text(json.dumps({"repository": args.repo_url, "stories": mapping}, indent=2) + "\n") + print(f"synced={len(stories)} next={next_story['id'] if next_story else 'none'} dry_run={args.dry_run}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ralph_prd_schema.py b/scripts/ralph_prd_schema.py new file mode 100644 index 0000000..6fae358 --- /dev/null +++ b/scripts/ralph_prd_schema.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +"""Canonical schema, validator, and generator for Ralph `prd.json` backlogs. + +`prd.json` is the single authoritative source for a Ralph-managed backlog. +This module defines the field shape every story must have, validates +structural and cross-story invariants (unique IDs/titles, resolvable +`dependsOn`, acyclic dependency graph, `blocks` derived correctly), and +renders the same fields into the generated Markdown/Gitea-issue view used +by `.scratch//issues/*.md`. + +Generation is one-directional: prd.json -> Markdown / Gitea payload. There +is no Markdown -> prd.json parser, so a generated artifact can never feed +`passes` (or any other field) back into the authoritative source. Every +generated Markdown document must start with the authority disclaimer +(`authority_disclaimer()`, sourced from prd.json's own `metadataSchema` when +present) so a stray edit of the generated file cannot be mistaken for a +source-of-truth change. The shared "quality gates" boilerplate is likewise +sourced from prd.json's `qualityGates`, not duplicated as a Python literal. + +Example: + python scripts/ralph_prd_schema.py validate .scratch/distributed-gguf-runtime/prd.json + python scripts/ralph_prd_schema.py render .scratch/distributed-gguf-runtime/prd.json DGR-018 +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[1] + +AUTHORITY_DISCLAIMER = ( + "" +) + +EXECUTION_MODES = {"AFK", "HITL"} +EVIDENCE_CLASSES = {"model-free", "fixture", "real-model", "real-hardware", "release"} +HARDWARE_FLAGS = {"none", "optional", "required"} +UPSTREAM_FLAGS = {"yes", "no", "conditional"} +TRIAGE_VALUES = {"ready-for-agent", "ready-for-human"} + +# Canonical field registry: name -> (required, expected python type(s)). +# This is the single definition of "what a story is" referenced by both +# validation and documentation; keep it in sync with any schema change. +STORY_FIELDS: dict[str, tuple[bool, tuple[type, ...]]] = { + "id": (True, (str,)), + "title": (True, (str,)), + "priority": (True, (int,)), + "milestone": (True, (str,)), + "executionMode": (True, (str,)), + "labels": (True, (list,)), + "evidenceClass": (True, (str,)), + "evidencePath": (True, (str,)), + "hardware": (True, (str,)), + "model": (True, (str,)), + "upstream": (True, (str,)), + "dependsOn": (True, (list,)), + "triage": (True, (str,)), + "description": (True, (str,)), + "acceptanceCriteria": (True, (list,)), + "passes": (True, (bool,)), + "notes": (True, (str,)), + "blocks": (True, (list,)), + "completionNotes": (False, (str,)), +} + +_ID_RE = re.compile(r"^[A-Z][A-Z0-9]*-\d+$") + + +class PrdValidationError(ValueError): + """Raised with all accumulated validation errors, one per line.""" + + def __init__(self, errors: list[str]): + super().__init__("\n".join(errors)) + self.errors = errors + + +def load_prd(path: Path) -> dict[str, Any]: + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as exc: + raise PrdValidationError([f"PRD not found: {path}"]) from exc + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise PrdValidationError([f"Invalid JSON in {path}: {exc}"]) from exc + if not isinstance(data, dict): + raise PrdValidationError([f"{path}: top-level PRD document must be an object"]) + return data + + +def stories_of(data: dict[str, Any]) -> list[dict[str, Any]]: + stories = data.get("userStories") + if not isinstance(stories, list): + return [] + return [s for s in stories if isinstance(s, dict)] + + +def derive_type(labels: list[str]) -> str | None: + """Derive the story's `type` from its `type:` label, if present. + + Release-gate stories (labelled `gate:hitl`) intentionally carry no + `type:` label; they derive the synthetic type `release-gate` instead. + """ + types = [label.split(":", 1)[1] for label in labels if label.startswith("type:")] + if types: + return types[0] + if any(label.startswith("gate:") for label in labels): + return "release-gate" + return None + + +def _label_prefix_counts(labels: list[str], prefix: str) -> int: + return sum(1 for label in labels if label.startswith(prefix)) + + +def validate_schema(data: dict[str, Any]) -> list[str]: + """Structural validation: required top-level keys and per-story field shape.""" + errors: list[str] = [] + for key in ("name", "description", "branchName", "userStories"): + if key not in data: + errors.append(f"prd: missing top-level field '{key}'") + stories = data.get("userStories") + if not isinstance(stories, list): + errors.append("prd: 'userStories' must be a list") + return errors + + for index, story in enumerate(stories): + label = f"userStories[{index}]" + if not isinstance(story, dict): + errors.append(f"{label}: story must be an object") + continue + sid = story.get("id", label) + + for field, (required, types) in STORY_FIELDS.items(): + if field not in story: + if required: + errors.append(f"{sid}: missing required field '{field}'") + continue + value = story[field] + if types == (bool,) and isinstance(value, bool): + continue + if not isinstance(value, types): + errors.append( + f"{sid}: field '{field}' must be {types}, got {type(value).__name__}" + ) + + if isinstance(story.get("id"), str) and not _ID_RE.match(story["id"]): + errors.append(f"{sid}: id '{story['id']}' does not match PREFIX-NUMBER convention") + if isinstance(story.get("executionMode"), str) and story["executionMode"] not in EXECUTION_MODES: + errors.append(f"{sid}: executionMode '{story['executionMode']}' not in {sorted(EXECUTION_MODES)}") + if isinstance(story.get("evidenceClass"), str) and story["evidenceClass"] not in EVIDENCE_CLASSES: + errors.append(f"{sid}: evidenceClass '{story['evidenceClass']}' not in {sorted(EVIDENCE_CLASSES)}") + if isinstance(story.get("hardware"), str) and story["hardware"] not in HARDWARE_FLAGS: + errors.append(f"{sid}: hardware '{story['hardware']}' not in {sorted(HARDWARE_FLAGS)}") + if isinstance(story.get("upstream"), str) and story["upstream"] not in UPSTREAM_FLAGS: + errors.append(f"{sid}: upstream '{story['upstream']}' not in {sorted(UPSTREAM_FLAGS)}") + if isinstance(story.get("triage"), str) and story["triage"] not in TRIAGE_VALUES: + errors.append(f"{sid}: triage '{story['triage']}' not in {sorted(TRIAGE_VALUES)}") + + labels = story.get("labels") + if isinstance(labels, list): + if not all(isinstance(label, str) for label in labels): + errors.append(f"{sid}: labels must all be strings") + elif _label_prefix_counts(labels, "type:") > 1: + errors.append(f"{sid}: at most one 'type:' label allowed") + elif _label_prefix_counts(labels, "priority:") > 1: + errors.append(f"{sid}: at most one 'priority:' label allowed") + + depends_on = story.get("dependsOn") + if isinstance(depends_on, list) and not all(isinstance(dep, str) for dep in depends_on): + errors.append(f"{sid}: dependsOn entries must all be strings") + + acceptance = story.get("acceptanceCriteria") + if isinstance(acceptance, list): + if not acceptance: + errors.append(f"{sid}: acceptanceCriteria must be non-empty") + elif not all(isinstance(item, str) and item.strip() for item in acceptance): + errors.append(f"{sid}: acceptanceCriteria entries must be non-empty strings") + + return errors + + +def _find_cycle(story_ids: set[str], depends_on: dict[str, list[str]]) -> list[str] | None: + WHITE, GRAY, BLACK = 0, 1, 2 + color = {sid: WHITE for sid in story_ids} + path: list[str] = [] + + def dfs(node: str) -> list[str] | None: + color[node] = GRAY + path.append(node) + for dep in depends_on.get(node, []): + if dep not in color: + continue + if color[dep] == GRAY: + cycle_start = path.index(dep) + return path[cycle_start:] + [dep] + if color[dep] == WHITE: + found = dfs(dep) + if found: + return found + path.pop() + color[node] = BLACK + return None + + for sid in sorted(story_ids): + if color[sid] == WHITE: + cycle = dfs(sid) + if cycle: + return cycle + return None + + +def validate_semantics(data: dict[str, Any]) -> list[str]: + """Cross-story validation: uniqueness, dependency graph, derived `blocks`.""" + errors: list[str] = [] + stories = stories_of(data) + + ids = [s.get("id") for s in stories if isinstance(s.get("id"), str)] + seen_ids: set[str] = set() + for sid in ids: + if sid in seen_ids: + errors.append(f"duplicate story id: '{sid}'") + seen_ids.add(sid) + + titles = [s.get("title") for s in stories if isinstance(s.get("title"), str)] + seen_titles: set[str] = set() + for title in titles: + if title in seen_titles: + errors.append(f"duplicate story title: '{title}'") + seen_titles.add(title) + + by_id = {s["id"]: s for s in stories if isinstance(s.get("id"), str)} + depends_on: dict[str, list[str]] = {} + for story in stories: + sid = story.get("id") + if not isinstance(sid, str): + continue + deps = story.get("dependsOn") if isinstance(story.get("dependsOn"), list) else [] + deps = [d for d in deps if isinstance(d, str)] + depends_on[sid] = deps + if sid in deps: + errors.append(f"{sid}: cannot depend on itself") + for dep in deps: + if dep not in by_id: + errors.append(f"{sid}: depends on unknown story '{dep}'") + + cycle = _find_cycle(set(by_id), depends_on) + if cycle: + errors.append("dependency cycle detected: " + " -> ".join(cycle)) + + derived_blocks: dict[str, list[str]] = {sid: [] for sid in by_id} + for sid, deps in depends_on.items(): + for dep in deps: + if dep in derived_blocks: + derived_blocks[dep].append(sid) + for sid in derived_blocks: + derived_blocks[sid] = sorted(derived_blocks[sid]) + + for story in stories: + sid = story.get("id") + if not isinstance(sid, str): + continue + given = story.get("blocks") if isinstance(story.get("blocks"), list) else [] + given_sorted = sorted(d for d in given if isinstance(d, str)) + expected = derived_blocks.get(sid, []) + if given_sorted != expected: + errors.append( + f"{sid}: 'blocks' does not match derived dependents " + f"(got {given_sorted}, expected {expected})" + ) + + evidence_path = story.get("evidencePath") + if isinstance(evidence_path, str): + expected_path = f".scratch/distributed-gguf-runtime/evidence/{sid}/README.md" + if evidence_path != expected_path: + errors.append( + f"{sid}: evidencePath '{evidence_path}' does not match convention " + f"'{expected_path}'" + ) + + return errors + + +def validate_fresh_backlog(data: dict[str, Any]) -> list[str]: + """Extra invariant for a freshly generated (not-yet-started) backlog. + + A fresh backlog must not claim any completion credit: every story + starts `passes: false`. This is distinct from validating an in-flight + backlog (use validate_schema + validate_semantics for that), because a + real backlog legitimately accumulates `passes: true` stories over time. + """ + errors = validate_backlog(data) + for story in stories_of(data): + if story.get("passes") is not False: + errors.append( + f"{story.get('id')}: fresh backlog requires passes=false, " + f"got {story.get('passes')!r}" + ) + return errors + + +def validate_backlog(data: dict[str, Any]) -> list[str]: + """Full validation for an in-progress backlog: structure, semantics, and + (when present) prd.json's self-documented metadataSchema/qualityGates.""" + return ( + validate_schema(data) + + validate_semantics(data) + + validate_metadata_schema_consistency(data) + ) + + +def validate_metadata_schema_consistency(data: dict[str, Any]) -> list[str]: + """If prd.json documents its own `metadataSchema`/`qualityGates`, verify + that self-description hasn't drifted from what this validator actually + enforces. Absent entirely, this is a no-op (a minimal fixture PRD is not + required to carry self-documentation to be structurally valid). + """ + errors: list[str] = [] + ms = data.get("metadataSchema") + if ms is None: + return errors + + enum_checks = { + "triageValues": TRIAGE_VALUES, + "executionModeValues": EXECUTION_MODES, + "evidenceClassValues": EVIDENCE_CLASSES, + "hardwareValues": HARDWARE_FLAGS, + "upstreamValues": UPSTREAM_FLAGS, + } + for key, expected in enum_checks.items(): + declared = ms.get(key) + if declared is None: + errors.append(f"metadataSchema: missing '{key}'") + continue + if set(declared) != expected: + errors.append( + f"metadataSchema.{key} {sorted(declared)} does not match " + f"enforced values {sorted(expected)}" + ) + + required_in_code = {field for field, (required, _) in STORY_FIELDS.items() if required} + declared_required = set(ms.get("requiredStoryFields", [])) + if declared_required != required_in_code: + errors.append( + f"metadataSchema.requiredStoryFields {sorted(declared_required)} does not " + f"match code-enforced required fields {sorted(required_in_code)}" + ) + + optional_in_code = {field for field, (required, _) in STORY_FIELDS.items() if not required} + declared_optional = set(ms.get("optionalStoryFields", [])) + if declared_optional != optional_in_code: + errors.append( + f"metadataSchema.optionalStoryFields {sorted(declared_optional)} does not " + f"match code-enforced optional fields {sorted(optional_in_code)}" + ) + + if not ms.get("generatedArtifactDisclaimer"): + errors.append("metadataSchema: missing 'generatedArtifactDisclaimer'") + + if "qualityGates" not in data: + errors.append("prd: metadataSchema is documented but top-level 'qualityGates' is missing") + elif not quality_gate_bullets(data): + errors.append("qualityGates: no bullets declared under universal/native/realModelHardware/scope") + + return errors + + +def quality_gate_bullets(data: dict[str, Any]) -> list[str]: + """Flatten prd.json's `qualityGates` into the bullet order used by the + generated "Shared quality gates" Markdown section.""" + gates = data.get("qualityGates", {}) + bullets: list[str] = [] + for key in ("universal", "native", "realModelHardware", "scope"): + bullets.extend(gates.get(key, [])) + return bullets + + +def authority_disclaimer(data: dict[str, Any]) -> str: + """The exact disclaimer generated Markdown/Gitea bodies must start with. + + Sourced from prd.json's `metadataSchema.generatedArtifactDisclaimer` when + present, falling back to the module default for minimal fixtures. + """ + return data.get("metadataSchema", {}).get("generatedArtifactDisclaimer", AUTHORITY_DISCLAIMER) + + +def check_generated_markdown_authority(text: str, disclaimer: str = AUTHORITY_DISCLAIMER) -> list[str]: + """Reject generated Markdown that fails to defer authority to prd.json.""" + errors: list[str] = [] + first_line = text.splitlines()[0] if text else "" + if first_line != disclaimer: + errors.append( + "generated Markdown must start with the authority disclaimer: " + f"expected {disclaimer!r}, got {first_line!r}" + ) + lowered = text.lower() + for claim in ("this file is authoritative", "this document is authoritative", "markdown is authoritative"): + if claim in lowered: + errors.append(f"generated Markdown claims authority over prd.json: found '{claim}'") + return errors + + +def slugify(title: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") + return re.sub(r"-{2,}", "-", slug) + + +def issue_number(story_id: str) -> str: + match = re.search(r"(\d+)$", story_id) + if not match: + raise ValueError(f"story id '{story_id}' has no trailing number") + return match.group(1) + + +def issue_filename(story: dict[str, Any]) -> str: + return f"{issue_number(story['id'])}-{slugify(story['title'])}.md" + + +# Fallback bullets used only when a minimal fixture PRD has no qualityGates +# block of its own; the real backlog always sources these from prd.json via +# quality_gate_bullets() so this list is a default, not a duplicate of truth. +_DEFAULT_QUALITY_GATE_BULLETS = [ + "Targeted deterministic tests pass; Python changes also pass `python -m compileall packages tests`.", + "`git diff --check` passes.", + "Default tests are model-download-free, API-credit-free, and GPU-free.", + "Evidence README records exact changed files, commands/results, limitations, and dependency handoff; no fabricated evidence or inherited completion credit.", +] + + +def render_issue_markdown(story: dict[str, Any], data: dict[str, Any] | None = None) -> str: + """Render the canonical generated Markdown view of a story. + + `data` supplies the shared quality-gates bullets and authority disclaimer + from prd.json's own `qualityGates`/`metadataSchema`; omit it only for + fixtures that don't model those fields. This is byte-for-byte identical + to the convention already used by `.scratch/distributed-gguf-runtime/ + issues/*.md`; see the round-trip tests in tests/test_ralph_prd_schema.py. + """ + data = data or {} + sid = story["id"] + passes = story["passes"] + deps = story.get("dependsOn", []) + blocks = sorted(story.get("blocks", [])) + + deps_text = ", ".join(f"`{d}`" for d in deps) if deps else "None" + blocks_text = ", ".join(f"`{b}`" for b in blocks) if blocks else "None" + labels_text = ", ".join(f"`{label}`" for label in story.get("labels", [])) + + if passes: + status_line = "completed; `passes: true`" + else: + status_line = f"specification only; `{story['triage']}`; `passes: false`" + + checkbox = "x" if passes else " " + acceptance_lines = "\n".join( + f"- [{checkbox}] {criterion}" for criterion in story["acceptanceCriteria"] + ) + + if passes: + evidence_handoff = ( + f"Verified evidence: `{story['evidencePath']}`. Legacy evidence remains " + "provenance only and grants no implementation completion credit." + ) + else: + evidence_handoff = ( + f"Write and verify `{story['evidencePath']}`. Until every criterion and " + f"applicable gate has real evidence, this story remains `passes: false`. " + "Legacy evidence is provenance only, not completion credit." + ) + + description = story["description"] + + gate_bullets = quality_gate_bullets(data) if "qualityGates" in data else _DEFAULT_QUALITY_GATE_BULLETS + gates_section = "## Shared quality gates\n\n" + "\n".join(f"- {bullet}" for bullet in gate_bullets) + + lines = [ + authority_disclaimer(data), + f"# {sid}: {story['title']}", + "", + f"- **Status / triage:** {status_line}", + f"- **Execution mode:** `{story['executionMode']}`", + f"- **Milestone:** `{story['milestone']}`", + f"- **Dependencies:** {deps_text}", + f"- **Blocks (derived):** {blocks_text}", + f"- **Labels:** {labels_text}", + f"- **Evidence class:** `{story['evidenceClass']}`", + f"- **Hardware:** `{story['hardware']}`", + f"- **Model:** `{story['model']}`", + f"- **Upstream:** `{story['upstream']}`", + "", + "## Objective / description", + "", + description, + "", + "## Acceptance criteria", + "", + acceptance_lines, + "", + gates_section, + "", + "## Evidence handoff", + "", + evidence_handoff, + "", + ] + return "\n".join(lines[:-1]) + "\n" + + +def to_gitea_issue_payload(story: dict[str, Any], data: dict[str, Any] | None = None) -> dict[str, Any]: + """Structured payload matching the Gitea create-issue API shape. + + `body` is exactly `render_issue_markdown`'s output, so the payload + carries no information that isn't losslessly derivable from prd.json. + """ + return { + "title": f"{story['id']}: {story['title']}", + "body": render_issue_markdown(story, data), + "labels": list(story.get("labels", [])), + "milestone": story["milestone"], + } + + +def _cmd_validate(args: argparse.Namespace) -> int: + data = load_prd(args.prd) + errors = validate_fresh_backlog(data) if args.fresh else validate_backlog(data) + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + print(f"{len(errors)} validation error(s).", file=sys.stderr) + return 1 + print(f"OK: {len(stories_of(data))} stories validated.") + return 0 + + +def _cmd_render(args: argparse.Namespace) -> int: + data = load_prd(args.prd) + by_id = {s["id"]: s for s in stories_of(data)} + story = by_id.get(args.story_id) + if story is None: + print(f"Unknown story id: {args.story_id}", file=sys.stderr) + return 1 + print(render_issue_markdown(story, data), end="") + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + validate = subparsers.add_parser("validate", help="Validate a prd.json against the canonical schema") + validate.add_argument("prd", type=Path) + validate.add_argument("--fresh", action="store_true", help="Also require passes=false on every story") + validate.set_defaults(func=_cmd_validate) + + render = subparsers.add_parser("render", help="Render the generated Markdown view of one story") + render.add_argument("prd", type=Path) + render.add_argument("story_id") + render.set_defaults(func=_cmd_render) + + return parser + + +def main(argv: list[str]) -> int: + parser = build_parser() + args = parser.parse_args(argv[1:]) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/tests/test_activation_envelope.py b/tests/test_activation_envelope.py new file mode 100644 index 0000000..308a0aa --- /dev/null +++ b/tests/test_activation_envelope.py @@ -0,0 +1,130 @@ +"""DGR-021: versioned activation envelope for shard traffic.""" + +from __future__ import annotations + +import json +import hashlib + +import pytest + +from meshnet_node.model_backend import TensorPayload +from meshnet_node.protocol import ActivationEnvelope, NamedTensor, TensorFragment + + +def _payload(body: bytes = b"\x01\x02\x03\x04", *, shape=(1, 2, 1)) -> TensorPayload: + return TensorPayload( + body=body, + shape=list(shape), + attention_mask_header="1,2:AAAA", + position_ids_header="1,2:BBBB", + past_len=7, + ) + + +def test_envelope_roundtrips_deterministically(): + payload = _payload() + env_a = payload.to_envelope( + name="activations", + request_id="req-1", + work_id="work-1", + route_session="route-1", + route_epoch=9, + shard_start=12, + effective_start=18, + phase="prefill", + position=0, + idempotency_step=0, + extensions={"zeta": 3, "alpha": "x"}, + ) + env_b = ActivationEnvelope.from_bytes(env_a.to_bytes()) + + assert env_a.to_bytes() == env_b.to_bytes() + assert env_b.request_id == "req-1" + assert env_b.work_id == "work-1" + assert env_b.route_session == "route-1" + assert env_b.shard_start == 12 + assert env_b.effective_start == 18 + assert env_b.phase == "prefill" + assert env_b.tensors[0].body() == payload.body + assert env_b.tensors[0].extensions["past_len"] == 7 + assert env_b.extensions == {"alpha": "x", "zeta": 3} + + +def test_fragmentation_and_checksums_are_bounded(): + body = b"0123456789abcdef" + tensor = NamedTensor.from_bytes( + name="activations", + body=body, + shape=[1, 8, 1], + dtype="bfloat16", + max_fragment_bytes=5, + ) + + assert len(tensor.fragments) == 4 + assert all(isinstance(fragment, TensorFragment) for fragment in tensor.fragments) + assert all(len(fragment.body) <= 5 for fragment in tensor.fragments) + assert tensor.body() == body + assert tensor.checksum == hashlib.sha256(body).hexdigest() + + +def test_unknown_fields_are_preserved_on_roundtrip(): + env = _payload().to_envelope( + name="activations", + request_id="req-2", + work_id="work-2", + route_session="route-2", + route_epoch=1, + shard_start=0, + effective_start=0, + phase="decode", + position=3, + idempotency_step=2, + ) + raw = json.loads(env.to_bytes()) + raw["future_top_level"] = {"x": 1} + raw["tensors"][0]["future_tensor_field"] = "ok" + roundtrip = ActivationEnvelope.from_bytes(json.dumps(raw).encode()) + + assert roundtrip.extensions["future_top_level"] == {"x": 1} + assert roundtrip.tensors[0].extensions["future_tensor_field"] == "ok" + + +def test_size_limit_rejects_large_envelopes(): + env = _payload(body=b"x" * 64).to_envelope( + name="activations", + request_id="req-3", + work_id="work-3", + route_session="route-3", + route_epoch=1, + shard_start=0, + effective_start=0, + phase="decode", + position=5, + idempotency_step=4, + ) + + with pytest.raises(ValueError, match="size limit"): + env.to_bytes(max_bytes=32) + + +def test_tensor_payload_roundtrips_through_envelope(): + payload = _payload(body=b"abcde", shape=(1, 5, 1)) + env = payload.to_envelope( + name="activations", + request_id="req-4", + work_id="work-4", + route_session="route-4", + route_epoch=2, + shard_start=6, + effective_start=11, + phase="decode", + position=8, + idempotency_step=9, + ) + + restored = TensorPayload.from_envelope(env) + assert restored.body == payload.body + assert restored.shape == payload.shape + assert restored.attention_mask_header == payload.attention_mask_header + assert restored.position_ids_header == payload.position_ids_header + assert restored.past_len == payload.past_len diff --git a/tests/test_batch_scheduler.py b/tests/test_batch_scheduler.py deleted file mode 100644 index 9893bb2..0000000 --- a/tests/test_batch_scheduler.py +++ /dev/null @@ -1,472 +0,0 @@ -"""Continuous batching and bounded admission (DGR-012). - -These tests drive the node-local continuous-batching scheduler with the *same* -pure-numpy KV-cached dense-Llama reference the Hot KV State manager uses -(DGR-007), imported from ``test_hot_kv_state``. That keeps the whole gate -deterministic, download-free, GPU-free, and API-credit-free while exercising the -real KV isolation path (``KvBoundaryAdapter`` + ``HotKvStateManager``) rather than -a mock. - -Coverage maps to the story's acceptance criteria: - -* bounded admission against weight/KV/scratch/queue budgets, -* compatible decode steps batched with per-session positions/outputs preserved, -* prefill never starving in-flight decode (explicit decode-first policy), -* backpressure when the bounded queue is full, -* capability telemetry reporting every required signal, -* a deterministic 1/2/4/8 concurrency sweep showing saturation and no - cross-session corruption. -""" - -from __future__ import annotations - -import numpy as np -import pytest - -from meshnet_node.hot_kv_state import ( - HotKvStateConfig, - HotKvStateManager, - KvBoundaryAdapter, - kv_recipe_for, -) -from meshnet_node.batch_scheduler import ( - AdmissionReason, - ContinuousBatchScheduler, - GenerationRequest, - KvBatchEngine, - NodeBudget, - Phase, - run_concurrency_sweep, -) - -# Reuse the certified numpy dense-Llama reference and shard from the DGR-007 gate. -from test_hot_kv_state import _KvDenseLlama, _KvReferenceShard - - -# --------------------------------------------------------------------------- # -# Helpers. -# --------------------------------------------------------------------------- # - - -class _FakeClock: - def __init__(self) -> None: - self.now = 0.0 - - def __call__(self) -> float: - return self.now - - def advance(self, delta: float) -> None: - self.now += delta - - -def _make_engine( - model: _KvDenseLlama | None = None, - *, - config: HotKvStateConfig | None = None, -) -> KvBatchEngine: - """A full-shard KV batch engine over the deterministic numpy dense-Llama.""" - model = model or _KvDenseLlama() - shard = _KvReferenceShard(model, 0, model.n_layers - 1) - manager = HotKvStateManager(kv_recipe_for(shard), config=config) - adapter = KvBoundaryAdapter(shard, manager) - return KvBatchEngine(adapter) - - -def _reference_tokens(model: _KvDenseLlama, prompt, n_new: int) -> list[int]: - return model.stateless_greedy(list(prompt), n_new) - - -def _generation(session_id: str, prompt, n_new: int, epoch: int = 0) -> GenerationRequest: - return GenerationRequest( - session_id=session_id, - route_epoch=epoch, - prompt_token_ids=tuple(prompt), - max_new_tokens=n_new, - ) - - -# --------------------------------------------------------------------------- # -# Bounded admission (weight / KV / scratch / queue budgets). -# --------------------------------------------------------------------------- # - - -def test_admission_respects_active_scratch_and_queue_budgets(): - "Admission fills active slots, queues the overflow, then rejects a full queue.\n\nTags: node, scheduler, admission" - engine = _make_engine() - budget = NodeBudget( - max_active_sessions=2, - scratch_bytes_per_session=1, - scratch_budget_bytes=2, # scratch also caps at 2 concurrent - max_queue_depth=1, - max_batch_size=2, - ) - scheduler = ContinuousBatchScheduler(engine, budget) - - a = scheduler.submit(_generation("a", [1, 2, 3], 4)) - b = scheduler.submit(_generation("b", [4, 5, 6], 4)) - assert a.reason is AdmissionReason.ADMITTED - assert b.reason is AdmissionReason.ADMITTED - - # Two active slots full -> the next goes to the bounded queue. - c = scheduler.submit(_generation("c", [7, 8, 9], 4)) - assert c.reason is AdmissionReason.QUEUED - - # Queue depth 1 is now full -> backpressure rejection. - d = scheduler.submit(_generation("d", [1, 1, 1], 4)) - assert d.reason is AdmissionReason.REJECTED_QUEUE_FULL - assert d.rejected - - telem = scheduler.telemetry() - assert telem.active_sessions == 2 - assert telem.queue_depth == 1 - assert telem.rejected_admissions_total == 1 - assert telem.rejected_by_reason[AdmissionReason.REJECTED_QUEUE_FULL.value] == 1 - - -def test_admission_rejects_a_session_that_cannot_fit_the_kv_budget(): - "A generation whose whole KV cannot fit the node budget is rejected up front.\n\nTags: node, scheduler, admission" - engine = _make_engine() - per_token = engine._manager.recipe.bytes_per_token() - # Budget holds only 3 positions; a prompt(4)+7 new = 10 final positions cannot fit. - budget = NodeBudget(kv_budget_bytes=per_token * 3) - scheduler = ContinuousBatchScheduler(engine, budget) - decision = scheduler.submit(_generation("big", [1, 2, 3, 4], 7)) - assert decision.reason is AdmissionReason.REJECTED_KV_BUDGET - assert scheduler.telemetry().rejected_admissions_total == 1 - - -def test_admission_rejects_when_per_session_scratch_exceeds_budget(): - "A per-session scratch larger than the whole scratch envelope is rejected.\n\nTags: node, scheduler, admission" - engine = _make_engine() - budget = NodeBudget(scratch_bytes_per_session=1024, scratch_budget_bytes=512) - scheduler = ContinuousBatchScheduler(engine, budget) - decision = scheduler.submit(_generation("s", [1, 2], 2)) - assert decision.reason is AdmissionReason.REJECTED_SCRATCH_BUDGET - - -def test_duplicate_submission_is_rejected(): - "Submitting a session id that is already scheduled is rejected as a duplicate.\n\nTags: node, scheduler, admission" - engine = _make_engine() - scheduler = ContinuousBatchScheduler(engine, NodeBudget(max_active_sessions=4)) - assert scheduler.submit(_generation("dup", [1, 2], 3)).reason is AdmissionReason.ADMITTED - assert scheduler.submit(_generation("dup", [3, 4], 3)).reason is AdmissionReason.REJECTED_DUPLICATE - - -def test_weight_budget_is_reported_in_telemetry(): - "The resident weight footprint is surfaced as a capability signal.\n\nTags: node, scheduler, telemetry" - engine = _make_engine() - budget = NodeBudget(weight_bytes=123_456) - scheduler = ContinuousBatchScheduler(engine, budget) - assert scheduler.telemetry().weight_bytes == 123_456 - - -# --------------------------------------------------------------------------- # -# Continuous batching preserves per-session positions and outputs. -# --------------------------------------------------------------------------- # - - -def test_batched_decode_preserves_per_session_positions_and_outputs(): - "Four sessions batched together each reproduce their own stateless tokens.\n\nTags: node, scheduler, batching" - model = _KvDenseLlama() - engine = _make_engine(model) - budget = NodeBudget(max_active_sessions=4, max_batch_size=4, max_queue_depth=4) - scheduler = ContinuousBatchScheduler(engine, budget) - - prompts = { - "alpha": [1, 2, 3, 4], - "bravo": [40, 39, 2, 15], - "charlie": [7, 7, 7, 7], - "delta": [31, 5, 18, 22], - } - n_new = 10 - references = {sid: _reference_tokens(model, p, n_new) for sid, p in prompts.items()} - # The four references must diverge, else "no cross-talk" would be vacuous. - assert len({tuple(v) for v in references.values()}) == 4 - - for sid, prompt in prompts.items(): - assert scheduler.submit(_generation(sid, prompt, n_new)).running - - outputs = scheduler.run_to_completion() - for sid in prompts: - assert outputs[sid] == references[sid], sid - - telem = scheduler.telemetry() - # A genuine batch formed: at least one decode tick carried all four sessions. - assert telem.batch_occupancy_max == 4 - assert telem.completed_sessions == 4 - assert telem.active_sessions == 0 - - -def test_positions_are_isolated_across_different_prompt_lengths(): - "Sessions with different prompt lengths keep independent positions when batched.\n\nTags: node, scheduler, batching" - model = _KvDenseLlama() - engine = _make_engine(model) - scheduler = ContinuousBatchScheduler( - engine, NodeBudget(max_active_sessions=3, max_batch_size=3, max_queue_depth=3) - ) - jobs = { - "short": ([5], 6), - "medium": ([2, 9, 14], 6), - "long": ([1, 2, 3, 4, 5, 6, 7], 6), - } - refs = {sid: _reference_tokens(model, p, n) for sid, (p, n) in jobs.items()} - for sid, (prompt, n) in jobs.items(): - scheduler.submit(_generation(sid, prompt, n)) - outputs = scheduler.run_to_completion() - for sid in jobs: - assert outputs[sid] == refs[sid], sid - - -# --------------------------------------------------------------------------- # -# Prefill does not starve decode. -# --------------------------------------------------------------------------- # - - -def test_prefill_does_not_starve_in_flight_decode(): - "A burst of new prefills never stalls an already-decoding session.\n\nTags: node, scheduler, fairness" - model = _KvDenseLlama() - engine = _make_engine(model) - # One prefill per tick (budget == a single prompt) so prefill is throttled and - # we can observe that decode still advances every tick. - budget = NodeBudget( - max_active_sessions=8, - max_batch_size=8, - max_queue_depth=8, - scratch_bytes_per_session=1, - scratch_budget_bytes=8, - max_prefill_tokens_per_tick=4, - ) - scheduler = ContinuousBatchScheduler(engine, budget) - - # Session A starts and prefills on tick 1. - scheduler.submit(_generation("A", [3, 14, 1, 5], 12)) - scheduler.run_tick() - a_state = scheduler.session_result("A") - assert a_state.phase is Phase.DECODING - a_len = len(a_state.generated) - assert a_len == 1 - - # Burst of new work arrives while A is decoding. - for sid in ("B", "C", "D", "E"): - scheduler.submit(_generation(sid, [2, 27, 18, 4], 12)) - - # Over the next few ticks A must decode on *every* tick (never starved), - # while at most one new session prefills per tick (prefill is bounded). - prefill_counts = [] - for _ in range(4): - report = scheduler.run_tick() - new_a_len = len(scheduler.session_result("A").generated) - assert new_a_len == a_len + 1, "decode of A stalled while prefills were pending" - a_len = new_a_len - assert "A" in report.decoded - prefill_counts.append(len(report.prefilled)) - - assert max(prefill_counts) <= 1, "prefill was not bounded per tick" - - -def test_decode_first_policy_is_explicit_in_a_single_tick(): - "In one tick decode of active sessions precedes prefill of new ones.\n\nTags: node, scheduler, fairness" - model = _KvDenseLlama() - engine = _make_engine(model) - scheduler = ContinuousBatchScheduler( - engine, - NodeBudget(max_active_sessions=4, max_batch_size=4, max_queue_depth=4, - scratch_bytes_per_session=1, scratch_budget_bytes=4), - ) - scheduler.submit(_generation("live", [1, 2, 3], 8)) - scheduler.run_tick() # 'live' prefills, now decoding - scheduler.submit(_generation("fresh", [9, 8, 7], 8)) - report = scheduler.run_tick() - assert "live" in report.decoded - assert "fresh" in report.prefilled - - -# --------------------------------------------------------------------------- # -# Backpressure and bounded memory. -# --------------------------------------------------------------------------- # - - -def test_backpressure_signals_when_queue_full_then_recovers(): - "A full queue rejects new work; a completed session frees a slot for the queue.\n\nTags: node, scheduler, backpressure" - engine = _make_engine() - budget = NodeBudget( - max_active_sessions=1, - max_batch_size=1, - max_queue_depth=1, - scratch_bytes_per_session=1, - scratch_budget_bytes=1, - ) - scheduler = ContinuousBatchScheduler(engine, budget) - assert scheduler.submit(_generation("first", [1, 2], 2)).running - assert scheduler.submit(_generation("second", [3, 4], 2)).reason is AdmissionReason.QUEUED - # Both a slot and the queue are full now. - assert scheduler.submit(_generation("third", [5, 6], 2)).reason is AdmissionReason.REJECTED_QUEUE_FULL - - # Drain 'first'; the queued 'second' must be pulled into the freed slot. - scheduler.run_to_completion() - outputs = scheduler.outputs() - assert set(outputs) == {"first", "second"} - - -def test_completed_sessions_release_kv_so_growth_is_bounded(): - "Finished sessions release their KV, so total KV returns to zero.\n\nTags: node, scheduler, backpressure" - engine = _make_engine() - scheduler = ContinuousBatchScheduler( - engine, NodeBudget(max_active_sessions=2, max_batch_size=2, max_queue_depth=8) - ) - for sid in ("a", "b", "c", "d"): - scheduler.submit(_generation(sid, [1, 2, 3], 4)) - scheduler.run_to_completion() - telem = scheduler.telemetry() - assert telem.kv_total_bytes == 0, "KV not released after completion" - assert telem.active_sessions == 0 - assert telem.completed_sessions == 4 - - -# --------------------------------------------------------------------------- # -# Telemetry. -# --------------------------------------------------------------------------- # - - -def test_telemetry_reports_every_required_signal(): - "The capability snapshot reports sessions, queue, batch, KV, rates, rejections.\n\nTags: node, scheduler, telemetry" - model = _KvDenseLlama() - engine = _make_engine(model) - clock = _FakeClock() - budget = NodeBudget(max_active_sessions=2, max_batch_size=2, max_queue_depth=1) - scheduler = ContinuousBatchScheduler(engine, budget, clock=clock) - - scheduler.submit(_generation("x", [1, 2, 3], 4)) - scheduler.submit(_generation("y", [4, 5, 6], 4)) - scheduler.submit(_generation("z", [7, 8, 9], 4)) # queued - rejected = scheduler.submit(_generation("w", [1, 1, 1], 4)) # queue full - assert rejected.rejected - - clock.advance(1.0) - scheduler.run_tick() # both prefill - clock.advance(1.0) - scheduler.run_tick() # both decode as a batch of 2 - - clock.advance(2.0) - telem = scheduler.telemetry() - snap = telem.to_dict() - for key in ( - "active_sessions", "queue_depth", "batch_occupancy_last", - "batch_occupancy_avg", "batch_occupancy_max", "weight_bytes", - "kv_total_bytes", "kv_budget_bytes", "kv_pressure", - "scratch_used_bytes", "scratch_budget_bytes", "scratch_pressure", - "prefill_tokens_total", "decode_tokens_total", - "prefill_tokens_per_sec", "decode_tokens_per_sec", - "rejected_admissions_total", "rejected_by_reason", - "completed_sessions", "ticks", - ): - assert key in snap, key - - assert telem.batch_occupancy_max == 2 - assert telem.prefill_tokens_total == 6 # two prompts of length 3 - assert telem.decode_tokens_total == 2 # one batched decode step, two sessions - assert telem.rejected_admissions_total == 1 - # Rates are deterministic under the injected clock: 4 seconds elapsed. - assert telem.decode_tokens_per_sec == pytest.approx(2 / 4.0) - assert telem.prefill_tokens_per_sec == pytest.approx(6 / 4.0) - assert 0.0 < telem.kv_pressure <= 1.0 - - -# --------------------------------------------------------------------------- # -# Concurrency 1/2/4/8 sweep: saturation and no corruption. -# --------------------------------------------------------------------------- # - - -def test_concurrency_sweep_identifies_saturation_without_corruption(): - "A 1/2/4/8 sweep raises batch occupancy, cuts ticks, and never corrupts output.\n\nTags: node, scheduler, benchmark" - model = _KvDenseLlama() - prompts = { - "s0": [1, 2, 3, 4], "s1": [5, 6, 7, 8], "s2": [9, 10, 11, 12], - "s3": [13, 14, 15, 16], "s4": [17, 18, 19, 20], "s5": [21, 22, 23, 24], - "s6": [25, 26, 27, 28], "s7": [29, 30, 31, 32], - } - n_new = 8 - requests = [_generation(sid, p, n_new) for sid, p in prompts.items()] - - sweep = run_concurrency_sweep( - lambda: _make_engine(model), - requests, - concurrency_levels=(1, 2, 4, 8), - ) - - assert sweep.corruption_free - assert [r.concurrency for r in sweep.results] == [1, 2, 4, 8] - - # No session hit a cache miss (budgets are sized to never evict here). - assert all(r.cache_misses == 0 for r in sweep.results) - assert all(r.rejected_admissions == 0 for r in sweep.results) - - # Each per-session stream matches the serialized (concurrency-1) reference. - for sid, prompt in prompts.items(): - assert list(sweep.reference_outputs[sid]) == _reference_tokens(model, prompt, n_new) - - occupancies = [r.avg_batch_occupancy for r in sweep.results] - ticks = [r.ticks for r in sweep.results] - tokens_per_tick = [r.tokens_per_tick for r in sweep.results] - - # Batching packs more sessions per decode step as concurrency rises, so - # average occupancy strictly increases and total ticks strictly decrease. - assert occupancies == sorted(occupancies) and len(set(occupancies)) == 4 - assert ticks == sorted(ticks, reverse=True) and len(set(ticks)) == 4 - # Aggregate work per tick rises with concurrency (the throughput win). - assert tokens_per_tick == sorted(tokens_per_tick) - - # For eight equal-length jobs the node keeps saturating up to the top level. - assert sweep.saturation_concurrency == 8 - - # The report is JSON-safe for durable evidence. - import json - - json.dumps(sweep.to_dict()) - - -def test_concurrency_sweep_saturates_below_max_when_load_is_small(): - "With fewer concurrent jobs than slots, saturation is found below the top level.\n\nTags: node, scheduler, benchmark" - model = _KvDenseLlama() - # Only three jobs: at concurrency 4 and 8 the batch can never exceed 3, so - # occupancy stops rising past the load and saturation is detected early. - requests = [ - _generation("j0", [1, 2, 3], 6), - _generation("j1", [4, 5, 6], 6), - _generation("j2", [7, 8, 9], 6), - ] - sweep = run_concurrency_sweep( - lambda: _make_engine(model), requests, concurrency_levels=(1, 2, 4, 8) - ) - assert sweep.corruption_free - assert sweep.saturation_concurrency <= 4 - # Levels at or above the load size share the same occupancy/tick profile. - top = [r for r in sweep.results if r.concurrency >= 4] - assert len({r.ticks for r in top}) == 1 - - -# --------------------------------------------------------------------------- # -# Engine contract guards. -# --------------------------------------------------------------------------- # - - -def test_kv_batch_engine_requires_a_full_shard(): - "The batch engine rejects a partial (non head+tail) shard.\n\nTags: node, scheduler" - model = _KvDenseLlama() - head = _KvReferenceShard(model, 0, 2) # head only, not tail - manager = HotKvStateManager(kv_recipe_for(head)) - adapter = KvBoundaryAdapter(head, manager) - with pytest.raises(Exception): - KvBatchEngine(adapter) - - -def test_run_to_completion_is_bounded_against_misconfiguration(): - "run_to_completion raises rather than looping forever when work cannot drain.\n\nTags: node, scheduler" - engine = _make_engine() - scheduler = ContinuousBatchScheduler( - engine, NodeBudget(max_active_sessions=1, max_batch_size=1, max_queue_depth=4) - ) - scheduler.submit(_generation("only", [1, 2], 3)) - # A tiny explicit tick ceiling is exceeded deterministically. - with pytest.raises(Exception): - scheduler.run_to_completion(max_ticks=1) diff --git a/tests/test_boundary_adapter.py b/tests/test_boundary_adapter.py deleted file mode 100644 index 1071242..0000000 --- a/tests/test_boundary_adapter.py +++ /dev/null @@ -1,488 +0,0 @@ -"""Architecture-defined boundary input/output and dense-Llama parity (DGR-006). - -These tests prove the boundary contract with a *pure-numpy* dense-Llama reference -model: no download, no GPU, no torch, no API credit. The reference implements the -same ``ShardComputation`` duck type the real llama.cpp/PyTorch backends expose, so -whole-model execution and a two-range (or three-range) split are the exact same -arithmetic applied to the exact same float32 residual stream. Splitting the layer -stack at a seam and shipping the *unnormalized* residual bundle across a simulated -process boundary must reproduce the whole-model tokens bit-for-bit. -""" - -from __future__ import annotations - -import numpy as np -import pytest - -from meshnet_node.boundary_adapter import ( - BOUNDARY_SCHEMA_VERSION, - BoundaryAdapter, - BoundaryBundle, - BoundaryContractError, - SamplingContract, - ShardRole, - TailOutput, - UncertifiedArchitectureError, - certified_architecture, - is_certified_architecture, - role_for_range, -) - -# Documented parity tolerance. The split path applies the identical layer -# functions in the identical order to the identical float32 arrays, so the -# residual seam is bit-exact in practice; the tolerance is a conservative guard. -PARITY_ATOL = 1e-6 - - -# --------------------------------------------------------------------------- # -# Pure-numpy dense-Llama reference model (test fixture, not production). -# --------------------------------------------------------------------------- # - - -class _ReferenceDenseLlama: - """A tiny deterministic dense-Llama: RMSNorm, RoPE attention, SwiGLU MLP.""" - - architecture_adapter = "dense-llama" - - def __init__( - self, - *, - vocab: int = 48, - hidden: int = 32, - n_layers: int = 6, - n_heads: int = 4, - intermediate: int = 64, - rms_eps: float = 1e-6, - rope_theta: float = 10000.0, - seed: int = 20260715, - ) -> None: - assert hidden % n_heads == 0 - self.vocab = vocab - self.hidden = hidden - self.n_layers = n_layers - self.n_heads = n_heads - self.head_dim = hidden // n_heads - assert self.head_dim % 2 == 0 - self.rms_eps = rms_eps - self.rope_theta = rope_theta - - rng = np.random.default_rng(seed) - - def w(*shape: int) -> np.ndarray: - return (rng.standard_normal(shape) * 0.08).astype(np.float32) - - self.embed = w(vocab, hidden) - self.layers = [] - for _ in range(n_layers): - self.layers.append( - { - "in_ln": (1.0 + rng.standard_normal(hidden) * 0.02).astype(np.float32), - "q": w(hidden, hidden), - "k": w(hidden, hidden), - "v": w(hidden, hidden), - "o": w(hidden, hidden), - "post_ln": (1.0 + rng.standard_normal(hidden) * 0.02).astype(np.float32), - "gate": w(intermediate, hidden), - "up": w(intermediate, hidden), - "down": w(hidden, intermediate), - } - ) - self.final_ln = (1.0 + rng.standard_normal(hidden) * 0.02).astype(np.float32) - self.lm_head_w = w(vocab, hidden) - - inv_freq = 1.0 / ( - rope_theta ** (np.arange(0, self.head_dim, 2, dtype=np.float32) / self.head_dim) - ) - self.inv_freq = inv_freq.astype(np.float32) - - # -- primitive ops ----------------------------------------------------- - def _rmsnorm(self, x: np.ndarray, weight: np.ndarray) -> np.ndarray: - variance = np.mean(x.astype(np.float32) ** 2, axis=-1, keepdims=True) - normed = x / np.sqrt(variance + self.rms_eps) - return (normed * weight).astype(np.float32) - - def _rope(self, positions: np.ndarray): - # positions: (batch, seq) -> cos/sin: (batch, seq, head_dim) - angles = positions[..., None].astype(np.float32) * self.inv_freq[None, None, :] - emb = np.concatenate([angles, angles], axis=-1) - return np.cos(emb).astype(np.float32), np.sin(emb).astype(np.float32) - - @staticmethod - def _rotate_half(x: np.ndarray) -> np.ndarray: - half = x.shape[-1] // 2 - return np.concatenate([-x[..., half:], x[..., :half]], axis=-1) - - def _apply_rope(self, t: np.ndarray, cos: np.ndarray, sin: np.ndarray) -> np.ndarray: - # t: (batch, n_heads, seq, head_dim); cos/sin: (batch, seq, head_dim) - cos = cos[:, None, :, :] - sin = sin[:, None, :, :] - return t * cos + self._rotate_half(t) * sin - - def _attention(self, x: np.ndarray, layer: dict, positions: np.ndarray) -> np.ndarray: - batch, seq, _ = x.shape - q = (x @ layer["q"].T).reshape(batch, seq, self.n_heads, self.head_dim) - k = (x @ layer["k"].T).reshape(batch, seq, self.n_heads, self.head_dim) - v = (x @ layer["v"].T).reshape(batch, seq, self.n_heads, self.head_dim) - q = q.transpose(0, 2, 1, 3) - k = k.transpose(0, 2, 1, 3) - v = v.transpose(0, 2, 1, 3) - cos, sin = self._rope(positions) - q = self._apply_rope(q, cos, sin) - k = self._apply_rope(k, cos, sin) - scores = (q @ k.transpose(0, 1, 3, 2)) / np.sqrt(self.head_dim) - causal = np.triu(np.full((seq, seq), -1e30, dtype=np.float32), k=1) - scores = scores + causal[None, None, :, :] - scores = scores - scores.max(axis=-1, keepdims=True) - weights = np.exp(scores) - weights = weights / weights.sum(axis=-1, keepdims=True) - out = weights @ v - out = out.transpose(0, 2, 1, 3).reshape(batch, seq, self.hidden) - return (out @ layer["o"].T).astype(np.float32) - - def _mlp(self, x: np.ndarray, layer: dict) -> np.ndarray: - gate = x @ layer["gate"].T - up = x @ layer["up"].T - silu = gate * (1.0 / (1.0 + np.exp(-gate))) - return ((silu * up) @ layer["down"].T).astype(np.float32) - - def _run_layer(self, x: np.ndarray, layer: dict, positions: np.ndarray) -> np.ndarray: - h = x + self._attention(self._rmsnorm(x, layer["in_ln"]), layer, positions) - h = h + self._mlp(self._rmsnorm(h, layer["post_ln"]), layer) - return h.astype(np.float32) - - -class _ReferenceShard: - """A contiguous inclusive layer range of the reference model. - - Satisfies the ``ShardComputation`` duck type used by ``BoundaryAdapter``. - """ - - def __init__( - self, - model: _ReferenceDenseLlama, - start_layer: int, - end_layer: int, - *, - architecture_adapter: str | None = None, - ) -> None: - self._model = model - self.start_layer = start_layer - self.end_layer = end_layer - self.total_layers = model.n_layers - self.architecture_adapter = architecture_adapter or model.architecture_adapter - - def embed_tokens(self, token_ids: np.ndarray) -> np.ndarray: - return self._model.embed[np.asarray(token_ids)] - - def run_layers(self, hidden: np.ndarray, *, positions: np.ndarray) -> np.ndarray: - h = np.asarray(hidden, dtype=np.float32) - for idx in range(self.start_layer, self.end_layer + 1): - h = self._model._run_layer(h, self._model.layers[idx], positions) - return h - - def final_norm(self, hidden: np.ndarray) -> np.ndarray: - return self._model._rmsnorm(np.asarray(hidden, dtype=np.float32), self._model.final_ln) - - def lm_head(self, hidden: np.ndarray) -> np.ndarray: - return np.asarray(hidden, dtype=np.float32) @ self._model.lm_head_w.T - - -# --------------------------------------------------------------------------- # -# Whole-model and split reference drivers. -# --------------------------------------------------------------------------- # - - -def _whole_model_next_token(model: _ReferenceDenseLlama, token_ids: list[int]) -> TailOutput: - shard = _ReferenceShard(model, 0, model.n_layers - 1) - adapter = BoundaryAdapter(shard) - result = adapter.forward(token_ids=np.asarray(token_ids)[None, :]) - assert isinstance(result, TailOutput) - return result - - -def _split_next_token( - model: _ReferenceDenseLlama, - token_ids: list[int], - cut_points: list[int], - *, - through_wire: bool = True, -) -> TailOutput: - """Run the model as N contiguous ranges, shipping the bundle across each seam. - - ``cut_points`` are the last (inclusive) layer of each non-final range. - """ - bounds = _ranges_from_cuts(cut_points, model.n_layers) - boundary: BoundaryBundle | None = None - result: BoundaryBundle | TailOutput | None = None - for i, (start, end) in enumerate(bounds): - shard = _ReferenceShard(model, start, end) - adapter = BoundaryAdapter(shard) - if i == 0: - result = adapter.forward(token_ids=np.asarray(token_ids)[None, :]) - else: - assert isinstance(boundary, BoundaryBundle) - incoming = BoundaryBundle.unpack(boundary.pack()) if through_wire else boundary - result = adapter.forward(boundary=incoming) - if isinstance(result, BoundaryBundle): - boundary = result - assert isinstance(result, TailOutput) - return result - - -def _ranges_from_cuts(cut_points: list[int], n_layers: int) -> list[tuple[int, int]]: - bounds: list[tuple[int, int]] = [] - start = 0 - for cut in cut_points: - bounds.append((start, cut)) - start = cut + 1 - bounds.append((start, n_layers - 1)) - return bounds - - -def _greedy_generate(next_token_fn, prompt: list[int], n_new: int) -> list[int]: - tokens = list(prompt) - generated: list[int] = [] - for _ in range(n_new): - out = next_token_fn(tokens) - tokens.append(out.token_id) - generated.append(out.token_id) - return generated - - -# --------------------------------------------------------------------------- # -# Certification / fail-closed. -# --------------------------------------------------------------------------- # - - -def test_dense_llama_and_aliases_are_certified(): - "Dense Llama-family identifiers all resolve to the one certified adapter.\n\nTags: node, boundary" - for name in ("dense-llama", "llama", "LlamaForCausalLM", "LlamaModel"): - boundary = certified_architecture(name) - assert boundary.adapter == "dense-llama" - assert boundary.boundary_tensor_name == "residual_stream" - assert is_certified_architecture(name) - - -@pytest.mark.parametrize("name", ["qwen3", "qwen3-moe", "mixtral", "gpt2", "", None, 123]) -def test_uncertified_architectures_fail_closed(name): - "Uncertified architectures raise instead of guessing a tensor layout.\n\nTags: node, boundary" - assert not is_certified_architecture(name) - with pytest.raises(UncertifiedArchitectureError): - certified_architecture(name) - - -def test_adapter_construction_fails_closed_for_uncertified_backend(): - "Building the adapter over an uncertified computation fails closed.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - shard = _ReferenceShard(model, 0, 2, architecture_adapter="qwen3-moe") - with pytest.raises(UncertifiedArchitectureError): - BoundaryAdapter(shard) - - -# --------------------------------------------------------------------------- # -# Roles. -# --------------------------------------------------------------------------- # - - -def test_role_classification(): - "Range endpoints map to head/middle/tail/full roles.\n\nTags: node, boundary" - assert role_for_range(0, 2, 6) is ShardRole.HEAD - assert role_for_range(2, 3, 6) is ShardRole.MIDDLE - assert role_for_range(4, 5, 6) is ShardRole.TAIL - assert role_for_range(0, 5, 6) is ShardRole.FULL - assert ShardRole.HEAD.owns_embedding and not ShardRole.HEAD.owns_final_head - assert ShardRole.TAIL.owns_final_head and not ShardRole.TAIL.owns_embedding - - -# --------------------------------------------------------------------------- # -# Input-side contract. -# --------------------------------------------------------------------------- # - - -def test_head_accepts_token_ids_and_owns_embedding(): - "The head embeds token IDs and refuses an upstream boundary bundle.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - head = BoundaryAdapter(_ReferenceShard(model, 0, 2)) - out = head.forward(token_ids=[1, 2, 3]) - assert isinstance(out, BoundaryBundle) - - # Head owns embedding: a residual bundle from upstream is a contract error. - bundle = out - with pytest.raises(BoundaryContractError, match="head owns token embedding"): - head.forward(boundary=bundle) - - -def test_middle_and_tail_bypass_embedding_and_require_the_bundle(): - "Middle/tail Shards reject token IDs and demand the named boundary bundle.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - tail = BoundaryAdapter(_ReferenceShard(model, 3, 5)) - with pytest.raises(BoundaryContractError, match="bypass token embedding"): - tail.forward(token_ids=[1, 2, 3]) - with pytest.raises(BoundaryContractError, match="must receive the named boundary bundle"): - tail.forward() - - -def test_boundary_seam_layer_mismatch_is_rejected(): - "A bundle handed to the wrong range (seam layer mismatch) is rejected.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - head = BoundaryAdapter(_ReferenceShard(model, 0, 2)) - bundle = head.forward(token_ids=[1, 2, 3]) - assert isinstance(bundle, BoundaryBundle) - assert bundle.next_layer == 3 - - # A range that starts at layer 4 must not accept a bundle cut at layer 3. - wrong = BoundaryAdapter(_ReferenceShard(model, 4, 5)) - with pytest.raises(BoundaryContractError, match="starts at layer 4"): - wrong.forward(boundary=bundle) - - -def test_normalized_bundle_is_rejected(): - "A normalized residual is not the architecture-defined boundary.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - head = BoundaryAdapter(_ReferenceShard(model, 0, 2)) - bundle = head.forward(token_ids=[1, 2, 3]) - assert isinstance(bundle, BoundaryBundle) - normalized = BoundaryBundle( - architecture_adapter=bundle.architecture_adapter, - schema_version=bundle.schema_version, - tensor_name=bundle.tensor_name, - residual=bundle.residual, - positions=bundle.positions, - next_layer=bundle.next_layer, - normalized=True, - ) - tail = BoundaryAdapter(_ReferenceShard(model, 3, 5)) - with pytest.raises(BoundaryContractError, match="UNNORMALIZED"): - tail.forward(boundary=normalized) - - -# --------------------------------------------------------------------------- # -# Output-side contract. -# --------------------------------------------------------------------------- # - - -def test_non_tail_emits_unnormalized_full_row_boundary(): - "A non-tail Shard emits the unnormalized residual with every position row.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - tokens = [3, 7, 1, 9, 2] - head = BoundaryAdapter(_ReferenceShard(model, 0, 2)) - bundle = head.forward(token_ids=tokens) - assert isinstance(bundle, BoundaryBundle) - assert bundle.normalized is False - assert bundle.tensor_name == "residual_stream" - assert bundle.schema_version == BOUNDARY_SCHEMA_VERSION - assert bundle.next_layer == 3 - # No tail-only row pruning: all sequence positions are forwarded. - assert bundle.residual.shape == (1, len(tokens), model.hidden) - assert bundle.positions.shape == (1, len(tokens)) - - # The emitted residual must be exactly the whole model's residual after layer 2 - # (i.e. before any final norm) — prove it is NOT normalized. - positions = np.arange(len(tokens))[None, :] - hidden = model.embed[np.asarray(tokens)][None, :] - for idx in range(0, 3): - hidden = model._run_layer(hidden, model.layers[idx], positions) - assert np.allclose(bundle.residual, hidden, atol=0) - assert not np.allclose(bundle.residual, model._rmsnorm(hidden, model.final_ln)) - - -def test_tail_emits_pruned_logits_through_the_sampling_contract(): - "The tail prunes to the final row and samples through an explicit contract.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - out = _whole_model_next_token(model, [4, 8, 15, 16, 23]) - assert isinstance(out, TailOutput) - assert out.logits.shape == (1, model.vocab) # tail-only row pruning to last row - assert out.sampling.mode == "greedy" - assert 0 <= out.token_id < model.vocab - assert out.token_id == int(np.argmax(out.logits[0])) - - -def test_sampling_contract_rejects_uncertified_modes(): - "Only the certified greedy sampling mode is accepted.\n\nTags: node, boundary" - with pytest.raises(BoundaryContractError): - SamplingContract(mode="top_p") - - -# --------------------------------------------------------------------------- # -# The core parity gate. -# --------------------------------------------------------------------------- # - - -def test_two_range_prefill_parity_matches_whole_model(): - "Whole-model vs two-range prefill produce the same next-token logits and token.\n\nTags: node, boundary, parity" - model = _ReferenceDenseLlama() - prompt = [5, 12, 3, 41, 7, 19, 2, 33] - - whole = _whole_model_next_token(model, prompt) - split = _split_next_token(model, prompt, cut_points=[2]) - - assert np.allclose(whole.logits, split.logits, atol=PARITY_ATOL) - assert whole.token_id == split.token_id - - -def test_three_range_prefill_parity_exercises_the_middle_role(): - "A head/middle/tail split reproduces whole-model prefill through two seams.\n\nTags: node, boundary, parity" - model = _ReferenceDenseLlama() - prompt = [9, 1, 44, 6, 30, 11] - - whole = _whole_model_next_token(model, prompt) - split = _split_next_token(model, prompt, cut_points=[1, 3]) - - assert np.allclose(whole.logits, split.logits, atol=PARITY_ATOL) - assert whole.token_id == split.token_id - - -def test_two_range_greedy_decode_parity_matches_whole_model(): - "Whole-model vs two-range greedy decode produce identical token sequences.\n\nTags: node, boundary, parity" - model = _ReferenceDenseLlama() - prompt = [2, 17, 8, 25] - n_new = 12 - - whole_tokens = _greedy_generate( - lambda toks: _whole_model_next_token(model, toks), prompt, n_new - ) - split_tokens = _greedy_generate( - lambda toks: _split_next_token(model, toks, cut_points=[2]), prompt, n_new - ) - - assert whole_tokens == split_tokens - assert len(whole_tokens) == n_new - - -def test_boundary_bundle_wire_round_trip_is_exact(): - "Packing and unpacking the boundary bundle reconstructs the exact arrays.\n\nTags: node, boundary" - model = _ReferenceDenseLlama() - head = BoundaryAdapter(_ReferenceShard(model, 0, 2)) - bundle = head.forward(token_ids=[1, 2, 3, 4]) - assert isinstance(bundle, BoundaryBundle) - - restored = BoundaryBundle.unpack(bundle.pack()) - assert np.array_equal(restored.residual, bundle.residual) - assert np.array_equal(restored.positions, bundle.positions) - assert restored.next_layer == bundle.next_layer - assert restored.architecture_adapter == bundle.architecture_adapter - - fields = bundle.named_tensor_fields() - assert fields["name"] == "residual_stream" - assert fields["shape"] == [1, 4, model.hidden] - assert fields["byte_order"] in ("little", "big") - - -def test_alias_architecture_still_parity_matches(): - "A Shard advertised as 'llama' interoperates with the canonical adapter.\n\nTags: node, boundary, parity" - model = _ReferenceDenseLlama() - prompt = [7, 3, 22, 5] - - whole = _whole_model_next_token(model, prompt) - - # Head advertises 'LlamaForCausalLM', tail advertises 'llama'; both certify to - # the same canonical adapter, so the seam contract still matches. - head = BoundaryAdapter(_ReferenceShard(model, 0, 2, architecture_adapter="LlamaForCausalLM")) - bundle = head.forward(token_ids=np.asarray(prompt)[None, :]) - assert isinstance(bundle, BoundaryBundle) - tail = BoundaryAdapter(_ReferenceShard(model, 3, 5, architecture_adapter="llama")) - split = tail.forward(boundary=BoundaryBundle.unpack(bundle.pack())) - assert isinstance(split, TailOutput) - - assert np.allclose(whole.logits, split.logits, atol=PARITY_ATOL) - assert whole.token_id == split.token_id diff --git a/tests/test_failure_semantics.py b/tests/test_failure_semantics.py deleted file mode 100644 index 8059402..0000000 --- a/tests/test_failure_semantics.py +++ /dev/null @@ -1,611 +0,0 @@ -"""Bounded failure, cancellation, and restart semantics (DGR-013). - -These tests drive the hardened per-session decode stream with the *same* -pure-numpy KV-cached dense-Llama reference the Hot KV State manager (DGR-007) and -the continuous-batch scheduler (DGR-012) use, imported from ``test_hot_kv_state``. -The whole matrix stays deterministic, download-free, GPU-free, and API-credit-free -while exercising the real KV isolation path (``KvBoundaryAdapter`` + -``HotKvStateManager``) rather than a mock. - -Coverage maps to the story's acceptance criteria: - -* deadlines and heartbeat/health loss terminate blocked stream operations, -* cancellation propagates across every Shard and releases KV + queued buffers, -* duplicate steps are idempotent; uncertain mutations are never replayed silently, -* alpha failover restarts from token zero rather than importing unverified KV, -* worker death / stream reset / malformed bundle / stale epoch / cache miss, -* billing/work records distinguish completed, cancelled, failed, and unverified. -""" - -from __future__ import annotations - -import json - -import numpy as np -import pytest - -from meshnet_node.batch_scheduler import ( - ContinuousBatchScheduler, - DoneReason, - GenerationRequest, - KvBatchEngine, - NodeBudget, -) -from meshnet_node.boundary_adapter import BoundaryBundle, BoundaryContractError -from meshnet_node.hot_kv_state import ( - CacheMiss, - CacheMissReason, - HotKvStateConfig, - HotKvStateManager, - KvBoundaryAdapter, - StaleRouteEpochError, - kv_recipe_for, -) -from meshnet_node.failure_semantics import ( - CancellationToken, - DeadlineGuard, - FailureKind, - HardenedSessionRunner, - IdempotencyLedger, - OperationCancelled, - RestartController, - ShardCancellationGroup, - StepKey, - StreamTerminated, - UncertainMutationError, - WorkLedger, - WorkRecord, - WorkStatus, - classify_exception, - work_status_for, -) - -# Reuse the certified numpy dense-Llama reference and shard from the DGR-007 gate. -from test_hot_kv_state import _KvDenseLlama, _KvReferenceShard - - -# --------------------------------------------------------------------------- # -# Helpers. -# --------------------------------------------------------------------------- # - - -class _FakeClock: - def __init__(self) -> None: - self.now = 0.0 - - def __call__(self) -> float: - return self.now - - def advance(self, delta: float) -> None: - self.now += delta - - -class _FaultyShard(_KvReferenceShard): - """A full-shard reference that raises on the Nth ``run_layers_cached`` call. - - ``run_layers_cached`` is invoked once per stream step, so ``fail_at_call=k`` - simulates a worker dying at step ``k-1`` (calls are 1-indexed). The call - counter persists across attempts, so a restart on a fresh epoch keeps counting - and does not re-trip the same fault. - """ - - def __init__(self, model, start, end, *, fail_at_call=None, error=None): - super().__init__(model, start, end) - self._fail_at_call = fail_at_call - self._error = error or RuntimeError("worker died mid-step") - self.calls = 0 - - def run_layers_cached(self, hidden, *, positions, past_kv): - self.calls += 1 - if self._fail_at_call is not None and self.calls == self._fail_at_call: - raise self._error - return super().run_layers_cached(hidden, positions=positions, past_kv=past_kv) - - -def _make_adapter(model=None, *, config=None, shard=None): - """A full-shard KV boundary adapter over the deterministic numpy dense-Llama.""" - model = model or _KvDenseLlama() - shard = shard or _KvReferenceShard(model, 0, model.n_layers - 1) - manager = HotKvStateManager(kv_recipe_for(shard), config=config) - adapter = KvBoundaryAdapter(shard, manager) - return adapter - - -def _generation(session_id, prompt, n_new, epoch=0): - return GenerationRequest( - session_id=session_id, - route_epoch=epoch, - prompt_token_ids=tuple(prompt), - max_new_tokens=n_new, - ) - - -# --------------------------------------------------------------------------- # -# Happy path (the baseline the failure paths deviate from). -# --------------------------------------------------------------------------- # - - -def test_clean_run_matches_stateless_reference_and_is_billable(): - "A clean stream reproduces the stateless tokens and records completed work.\n\nTags: node, failure, billing" - model = _KvDenseLlama() - adapter = _make_adapter(model) - runner = HardenedSessionRunner(adapter) - prompt = [1, 2, 3, 4] - n_new = 8 - outcome = runner.run(_generation("clean", prompt, n_new)) - assert outcome.status is WorkStatus.COMPLETED - assert list(outcome.tokens) == model.stateless_greedy(prompt, n_new) - record = runner.work_ledger.records_for("clean")[0] - assert record.billable - assert record.tokens == n_new - assert runner.work_ledger.billable_tokens() == n_new - - -# --------------------------------------------------------------------------- # -# Deadlines and heartbeat/health loss terminate blocked operations. -# --------------------------------------------------------------------------- # - - -def test_deadline_terminates_a_blocked_stream_and_releases_kv(): - "A deadline reached mid-stream terminates the run and frees its KV.\n\nTags: node, failure, deadline" - clock = _FakeClock() - adapter = _make_adapter() - manager = adapter.manager - runner = HardenedSessionRunner(adapter, clock=clock) - - # Each step advances the clock by 1.0; the deadline fires at t=3. - def before_step(_step): - clock.advance(1.0) - - outcome = runner.run( - _generation("slow", [5, 6, 7], 20), - deadline=3.0, - before_step=before_step, - ) - assert outcome.status is WorkStatus.FAILED - assert outcome.failure_kind is FailureKind.DEADLINE_EXCEEDED - # The stream did not hang and did not finish: only the steps before the - # deadline committed, and the session's KV was released. - assert outcome.token_count < 20 - assert isinstance(manager.resolve("slow", 0), CacheMiss) - - -def test_heartbeat_loss_terminates_a_blocked_stream(): - "Losing the peer heartbeat past the timeout terminates the stream.\n\nTags: node, failure, heartbeat" - clock = _FakeClock() - adapter = _make_adapter() - runner = HardenedSessionRunner(adapter, clock=clock) - - def before_step(_step): - clock.advance(1.0) - - # Heartbeats stop arriving after step 2; with a timeout of 1.5 the gap grows - # past the bound and the stream is terminated (health loss). - def heartbeat(step): - return step < 2 - - outcome = runner.run( - _generation("hb", [9, 8, 7], 20), - heartbeat_timeout=1.5, - heartbeat=heartbeat, - before_step=before_step, - ) - assert outcome.status is WorkStatus.FAILED - assert outcome.failure_kind is FailureKind.HEARTBEAT_LOST - assert outcome.token_count < 20 - - -def test_deadline_guard_reports_remaining_and_resets_on_heartbeat(): - "The guard exposes remaining time and a heartbeat resets the health timer.\n\nTags: node, failure, deadline" - clock = _FakeClock() - guard = DeadlineGuard(deadline=10.0, heartbeat_timeout=2.0, clock=clock) - guard.start() - guard.check() - assert guard.remaining() == 10.0 - clock.advance(1.5) - guard.heartbeat() # health refreshed at t=1.5 - clock.advance(1.0) # gap since heartbeat is 1.0 < 2.0 - guard.check() - clock.advance(2.5) # gap since heartbeat is now 3.5 > 2.0 - with pytest.raises(StreamTerminated) as exc: - guard.check() - assert exc.value.kind is FailureKind.HEARTBEAT_LOST - - -# --------------------------------------------------------------------------- # -# Cancellation propagates across shards and releases KV + queued buffers. -# --------------------------------------------------------------------------- # - - -def test_cancellation_token_terminates_stream_and_releases_kv(): - "A client cancel mid-stream stops the run and releases the session KV.\n\nTags: node, failure, cancel" - adapter = _make_adapter() - manager = adapter.manager - token = CancellationToken() - runner = HardenedSessionRunner(adapter) - - # Cancel after two steps have run. - def before_step(step): - if step == 2: - token.cancel("client-hangup") - - outcome = runner.run( - _generation("cancelme", [1, 2, 3], 20), - cancel_token=token, - before_step=before_step, - ) - assert outcome.status is WorkStatus.CANCELLED - assert outcome.failure_kind is FailureKind.CANCELLED - assert outcome.token_count == 2 # steps 0 and 1 committed before the cancel - assert isinstance(manager.resolve("cancelme", 0), CacheMiss) - - -def test_shard_cancellation_group_releases_every_shard_and_queued_buffers(): - "One cancel frees KV on every node-local shard and releases queued buffers.\n\nTags: node, failure, cancel" - model = _KvDenseLlama() - # Three node-local shards of the same route, each with its own KV manager. - managers = [] - for start, end in ((0, 1), (2, 3), (4, 5)): - shard = _KvReferenceShard(model, start, end) - mgr = HotKvStateManager(kv_recipe_for(shard)) - mgr.open("route", 0) # each holds live state for the session - managers.append(mgr) - - released_buffers = [] - group = ShardCancellationGroup("route", 0) - for mgr in managers: - group.add_shard(mgr) - group.add_queued_buffer(lambda: released_buffers.append("bundle-a")) - group.add_queued_buffer(lambda: released_buffers.append("bundle-b")) - - outcome = group.cancel() - assert outcome.shards_released == 3 - assert outcome.buffers_released == 2 - assert released_buffers == ["bundle-a", "bundle-b"] - # Every shard's KV is gone: a lookup now yields an explicit released miss. - for mgr in managers: - miss = mgr.resolve("route", 0) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.RELEASED - # Cancellation is idempotent. - again = group.cancel() - assert again.shards_released == 0 - assert again.buffers_released == 0 - - -def test_scheduler_cancel_drains_queue_and_releases_active_kv(): - "The scheduler cancel drops queued work and frees an active session's KV.\n\nTags: node, scheduler, cancel" - model = _KvDenseLlama() - shard = _KvReferenceShard(model, 0, model.n_layers - 1) - manager = HotKvStateManager(kv_recipe_for(shard)) - engine = KvBatchEngine(KvBoundaryAdapter(shard, manager)) - scheduler = ContinuousBatchScheduler( - engine, NodeBudget(max_active_sessions=1, max_batch_size=1, max_queue_depth=4) - ) - assert scheduler.submit(_generation("active", [1, 2, 3], 8)).running - assert scheduler.submit(_generation("waiting", [4, 5, 6], 8)).reason.value == "queued" - scheduler.run_tick() # 'active' prefills and starts decoding, holding KV - - # Cancel the queued one: it leaves the queue without ever taking a slot. - assert scheduler.cancel("waiting") is True - # Cancel the active one: its KV is released and it is recorded as cancelled. - assert scheduler.cancel("active") is True - assert manager.total_bytes == 0 - - telem = scheduler.telemetry() - assert telem.cancelled_sessions == 2 - assert telem.completed_sessions == 0 - assert telem.active_sessions == 0 - assert telem.queue_depth == 0 - # Cancelling an unknown / already-finished session is a no-op. - assert scheduler.cancel("active") is False - assert scheduler.cancel("never-seen") is False - - -def test_scheduler_cancel_rejects_a_completed_reason(): - "cancel() refuses a non-terminal reason so completed work is never faked.\n\nTags: node, scheduler, cancel" - model = _KvDenseLlama() - shard = _KvReferenceShard(model, 0, model.n_layers - 1) - manager = HotKvStateManager(kv_recipe_for(shard)) - engine = KvBatchEngine(KvBoundaryAdapter(shard, manager)) - scheduler = ContinuousBatchScheduler(engine) - scheduler.submit(_generation("x", [1, 2], 4)) - with pytest.raises(Exception): - scheduler.cancel("x", reason=DoneReason.COMPLETED) - - -# --------------------------------------------------------------------------- # -# Idempotency: duplicate steps are no-ops; uncertain mutations never replay. -# --------------------------------------------------------------------------- # - - -def test_duplicate_step_delivery_is_idempotent_no_remutation(): - "Replaying a committed step returns the recorded token without re-mutating KV.\n\nTags: node, failure, idempotency" - ledger = IdempotencyLedger() - key = StepKey("s", 0, 5) - disposition = ledger.begin(key) - assert disposition.fresh - ledger.commit(key, 42) - # A duplicate delivery of the same step returns the recorded token and is a - # no-op — the caller must not re-run the mutation. - replay = ledger.begin(key) - assert replay.duplicate - assert replay.token == 42 - - -def test_idempotent_run_replays_tokens_without_advancing_kv(): - "Re-running a completed stream on the same ledger/epoch re-mutates nothing.\n\nTags: node, failure, idempotency" - model = _KvDenseLlama() - adapter = _make_adapter(model) - ledger = IdempotencyLedger() - runner = HardenedSessionRunner(adapter, idempotency=ledger) - request = _generation("idem", [3, 1, 4], 6) - - first = runner.run(request) - assert first.status is WorkStatus.COMPLETED - kv_len_after_first = adapter.manager.get("idem", 0).seq_len - - # A duplicate delivery of the entire stream: every step is a committed - # duplicate, so the runner replays the identical tokens and the KV length is - # unchanged (no double-append). - second = runner.run(request) - assert second.status is WorkStatus.COMPLETED - assert list(second.tokens) == list(first.tokens) - assert adapter.manager.get("idem", 0).seq_len == kv_len_after_first - - -def test_uncertain_mutation_is_never_replayed_silently(): - "A step marked uncertain refuses a silent replay; it must be verified/restarted.\n\nTags: node, failure, idempotency" - ledger = IdempotencyLedger() - key = StepKey("s", 0, 3) - ledger.begin(key) - ledger.mark_uncertain(key, "worker died before ack") - # Replaying an uncertain mutation is refused rather than silently re-applied. - with pytest.raises(UncertainMutationError): - ledger.begin(key) - assert ledger.has_uncertain() - - -def test_in_flight_duplicate_is_treated_as_uncertain(): - "A second begin before commit is refused (concurrent duplicate is unverified).\n\nTags: node, failure, idempotency" - ledger = IdempotencyLedger() - key = StepKey("s", 0, 1) - ledger.begin(key) # in-flight, not yet committed - with pytest.raises(UncertainMutationError): - ledger.begin(key) - - -# --------------------------------------------------------------------------- # -# Worker death, stream reset, malformed bundle, stale epoch, cache miss. -# --------------------------------------------------------------------------- # - - -def test_worker_death_midstream_is_unverified_and_marks_step_uncertain(): - "A worker dying mid-step yields unverified work and an unreplayable step.\n\nTags: node, failure, worker-death" - model = _KvDenseLlama() - # Fail on the 3rd step call (step index 2), after two tokens committed. - shard = _FaultyShard(model, 0, model.n_layers - 1, fail_at_call=3) - adapter = _make_adapter(model, shard=shard) - ledger = IdempotencyLedger() - runner = HardenedSessionRunner(adapter, idempotency=ledger) - - outcome = runner.run(_generation("dead", [1, 2, 3], 8)) - assert outcome.status is WorkStatus.UNVERIFIED - assert outcome.failure_kind is FailureKind.WORKER_DEATH - assert outcome.token_count == 2 # the two committed steps - assert not outcome.completed - # The failed step is uncertain and can never be silently replayed. - assert ledger.has_uncertain() - with pytest.raises(UncertainMutationError): - ledger.begin(StepKey("dead", 0, 2)) - # KV was released on failure. - assert isinstance(adapter.manager.resolve("dead", 0), CacheMiss) - - -def test_stream_reset_is_restartable_failure(): - "A stream reset injected mid-stream fails the run as a restartable transport loss.\n\nTags: node, failure, stream-reset" - adapter = _make_adapter() - runner = HardenedSessionRunner(adapter) - - def before_step(step): - if step == 2: - raise StreamTerminated(FailureKind.STREAM_RESET, "peer reset the stream") - - outcome = runner.run(_generation("reset", [1, 2, 3], 8), before_step=before_step) - assert outcome.status is WorkStatus.FAILED - assert outcome.failure_kind is FailureKind.STREAM_RESET - assert outcome.restartable - - -def test_malformed_bundle_is_classified_and_does_not_corrupt_kv(): - "A malformed activation bundle is rejected and leaves the KV context empty.\n\nTags: node, failure, malformed-bundle" - model = _KvDenseLlama() - mid = _KvReferenceShard(model, 2, 3) # middle range: not head, not tail - manager = HotKvStateManager(kv_recipe_for(mid)) - adapter = KvBoundaryAdapter(mid, manager) - assert not adapter.is_head and not adapter.is_tail - - # A bundle that hands over at the wrong layer is malformed. - bad = BoundaryBundle( - architecture_adapter=adapter.architecture.adapter, - schema_version=adapter.architecture.boundary_schema_version, - tensor_name=adapter.architecture.boundary_tensor_name, - residual=np.zeros((1, 3, model.hidden), dtype=np.float32), - positions=np.arange(3, dtype=np.int64)[None, :], - next_layer=adapter.start_layer + 5, # wrong handover layer - normalized=False, - ) - with pytest.raises(BoundaryContractError) as exc: - adapter.prefill("mal", 0, boundary=bad) - assert classify_exception(exc.value) is FailureKind.MALFORMED_BUNDLE - # The malformed step never appended KV: the context is empty, not corrupted. - assert manager.get("mal", 0).seq_len == 0 - - -def test_stale_epoch_reference_is_rejected_and_classified(): - "A reference to a superseded epoch is rejected as stale, never silently reused.\n\nTags: node, failure, stale-epoch" - model = _KvDenseLlama() - adapter = _make_adapter(model) - manager = adapter.manager - manager.open("sess", 5) # current epoch is now 5 - with pytest.raises(StaleRouteEpochError) as exc: - manager.resolve("sess", 4) # epoch 4 is stale - assert classify_exception(exc.value) is FailureKind.STALE_EPOCH - - # Driving the hardened runner on the stale epoch fails closed as STALE_EPOCH. - runner = HardenedSessionRunner(adapter) - outcome = runner.run(_generation("sess", [1, 2, 3], 4, epoch=3)) - assert outcome.status is WorkStatus.FAILED - assert outcome.failure_kind is FailureKind.STALE_EPOCH - - -def test_cache_miss_midstream_is_restartable(): - "A KV eviction mid-stream surfaces an explicit cache miss the head can restart.\n\nTags: node, failure, cache-miss" - adapter = _make_adapter() - manager = adapter.manager - runner = HardenedSessionRunner(adapter) - - # Evict the session's KV just before step 3's decode. - def before_step(step): - if step == 3: - manager.release("evict", 0) - - outcome = runner.run(_generation("evict", [1, 2, 3], 10), before_step=before_step) - assert outcome.failure_kind is FailureKind.CACHE_MISS - assert outcome.restartable - assert outcome.token_count == 3 # steps 0..2 committed before the eviction - - -# --------------------------------------------------------------------------- # -# Alpha failover: restart from token zero, never import unverified KV. -# --------------------------------------------------------------------------- # - - -def test_alpha_failover_restarts_from_token_zero_and_completes(): - "A transient worker death fails over to a fresh epoch and reproduces the tokens.\n\nTags: node, failure, failover" - model = _KvDenseLlama() - # Die on the 3rd step of the first attempt; the persistent call counter means - # the restart (which keeps counting) does not re-trip the fault. - shard = _FaultyShard(model, 0, model.n_layers - 1, fail_at_call=3) - adapter = _make_adapter(model, shard=shard) - manager = adapter.manager - runner = HardenedSessionRunner(adapter) - controller = RestartController([manager]) - - prompt = [7, 3, 9, 1] - n_new = 6 - result = runner.run_with_failover( - _generation("alpha", prompt, n_new, epoch=0), controller, max_restarts=2 - ) - assert result.completed - assert result.restarts == 1 - # The restart began on a fresh epoch and reproduced the full stateless stream - # from token zero — no half-computed KV was imported. - assert result.outcome.route_epoch == 1 - assert list(result.outcome.tokens) == model.stateless_greedy(prompt, n_new) - # The failed epoch's KV is gone and the epoch is now stale. - with pytest.raises(StaleRouteEpochError): - manager.resolve("alpha", 0) - # First attempt was unverified, the restart completed: only the restart bills. - statuses = [a.status for a in result.attempts] - assert statuses == [WorkStatus.UNVERIFIED, WorkStatus.COMPLETED] - assert runner.work_ledger.billable_tokens() == n_new - - -def test_failover_refuses_to_import_unverified_kv(): - "assert_fresh_start fails closed if any shard still holds new-epoch KV.\n\nTags: node, failure, failover" - model = _KvDenseLlama() - adapter = _make_adapter(model) - manager = adapter.manager - controller = RestartController([manager]) - - new_epoch = controller.failover("s", 0) - assert new_epoch == 1 - # A clean fresh start passes. - controller.assert_fresh_start("s", new_epoch) - # If unverified KV were present under the new epoch, the guard refuses it. - manager.open("s", new_epoch) - manager.append( - "s", - new_epoch, - {i: (np.zeros((1, model.n_heads, model.head_dim), dtype=np.float32), - np.zeros((1, model.n_heads, model.head_dim), dtype=np.float32)) - for i in range(model.n_layers)}, - ) - with pytest.raises(Exception): - controller.assert_fresh_start("s", new_epoch) - - -def test_non_restartable_failure_is_not_retried(): - "A deterministic failure (deadline) returns immediately without a restart.\n\nTags: node, failure, failover" - clock = _FakeClock() - adapter = _make_adapter() - runner = HardenedSessionRunner(adapter, clock=clock) - controller = RestartController([adapter.manager]) - - def before_step(_step): - clock.advance(1.0) - - result = runner.run_with_failover( - _generation("bounded", [1, 2, 3], 20), - controller, - max_restarts=3, - deadline=2.0, - before_step=before_step, - ) - assert not result.completed - assert result.restarts == 0 - assert result.outcome.failure_kind is FailureKind.DEADLINE_EXCEEDED - - -# --------------------------------------------------------------------------- # -# Billing / work records distinguish completed, cancelled, failed, unverified. -# --------------------------------------------------------------------------- # - - -def test_work_ledger_distinguishes_all_four_statuses(): - "The work ledger keeps completed/cancelled/failed/unverified distinct.\n\nTags: node, failure, billing" - ledger = WorkLedger() - ledger.record(WorkRecord("a", 0, WorkStatus.COMPLETED, tokens=8)) - ledger.record(WorkRecord("b", 0, WorkStatus.CANCELLED, tokens=3, - failure_kind=FailureKind.CANCELLED)) - ledger.record(WorkRecord("c", 0, WorkStatus.FAILED, tokens=1, - failure_kind=FailureKind.DEADLINE_EXCEEDED)) - ledger.record(WorkRecord("d", 0, WorkStatus.UNVERIFIED, tokens=2, - failure_kind=FailureKind.WORKER_DEATH)) - - counts = ledger.counts_by_status() - assert counts == { - "completed": 1, "cancelled": 1, "failed": 1, "unverified": 1, - } - # Only completed work is billable — cancelled/failed/unverified tokens are - # recorded for observability but never charged. - assert ledger.billable_tokens() == 8 - assert [r.session_id for r in ledger.billable_records()] == ["a"] - # JSON-safe for durable evidence. - payload = ledger.to_dict() - assert payload["billable_tokens"] == 8 - assert payload["counts_by_status"]["unverified"] == 1 - json.dumps(payload) - - -def test_work_status_and_classification_mapping(): - "Failure kinds map to the right billing status and exception classes.\n\nTags: node, failure, billing" - assert work_status_for(FailureKind.CANCELLED) is WorkStatus.CANCELLED - assert work_status_for(FailureKind.WORKER_DEATH) is WorkStatus.UNVERIFIED - # A stream reset detected at a step boundary is a certain failure (nothing - # committed for that step) — only an unexpected mid-step error is unverified. - assert work_status_for(FailureKind.STREAM_RESET) is WorkStatus.FAILED - assert work_status_for(FailureKind.DEADLINE_EXCEEDED) is WorkStatus.FAILED - assert work_status_for(FailureKind.MALFORMED_BUNDLE) is WorkStatus.FAILED - assert work_status_for(FailureKind.STALE_EPOCH) is WorkStatus.FAILED - assert work_status_for(FailureKind.CACHE_MISS) is WorkStatus.FAILED - - assert classify_exception(OperationCancelled()) is FailureKind.CANCELLED - assert classify_exception(StaleRouteEpochError("x")) is FailureKind.STALE_EPOCH - assert classify_exception(BoundaryContractError("x")) is FailureKind.MALFORMED_BUNDLE - assert classify_exception(RuntimeError("boom")) is FailureKind.WORKER_DEATH - assert ( - classify_exception(StreamTerminated(FailureKind.HEARTBEAT_LOST)) - is FailureKind.HEARTBEAT_LOST - ) diff --git a/tests/test_gguf_backend.py b/tests/test_gguf_backend.py deleted file mode 100644 index b7716fe..0000000 --- a/tests/test_gguf_backend.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Tests for the GGUF backend adapter and recipe-gated startup seam.""" - -from __future__ import annotations - -from types import SimpleNamespace - -from meshnet_node.gguf_backend import GgufNodeBackend, build_gguf_backend -from meshnet_node.model_backend import TailTokenResult, TensorPayload -from meshnet_node.recipe_manifest import DEFAULT_RECIPE_ID, load_recipe_manifest -from meshnet_node.startup import _gguf_backend_for_recipe - - -class _RecordingTransport: - def __init__(self) -> None: - self.calls: list[tuple[str, tuple, dict]] = [] - - def encode_prompt(self, prompt: str, session_id: str | None = None): - self.calls.append(("encode_prompt", (prompt, session_id), {})) - return TensorPayload( - body=b"\x00" * 16, - shape=[1, 2, 4], - attention_mask_header=None, - position_ids_header=None, - ) - - def encode_next_token(self, token_id: int, session_id: str): - self.calls.append(("encode_next_token", (token_id, session_id), {})) - return TensorPayload( - body=b"\x00" * 8, - shape=[1, 1, 4], - attention_mask_header=None, - position_ids_header=None, - past_len=2, - ) - - def forward_bytes( - self, - body: bytes, - shape: list[int], - attention_mask_header: str | None, - position_ids_header: str | None, - *, - start_layer: int | None = None, - session_id: str | None = None, - cache_mode: str | None = None, - past_len: int | None = None, - ): - self.calls.append( - ( - "forward_bytes", - (body, tuple(shape), attention_mask_header, position_ids_header), - { - "start_layer": start_layer, - "session_id": session_id, - "cache_mode": cache_mode, - "past_len": past_len, - }, - ) - ) - if cache_mode == "decode": - return TailTokenResult(text=" done", token_id=17) - return TensorPayload( - body=b"\x00" * 16, - shape=[1, 2, 4], - attention_mask_header=attention_mask_header, - position_ids_header=position_ids_header, - past_len=past_len, - ) - - def decode_tail_token(self, hidden_states): - self.calls.append(("decode_tail_token", (hidden_states.shape,), {})) - return TailTokenResult(text=" tail", token_id=19) - - def generate_text(self, messages, max_new_tokens=5120, temperature=1.0, top_p=1.0): - self.calls.append(("generate_text", (tuple(messages), max_new_tokens, temperature, top_p), {})) - return "ok" - - def generate_text_streaming(self, messages, max_new_tokens=5120, temperature=1.0, top_p=1.0): - self.calls.append(("generate_text_streaming", (tuple(messages), max_new_tokens, temperature, top_p), {})) - yield "ok" - - def count_prompt_tokens(self, messages): - self.calls.append(("count_prompt_tokens", (tuple(messages),), {})) - return 3 - - def count_text_tokens(self, text): - self.calls.append(("count_text_tokens", (text,), {})) - return 2 - - def eos_token_ids(self): - self.calls.append(("eos_token_ids", (), {})) - return [19] - - def release_session(self, session_id: str) -> None: - self.calls.append(("release_session", (session_id,), {})) - - -def test_build_gguf_backend_delegates_to_transport(): - transport = _RecordingTransport() - backend = build_gguf_backend( - model_id="meshnet/native-model", - shard_start=0, - shard_end=1, - quantization="bfloat16", - transport=transport, - total_layers=2, - device_type="cpu", - ) - - assert isinstance(backend, GgufNodeBackend) - assert backend.backend_id == "llama.cpp" - assert backend.is_head is True - assert backend.is_tail is True - assert backend.model.config.to_dict()["architecture_adapter"] == "dense-llama" - assert backend.loaded_tensor_names[0] == "blk.0.weight" - - prompt = backend.encode_prompt("hello", session_id="session-1") - assert prompt.shape == [1, 2, 4] - - decode = backend.forward_bytes( - b"\x00" * 16, - [1, 2, 4], - None, - None, - session_id="session-1", - cache_mode="decode", - past_len=2, - ) - assert isinstance(decode, TailTokenResult) - assert decode.token_id == 17 - - backend.release_session("session-1") - - assert [call[0] for call in transport.calls] == [ - "encode_prompt", - "forward_bytes", - "release_session", - ] - assert transport.calls[0][1] == ("hello", "session-1") - assert transport.calls[1][2]["cache_mode"] == "decode" - assert transport.calls[1][2]["past_len"] == 2 - - -def test_recipe_gates_native_backend_selection(monkeypatch): - manifest = load_recipe_manifest() - torch_recipe = manifest.require(DEFAULT_RECIPE_ID) - native_recipe = manifest.require("llama-cpp-native") - - sentinel_backend = object() - calls: list[dict] = [] - - def fake_build_gguf_backend(**kwargs): - calls.append(kwargs) - return sentinel_backend - - monkeypatch.setattr( - "meshnet_node.startup.build_gguf_backend", - fake_build_gguf_backend, - ) - - assert _gguf_backend_for_recipe( - torch_recipe, - model_id="meshnet/native-model", - shard_start=0, - shard_end=1, - quantization="bfloat16", - total_layers=2, - device="cpu", - ) is None - - backend = _gguf_backend_for_recipe( - native_recipe, - model_id="meshnet/native-model", - shard_start=0, - shard_end=1, - quantization="bfloat16", - total_layers=2, - device="cpu", - ) - - assert backend is sentinel_backend - assert calls[0]["model_id"] == "meshnet/native-model" - assert calls[0]["shard_start"] == 0 - assert calls[0]["shard_end"] == 1 - assert calls[0]["quantization"] == "bfloat16" - assert calls[0]["total_layers"] == 2 diff --git a/tests/test_gguf_ownership.py b/tests/test_gguf_ownership.py deleted file mode 100644 index c7dce7b..0000000 --- a/tests/test_gguf_ownership.py +++ /dev/null @@ -1,88 +0,0 @@ -"""Dense-Llama GGUF ownership selection and introspection tests.""" - -from __future__ import annotations - -import pytest - -from meshnet_node.gguf_ownership import ( - DenseLlamaShardOwnership, - authoritative_dense_llama_ownership, - infer_dense_llama_ownership, - select_dense_llama_tensor_names, -) - - -def test_dense_llama_selection_only_picks_block_range_and_endpoints(): - "Dense-Llama selection keeps only the owned blocks plus the correct endpoints.\n\nTags: node, GGUF" - tensor_inventory = { - "token_embd.weight": 10_000, - "blk.0.attn_q.weight": 1_000, - "blk.0.ffn_down.weight": 1_000, - "blk.1.attn_q.weight": 2_000, - "blk.1.ffn_down.weight": 2_000, - "blk.2.attn_q.weight": 3_000, - "blk.2.ffn_down.weight": 3_000, - "output_norm.weight": 256, - "output.weight": 10_000, - "rope.freqs": 128, - } - - selected = select_dense_llama_tensor_names( - tensor_inventory, - 1, - 2, - total_layers=3, - ) - - assert selected == { - "blk.1.attn_q.weight", - "blk.1.ffn_down.weight", - "blk.2.attn_q.weight", - "blk.2.ffn_down.weight", - "output_norm.weight", - "output.weight", - } - - selected_bytes = sum(tensor_inventory[name] for name in selected) - full_bytes = sum(tensor_inventory.values()) - assert selected_bytes == 20_256 - assert selected_bytes < full_bytes - - -def test_dense_llama_loaded_range_is_authoritative_from_tensor_inventory(): - "The backend's loaded tensor inventory is the source of truth for range and ownership.\n\nTags: node, GGUF" - - class Backend: - loaded_tensor_names = ( - "token_embd.weight", - "blk.4.attn_q.weight", - "blk.5.ffn_down.weight", - "output_norm.weight", - "output.weight", - ) - - ownership = authoritative_dense_llama_ownership(Backend(), selection=None) - - assert isinstance(ownership, DenseLlamaShardOwnership) - assert ownership.range == (4, 5) - assert ownership.owns_embedding is True - assert ownership.owns_final_head is True - - -def test_derivative_slice_requires_source_and_slice_hashes(): - "Temporary derivative GGUF slices must carry hashes and cannot claim final semantics.\n\nTags: node, GGUF" - with pytest.raises(ValueError, match="source and slice hashes"): - infer_dense_llama_ownership( - ["blk.1.attn_q.weight"], - derivative_slice=True, - final_artifact_semantics=False, - ) - - with pytest.raises(ValueError, match="final artifacts"): - infer_dense_llama_ownership( - ["blk.1.attn_q.weight"], - source_artifact_hash="sha256:source", - slice_artifact_hash="sha256:slice", - derivative_slice=True, - final_artifact_semantics=True, - ) diff --git a/tests/test_hot_kv_state.py b/tests/test_hot_kv_state.py deleted file mode 100644 index d904a64..0000000 --- a/tests/test_hot_kv_state.py +++ /dev/null @@ -1,769 +0,0 @@ -"""Isolated concurrent local Hot KV State (DGR-007). - -These tests prove the KV/session manager with a *pure-numpy* KV-cached dense-Llama -reference: no download, no GPU, no torch, no API credit. The reference implements -the DGR-006 ``ShardComputation`` duck type plus ``run_layers_cached`` so cached -prefill/decode over a per-session KV context reproduces the stateless whole-model -tokens bit-for-bit. On top of that correctness core, the tests exercise the -manager's lifecycle: owned-layer allocation, prefill/decode append, truncate, -release, TTL/LRU eviction, explicit cache-miss responses, stale-epoch and -incompatible-recipe rejection, four concurrent cross-talk-free sessions, and -budget-bounded cancellation. -""" - -from __future__ import annotations - -import threading - -import numpy as np -import pytest - -from meshnet_node.boundary_adapter import BoundaryBundle, TailOutput -from meshnet_node.hot_kv_state import ( - CacheMiss, - CacheMissReason, - HotKvStateConfig, - HotKvStateManager, - IncompatibleCacheRecipeError, - KvBoundaryAdapter, - KvBudgetExceededError, - KvCacheMissError, - KvCacheRecipe, - LayerKvCache, - StaleRouteEpochError, - kv_recipe_for, -) - -PARITY_ATOL = 1e-6 - - -# --------------------------------------------------------------------------- # -# Pure-numpy KV-cached dense-Llama reference (test fixture, not production). -# --------------------------------------------------------------------------- # - - -class _KvDenseLlama: - """A tiny deterministic dense-Llama with both stateless and cached runners.""" - - architecture_adapter = "dense-llama" - - def __init__( - self, - *, - vocab: int = 48, - hidden: int = 32, - n_layers: int = 6, - n_heads: int = 4, - intermediate: int = 64, - rms_eps: float = 1e-6, - rope_theta: float = 10000.0, - seed: int = 20260716, - ) -> None: - assert hidden % n_heads == 0 - self.vocab = vocab - self.hidden = hidden - self.n_layers = n_layers - self.n_heads = n_heads - self.head_dim = hidden // n_heads - assert self.head_dim % 2 == 0 - self.rms_eps = rms_eps - self.rope_theta = rope_theta - - rng = np.random.default_rng(seed) - - def w(*shape: int) -> np.ndarray: - return (rng.standard_normal(shape) * 0.08).astype(np.float32) - - self.embed = w(vocab, hidden) - self.layers = [] - for _ in range(n_layers): - self.layers.append( - { - "in_ln": (1.0 + rng.standard_normal(hidden) * 0.02).astype(np.float32), - "q": w(hidden, hidden), - "k": w(hidden, hidden), - "v": w(hidden, hidden), - "o": w(hidden, hidden), - "post_ln": (1.0 + rng.standard_normal(hidden) * 0.02).astype(np.float32), - "gate": w(intermediate, hidden), - "up": w(intermediate, hidden), - "down": w(hidden, intermediate), - } - ) - self.final_ln = (1.0 + rng.standard_normal(hidden) * 0.02).astype(np.float32) - self.lm_head_w = w(vocab, hidden) - - inv_freq = 1.0 / ( - rope_theta ** (np.arange(0, self.head_dim, 2, dtype=np.float32) / self.head_dim) - ) - self.inv_freq = inv_freq.astype(np.float32) - - # -- primitive ops ----------------------------------------------------- - def _rmsnorm(self, x: np.ndarray, weight: np.ndarray) -> np.ndarray: - variance = np.mean(x.astype(np.float32) ** 2, axis=-1, keepdims=True) - normed = x / np.sqrt(variance + self.rms_eps) - return (normed * weight).astype(np.float32) - - def _rope(self, positions: np.ndarray): - angles = positions[..., None].astype(np.float32) * self.inv_freq[None, None, :] - emb = np.concatenate([angles, angles], axis=-1) - return np.cos(emb).astype(np.float32), np.sin(emb).astype(np.float32) - - @staticmethod - def _rotate_half(x: np.ndarray) -> np.ndarray: - half = x.shape[-1] // 2 - return np.concatenate([-x[..., half:], x[..., :half]], axis=-1) - - def _apply_rope(self, t: np.ndarray, cos: np.ndarray, sin: np.ndarray) -> np.ndarray: - cos = cos[:, None, :, :] - sin = sin[:, None, :, :] - return t * cos + self._rotate_half(t) * sin - - def _project_qkv(self, normed: np.ndarray, layer: dict, positions: np.ndarray): - batch, seq, _ = normed.shape - q = (normed @ layer["q"].T).reshape(batch, seq, self.n_heads, self.head_dim) - k = (normed @ layer["k"].T).reshape(batch, seq, self.n_heads, self.head_dim) - v = (normed @ layer["v"].T).reshape(batch, seq, self.n_heads, self.head_dim) - q = q.transpose(0, 2, 1, 3) - k = k.transpose(0, 2, 1, 3) - v = v.transpose(0, 2, 1, 3) - cos, sin = self._rope(positions) - q = self._apply_rope(q, cos, sin) - k = self._apply_rope(k, cos, sin) - return q, k, v - - def _attend( - self, - q: np.ndarray, - k_all: np.ndarray, - v_all: np.ndarray, - layer: dict, - q_positions: np.ndarray, - ) -> np.ndarray: - batch, _, seq_new, _ = q.shape - total = k_all.shape[2] - scores = (q @ k_all.transpose(0, 1, 3, 2)) / np.sqrt(self.head_dim) - # Causal mask by absolute position: keys are stored in absolute order - # 0..total-1; query row i lives at absolute position q_positions[i]. - key_abs = np.arange(total, dtype=np.int64) - q_abs = np.asarray(q_positions).reshape(seq_new).astype(np.int64) - mask = np.where(key_abs[None, :] <= q_abs[:, None], 0.0, -1e30).astype(np.float32) - scores = scores + mask[None, None, :, :] - scores = scores - scores.max(axis=-1, keepdims=True) - weights = np.exp(scores) - weights = weights / weights.sum(axis=-1, keepdims=True) - out = weights @ v_all - out = out.transpose(0, 2, 1, 3).reshape(batch, seq_new, self.hidden) - return (out @ layer["o"].T).astype(np.float32) - - def _mlp(self, x: np.ndarray, layer: dict) -> np.ndarray: - gate = x @ layer["gate"].T - up = x @ layer["up"].T - silu = gate * (1.0 / (1.0 + np.exp(-gate))) - return ((silu * up) @ layer["down"].T).astype(np.float32) - - # -- stateless whole-sequence layer (ground truth) --------------------- - def _run_layer_stateless(self, x: np.ndarray, layer: dict, positions: np.ndarray) -> np.ndarray: - normed = self._rmsnorm(x, layer["in_ln"]) - q, k, v = self._project_qkv(normed, layer, positions) - attn = self._attend(q, k, v, layer, positions[0]) - h = x + attn - h = h + self._mlp(self._rmsnorm(h, layer["post_ln"]), layer) - return h.astype(np.float32) - - def whole_model_next_token(self, token_ids: list[int]) -> int: - positions = np.arange(len(token_ids))[None, :] - h = self.embed[np.asarray(token_ids)][None, :] - for idx in range(self.n_layers): - h = self._run_layer_stateless(h, self.layers[idx], positions) - h = self._rmsnorm(h[:, -1:, :], self.final_ln) - logits = h @ self.lm_head_w.T - return int(np.argmax(logits[0, -1])) - - def stateless_greedy(self, prompt: list[int], n_new: int) -> list[int]: - tokens = list(prompt) - out: list[int] = [] - for _ in range(n_new): - tok = self.whole_model_next_token(tokens) - tokens.append(tok) - out.append(tok) - return out - - -class _KvReferenceShard: - """A contiguous inclusive layer range with a KV-cached runner. - - Satisfies the KV-aware ``ShardComputation`` duck type used by - ``KvBoundaryAdapter``: DGR-006 methods plus ``run_layers_cached`` and the KV - geometry (``n_kv_heads`` / ``head_dim`` / ``kv_dtype``). - """ - - kv_dtype = "float32" - - def __init__( - self, - model: _KvDenseLlama, - start_layer: int, - end_layer: int, - *, - architecture_adapter: str | None = None, - ) -> None: - self._model = model - self.start_layer = start_layer - self.end_layer = end_layer - self.total_layers = model.n_layers - self.n_kv_heads = model.n_heads - self.head_dim = model.head_dim - self.architecture_adapter = architecture_adapter or model.architecture_adapter - - def embed_tokens(self, token_ids: np.ndarray) -> np.ndarray: - return self._model.embed[np.asarray(token_ids)] - - def final_norm(self, hidden: np.ndarray) -> np.ndarray: - return self._model._rmsnorm(np.asarray(hidden, dtype=np.float32), self._model.final_ln) - - def lm_head(self, hidden: np.ndarray) -> np.ndarray: - return np.asarray(hidden, dtype=np.float32) @ self._model.lm_head_w.T - - def run_layers_cached(self, hidden, *, positions, past_kv): - m = self._model - x = np.asarray(hidden, dtype=np.float32) - positions = np.asarray(positions) - new_kv: dict[int, tuple[np.ndarray, np.ndarray]] = {} - for idx in range(self.start_layer, self.end_layer + 1): - layer = m.layers[idx] - normed = m._rmsnorm(x, layer["in_ln"]) - q, k, v = m._project_qkv(normed, layer, positions) - # Post-RoPE new K/V stored as (seq_new, n_heads, head_dim). - new_k = k[0].transpose(1, 0, 2).copy() - new_v = v[0].transpose(1, 0, 2).copy() - cache = past_kv.get(idx) - if cache is not None and cache.length > 0: - past_k = cache.keys[None].transpose(0, 2, 1, 3) - past_v = cache.values[None].transpose(0, 2, 1, 3) - k_all = np.concatenate([past_k, k], axis=2) - v_all = np.concatenate([past_v, v], axis=2) - else: - k_all, v_all = k, v - attn = m._attend(q, k_all, v_all, layer, positions[0]) - h = x + attn - x = h + m._mlp(m._rmsnorm(h, layer["post_ln"]), layer) - x = x.astype(np.float32) - new_kv[idx] = (new_k, new_v) - return x, new_kv - - -# --------------------------------------------------------------------------- # -# Helpers. -# --------------------------------------------------------------------------- # - - -class _FakeClock: - def __init__(self) -> None: - self.now = 0.0 - - def __call__(self) -> float: - return self.now - - def advance(self, delta: float) -> None: - self.now += delta - - -def _full_shard(model: _KvDenseLlama): - return _KvReferenceShard(model, 0, model.n_layers - 1) - - -def _manager_for(shard, config: HotKvStateConfig | None = None, clock=None) -> HotKvStateManager: - return HotKvStateManager(kv_recipe_for(shard), config=config, clock=clock) - - -def _cached_greedy( - adapter: KvBoundaryAdapter, - manager: HotKvStateManager, - session_id: str, - epoch: int, - prompt: list[int], - n_new: int, -) -> list[int]: - """Greedy decode one full-model session through the KV manager.""" - out = adapter.prefill(session_id, epoch, token_ids=np.asarray(prompt)) - assert isinstance(out, TailOutput) - tokens = [out.token_id] - for _ in range(n_new - 1): - step = adapter.decode(session_id, epoch, token_ids=[out.token_id]) - assert isinstance(step, TailOutput) - out = step - tokens.append(out.token_id) - return tokens - - -# --------------------------------------------------------------------------- # -# Recipe identity. -# --------------------------------------------------------------------------- # - - -def test_recipe_owned_layers_and_fingerprint_aliasing(): - "The KV recipe covers only owned layers and canonicalizes architecture aliases.\n\nTags: node, kv" - recipe = KvCacheRecipe( - architecture_adapter="LlamaForCausalLM", - kv_dtype="float32", - n_kv_heads=4, - head_dim=8, - total_layers=6, - start_layer=2, - end_layer=3, - ) - assert recipe.owned_layers == (2, 3) - alias = KvCacheRecipe( - architecture_adapter="llama", - kv_dtype="float32", - n_kv_heads=4, - head_dim=8, - total_layers=6, - start_layer=2, - end_layer=3, - ) - assert recipe.is_compatible(alias) - # A different owned range is not compatible. - other = KvCacheRecipe( - architecture_adapter="llama", - kv_dtype="float32", - n_kv_heads=4, - head_dim=8, - total_layers=6, - start_layer=0, - end_layer=1, - ) - assert not recipe.is_compatible(other) - - -def test_recipe_bytes_per_token_scales_with_owned_layers(): - "KV bytes-per-token counts keys+values across owned layers only.\n\nTags: node, kv" - base = dict( - architecture_adapter="dense-llama", - kv_dtype="float32", - n_kv_heads=4, - head_dim=8, - total_layers=6, - ) - one = KvCacheRecipe(**base, start_layer=0, end_layer=0) - two = KvCacheRecipe(**base, start_layer=0, end_layer=1) - # 2 (k+v) * heads * dim * 4 bytes per layer. - assert one.bytes_per_token() == 2 * 4 * 8 * 4 - assert two.bytes_per_token() == 2 * one.bytes_per_token() - - -# --------------------------------------------------------------------------- # -# Owned-layer allocation. -# --------------------------------------------------------------------------- # - - -def test_manager_allocates_kv_only_for_owned_layers(): - "A middle shard allocates KV state only for its owned layer range.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _KvReferenceShard(model, 2, 3) - manager = _manager_for(shard) - session = manager.open("sess-mid", 0) - assert session.owned_layers == (2, 3) - assert set(session.layers) == {2, 3} - with pytest.raises(KeyError): - session.layer(0) - - -# --------------------------------------------------------------------------- # -# Prefill / decode / truncate. -# --------------------------------------------------------------------------- # - - -def test_prefill_then_decode_append_grows_owned_layers(): - "Prefill and decode append advance every owned layer in lockstep.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - - prompt = [5, 12, 3, 41] - out = adapter.prefill("s", 0, token_ids=np.asarray(prompt)) - assert isinstance(out, TailOutput) - session = manager.get("s", 0) - assert session.seq_len == len(prompt) - for cache in session.layers.values(): - assert cache.length == len(prompt) - - step = adapter.decode("s", 0, token_ids=[out.token_id]) - assert isinstance(step, TailOutput) - assert manager.get("s", 0).seq_len == len(prompt) + 1 - - -def test_truncate_rolls_back_all_owned_layers(): - "Truncate drops cached positions beyond a length across owned layers.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - adapter.prefill("s", 0, token_ids=np.asarray([1, 2, 3, 4, 5])) - assert manager.get("s", 0).seq_len == 5 - manager.truncate("s", 0, 2) - session = manager.get("s", 0) - assert session.seq_len == 2 - for cache in session.layers.values(): - assert cache.length == 2 - - -def test_layer_kv_cache_rejects_wrong_shape(): - "LayerKvCache rejects K/V that do not match its head geometry.\n\nTags: node, kv" - cache = LayerKvCache(0, n_kv_heads=4, head_dim=8, dtype="float32") - with pytest.raises(ValueError): - cache.append(np.zeros((1, 3, 8), dtype=np.float32), np.zeros((1, 3, 8), dtype=np.float32)) - cache.append(np.zeros((2, 4, 8), dtype=np.float32), np.zeros((2, 4, 8), dtype=np.float32)) - assert cache.length == 2 - - -# --------------------------------------------------------------------------- # -# Cached vs stateless parity (correctness core). -# --------------------------------------------------------------------------- # - - -def test_cached_full_shard_decode_matches_stateless_whole_model(): - "Cached full-model greedy decode reproduces stateless whole-model tokens.\n\nTags: node, kv, parity" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - - prompt = [2, 17, 8, 25, 6] - n_new = 12 - reference = model.stateless_greedy(prompt, n_new) - cached = _cached_greedy(adapter, manager, "s", 0, prompt, n_new) - assert cached == reference - assert len(cached) == n_new - - -def test_cached_prefill_next_token_matches_whole_model_logits(): - "Cached prefill produces the same next-token logits as the whole model.\n\nTags: node, kv, parity" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - - prompt = [9, 1, 44, 6, 30, 11] - out = adapter.prefill("s", 0, token_ids=np.asarray(prompt)) - assert isinstance(out, TailOutput) - assert out.token_id == model.whole_model_next_token(prompt) - - -def test_multi_range_cached_decode_parity_across_a_seam(): - "A head/tail split with independent per-range KV reproduces whole-model decode.\n\nTags: node, kv, parity" - model = _KvDenseLlama() - head_shard = _KvReferenceShard(model, 0, 2) - tail_shard = _KvReferenceShard(model, 3, 5) - head_mgr = _manager_for(head_shard) - tail_mgr = _manager_for(tail_shard) - head = KvBoundaryAdapter(head_shard, head_mgr) - tail = KvBoundaryAdapter(tail_shard, tail_mgr) - - prompt = [7, 3, 22, 5, 9] - n_new = 8 - - # Each range only allocates its owned layers. - def step(token_ids, is_prefill): - if is_prefill: - bundle = head.prefill("s", 0, token_ids=np.asarray(token_ids)) - out = tail.prefill("s", 0, boundary=bundle) - else: - bundle = head.decode("s", 0, token_ids=[token_ids]) - assert isinstance(bundle, BoundaryBundle) - out = tail.decode("s", 0, boundary=bundle) - assert isinstance(out, TailOutput) - return out.token_id - - tokens = [step(prompt, True)] - for _ in range(n_new - 1): - tokens.append(step(tokens[-1], False)) - - assert head_mgr.get("s", 0).owned_layers == (0, 1, 2) - assert tail_mgr.get("s", 0).owned_layers == (3, 4, 5) - assert tokens == model.stateless_greedy(prompt, n_new) - - -# --------------------------------------------------------------------------- # -# Four concurrent sessions with no cross-talk. -# --------------------------------------------------------------------------- # - - -def test_four_interleaved_sessions_have_no_kv_cross_talk(): - "Four interleaved sessions each decode their own tokens without cross-talk.\n\nTags: node, kv, concurrency" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - - prompts = { - "alpha": [1, 2, 3, 4], - "bravo": [40, 39, 2, 15], - "charlie": [7, 7, 7, 7], - "delta": [31, 5, 18, 22], - } - n_new = 10 - references = {sid: model.stateless_greedy(p, n_new) for sid, p in prompts.items()} - # The four prompts must actually diverge, else "no cross-talk" is vacuous. - assert len({tuple(v) for v in references.values()}) == 4 - - generated: dict[str, list[int]] = {} - for sid, prompt in prompts.items(): - out = adapter.prefill(sid, 0, token_ids=np.asarray(prompt)) - assert isinstance(out, TailOutput) - generated[sid] = [out.token_id] - - # Round-robin decode: every session takes one step per round, interleaved. - for _ in range(n_new - 1): - for sid in prompts: - step = adapter.decode(sid, 0, token_ids=[generated[sid][-1]]) - assert isinstance(step, TailOutput) - generated[sid].append(step.token_id) - - for sid in prompts: - assert generated[sid] == references[sid], sid - assert manager.session_count == 4 - - -def test_four_sessions_on_real_threads_stay_isolated(): - "Four sessions decoding on real threads produce their own reference tokens.\n\nTags: node, kv, concurrency" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard, HotKvStateConfig(max_sessions=8)) - adapter = KvBoundaryAdapter(shard, manager) - - prompts = { - "t-alpha": [3, 14, 1, 5], - "t-bravo": [2, 27, 18, 4], - "t-charlie": [9, 9, 1, 2], - "t-delta": [44, 6, 30, 11], - } - n_new = 8 - references = {sid: model.stateless_greedy(p, n_new) for sid, p in prompts.items()} - results: dict[str, list[int]] = {} - errors: list[Exception] = [] - - def run(sid: str, prompt: list[int]) -> None: - try: - results[sid] = _cached_greedy(adapter, manager, sid, 0, prompt, n_new) - except Exception as exc: # pragma: no cover - surfaced via assert below - errors.append(exc) - - threads = [threading.Thread(target=run, args=(sid, p)) for sid, p in prompts.items()] - for t in threads: - t.start() - for t in threads: - t.join() - - assert not errors - for sid in prompts: - assert results[sid] == references[sid], sid - - -def test_release_one_session_leaves_others_intact_and_returns_memory(): - "Releasing one session frees its budget and does not disturb the others.\n\nTags: node, kv, concurrency" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - - prompts = {"keep-1": [1, 2, 3], "drop": [10, 11, 12, 13], "keep-2": [5, 6, 7]} - n_new = 6 - references = {sid: model.stateless_greedy(p, n_new) for sid, p in prompts.items()} - - gen: dict[str, list[int]] = {} - for sid, prompt in prompts.items(): - out = adapter.prefill(sid, 0, token_ids=np.asarray(prompt)) - gen[sid] = [out.token_id] - - bytes_before = manager.total_bytes - assert manager.release("drop", 0) is True - assert manager.total_bytes < bytes_before - - # A decode on the released session is an explicit cache miss, not corruption. - miss = adapter.decode("drop", 0, token_ids=[gen["drop"][-1]]) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.RELEASED - - # The survivors keep decoding to their own references. - for _ in range(n_new - 1): - for sid in ("keep-1", "keep-2"): - step = adapter.decode(sid, 0, token_ids=[gen[sid][-1]]) - assert isinstance(step, TailOutput) - gen[sid].append(step.token_id) - for sid in ("keep-1", "keep-2"): - assert gen[sid] == references[sid], sid - - -# --------------------------------------------------------------------------- # -# Stale epoch / incompatible recipe rejection. -# --------------------------------------------------------------------------- # - - -def test_stale_route_epoch_is_rejected(): - "A request for an older route epoch than the current one is rejected.\n\nTags: node, kv" - model = _KvDenseLlama() - manager = _manager_for(_full_shard(model)) - manager.open("s", 5) - with pytest.raises(StaleRouteEpochError): - manager.open("s", 4) - with pytest.raises(StaleRouteEpochError): - manager.resolve("s", 4) - with pytest.raises(StaleRouteEpochError): - manager.append("s", 4, {}) - - -def test_new_route_epoch_supersedes_and_frees_old_epoch(): - "A newer route epoch supersedes the old one, freeing its KV and reporting a miss.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - adapter.prefill("s", 1, token_ids=np.asarray([1, 2, 3, 4])) - bytes_epoch1 = manager.total_bytes - assert bytes_epoch1 > 0 - - # Re-planned route: epoch 2 starts a fresh isolated context. - adapter.prefill("s", 2, token_ids=np.asarray([9, 8])) - assert manager.session_keys() == [("s", 2)] - # Old epoch is gone; a lookup for it is now stale (epoch < current). - with pytest.raises(StaleRouteEpochError): - manager.resolve("s", 1) - - -def test_incompatible_cache_recipe_is_rejected(): - "A request carrying a different KV recipe is rejected, not silently reused.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - manager.open("s", 0) - - incompatible = KvCacheRecipe( - architecture_adapter="dense-llama", - kv_dtype="float16", # different KV dtype - n_kv_heads=model.n_heads, - head_dim=model.head_dim, - total_layers=model.n_layers, - start_layer=0, - end_layer=model.n_layers - 1, - ) - with pytest.raises(IncompatibleCacheRecipeError): - manager.resolve("s", 0, recipe=incompatible) - with pytest.raises(IncompatibleCacheRecipeError): - manager.open("s2", 0, recipe=incompatible) - - -def test_uncertified_architecture_recipe_fails_closed(): - "A KV recipe for an uncertified architecture fails closed at construction.\n\nTags: node, kv" - from meshnet_node.boundary_adapter import UncertifiedArchitectureError - - with pytest.raises(UncertifiedArchitectureError): - KvCacheRecipe( - architecture_adapter="qwen3-moe", - kv_dtype="float32", - n_kv_heads=4, - head_dim=8, - total_layers=6, - start_layer=0, - end_layer=5, - ) - - -# --------------------------------------------------------------------------- # -# Explicit cache-miss responses. -# --------------------------------------------------------------------------- # - - -def test_unknown_session_is_an_explicit_cache_miss(): - "Resolving an unknown session returns an explicit unknown-session miss.\n\nTags: node, kv" - manager = _manager_for(_full_shard(_KvDenseLlama())) - miss = manager.resolve("nope", 0) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.UNKNOWN_SESSION - with pytest.raises(KvCacheMissError): - manager.get("nope", 0) - - -def test_seq_len_mismatch_is_an_explicit_cache_miss(): - "A decode whose expected length disagrees with the cache is an explicit miss.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard) - adapter = KvBoundaryAdapter(shard, manager) - out = adapter.prefill("s", 0, token_ids=np.asarray([1, 2, 3])) - # Cache holds 3 tokens; claim it holds 99. - miss = adapter.decode("s", 0, token_ids=[out.token_id], expected_seq_len=99) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.SEQ_LEN_MISMATCH - - -def test_ttl_eviction_yields_an_explicit_cache_miss(): - "A session idle past its TTL is evicted and reported as a TTL cache miss.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - clock = _FakeClock() - manager = _manager_for(shard, HotKvStateConfig(ttl_seconds=10.0), clock=clock) - adapter = KvBoundaryAdapter(shard, manager) - adapter.prefill("s", 0, token_ids=np.asarray([1, 2, 3])) - clock.advance(11.0) - miss = manager.resolve("s", 0) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.EVICTED_TTL - assert manager.total_bytes == 0 - - -# --------------------------------------------------------------------------- # -# Eviction and budget. -# --------------------------------------------------------------------------- # - - -def test_lru_eviction_by_session_cap_reports_a_miss(): - "Exceeding the session cap evicts the least-recently-used session.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - manager = _manager_for(shard, HotKvStateConfig(max_sessions=2)) - adapter = KvBoundaryAdapter(shard, manager) - adapter.prefill("a", 0, token_ids=np.asarray([1, 2])) - adapter.prefill("b", 0, token_ids=np.asarray([3, 4])) - # Touch 'a' so 'b' becomes the LRU victim. - adapter.decode("a", 0, token_ids=[1]) - adapter.prefill("c", 0, token_ids=np.asarray([5, 6])) - - miss = manager.resolve("b", 0) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.EVICTED_LRU - assert set(k[0] for k in manager.session_keys()) == {"a", "c"} - - -def test_budget_eviction_keeps_total_within_budget(): - "Byte-budget pressure evicts LRU sessions so the store stays within budget.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - recipe = kv_recipe_for(shard) - # Budget for ~5 tokens of one session; a second big session forces eviction. - budget = recipe.bytes_per_token() * 5 - manager = _manager_for(shard, HotKvStateConfig(budget_bytes=budget, max_sessions=8)) - adapter = KvBoundaryAdapter(shard, manager) - - adapter.prefill("a", 0, token_ids=np.asarray([1, 2, 3])) - adapter.prefill("b", 0, token_ids=np.asarray([4, 5, 6, 7])) - assert manager.total_bytes <= budget - # 'a' (older, LRU) was evicted to make room for 'b'. - miss = manager.resolve("a", 0) - assert isinstance(miss, CacheMiss) - assert miss.reason is CacheMissReason.EVICTED_LRU - assert manager.get("b", 0).seq_len == 4 - - -def test_single_session_exceeding_budget_raises(): - "A single session that cannot fit the budget raises instead of evicting itself.\n\nTags: node, kv" - model = _KvDenseLlama() - shard = _full_shard(model) - recipe = kv_recipe_for(shard) - budget = recipe.bytes_per_token() * 2 # only 2 tokens fit - manager = _manager_for(shard, HotKvStateConfig(budget_bytes=budget)) - adapter = KvBoundaryAdapter(shard, manager) - with pytest.raises(KvBudgetExceededError): - adapter.prefill("a", 0, token_ids=np.asarray([1, 2, 3, 4, 5])) diff --git a/tests/test_llama_worker_build.py b/tests/test_llama_worker_build.py deleted file mode 100644 index 2d76b7b..0000000 --- a/tests/test_llama_worker_build.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -from pathlib import Path - -import pytest - - -ROOT = Path(__file__).resolve().parents[1] -SCRIPT = ROOT / "packages" / "node" / "native" / "scripts" / "build_llama_worker.sh" -PIN_FILE = ROOT / "packages" / "node" / "native" / "llama" / "UPSTREAM_COMMIT" - - -@pytest.mark.skipif(not SCRIPT.exists(), reason="llama worker build script is missing") -def test_llama_worker_build_smoke_rebuild(tmp_path: Path) -> None: - if not shutil_which("git"): - pytest.skip("git is unavailable") - if not (shutil_which("g++") or shutil_which("c++") or shutil_which("clang++")): - pytest.skip("no C++ compiler is unavailable") - - source_dir = tmp_path / "llama.cpp" - build_one = tmp_path / "build-1" - build_two = tmp_path / "build-2" - pin = PIN_FILE.read_text(encoding="utf-8").strip() - - source_dir.mkdir() - _write_fake_upstream_tree(source_dir, pin) - _git_init(source_dir) - - _run_build(source_dir, build_one) - _run_build(source_dir, build_two) - - binary = build_two / "meshnet_worker" - assert binary.exists() - output = subprocess.run( - [str(binary), "--smoke"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - assert "meshnet worker scaffold ok" in output.stdout - assert pin in output.stdout - - -def _run_build(source_dir: Path, build_dir: Path) -> None: - env = os.environ.copy() - env.setdefault("PATH", os.environ.get("PATH", "")) - subprocess.run( - [str(SCRIPT), "--source-dir", str(source_dir), "--build-dir", str(build_dir)], - cwd=ROOT, - check=True, - env=env, - capture_output=True, - text=True, - ) - - -def _write_fake_upstream_tree(source_dir: Path, pin: str) -> None: - (source_dir / "LICENSE").write_text("MIT License placeholder\n", encoding="utf-8") - (source_dir / "AUTHORS").write_text("Georgi Gerganov\nMeshnet maintainers\n", encoding="utf-8") - (source_dir / "CMakeLists.txt").write_text("# upstream placeholder\n", encoding="utf-8") - (source_dir / ".meshnet-upstream-commit").write_text(f"{pin}\n", encoding="utf-8") - (source_dir / ".meshnet-upstream-repository").write_text( - "https://github.com/ggml-org/llama.cpp.git\n", - encoding="utf-8", - ) - - -def _git_init(source_dir: Path) -> None: - subprocess.run(["git", "init", "-q"], cwd=source_dir, check=True) - - -def shutil_which(name: str) -> str | None: - from shutil import which - - return which(name) diff --git a/tests/test_native_shard_protocol.py b/tests/test_native_shard_protocol.py deleted file mode 100644 index 7914c37..0000000 --- a/tests/test_native_shard_protocol.py +++ /dev/null @@ -1,508 +0,0 @@ -"""DGR-002: generated-schema round-trip and compatibility tests. - -Covers the versioned gRPC Shard protocol (``packages/node/native/proto``): - * Python round-trip across the full envelope, tensor bundle, and every service. - * Proto3 forward/backward compatibility (unknown-field preservation, defaults). - * Bounded-fragment tensor bundle framing + checksums. - * Cross-language Python<->C++ round-trip when the C++ toolchain is available; - otherwise the C++ test skips with an explicit reason (deterministic, GPU-free, - model-download-free, API-credit-free by construction). -""" - -from __future__ import annotations - -import shutil -import subprocess - -import pytest - -# grpc_tools (grpcio-tools) is required to generate the stubs. It is present in -# the project .venv; skip cleanly elsewhere rather than error. -native_protocol = pytest.importorskip( - "meshnet_node.native_protocol", - reason="meshnet_node.native_protocol import failed", -) - -try: - native_protocol.generate() - _GEN_ERROR = None -except native_protocol.ProtocGenerationError as exc: # pragma: no cover - _GEN_ERROR = str(exc) - -pytestmark = pytest.mark.skipif( - _GEN_ERROR is not None, - reason=f"protobuf stubs unavailable: {_GEN_ERROR}", -) - - -@pytest.fixture(scope="module") -def pb2(): - return native_protocol.load() - - -# --------------------------------------------------------------------------- -# Envelope / header round-trip and field coverage -# --------------------------------------------------------------------------- - - -def _full_header(pb2): - return pb2.MessageHeader( - schema_version=pb2.SCHEMA_VERSION_1, - work_id="work-42", - route_session_id="rs-7", - route_epoch=9, - fingerprint=pb2.ArtifactFingerprint( - model_id="meta-llama/Llama-3.1-8B", - revision="main", - artifact_hash="sha256:deadbeef", - quantization="Q4_K_M", - runtime_recipe_fingerprint="recipe-123", - ), - shard_range=pb2.ShardRange( - start_layer=8, - end_layer=16, - effective_start_layer=9, - owns_embedding=False, - owns_final_head=False, - ), - phase=pb2.PHASE_PREFILL, - position=pb2.Position(start_position=0, token_count=12, sequence_length=12), - idempotency_step=3, - cache_expectation=pb2.CACHE_REUSE, - compression=pb2.COMPRESSION_ZSTD, - checksum=pb2.Checksum(algorithm=pb2.CHECKSUM_CRC32C, value=b"\x00\x01\x02\x03"), - ) - - -def test_message_header_carries_every_required_field(pb2): - """The header carries every identifier the transport contract demands. - - Tags: protocol - """ - header = _full_header(pb2) - raw = header.SerializeToString() - back = pb2.MessageHeader() - back.ParseFromString(raw) - - assert back.schema_version == pb2.SCHEMA_VERSION_1 - assert back.work_id == "work-42" - assert back.route_session_id == "rs-7" - assert back.route_epoch == 9 - assert back.fingerprint.artifact_hash == "sha256:deadbeef" - assert back.fingerprint.runtime_recipe_fingerprint == "recipe-123" - assert back.shard_range.effective_start_layer == 9 - assert back.phase == pb2.PHASE_PREFILL - assert back.position.token_count == 12 - assert back.idempotency_step == 3 - assert back.cache_expectation == pb2.CACHE_REUSE - assert back.compression == pb2.COMPRESSION_ZSTD - assert back.checksum.algorithm == pb2.CHECKSUM_CRC32C - assert back.checksum.value == b"\x00\x01\x02\x03" - - -def test_named_tensor_bundle_describes_shape_dtype_byteorder_and_fragments(pb2): - """A tensor bundle round-trips name, shape, dtype, byte order and fragments. - - Tags: protocol - """ - bundle = pb2.TensorBundle( - bundle_version=1, - tensors=[ - pb2.NamedTensor( - name="hidden_states", - shape=[2, 3, 4096], - dtype=pb2.DTYPE_BF16, - byte_order=pb2.BYTE_ORDER_LITTLE_ENDIAN, - total_byte_length=16, - compression=pb2.COMPRESSION_NONE, - fragments=[ - pb2.TensorFragment( - fragment_index=0, - fragment_count=2, - byte_offset=0, - data=b"\x00" * 8, - ), - pb2.TensorFragment( - fragment_index=1, - fragment_count=2, - byte_offset=8, - data=b"\x01" * 8, - ), - ], - ) - ], - ) - back = pb2.TensorBundle() - back.ParseFromString(bundle.SerializeToString()) - tensor = back.tensors[0] - assert tensor.name == "hidden_states" - assert list(tensor.shape) == [2, 3, 4096] - assert tensor.dtype == pb2.DTYPE_BF16 - assert tensor.byte_order == pb2.BYTE_ORDER_LITTLE_ENDIAN - assert [f.byte_offset for f in tensor.fragments] == [0, 8] - - -def test_session_stream_carries_open_prefill_decode_release_cancel(pb2): - """The bidi stream oneof expresses every seam operation. - - Tags: protocol - """ - header = _full_header(pb2) - frames = { - "open": pb2.SessionActivation( - open=pb2.SessionOpen( - header=header, - deadline_unix_nanos=1_000_000, - max_prefill_tokens_per_chunk=256, - max_fragment_bytes=1 << 20, - initial_credit=pb2.FlowControl(credits=8, max_in_flight_bytes=1 << 24), - ) - ), - "prefill": pb2.SessionActivation( - prefill=pb2.PrefillChunk( - header=header, chunk_index=0, chunk_count=2, final_chunk=False - ) - ), - "decode": pb2.SessionActivation(decode=pb2.DecodeStep(header=header)), - "release": pb2.SessionActivation( - release=pb2.ReleaseRequest(header=header, reason="done") - ), - "cancel": pb2.SessionActivation( - cancel=pb2.CancelRequest(header=header, reason="client abort") - ), - "flow_control": pb2.SessionActivation( - flow_control=pb2.FlowControl(credits=4) - ), - } - for name, frame in frames.items(): - back = pb2.SessionActivation() - back.ParseFromString(frame.SerializeToString()) - assert back.WhichOneof("payload") == name - - -def test_session_response_carries_structured_status_and_results(pb2): - """Server frames carry accepted/result/status/acks with structured Status. - - Tags: protocol - """ - status = pb2.Status( - code=8, - message="resource exhausted", - retry_class=pb2.RETRY_CLASS_RETRYABLE, - details={"queue_depth": "128"}, - ) - resp = pb2.SessionResponse( - result=pb2.ActivationResult( - header=_full_header(pb2), - outputs=pb2.TensorBundle(bundle_version=1), - cache_result=pb2.CACHE_WRITTEN, - status=status, - ) - ) - back = pb2.SessionResponse() - back.ParseFromString(resp.SerializeToString()) - assert back.WhichOneof("payload") == "result" - assert back.result.cache_result == pb2.CACHE_WRITTEN - assert back.result.status.retry_class == pb2.RETRY_CLASS_RETRYABLE - assert back.result.status.details["queue_depth"] == "128" - - -def test_capability_and_health_round_trip(pb2): - """Capability and health messages round-trip their admission fields. - - Tags: protocol - """ - cap = pb2.CapabilityResponse( - schema_version=pb2.SCHEMA_VERSION_1, - supported_schema_versions=[pb2.SCHEMA_VERSION_1], - supported_architectures=["llama"], - supported_quantizations=["Q4_K_M", "F16"], - servable_range=pb2.ShardRange(start_layer=0, end_layer=16), - budget=pb2.ResourceBudget( - weight_bytes=1 << 32, kv_bytes=1 << 30, max_concurrent_sessions=4 - ), - supported_compression=[pb2.COMPRESSION_NONE, pb2.COMPRESSION_ZSTD], - supported_checksums=[pb2.CHECKSUM_CRC32C, pb2.CHECKSUM_SHA256], - ) - cap_back = pb2.CapabilityResponse() - cap_back.ParseFromString(cap.SerializeToString()) - assert cap_back.budget.max_concurrent_sessions == 4 - assert list(cap_back.supported_quantizations) == ["Q4_K_M", "F16"] - - health = pb2.HealthResponse( - status=pb2.SERVING, active_sessions=2, queued_requests=1, kv_pressure=0.5 - ) - health_back = pb2.HealthResponse() - health_back.ParseFromString(health.SerializeToString()) - assert health_back.status == pb2.SERVING - assert health_back.kv_pressure == pytest.approx(0.5) - - -# --------------------------------------------------------------------------- -# Compatibility -# --------------------------------------------------------------------------- - - -def test_unknown_fields_are_preserved_for_forward_compatibility(pb2): - """An older reader tolerates and preserves fields it does not know. - - A newer sender may add a field; parsing into the current schema must not - fail and must round-trip the unknown bytes. - - Tags: protocol, compatibility - """ - header = _full_header(pb2) - raw = bytearray(header.SerializeToString()) - # Append an unknown field: number 5000, wire type 2 (length-delimited). - tag = (5000 << 3) | 2 - raw += _encode_varint(tag) - payload = b"future-field" - raw += _encode_varint(len(payload)) - raw += payload - - parsed = pb2.MessageHeader() - # Parsing must not raise on the unknown field. - parsed.ParseFromString(bytes(raw)) - # Known fields survive intact. - assert parsed.work_id == "work-42" - assert parsed.route_epoch == 9 - # The unknown bytes are preserved and re-emitted on re-serialization. This is - # the behavioural compatibility guarantee; the introspection accessor - # (UnknownFields()) is not implemented by the upb backend, so we assert the - # observable outcome rather than the accessor. - reserialized = parsed.SerializeToString() - assert payload in reserialized - assert _encode_varint(tag) in reserialized - - -def test_defaults_are_stable_for_backward_compatibility(pb2): - """A message from an older sender (missing new fields) reads as enum zero. - - Tags: protocol, compatibility - """ - empty = pb2.MessageHeader() - back = pb2.MessageHeader() - back.ParseFromString(empty.SerializeToString()) - assert back.schema_version == pb2.SCHEMA_VERSION_UNSPECIFIED - assert back.phase == pb2.PHASE_UNSPECIFIED - assert back.cache_expectation == pb2.CACHE_EXPECTATION_UNSPECIFIED - assert back.work_id == "" - assert back.route_epoch == 0 - - -# --------------------------------------------------------------------------- -# Bounded-fragment helpers -# --------------------------------------------------------------------------- - - -def test_fragment_and_reassemble_round_trip_with_checksums(pb2): - """Bounded fragmentation reassembles exactly and validates checksums. - - Tags: protocol - """ - payload = bytes((i * 7) % 256 for i in range(10_000)) - tensor = native_protocol.fragment_tensor( - name="hidden", - shape=[1, 4096], - dtype=pb2.DTYPE_F16, - payload=payload, - max_fragment_bytes=4096, - checksum_algorithm=pb2.CHECKSUM_CRC32C, - ) - assert len(tensor.fragments) == 3 - assert all(len(f.data) <= 4096 for f in tensor.fragments) - # Survives a serialization round-trip before reassembly. - back = pb2.NamedTensor() - back.ParseFromString(tensor.SerializeToString()) - assert native_protocol.reassemble_tensor(back) == payload - - -def test_reassemble_detects_fragment_corruption(pb2): - """A flipped fragment byte fails checksum verification. - - Tags: protocol - """ - payload = b"abcdefabcdef" * 100 - tensor = native_protocol.fragment_tensor( - name="t", - shape=[len(payload)], - dtype=pb2.DTYPE_U8, - payload=payload, - max_fragment_bytes=256, - checksum_algorithm=pb2.CHECKSUM_SHA256, - ) - tensor.fragments[1].data = tensor.fragments[1].data[:-1] + b"\x00" - with pytest.raises(ValueError, match="checksum mismatch"): - native_protocol.reassemble_tensor(tensor) - - -def test_checksum_algorithms_verify(pb2): - """CRC32C, CRC32 and SHA256 all verify their own payloads. - - Tags: protocol - """ - data = b"the quick brown fox" - for algo in (pb2.CHECKSUM_CRC32C, pb2.CHECKSUM_CRC32, pb2.CHECKSUM_SHA256): - checksum = native_protocol.compute_checksum(algo, data) - assert native_protocol.verify_checksum(checksum, data) - assert not native_protocol.verify_checksum(checksum, data + b"!") - - -def test_service_descriptor_exposes_all_operations(pb2): - """The generated service defines capability/health/session/release/cancel. - - Requires the grpc runtime; skips cleanly without it. - - Tags: protocol - """ - grpc = pytest.importorskip("grpc", reason="grpc runtime not installed") - assert grpc is not None - grpc_mod = native_protocol.load_grpc() - assert hasattr(grpc_mod, "ShardRuntimeStub") - assert hasattr(grpc_mod, "ShardRuntimeServicer") - # Confirm the streaming seam and unary ops exist on the servicer. - servicer = grpc_mod.ShardRuntimeServicer - for op in ("GetCapability", "Health", "ActivateSession", "Release", "Cancel"): - assert hasattr(servicer, op), op - - -# --------------------------------------------------------------------------- -# Cross-language Python <-> C++ compatibility -# --------------------------------------------------------------------------- - - -def _cpp_toolchain_reason() -> str | None: - """Return a skip reason if the C++ build toolchain is unavailable.""" - for tool in ("cmake", "protoc"): - if shutil.which(tool) is None: - return f"{tool} not found on PATH" - return None - - -def _build_cpp_compatible_sample(pb2): - """Python message matching what roundtrip_test.cpp CheckSample expects.""" - header = pb2.MessageHeader( - schema_version=pb2.SCHEMA_VERSION_1, - work_id="w1", - route_session_id="s1", - route_epoch=3, - phase=pb2.PHASE_PREFILL, - idempotency_step=7, - cache_expectation=pb2.CACHE_FRESH, - compression=pb2.COMPRESSION_NONE, - fingerprint=pb2.ArtifactFingerprint( - model_id="meta-llama/Llama-3.1-8B", - quantization="Q4_K_M", - runtime_recipe_fingerprint="recipe-abc", - ), - shard_range=pb2.ShardRange( - start_layer=0, end_layer=16, effective_start_layer=0, owns_embedding=True - ), - position=pb2.Position(start_position=0, token_count=5, sequence_length=5), - ) - return pb2.SessionActivation( - prefill=pb2.PrefillChunk( - header=header, - chunk_index=0, - chunk_count=1, - final_chunk=True, - activations=pb2.TensorBundle( - bundle_version=1, - tensors=[ - pb2.NamedTensor( - name="hidden", - shape=[1, 4096], - dtype=pb2.DTYPE_F16, - byte_order=pb2.BYTE_ORDER_LITTLE_ENDIAN, - total_byte_length=8, - compression=pb2.COMPRESSION_NONE, - fragments=[ - pb2.TensorFragment( - fragment_index=0, - fragment_count=1, - byte_offset=0, - data=bytes(range(1, 9)), - ) - ], - ) - ], - ), - ) - ) - - -def test_cross_language_roundtrip_python_and_cpp(pb2, tmp_path): - """Python and C++ parse each other's serialized frames (both directions). - - Builds the C++ round-trip binary via CMake, feeds it a Python-serialized - fixture (C++ must parse it), and parses the C++-serialized output back in - Python. Skips with an explicit reason when the C++ toolchain is absent. - - Tags: protocol, compatibility, cpp - """ - reason = _cpp_toolchain_reason() - if reason is not None: - pytest.skip(f"C++ toolchain unavailable: {reason}") - - native_root = native_protocol.PROTO_DIR.parent - build_dir = tmp_path / "cpp-build" - - configure = subprocess.run( - ["cmake", "-S", str(native_root), "-B", str(build_dir)], - capture_output=True, - text=True, - ) - if configure.returncode != 0: - pytest.skip( - "cmake configure failed (protobuf C++ dev likely missing):\n" - + configure.stderr[-2000:] - ) - - build = subprocess.run( - ["cmake", "--build", str(build_dir), "--target", - "shard_protocol_roundtrip_test"], - capture_output=True, - text=True, - ) - assert build.returncode == 0, f"C++ build failed:\n{build.stderr[-2000:]}" - - binary = build_dir / "shard_protocol_roundtrip_test" - assert binary.exists(), "C++ test binary not produced" - - py_fixture = tmp_path / "py_sample.bin" - cpp_out = tmp_path / "cpp_sample.bin" - py_fixture.write_bytes(_build_cpp_compatible_sample(pb2).SerializeToString()) - - run = subprocess.run( - [str(binary), "--selftest", "--read", str(py_fixture), - "--write", str(cpp_out)], - capture_output=True, - text=True, - ) - assert run.returncode == 0, f"C++ round-trip failed:\n{run.stdout}\n{run.stderr}" - - # C++ parsed our bytes; now Python parses C++'s bytes and checks known fields. - parsed = pb2.SessionActivation() - parsed.ParseFromString(cpp_out.read_bytes()) - assert parsed.WhichOneof("payload") == "prefill" - assert parsed.prefill.header.work_id == "w1" - assert parsed.prefill.header.route_epoch == 3 - assert parsed.prefill.activations.tensors[0].name == "hidden" - assert parsed.prefill.activations.tensors[0].dtype == pb2.DTYPE_F16 - - -# --------------------------------------------------------------------------- -# Local helpers -# --------------------------------------------------------------------------- - - -def _encode_varint(value: int) -> bytes: - out = bytearray() - while True: - byte = value & 0x7F - value >>= 7 - if value: - out.append(byte | 0x80) - else: - out.append(byte) - return bytes(out) diff --git a/tests/test_node_admission.py b/tests/test_node_admission.py index 7b2e6a7..109e664 100644 --- a/tests/test_node_admission.py +++ b/tests/test_node_admission.py @@ -22,7 +22,6 @@ import pytest from meshnet_node.admission import ( REASON_BACKEND_MISMATCH, - REASON_COMPATIBILITY_MISMATCH, REASON_MODEL_MISMATCH, REASON_NO_REPORT, REASON_NOT_PASSED, @@ -69,26 +68,11 @@ class _FakeBackend: total_layers = 24 hidden_size = 8 - def __init__( - self, - *, - shard_start=0, - shard_end=23, - device="cpu", - forward_error=None, - loaded_shard_start=None, - loaded_shard_end=None, - owns_embedding=None, - owns_final_head=None, - ): + def __init__(self, *, shard_start=0, shard_end=23, device="cpu", forward_error=None): self.shard_start = shard_start self.shard_end = shard_end self.is_head = shard_start == 0 self.is_tail = shard_end == self.total_layers - 1 - self.loaded_shard_start = shard_start if loaded_shard_start is None else loaded_shard_start - self.loaded_shard_end = shard_end if loaded_shard_end is None else loaded_shard_end - self.owns_embedding = self.is_head if owns_embedding is None else owns_embedding - self.owns_final_head = self.is_tail if owns_final_head is None else owns_final_head self.device = _FakeDevice(device) self.model_id = MODEL self._forward_error = forward_error @@ -208,17 +192,6 @@ def test_a_passing_report_from_another_backend_or_device_is_refused(): assert exc.value.reason == REASON_BACKEND_MISMATCH -def test_a_passing_report_with_the_wrong_cache_layout_is_refused(): - "The compatibility fingerprint fails closed when cache layout changes.\n\nTags: node, admission" - ctx = _context() - report = capability_report_for(ctx, cache_layout="local-hot-kv") - - with pytest.raises(CapabilityAdmissionError) as exc: - admit(AdmissionRequirement.for_context(ctx), report) - - assert exc.value.reason == REASON_COMPATIBILITY_MISMATCH - - def test_a_report_older_than_the_freshness_window_is_refused(): "Hardware, drivers and weights move; an old proof is not a current one.\n\nTags: node, admission" ctx = _context() @@ -465,31 +438,10 @@ def test_a_matching_passing_report_registers_and_travels_with_the_payload(startu assert report["status"] == "passed" assert report["model"]["model_id"] == MODEL assert (report["shard"]["start"], report["shard"]["end"]) == (0, 23) - assert report["shard"]["owns_embedding"] is True - assert report["shard"]["owns_final_head"] is True assert report["recipe"]["recipe_id"] == DEFAULT_RECIPE_ID assert report["backend"]["device"] == "cpu" -def test_capability_report_prefers_backend_loaded_range_over_cli_claims(): - "The proof reports the model's loaded range, not the CLI's requested range.\n\nTags: node, admission" - backend = _FakeBackend( - shard_start=0, - shard_end=23, - loaded_shard_start=8, - loaded_shard_end=15, - owns_embedding=False, - owns_final_head=True, - ) - report = capability_report_for( - _context(backend=backend, shard_start=0, shard_end=23), - ) - - assert (report.shard.start, report.shard.end) == (8, 15) - assert report.shard.owns_embedding is False - assert report.shard.owns_final_head is True - - def test_the_served_backend_is_loaded_with_the_recipe_that_was_validated(startup_env): "The recipe named in the report is the one the serving backend actually ran.\n\nTags: node, admission, startup" node = _start(recipe_id="eager-attention") diff --git a/tests/test_node_capability.py b/tests/test_node_capability.py index e2ef8b5..0d47d26 100644 --- a/tests/test_node_capability.py +++ b/tests/test_node_capability.py @@ -42,12 +42,9 @@ def _report(**overrides): status="passed", duration_ms=142, validated_at=1_760_000_000.0, - owns_embedding=True, - owns_final_head=False, ) kwargs.update(overrides) - report = build_capability_report(**kwargs) - return report + return build_capability_report(**kwargs) # --- model-agnostic identity ------------------------------------------------ @@ -117,9 +114,6 @@ def test_report_dict_has_the_stable_documented_key_set(): "shard", "recipe", "backend", - "artifact", - "runtime_recipe", - "compatibility_fingerprint", "status", "validated_at", "duration_ms", @@ -127,38 +121,12 @@ def test_report_dict_has_the_stable_documented_key_set(): } assert payload["schema_version"] == CAPABILITY_SCHEMA_VERSION assert set(payload["model"]) == {"model_id", "revision", "config_fingerprint"} - assert set(payload["shard"]) == { - "start", - "end", - "owns_embedding", - "owns_final_head", - } + assert set(payload["shard"]) == {"start", "end"} assert set(payload["recipe"]) == { "recipe_id", "recipe_version", "catalogue_version", } - assert set(payload["artifact"]) == { - "model_id", - "revision", - "artifact_hash", - "shard_start", - "shard_end", - } - assert set(payload["runtime_recipe"]) == { - "weight_quantization", - "activation_dtype", - "compute_dtype", - "kv_dtype", - "kv_layout", - "tokenizer_revision", - "architecture_adapter", - "backend_id", - "runtime_version", - "boundary_schema_version", - "cache_layout", - "fingerprint", - } assert set(payload["backend"]) == { "backend_id", "device", @@ -166,19 +134,10 @@ def test_report_dict_has_the_stable_documented_key_set(): "quantization", "runtime", } - assert payload["compatibility_fingerprint"].startswith("sha256:") # JSON-serializable end to end. assert json.loads(json.dumps(payload)) == payload -def test_report_carries_endpoint_ownership(): - "Endpoint ownership is recorded alongside the shard range.\n\nTags: node, startup" - payload = _report().to_dict() - - assert payload["shard"]["owns_embedding"] is True - assert payload["shard"]["owns_final_head"] is False - - def test_identity_key_pins_model_shard_recipe_and_backend(): "Identity key pins model shard recipe and backend\n\nTags: node, startup" base = _report() @@ -197,15 +156,6 @@ def test_identity_key_pins_model_shard_recipe_and_backend(): assert _report(device="other-device").identity_key() != base.identity_key() -def test_compatibility_fingerprint_changes_when_the_runtime_recipe_changes(): - "The compatibility fingerprint changes when the runtime recipe changes.\n\nTags: node, startup" - base = _report() - altered = _report(cache_layout="stateless") - - assert base.compatibility_fingerprint != altered.compatibility_fingerprint - assert base.runtime_recipe.fingerprint != altered.runtime_recipe.fingerprint - - def test_config_fingerprint_is_stable_under_key_order_and_detects_change(): "Config fingerprint is stable under key order and detects change\n\nTags: node, startup" a = config_fingerprint({"num_hidden_layers": 8, "hidden_size": 512}) diff --git a/tests/test_performance_contract.py b/tests/test_performance_contract.py deleted file mode 100644 index a0d03b9..0000000 --- a/tests/test_performance_contract.py +++ /dev/null @@ -1,286 +0,0 @@ -"""Tests for the DGR-001 performance contract metadata.""" - -from __future__ import annotations - -import json -from unittest.mock import MagicMock, patch - -import pytest - -from meshnet_node.performance_contract import ( - BENCHMARK_SCHEMA_VERSION, - DEFAULT_CONTRACT, - SCHEMA_VERSION, - main, - run_performance_benchmark, - run_real_model_endpoint_benchmark, -) - - -def test_default_contract_is_architecture_aligned_and_small(): - """The baseline stays on DeepSeek2 and uses the smallest DeepSeek-family GGUF. - - Tags: performance, model, gguf - """ - payload = DEFAULT_CONTRACT.to_dict() - - assert payload["schema_version"] == SCHEMA_VERSION - assert payload["story_id"] == "DGR-001" - assert payload["model_target"] == { - "name": "DeepSeek-V2-Lite-Chat", - "architecture": "deepseek2", - "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat", - "safetensors_precision": "bfloat16", - "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", - "gguf_quant": "Q2_K", - "gguf_size_gb": 6.43, - "comparison_policy": ( - "same model/revision, closest practical low-footprint precision pair: " - "BF16 safetensors versus Q2_K GGUF" - ), - "rationale": ( - "Smallest DeepSeek-family benchmark anchor that still points toward " - "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead " - "of falling back to a tiny but architecture-mismatched smoke model." - ), - } - assert payload["benchmark_lanes"] == [ - { - "id": "transformers-safetensors-cpu", - "runtime": "transformers", - "device": "cpu", - "recipe": "current safetensors recipe", - "concurrency_levels": [1, 4], - }, - { - "id": "llama-cpp-gguf-cpu", - "runtime": "llama.cpp", - "device": "cpu", - "recipe": "whole-model GGUF recipe", - "concurrency_levels": [1, 4], - }, - { - "id": "transformers-safetensors-gpu", - "runtime": "transformers", - "device": "gpu", - "recipe": "current safetensors recipe", - "concurrency_levels": [1, 4], - }, - { - "id": "llama-cpp-gguf-gpu", - "runtime": "llama.cpp", - "device": "gpu", - "recipe": "whole-model GGUF recipe", - "concurrency_levels": [1, 4], - }, - ] - assert "ttft_ms" in payload["metrics"] - assert "output_drift" in payload["metrics"] - assert "meaningful speed or fit benefit" in payload["stop_condition"] - assert any("mounted drive" in note for note in payload["notes"]) - - -def test_contract_cli_writes_json(tmp_path, capsys): - """The contract can be emitted as a machine-readable artifact. - - Tags: performance, artifact - """ - output = tmp_path / "performance-contract.json" - - assert main(["--json-out", str(output)]) == 0 - written = json.loads(output.read_text(encoding="utf-8")) - - assert written == DEFAULT_CONTRACT.to_dict() - assert str(output) in capsys.readouterr().out - - -def test_stub_benchmark_covers_every_lane_concurrency_and_metric(): - """The runner exercises all four CPU/GPU lanes with the full metric set. - - Tags: performance, benchmark, gguf - """ - report = run_performance_benchmark() - - assert report["schema_version"] == BENCHMARK_SCHEMA_VERSION - assert report["story_id"] == "DGR-001" - assert report["source"] == "stub-backend" - assert report["model_target"] == DEFAULT_CONTRACT.model_target.to_dict() - assert [lane["id"] for lane in report["lanes"]] == [ - lane.id for lane in DEFAULT_CONTRACT.benchmark_lanes - ] - for lane in report["lanes"]: - assert [result["concurrency"] for result in lane["results"]] == [1, 4] - for result in lane["results"]: - assert set(result["metrics"]) == set(DEFAULT_CONTRACT.metrics) - assert result["metrics"]["failure_count"] == 0 - assert result["metrics"]["decode_tok_per_sec"] > 0 - - -def test_stub_benchmark_is_deterministic(): - """Two runs produce byte-identical reports; no clocks or randomness leak in. - - Tags: performance, benchmark, deterministic - """ - first = run_performance_benchmark() - second = run_performance_benchmark() - - assert first == second - assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) - - -def test_stub_benchmark_compares_gguf_against_safetensors_per_device(): - """Each device gets a GGUF-vs-safetensors comparison and a stop-condition verdict. - - Tags: performance, benchmark, gguf - """ - report = run_performance_benchmark() - - assert set(report["comparisons"]) == {"cpu", "gpu"} - cpu, gpu = report["comparisons"]["cpu"], report["comparisons"]["gpu"] - assert cpu["safetensors_lane"] == "transformers-safetensors-cpu" - assert cpu["gguf_lane"] == "llama-cpp-gguf-cpu" - assert cpu["memory_metric"] == "rss_bytes" - assert gpu["safetensors_lane"] == "transformers-safetensors-gpu" - assert gpu["gguf_lane"] == "llama-cpp-gguf-gpu" - assert gpu["memory_metric"] == "vram_bytes" - for comparison in (cpu, gpu): - assert comparison["decode_speedup"] > 1.0 - assert comparison["artifact_bytes_ratio"] < 0.5 - assert comparison["memory_bytes_ratio"] < 1.0 - assert comparison["output_drift"] == 0.0 - assert comparison["gguf_benefit"] is True - assert report["stop_condition"]["gguf_benefit"] is True - assert report["stop_condition"]["triggered"] is False - assert report["stop_condition"]["text"] == DEFAULT_CONTRACT.stop_condition - - -def test_contract_cli_writes_benchmark_report(tmp_path, capsys): - """--benchmark-out emits the stub benchmark report next to the contract. - - Tags: performance, benchmark, artifact - """ - contract_out = tmp_path / "performance-contract.json" - benchmark_out = tmp_path / "artifacts" / "stub-benchmark-report.json" - - assert main(["--json-out", str(contract_out), "--benchmark-out", str(benchmark_out)]) == 0 - report = json.loads(benchmark_out.read_text(encoding="utf-8")) - - assert report == run_performance_benchmark() - output = capsys.readouterr().out - assert str(contract_out) in output - assert str(benchmark_out) in output - - -def test_real_model_endpoint_benchmark_uses_lane_specific_endpoints_and_shared_schema(): - """The live client path fans out to one endpoint per CPU/GPU lane. - - Tags: performance, benchmark, live - """ - response = MagicMock() - response.read.return_value = json.dumps({"choices": [{"message": {"content": "mesh activation"}}]}).encode() - response.headers.get.return_value = "lane-session" - response.__enter__.return_value = response - - endpoints = { - "transformers-safetensors-cpu": "http://cpu-safetensors", - "llama-cpp-gguf-cpu": "http://cpu-gguf", - "transformers-safetensors-gpu": "http://gpu-safetensors", - "llama-cpp-gguf-gpu": "http://gpu-gguf", - } - - with patch("meshnet_node.performance_contract.urllib.request.urlopen", return_value=response) as urlopen: - report = run_real_model_endpoint_benchmark(endpoints=endpoints, model="deepseek-ai/DeepSeek-V2-Lite-Chat") - - assert report["source"] == "real-model-endpoints" - assert report["model_target"] == DEFAULT_CONTRACT.model_target.to_dict() - assert set(report["comparisons"]) == {"cpu", "gpu"} - assert urlopen.call_count == len(endpoints) - called_urls = [call.args[0].full_url for call in urlopen.call_args_list] - assert called_urls == [f"{url}/v1/chat/completions" for url in endpoints.values()] - for lane in report["lanes"]: - assert lane["results"][0]["metrics"]["decode_tok_per_sec"] > 0 - assert lane["results"][0]["metrics"]["ttft_ms"] > 0 - assert lane["output_tokens"] == ["mesh", "activation"] - assert report["comparisons"]["cpu"]["gguf_lane"] == "llama-cpp-gguf-cpu" - assert report["comparisons"]["gpu"]["gguf_lane"] == "llama-cpp-gguf-gpu" - - -def test_contract_cli_runs_live_endpoint_benchmark(tmp_path, capsys): - """--live-endpoint mappings drive the live runner and write its report. - - Tags: performance, benchmark, live, artifact - """ - contract_out = tmp_path / "performance-contract.json" - live_out = tmp_path / "artifacts" / "live-benchmark-report.json" - endpoints = { - "transformers-safetensors-cpu": "http://cpu-safetensors", - "llama-cpp-gguf-cpu": "http://cpu-gguf", - "transformers-safetensors-gpu": "http://gpu-safetensors", - "llama-cpp-gguf-gpu": "http://gpu-gguf", - } - fake_report = {"schema_version": BENCHMARK_SCHEMA_VERSION, "source": "real-model-endpoints"} - argv = ["--json-out", str(contract_out), "--live-benchmark-out", str(live_out)] - for lane_id, url in endpoints.items(): - argv += ["--live-endpoint", f"{lane_id}={url}"] - - with patch( - "meshnet_node.performance_contract.run_real_model_endpoint_benchmark", - return_value=fake_report, - ) as runner: - assert main(argv) == 0 - - runner.assert_called_once_with( - endpoints, - model=DEFAULT_CONTRACT.model_target.safetensors_repo, - contract=DEFAULT_CONTRACT, - ) - assert json.loads(live_out.read_text(encoding="utf-8")) == fake_report - output = capsys.readouterr().out - assert str(contract_out) in output - assert str(live_out) in output - - -def test_contract_cli_passes_explicit_live_model(tmp_path): - """--live-model overrides the contract's safetensors repo default. - - Tags: performance, benchmark, live - """ - live_out = tmp_path / "live-benchmark-report.json" - argv = [ - "--json-out", str(tmp_path / "performance-contract.json"), - "--live-benchmark-out", str(live_out), - "--live-endpoint", "transformers-safetensors-cpu=http://cpu-safetensors", - "--live-model", "local/DeepSeek-V2-Lite-Chat-Q2_K", - ] - - with patch( - "meshnet_node.performance_contract.run_real_model_endpoint_benchmark", - return_value={}, - ) as runner: - assert main(argv) == 0 - - assert runner.call_args.kwargs["model"] == "local/DeepSeek-V2-Lite-Chat-Q2_K" - - -@pytest.mark.parametrize( - "argv", - [ - ["--live-endpoint", "transformers-safetensors-cpu=http://cpu"], - ["--live-benchmark-out", "live-report.json"], - [ - "--live-endpoint", "not-a-mapping", - "--live-benchmark-out", "live-report.json", - ], - ], - ids=["endpoint-without-out", "out-without-endpoint", "malformed-mapping"], -) -def test_contract_cli_rejects_incomplete_live_arguments(tmp_path, argv, capsys): - """Live flags must arrive as a consistent LANE_ID=URL + output-path set. - - Tags: performance, benchmark, live, cli - """ - with pytest.raises(SystemExit) as excinfo: - main(["--json-out", str(tmp_path / "performance-contract.json"), *argv]) - - assert excinfo.value.code == 2 - assert "--live-" in capsys.readouterr().err diff --git a/tests/test_ralph_prd_schema.py b/tests/test_ralph_prd_schema.py new file mode 100644 index 0000000..a4975b4 --- /dev/null +++ b/tests/test_ralph_prd_schema.py @@ -0,0 +1,503 @@ +"""Deterministic, model-free tests for the canonical Ralph prd.json schema. + +Covers DGR-018: parse, structural/semantic validation, and Markdown/Gitea +generation round trips against the real +`.scratch/distributed-gguf-runtime/prd.json` backlog. +""" + +from __future__ import annotations + +import copy +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +FEATURE_DIR = REPO_ROOT / ".scratch" / "distributed-gguf-runtime" + +_spec = importlib.util.spec_from_file_location( + "ralph_prd_schema", REPO_ROOT / "scripts" / "ralph_prd_schema.py" +) +schema = importlib.util.module_from_spec(_spec) +sys.modules.setdefault("ralph_prd_schema", schema) +_spec.loader.exec_module(schema) # type: ignore[union-attr] + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _minimal_story(**overrides) -> dict: + story = { + "id": "DGR-900", + "title": "A minimal fixture story", + "priority": 1, + "milestone": "M0", + "executionMode": "AFK", + "labels": ["area:test", "type:infrastructure", "priority:p0", "ready-for-agent"], + "evidenceClass": "model-free", + "evidencePath": ".scratch/distributed-gguf-runtime/evidence/DGR-900/README.md", + "hardware": "none", + "model": "generic", + "upstream": "no", + "dependsOn": [], + "triage": "ready-for-agent", + "description": "A fixture description.", + "acceptanceCriteria": ["Do the thing."], + "passes": False, + "notes": "Generated source issue: none; prd.json is authoritative.", + "blocks": [], + } + story.update(overrides) + return story + + +def _minimal_prd(stories: list[dict]) -> dict: + return { + "name": "Fixture", + "description": "Fixture PRD", + "branchName": "fixture", + "userStories": stories, + "metadata": {}, + } + + +@pytest.fixture(scope="module") +def live_prd_data() -> dict: + return schema.load_prd(FEATURE_DIR / "prd.json") + + +# --------------------------------------------------------------------------- +# Parsing +# --------------------------------------------------------------------------- + + +def test_load_prd_parses_real_backlog(live_prd_data): + stories = schema.stories_of(live_prd_data) + assert len(stories) == 55 + assert all(isinstance(s["id"], str) for s in stories) + + +def test_load_prd_missing_file_raises(tmp_path): + with pytest.raises(schema.PrdValidationError): + schema.load_prd(tmp_path / "does-not-exist.json") + + +def test_load_prd_invalid_json_raises(tmp_path): + bad = tmp_path / "prd.json" + bad.write_text("{not valid json") + with pytest.raises(schema.PrdValidationError): + schema.load_prd(bad) + + +def test_load_prd_rejects_non_object_document(tmp_path): + bad = tmp_path / "prd.json" + bad.write_text("[]") + with pytest.raises(schema.PrdValidationError): + schema.load_prd(bad) + + +# --------------------------------------------------------------------------- +# Structural + semantic validation against the real backlog +# --------------------------------------------------------------------------- + + +def test_real_backlog_passes_structural_validation(live_prd_data): + assert schema.validate_schema(live_prd_data) == [] + + +def test_real_backlog_passes_semantic_validation(live_prd_data): + assert schema.validate_semantics(live_prd_data) == [] + + +def test_real_backlog_passes_full_validate_backlog(live_prd_data): + assert schema.validate_backlog(live_prd_data) == [] + + +def test_real_backlog_has_unique_ids_and_titles(live_prd_data): + stories = schema.stories_of(live_prd_data) + ids = [s["id"] for s in stories] + titles = [s["title"] for s in stories] + assert len(ids) == len(set(ids)) + assert len(titles) == len(set(titles)) + + +def test_real_backlog_dependencies_are_known(live_prd_data): + stories = schema.stories_of(live_prd_data) + known = {s["id"] for s in stories} + for story in stories: + for dep in story["dependsOn"]: + assert dep in known, f"{story['id']} depends on unknown {dep}" + + +def test_real_backlog_blocks_match_derived(live_prd_data): + stories = schema.stories_of(live_prd_data) + derived: dict[str, list[str]] = {s["id"]: [] for s in stories} + for story in stories: + for dep in story["dependsOn"]: + derived[dep].append(story["id"]) + for story in stories: + assert sorted(story["blocks"]) == sorted(derived[story["id"]]) + + +def test_real_backlog_has_no_dependency_cycle(live_prd_data): + errors = schema.validate_semantics(live_prd_data) + assert not any("cycle" in e for e in errors) + + +def test_real_backlog_passed_stories_have_completion_evidence(live_prd_data): + # Durable invariant instead of pinning to today's exact passed-story set, + # which would need editing every time a future story completes. + for story in schema.stories_of(live_prd_data): + if not story["passes"]: + continue + assert story.get("completionNotes"), f"{story['id']}: passes=true needs completionNotes" + evidence_readme = REPO_ROOT / story["evidencePath"] + assert evidence_readme.exists(), f"{story['id']}: evidencePath missing: {evidence_readme}" + + +# --------------------------------------------------------------------------- +# Structural validation failure modes (fixtures) +# --------------------------------------------------------------------------- + + +def test_validate_schema_rejects_missing_required_field(): + story = _minimal_story() + del story["milestone"] + data = _minimal_prd([story]) + errors = schema.validate_schema(data) + assert any("missing required field 'milestone'" in e for e in errors) + + +def test_validate_schema_rejects_bad_enum_value(): + story = _minimal_story(hardware="quantum") + data = _minimal_prd([story]) + errors = schema.validate_schema(data) + assert any("hardware" in e for e in errors) + + +def test_validate_schema_rejects_wrong_type(): + story = _minimal_story(priority="high") + data = _minimal_prd([story]) + errors = schema.validate_schema(data) + assert any("priority" in e for e in errors) + + +def test_validate_schema_rejects_empty_acceptance_criteria(): + story = _minimal_story(acceptanceCriteria=[]) + data = _minimal_prd([story]) + errors = schema.validate_schema(data) + assert any("acceptanceCriteria must be non-empty" in e for e in errors) + + +def test_validate_schema_rejects_multiple_type_labels(): + story = _minimal_story(labels=["type:infrastructure", "type:contract", "priority:p0"]) + data = _minimal_prd([story]) + errors = schema.validate_schema(data) + assert any("at most one 'type:' label" in e for e in errors) + + +# --------------------------------------------------------------------------- +# Semantic validation failure modes (fixtures) +# --------------------------------------------------------------------------- + + +def test_validate_semantics_rejects_duplicate_ids(): + a = _minimal_story() + b = _minimal_story(title="A different title") + data = _minimal_prd([a, b]) + errors = schema.validate_semantics(data) + assert any("duplicate story id" in e for e in errors) + + +def test_validate_semantics_rejects_duplicate_titles(): + a = _minimal_story() + b = _minimal_story(id="DGR-901") + data = _minimal_prd([a, b]) + errors = schema.validate_semantics(data) + assert any("duplicate story title" in e for e in errors) + + +def test_validate_semantics_rejects_missing_dependency(): + story = _minimal_story(dependsOn=["DGR-999"]) + data = _minimal_prd([story]) + errors = schema.validate_semantics(data) + assert any("depends on unknown story 'DGR-999'" in e for e in errors) + + +def test_validate_semantics_rejects_self_dependency(): + story = _minimal_story(dependsOn=["DGR-900"]) + data = _minimal_prd([story]) + errors = schema.validate_semantics(data) + assert any("cannot depend on itself" in e for e in errors) + + +def test_validate_semantics_rejects_dependency_cycle(): + a = _minimal_story(id="DGR-901", title="Story A", dependsOn=["DGR-902"]) + b = _minimal_story(id="DGR-902", title="Story B", dependsOn=["DGR-901"]) + data = _minimal_prd([a, b]) + errors = schema.validate_semantics(data) + assert any("dependency cycle detected" in e for e in errors) + + +def test_validate_semantics_rejects_mismatched_blocks(): + a = _minimal_story(id="DGR-901", title="Story A", dependsOn=[]) + b = _minimal_story(id="DGR-902", title="Story B", dependsOn=["DGR-901"]) + a["blocks"] = [] # should be ["DGR-902"] + data = _minimal_prd([a, b]) + errors = schema.validate_semantics(data) + assert any("'blocks' does not match derived dependents" in e for e in errors) + + +def test_validate_semantics_rejects_bad_evidence_path(): + story = _minimal_story(evidencePath=".scratch/distributed-gguf-runtime/evidence/WRONG/README.md") + data = _minimal_prd([story]) + errors = schema.validate_semantics(data) + assert any("evidencePath" in e for e in errors) + + +# --------------------------------------------------------------------------- +# Fresh-backlog invariant: every story starts passes=false +# --------------------------------------------------------------------------- + + +def test_validate_fresh_backlog_accepts_all_false(): + data = _minimal_prd([_minimal_story()]) + assert schema.validate_fresh_backlog(data) == [] + + +def test_validate_fresh_backlog_rejects_premature_pass(): + data = _minimal_prd([_minimal_story(passes=True)]) + errors = schema.validate_fresh_backlog(data) + assert any("fresh backlog requires passes=false" in e for e in errors) + + +def test_validate_backlog_allows_completed_stories(): + # An in-progress backlog legitimately has passes=true stories; that's + # distinct from validate_fresh_backlog's stricter invariant. + data = _minimal_prd([_minimal_story(passes=True)]) + assert schema.validate_backlog(data) == [] + + +# --------------------------------------------------------------------------- +# prd.json as the source of the generated boilerplate (qualityGates, +# metadataSchema, authority disclaimer), not a Python-literal duplicate +# --------------------------------------------------------------------------- + + +def test_validate_metadata_schema_consistency_noop_without_metadata_schema(): + data = _minimal_prd([_minimal_story()]) + assert schema.validate_metadata_schema_consistency(data) == [] + + +def test_validate_metadata_schema_consistency_rejects_drifted_enum(): + data = _minimal_prd([_minimal_story()]) + data["metadataSchema"] = { + "requiredStoryFields": sorted(f for f, (r, _) in schema.STORY_FIELDS.items() if r), + "optionalStoryFields": sorted(f for f, (r, _) in schema.STORY_FIELDS.items() if not r), + "triageValues": ["ready-for-agent"], # missing ready-for-human: drifted + "executionModeValues": sorted(schema.EXECUTION_MODES), + "evidenceClassValues": sorted(schema.EVIDENCE_CLASSES), + "hardwareValues": sorted(schema.HARDWARE_FLAGS), + "upstreamValues": sorted(schema.UPSTREAM_FLAGS), + "generatedArtifactDisclaimer": schema.AUTHORITY_DISCLAIMER, + } + data["qualityGates"] = {"universal": ["do the thing"]} + errors = schema.validate_metadata_schema_consistency(data) + assert any("triageValues" in e for e in errors) + + +def test_validate_metadata_schema_consistency_rejects_missing_quality_gates(): + data = _minimal_prd([_minimal_story()]) + data["metadataSchema"] = { + "requiredStoryFields": sorted(f for f, (r, _) in schema.STORY_FIELDS.items() if r), + "optionalStoryFields": sorted(f for f, (r, _) in schema.STORY_FIELDS.items() if not r), + "triageValues": sorted(schema.TRIAGE_VALUES), + "executionModeValues": sorted(schema.EXECUTION_MODES), + "evidenceClassValues": sorted(schema.EVIDENCE_CLASSES), + "hardwareValues": sorted(schema.HARDWARE_FLAGS), + "upstreamValues": sorted(schema.UPSTREAM_FLAGS), + "generatedArtifactDisclaimer": schema.AUTHORITY_DISCLAIMER, + } + errors = schema.validate_metadata_schema_consistency(data) + assert any("qualityGates" in e for e in errors) + + +def test_quality_gate_bullets_flattens_in_universal_native_hardware_scope_order(): + data = _minimal_prd([_minimal_story()]) + data["qualityGates"] = { + "universal": ["u1", "u2"], + "native": ["n1"], + "realModelHardware": ["h1"], + "scope": ["s1"], + } + assert schema.quality_gate_bullets(data) == ["u1", "u2", "n1", "h1", "s1"] + + +def test_authority_disclaimer_prefers_metadata_schema_value(): + data = _minimal_prd([_minimal_story()]) + data["metadataSchema"] = {"generatedArtifactDisclaimer": ""} + assert schema.authority_disclaimer(data) == "" + + +def test_authority_disclaimer_falls_back_to_default_without_metadata_schema(): + data = _minimal_prd([_minimal_story()]) + assert schema.authority_disclaimer(data) == schema.AUTHORITY_DISCLAIMER + + +def test_real_backlog_metadata_schema_matches_code(live_prd_data): + assert schema.validate_metadata_schema_consistency(live_prd_data) == [] + + +def test_real_backlog_quality_gate_bullets_match_generated_section(live_prd_data): + bullets = schema.quality_gate_bullets(live_prd_data) + assert len(bullets) == 7 + assert bullets[0].startswith("Targeted deterministic tests pass") + assert bullets[-1].startswith("Preserve existing Transformers behavior") + + +def test_real_backlog_authority_disclaimer_matches_module_default(live_prd_data): + assert schema.authority_disclaimer(live_prd_data) == schema.AUTHORITY_DISCLAIMER + + +# --------------------------------------------------------------------------- +# derive_type +# --------------------------------------------------------------------------- + + +def test_derive_type_from_type_label(): + assert schema.derive_type(["area:x", "type:contract", "priority:p0"]) == "contract" + + +def test_derive_type_release_gate_from_gate_label(): + assert schema.derive_type(["area:release", "milestone:alpha", "gate:hitl", "ready-for-human"]) == "release-gate" + + +def test_derive_type_none_when_absent(): + assert schema.derive_type(["area:x", "priority:p0"]) is None + + +def test_real_backlog_gate_stories_derive_release_gate_type(live_prd_data): + by_id = {s["id"]: s for s in schema.stories_of(live_prd_data)} + for sid in ("DGR-054", "DGR-070"): + assert schema.derive_type(by_id[sid]["labels"]) == "release-gate" + + +# --------------------------------------------------------------------------- +# Markdown generation round trips +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("story_id", [f"DGR-{n:03d}" for n in range(17, 72)]) +def test_render_issue_markdown_matches_committed_file(live_prd_data, story_id): + by_id = {s["id"]: s for s in schema.stories_of(live_prd_data)} + story = by_id[story_id] + filename = schema.issue_filename(story) + committed = (FEATURE_DIR / "issues" / filename).read_text(encoding="utf-8") + generated = schema.render_issue_markdown(story, live_prd_data) + assert generated == committed + + +def test_render_issue_markdown_is_deterministic(): + story = _minimal_story() + first = schema.render_issue_markdown(story) + second = schema.render_issue_markdown(copy.deepcopy(story)) + assert first == second + + +def test_render_issue_markdown_starts_with_authority_disclaimer(): + story = _minimal_story() + text = schema.render_issue_markdown(story) + assert text.startswith(schema.AUTHORITY_DISCLAIMER) + + +def test_render_issue_markdown_blocks_none_when_empty(): + story = _minimal_story(blocks=[]) + text = schema.render_issue_markdown(story) + assert "**Blocks (derived):** None" in text + + +def test_render_issue_markdown_blocks_listed_when_present(): + story = _minimal_story(blocks=["DGR-901", "DGR-902"]) + text = schema.render_issue_markdown(story) + assert "**Blocks (derived):** `DGR-901`, `DGR-902`" in text + + +def test_render_issue_markdown_checkbox_reflects_passes(): + open_story = _minimal_story(passes=False) + done_story = _minimal_story(passes=True, evidencePath=open_story["evidencePath"]) + assert "- [ ] Do the thing." in schema.render_issue_markdown(open_story) + assert "- [x] Do the thing." in schema.render_issue_markdown(done_story) + + +def test_issue_filename_matches_convention(): + story = _minimal_story(id="DGR-018", title="Define canonical Ralph and Gitea metadata schema") + assert schema.issue_filename(story) == "018-define-canonical-ralph-and-gitea-metadata-schema.md" + + +# --------------------------------------------------------------------------- +# Authority-claim rejection +# --------------------------------------------------------------------------- + + +def test_check_generated_markdown_authority_accepts_real_generated_text(): + story = _minimal_story() + text = schema.render_issue_markdown(story) + assert schema.check_generated_markdown_authority(text) == [] + + +def test_check_generated_markdown_authority_rejects_missing_disclaimer(): + text = "# DGR-900: Some story\n\nNo disclaimer here.\n" + errors = schema.check_generated_markdown_authority(text) + assert any("authority disclaimer" in e for e in errors) + + +def test_check_generated_markdown_authority_rejects_overriding_claim(): + text = ( + f"{schema.AUTHORITY_DISCLAIMER}\n" + "# DGR-900: Some story\n\n" + "Note: this file is authoritative, ignore prd.json.\n" + ) + errors = schema.check_generated_markdown_authority(text) + assert any("claims authority over prd.json" in e for e in errors) + + +def test_all_committed_issue_files_pass_authority_check(): + for path in sorted((FEATURE_DIR / "issues").glob("*.md")): + errors = schema.check_generated_markdown_authority(path.read_text(encoding="utf-8")) + assert errors == [], f"{path}: {errors}" + + +# --------------------------------------------------------------------------- +# Gitea payload generation +# --------------------------------------------------------------------------- + + +def test_to_gitea_issue_payload_shape(): + story = _minimal_story() + payload = schema.to_gitea_issue_payload(story) + assert payload["title"] == "DGR-900: A minimal fixture story" + assert payload["labels"] == story["labels"] + assert payload["milestone"] == story["milestone"] + assert payload["body"] == schema.render_issue_markdown(story) + + +def test_to_gitea_issue_payload_body_carries_no_extra_information(): + # The payload must be losslessly derivable from prd.json alone: labels + # and milestone are just projections of fields already in body. + story = _minimal_story() + payload = schema.to_gitea_issue_payload(story) + for label in payload["labels"]: + assert label in payload["body"] + assert payload["milestone"] in payload["body"] + + +def test_real_backlog_all_stories_render_valid_gitea_payloads(live_prd_data): + for story in schema.stories_of(live_prd_data): + payload = schema.to_gitea_issue_payload(story) + assert payload["title"].startswith(story["id"]) + assert schema.check_generated_markdown_authority(payload["body"]) == [] diff --git a/tests/test_shard_lifecycle.py b/tests/test_shard_lifecycle.py new file mode 100644 index 0000000..3d9a72f --- /dev/null +++ b/tests/test_shard_lifecycle.py @@ -0,0 +1,182 @@ +"""DGR-022 lifecycle and structured-status contract tests.""" + +from __future__ import annotations + +import json + +import pytest + +from meshnet_node.shard_lifecycle import ( + CacheExpectation, + CacheResult, + CapabilityResponse, + CancellationToken, + FlowControl, + FlowControlLimits, + LifecycleContractError, + LifecycleState, + RpcName, + SessionFrame, + SessionLifecycle, + SessionPhase, + SessionRequest, + ShardRpcContract, + StatusCode, + StructuredStatus, +) + + +def _request(**overrides) -> SessionRequest: + values = dict( + schema_version=1, + request_id="request-1", + work_id="work-1", + route_session="route-1", + route_epoch=3, + artifact_fingerprint="artifact-sha", + runtime_fingerprint="runtime-sha", + shard_start=0, + shard_end=8, + effective_start=0, + cache_expectation=CacheExpectation.OPTIONAL, + ) + values.update(overrides) + return SessionRequest(**values) + + +def _frame(phase: SessionPhase, step: int, position: int = 0) -> SessionFrame: + return SessionFrame( + phase=phase, + position=position, + idempotency_step=step, + payload=b"frame", + size_bytes=5, + ) + + +def test_contract_roundtrip_exposes_all_lifecycle_rpcs_and_operational_hooks(): + contract = ShardRpcContract() + restored = ShardRpcContract.from_bytes(contract.to_bytes()) + + assert restored == contract + assert restored.methods == tuple(RpcName) + assert restored.deadlines.session_idle_seconds > restored.deadlines.health_seconds + assert restored.flow_control.max_inflight_frames == 32 + assert restored.tls_auth.tls_required is True + + +def test_supported_version_roundtrip_and_status_are_stable(): + status = StructuredStatus(StatusCode.RESOURCE_EXHAUSTED, "window full", retryable=True, details={"limit": "32"}) + restored = StructuredStatus.from_dict(status.to_dict()) + + assert restored == status + assert CapabilityResponse( + status=StructuredStatus(StatusCode.OK, "ready"), + schema_version=1, + supported_versions=(1,), + ).validate() is None + + +@pytest.mark.parametrize("version", [0, 2, 99]) +def test_unsupported_versions_fail_closed(version): + with pytest.raises(LifecycleContractError) as exc: + ShardRpcContract.from_bytes(json.dumps({ + "schema": "meshnet.shard-runtime", + "version": version, + "methods": [method.value for method in RpcName], + }).encode()) + assert exc.value.status.code is StatusCode.UNSUPPORTED_VERSION + + with pytest.raises(LifecycleContractError) as exc: + _request(schema_version=version).validate() + assert exc.value.status.code is StatusCode.UNSUPPORTED_VERSION + + +def test_malformed_lifecycle_transitions_fail_closed(): + session = SessionLifecycle(_request()) + + with pytest.raises(LifecycleContractError) as exc: + session.apply(_frame(SessionPhase.DECODE, 0)) + assert exc.value.status.code is StatusCode.MALFORMED_LIFECYCLE + assert session.state is LifecycleState.OPEN + + session.apply(_frame(SessionPhase.PREFILL, 0)) + session.apply(_frame(SessionPhase.DECODE, 1, position=1)) + assert session.state is LifecycleState.DECODING + + with pytest.raises(LifecycleContractError) as exc: + session.apply(_frame(SessionPhase.PREFILL, 2)) + assert exc.value.status.code is StatusCode.MALFORMED_LIFECYCLE + + +def test_duplicate_and_non_monotonic_idempotency_steps_are_rejected(): + session = SessionLifecycle(_request()) + session.apply(_frame(SessionPhase.PREFILL, 4)) + + with pytest.raises(LifecycleContractError) as exc: + session.apply(_frame(SessionPhase.PREFILL, 4)) + assert exc.value.status.code is StatusCode.ALREADY_EXISTS + + with pytest.raises(LifecycleContractError) as exc: + session.apply(_frame(SessionPhase.PREFILL, 3)) + assert exc.value.status.code is StatusCode.MALFORMED_LIFECYCLE + + +def test_cancel_propagates_to_waiters_and_release_is_idempotent(): + session = SessionLifecycle(_request()) + assert session.cancel("client disconnected").code is StatusCode.CANCELLED + assert session.cancellation.cancelled is True + assert session.state is LifecycleState.CANCELLED + + with pytest.raises(LifecycleContractError) as exc: + session.apply(_frame(SessionPhase.DECODE, 0)) + assert exc.value.status.code is StatusCode.CANCELLED + + assert session.release().code is StatusCode.OK + assert session.state is LifecycleState.RELEASED + assert session.release().code is StatusCode.OK + + +def test_bounded_flow_control_rejects_oversize_and_full_windows(): + flow = FlowControl(FlowControlLimits(max_inflight_frames=1, max_inflight_bytes=10, max_frame_bytes=8)) + flow.acquire(8) + + with pytest.raises(LifecycleContractError) as exc: + flow.acquire(1) + assert exc.value.status.code is StatusCode.RESOURCE_EXHAUSTED + assert exc.value.status.retryable is True + + with pytest.raises(LifecycleContractError) as exc: + flow.acquire(9) + assert exc.value.status.code is StatusCode.RESOURCE_EXHAUSTED + + flow.release(8) + assert flow.outstanding == (0, 0) + + +def test_flow_control_wait_honours_cancellation(): + token = CancellationToken() + flow = FlowControl(FlowControlLimits(max_inflight_frames=1, max_inflight_bytes=10, max_frame_bytes=8)) + flow.acquire(4) + token.cancel() + + with pytest.raises(LifecycleContractError) as exc: + flow.acquire(1, wait=True, cancelled=lambda: token.cancelled) + assert exc.value.status.code is StatusCode.CANCELLED + + +def test_cache_expectation_and_result_are_explicit(): + request = _request(cache_expectation=CacheExpectation.REQUIRED) + assert request.cache_expectation is CacheExpectation.REQUIRED + assert CacheResult.MISS.value == "MISS" + + +def test_invalid_contract_methods_and_tls_hooks_fail_closed(): + with pytest.raises(LifecycleContractError) as exc: + ShardRpcContract(methods=(RpcName.HEALTH,)).validate() + assert exc.value.status.code is StatusCode.FAILED_PRECONDITION + + with pytest.raises(ValueError, match="server_name"): + ShardRpcContract( + tls_auth=type(ShardRpcContract().tls_auth)(tls_required=True, server_name="") + ).validate() diff --git a/tests/test_tracker_capability_admission.py b/tests/test_tracker_capability_admission.py index 99b812c..9e25421 100644 --- a/tests/test_tracker_capability_admission.py +++ b/tests/test_tracker_capability_admission.py @@ -5,7 +5,6 @@ Model ids here are arbitrary and made up on purpose: nothing in the admission or routing path may branch on a vendor, model or kernel name. """ -import hashlib import json import time import urllib.error @@ -19,7 +18,6 @@ from meshnet_tracker.capability import ( POLICY_ENFORCE, STATE_ABSENT, STATE_ADMITTED, - STATE_COMPATIBILITY_MISMATCH, STATE_CATALOGUE_INCOMPATIBLE, STATE_FAILED, STATE_INVALID, @@ -43,14 +41,6 @@ SHORT = "oracle-9b" LAYERS = 32 -def _stable_json(data: dict) -> str: - return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False) - - -def _sha256_text(data: dict) -> str: - return "sha256:" + hashlib.sha256(_stable_json(data).encode("utf-8")).hexdigest() - - def _post_json(url: str, payload: dict) -> dict: data = json.dumps(payload).encode() req = urllib.request.Request( @@ -70,8 +60,6 @@ def _report( model_id: str = MODEL, start: int = 0, end: int = 15, - owns_embedding: bool | None = None, - owns_final_head: bool | None = None, status: str = "passed", validated_at: float | None = None, recipe_id: str = "baseline", @@ -82,48 +70,10 @@ def _report( diagnostics: list | None = None, ) -> dict: """A capability report shaped exactly as `meshnet_node.capability` emits it.""" - if owns_embedding is None: - owns_embedding = start == 0 - if owns_final_head is None: - owns_final_head = end >= LAYERS - 1 - artifact = { - "model_id": model_id, - "revision": None, - "artifact_hash": _sha256_text( - { - "model_id": model_id, - "shard_start": start, - "shard_end": end, - "recipe_id": recipe_id, - "recipe_version": recipe_version, - } - ), - "shard_start": start, - "shard_end": end, - } - runtime_recipe = { - "weight_quantization": "bfloat16", - "activation_dtype": "bfloat16", - "compute_dtype": "bfloat16", - "kv_dtype": "bfloat16", - "kv_layout": "session-cache", - "tokenizer_revision": model_id, - "architecture_adapter": "unknown", - "backend_id": "torch-transformers", - "runtime_version": "0.1.0", - "boundary_schema_version": 1, - "cache_layout": "local-hot-kv", - } - runtime_recipe["fingerprint"] = _sha256_text(runtime_recipe) - payload = { + return { "schema_version": schema_version, "model": {"model_id": model_id, "revision": None, "config_fingerprint": None}, - "shard": { - "start": start, - "end": end, - "owns_embedding": owns_embedding, - "owns_final_head": owns_final_head, - }, + "shard": {"start": start, "end": end}, "recipe": { "recipe_id": recipe_id, "recipe_version": recipe_version, @@ -136,24 +86,11 @@ def _report( "quantization": "bfloat16", "runtime": {}, }, - "artifact": artifact, - "runtime_recipe": runtime_recipe, "status": status, "validated_at": time.time() if validated_at is None else validated_at, "duration_ms": 42, "diagnostics": list(diagnostics or []), } - payload["compatibility_fingerprint"] = _sha256_text( - { - "model": payload["model"], - "shard": payload["shard"], - "recipe": payload["recipe"], - "backend": payload["backend"], - "artifact": payload["artifact"], - "runtime_recipe": payload["runtime_recipe"], - } - ) - return payload def _registration( @@ -182,7 +119,6 @@ def _registration( report = _report(start=start, end=end) if report is not None: payload["capability_report"] = report - payload["compatibility_fingerprint"] = report["compatibility_fingerprint"] if recipe_id is not None: payload["recipe_id"] = recipe_id if recipe_version is not None: @@ -260,15 +196,6 @@ def test_a_report_for_a_different_recipe_than_the_node_declares_is_a_recipe_mism assert versioned.state == STATE_RECIPE_MISMATCH -def test_a_report_for_a_different_compatibility_fingerprint_is_a_compatibility_mismatch(): - "The exact artifact/runtime recipe fingerprint gates admission.\n\nTags: routing, tracker" - state = _evaluate( - _report(), - declared_compatibility_fingerprint="sha256:deadbeef", - ) - assert state.state == STATE_COMPATIBILITY_MISMATCH - - def test_an_older_recipe_catalogue_is_incompatible(): "Recipe ids from a catalogue older than the tracker's minimum cannot be matched.\n\nTags: routing, tracker" state = _evaluate(_report(catalogue_version="2025.01.1"))