neuron-tai/.scratch/distributed-inference-performance/prd.json

{
  "name": "Distributed inference performance",
  "description": "Measure and reduce avoidable transport, HTTP, telemetry, compression, buffering, and copy overhead around Route Session cached decode.",
  "branchName": "ralph/distributed-inference-performance",
  "userStories": [
    {
      "id": "DIP-001",
      "title": "01 — Baseline and profiling harness",
      "description": "Create a deterministic stub-backed direct and relay Route Session benchmark that reports per-token and per-seam timing, bytes, compression, queueing, and connection counts.",
      "acceptanceCriteria": [
        "Fixed prompt/token scenario runs in direct and relay modes",
        "Reports p50/p95 latency, payload bytes, compression ratio, connections, and queue wait",
        "Distinguishes prefill/decode and cached/stateless modes",
        "Produces machine-readable JSON and human-readable summary",
        "Can assert connection count and output token identity"
      ],
      "priority": 1,
      "passes": true,
      "notes": "Completed 2026-07-14. Deterministic direct/relay and cached/stateless stub benchmark with JSON/summary attribution; focused test suite passes (7). Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md",
      "dependsOn": []
    },
    {
      "id": "DIP-002",
      "title": "02 — Persistent relay compatibility hardening",
      "description": "Harden one persistent relay RPC connection per Route Session seam while preserving unique request IDs, legacy compatibility, cleanup, cancellation, and safe failure behavior.",
      "acceptanceCriteria": [
        "One healthy relay connection serves sequential cached decode requests",
        "Legacy one-request relays fail over without replaying uncertain mutations",
        "Pending RPC state is cleaned on close, cancellation, timeout, and disconnect",
        "Concurrent sessions do not share an unsafe socket",
        "Tests cover binary, JSON, timeout, disconnect, cancellation, and cleanup"
      ],
      "priority": 2,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md",
      "dependsOn": ["DIP-001"]
    },
    {
      "id": "DIP-003",
      "title": "03 — Direct and bridge HTTP keep-alive",
      "description": "Amortize TCP setup for direct node hops and relay bridge loopback forwarding with bounded connection ownership and correct HTTP framing.",
      "acceptanceCriteria": [
        "Direct cached decode reuses downstream HTTP connections",
        "Bridge loopback forwarding reuses connections without blocking unrelated workers",
        "HTTP/1.1 framing works for success, error, empty, stream, and cancellation",
        "Stale connections are invalidated under the existing fallback policy",
        "Benchmark shows healthy-session connection count independent of token count"
      ],
      "priority": 3,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/03-http-keepalive.md",
      "dependsOn": ["DIP-001"]
    },
    {
      "id": "DIP-004",
      "title": "04 — Activation Seam telemetry and bounded progress reporting",
      "description": "Expose per-seam timing and byte counters while aggregating progress work so telemetry does not become per-token hot-loop overhead.",
      "acceptanceCriteria": [
        "Telemetry includes seam latency and rolling tokens/sec",
        "Stable session and unique activation IDs remain correlated",
        "Counters are sampled or aggregated without network I/O in model execution",
        "Decode logs summarize by session with actionable failures",
        "Tests verify cadence and cleanup"
      ],
      "priority": 4,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/04-seam-telemetry.md",
      "dependsOn": ["DIP-001"]
    },
    {
      "id": "DIP-005",
      "title": "05 — Trace-driven activation compression",
      "description": "Choose zstd based on measured savings and CPU cost, preserving a raw fast path for small or incompressible decode activations.",
      "acceptanceCriteria": [
        "Compression policy is explicit and configurable per route condition",
        "Bodies below savings threshold are sent raw",
        "Compression timing and byte counters are reported",
        "Prefill and decode can use different policies",
        "Tests cover compressible, incompressible, threshold, malformed, and legacy bodies"
      ],
      "priority": 5,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/05-adaptive-compression.md",
      "dependsOn": ["DIP-001"]
    },
    {
      "id": "DIP-006",
      "title": "06 — Activation framing and copy reduction",
      "description": "Measure and reduce avoidable binary framing, metadata, CPU/GPU conversion, and decompression allocations without changing the wire contract.",
      "acceptanceCriteria": [
        "Benchmark attributes copy/allocation cost separately",
        "Decode hidden state avoids unnecessary float32 conversion",
        "Activation bodies remain binary and buffers have explicit ownership",
        "Metadata encoding remains semantically compatible",
        "Wire and token-output regression tests pass"
      ],
      "priority": 6,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/06-activation-framing-copies.md",
      "dependsOn": ["DIP-001"]
    },
    {
      "id": "DIP-007",
      "title": "07 — Bounded prefill chunk backpressure",
      "description": "Bound large prefill transfer with configurable chunking and in-flight limits, propagating downstream congestion without reordering or unbounded buffering.",
      "acceptanceCriteria": [
        "Chunk size and in-flight limit have safe defaults",
        "Peak buffered bytes stay within configured bounds",
        "Slow consumers apply backpressure and preserve order",
        "Cancellation and route failure release queued buffers",
        "Tests cover chunking, slow consumers, failure, and legacy peers"
      ],
      "priority": 7,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/07-prefill-backpressure.md",
      "dependsOn": ["DIP-001", "DIP-004"]
    },
    {
      "id": "DIP-008",
      "title": "08 — End-to-end distributed performance gate",
      "description": "Make direct, persistent-relay, cached, and stateless benchmark comparisons repeatable and fail on meaningful transport regressions while verifying output and cleanup.",
      "acceptanceCriteria": [
        "Deterministic stub benchmark runs locally and in CI",
        "Report compares throughput, latency, bytes, connections, compression CPU, and buffers",
        "Regression thresholds tolerate host variance and catch meaningful slowdowns",
        "Opt-in real-model LAN command emits the same metrics",
        "Gate verifies token identity, session stability, and resource cleanup"
      ],
      "priority": 8,
      "passes": false,
      "notes": "Source issue: .scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md",
      "dependsOn": ["DIP-002", "DIP-003", "DIP-004", "DIP-005", "DIP-006", "DIP-007"]
    }
  ]
}