diff --git a/.scratch/distributed-inference-network/issues/30-manual-route-and-hop-benchmark.md b/.scratch/distributed-inference-network/issues/30-manual-route-and-hop-benchmark.md new file mode 100644 index 0000000..c664fd9 --- /dev/null +++ b/.scratch/distributed-inference-network/issues/30-manual-route-and-hop-benchmark.md @@ -0,0 +1,48 @@ +# US-020 — Manual route selection + hop-penalty benchmarking + +## Context + +The tracker auto-selects inference routes based on synthetic benchmark scores. To measure +the real cost of adding hops (latency per node boundary), we need: + +1. A way to pin a request to a specific route so we control the variable. +2. A benchmark endpoint that runs the same prompt through 1-node, 2-node, and 3-node + routes and records per-hop latency. + +Results are stored to disk. Routing algorithm is **not** changed in this story — this is +data collection only. The data will inform a future routing optimisation story. + +## Design decisions (grilled 2026-07-01) + +| Decision | Choice | +|---|---| +| Route spec | Optional `route` field in JSON request body (list of node IDs) | +| Trigger | Explicit only — `POST /v1/benchmark/hop-penalty` endpoint | +| Auth | Header-presence stub (`Authorization` must be non-empty); real auth in future story | +| Routing integration | Store data only; routing algorithm unchanged | +| Persistence | Append to `benchmark_results.json` in tracker working dir; in-memory queryable | + +## Acceptance criteria + +- `POST /v1/chat/completions` accepts optional `"route": ["", ...]` in the + request body. If present, the tracker uses those nodes in order instead of auto-selecting. + If absent, existing routing is unchanged (no breaking change for unaware clients). +- Missing or invalid node IDs in `route` return HTTP 400 with a descriptive error. +- `POST /v1/benchmark/hop-penalty` is auth-gated: requests without a non-empty + `Authorization` header return HTTP 401. Body: `{"model": "...", "prompt": "...", + "max_new_tokens": 64}`. +- Benchmark fans out to up to three routes: 1-node (single node covering all layers), + 2-node (two consecutive shard nodes), 3-node (three nodes) — using whatever is + currently registered. Routes with insufficient coverage are skipped, not errored. +- Response includes per-route breakdown: `total_ms`, `per_hop_ms: [...]`, + `tokens_generated`, `route: [node_id, ...]`. +- Results are appended to `/benchmark_results.json` (created if + absent) as a JSON array. Each entry includes timestamp, model, prompt hash, and the + per-route breakdown. +- `GET /v1/benchmark/results` returns the stored results array. Also auth-gated. +- Clients that never send `route` or call `/v1/benchmark/*` are completely unaffected. +- Integration test: send the same prompt via a pinned 1-node route and a pinned 2-node + route; assert 2-node result has 2 entries in `per_hop_ms`; assert both records appear + in `benchmark_results.json`. +- `python -m pytest` passes from repo root. +- Commit only this story's changes. diff --git a/.scratch/distributed-inference-network/prd.json b/.scratch/distributed-inference-network/prd.json index 7be1bed..131bdd1 100644 --- a/.scratch/distributed-inference-network/prd.json +++ b/.scratch/distributed-inference-network/prd.json @@ -684,10 +684,36 @@ "US-022" ], "completionNotes": "_relay_hop() added to torch_server.py. _get_remaining_route returns list[dict]. relay_bridge.py updated with body_base64 support. Tracker injects relay_addr into downstream hop dicts. 157 tests pass." + }, + { + "id": "US-030", + "title": "30 — Manual route selection + hop-penalty benchmarking", + "description": "Two additions to the tracker. (1) Callers can pin an explicit inference route by passing an optional \"route\": [\"\", ...] field in the POST /v1/chat/completions body. The tracker uses those nodes in order instead of auto-selecting; clients that omit the field are unaffected. (2) A new privileged POST /v1/benchmark/hop-penalty endpoint runs the same prompt through up to three routes (1-node, 2-node, 3-node) using whatever nodes are registered, records per-hop latency, and appends results to benchmark_results.json in the tracker's working directory. The routing algorithm is not changed — this story is data collection only. Auth is a header-presence stub (non-empty Authorization header required for benchmark endpoints).", + "acceptanceCriteria": [ + "POST /v1/chat/completions accepts optional \"route\": [node_id, ...] in the request body; if present, tracker routes to those nodes in order; if absent, existing auto-routing is unchanged", + "Missing or invalid node IDs in route return HTTP 400 with a descriptive error message", + "POST /v1/benchmark/hop-penalty requires a non-empty Authorization header; missing/empty returns HTTP 401", + "Benchmark body: {\"model\": \"...\", \"prompt\": \"...\", \"max_new_tokens\": 64 (optional)}", + "Benchmark fans out to up to 3 routes (1-node, 2-node, 3-node) using currently registered nodes; routes with insufficient coverage are skipped, not errored", + "Benchmark response includes per-route entries: {\"route\": [node_id, ...], \"total_ms\": float, \"per_hop_ms\": [float, ...], \"tokens_generated\": int}", + "Results appended to /benchmark_results.json (created if absent) as a JSON array; each entry includes ISO timestamp, model, sha256 of prompt, and per-route breakdown", + "GET /v1/benchmark/results returns the stored results array; also requires non-empty Authorization header", + "Integration test: pin a 1-node route and a 2-node route for the same prompt; assert 2-node result has 2 per_hop_ms entries; assert both records appear in benchmark_results.json", + "Clients that never send route or call /v1/benchmark/* are completely unaffected (existing tests pass unchanged)", + "python -m pytest passes from repo root", + "Commit only this story's changes" + ], + "priority": 30, + "status": "open", + "notes": "Source issue: .scratch/distributed-inference-network/issues/30-manual-route-and-hop-benchmark.md. Design decisions grilled 2026-07-01: route via body field, explicit-only benchmark trigger, auth stub, routing algorithm unchanged, persist to JSON file.", + "dependsOn": [ + "US-014", + "US-019" + ] } ], "metadata": { - "updatedAt": "2026-06-29T15:35:00.000Z", + "updatedAt": "2026-07-01T00:00:00.000Z", "statusVocabulary": { "open": "Not started", "in-design": "Decisions pending before implementation can begin",