From 905ea16ce008d50d2b1b3a93b4508d4c2cef9f78 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 16:55:52 +0300 Subject: [PATCH] feat: complete route session baseline benchmark --- .claude/memory/project-status.md | 4 ++++ .../issues/01-baseline-profiling-harness.md | 15 ++++++------ .../prd.json | 4 ++-- .../meshnet_node/route_session_benchmark.py | 24 ++++++++++++++++--- tests/test_route_session_benchmark.py | 13 ++++++++-- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/.claude/memory/project-status.md b/.claude/memory/project-status.md index 8c26a8b..3fb0fc7 100644 --- a/.claude/memory/project-status.md +++ b/.claude/memory/project-status.md @@ -8,6 +8,10 @@ metadata: # Project Status (2026-07-13) +## Distributed inference performance (2026-07-14) + +`DIP-001` is done in `.scratch/distributed-inference-performance/`: the deterministic two-node Route Session stub benchmark covers direct/relay plus cached/stateless prefill and decode. Its JSON and concise summary explicitly attribute model execution, activation encode/decode, compression, connection setup, relay queueing, local HTTP forwarding, and end-to-end seam latency. `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` passed (7); the fixture assertion checks output-token identity and connection attempts. + > Doc reconciliation 2026-07-13: `docs/prd.json` tracks US-001…US-050 (048 memory budget, 049 mainnet pilot, 050 Qwen demand placement). ADRs 0025–0026 added (TAI phase B/C, assignment ownership). All 35 user stories in docs/prd.json are done (35/35), including the reward-system arc US-030…US-035 completed 2026-07-02: diff --git a/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md b/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md index 0847134..79d1170 100644 --- a/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md +++ b/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md @@ -1,4 +1,4 @@ -Status: ready-for-agent +Status: done (2026-07-14) # 01 — Baseline and profiling harness @@ -12,16 +12,15 @@ sizes and connection counts without requiring a real model or external host. ## Acceptance criteria -- [ ] The harness runs a fixed prompt and fixed generated-token count through a +- [x] The harness runs a fixed prompt and fixed generated-token count through a two-node route in direct and relay modes. -- [ ] It reports p50/p95 per-token latency, per-hop latency, payload bytes, +- [x] It reports p50/p95 per-token latency, per-hop latency, payload bytes, compression ratio, connection attempts, and queue wait. -- [ ] It distinguishes prefill from decode and cached from stateless mode. -- [ ] It emits machine-readable JSON suitable for CI artifacts and a concise +- [x] It distinguishes prefill from decode and cached from stateless mode. +- [x] It emits machine-readable JSON suitable for CI artifacts and a concise human-readable summary. -- [ ] A test fixture can assert connection attempts and output token identity. +- [x] A test fixture can assert connection attempts and output token identity. ## Blocked by -None - can start immediately. - +None - completed. Verified with `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` (7 passed). diff --git a/.scratch/distributed-inference-performance/prd.json b/.scratch/distributed-inference-performance/prd.json index a9000e7..7184324 100644 --- a/.scratch/distributed-inference-performance/prd.json +++ b/.scratch/distributed-inference-performance/prd.json @@ -15,8 +15,8 @@ "Can assert connection count and output token identity" ], "priority": 1, - "passes": false, - "notes": "Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md", + "passes": true, + "notes": "Completed 2026-07-14. Deterministic direct/relay and cached/stateless stub benchmark with JSON/summary attribution; focused test suite passes (7). Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md", "dependsOn": [] }, { diff --git a/packages/node/meshnet_node/route_session_benchmark.py b/packages/node/meshnet_node/route_session_benchmark.py index f83eb5f..fdcfef4 100644 --- a/packages/node/meshnet_node/route_session_benchmark.py +++ b/packages/node/meshnet_node/route_session_benchmark.py @@ -44,6 +44,7 @@ class SeamSample: cache_mode: CacheMode model_ms: float encode_ms: float + activation_decode_ms: float framing_ms: float metadata_ms: float copy_allocation_ms: float @@ -52,6 +53,7 @@ class SeamSample: decompression_ms: float connection_setup_ms: float queue_wait_ms: float + local_http_forwarding_ms: float transport_ms: float seam_latency_ms: float payload_bytes: int @@ -120,6 +122,10 @@ def _summary(samples: list[SeamSample]) -> dict[str, float | int]: "compression_cpu_ms": round( sum(sample.compression_ms + sample.decompression_ms for sample in samples), 4 ), + "model_execution_ms": round(sum(sample.model_ms for sample in samples), 4), + "activation_encoding_ms": round(sum(sample.encode_ms for sample in samples), 4), + "activation_decoding_ms": round(sum(sample.activation_decode_ms for sample in samples), 4), + "local_http_forwarding_ms": round(sum(sample.local_http_forwarding_ms for sample in samples), 4), "peak_buffered_bytes": max((sample.copy_allocation_bytes for sample in samples), default=0), } @@ -159,6 +165,7 @@ class _StubTransport: queue_wait_ms = 0.0 if self.mode == "direct" else 0.18 + (0.05 if token_index is not None and token_index % 2 else 0.0) model_ms = 1.6 if phase == "prefill" else 0.45 encode_ms = 0.16 if phase == "prefill" else 0.06 + activation_decode_ms = 0.055 if phase == "prefill" else 0.02 # Keep framing/metadata/copy costs explicit rather than hiding them in # serialization or transport time. The stub owns one binary frame and # one response body per hop; no base64 body is modeled. @@ -168,20 +175,26 @@ class _StubTransport: copy_allocation_bytes = wire_bytes + payload_bytes compression_ms = 0.09 if self.scenario.compression else 0.0 decompression_ms = 0.07 if self.scenario.compression else 0.0 + # Both routes finish by forwarding the decoded activation to the local + # tail-node HTTP handler; relay adds its own queue before that hop. + local_http_forwarding_ms = 0.11 if self.mode == "direct" else 0.16 transport_ms = (0.32 if self.mode == "direct" else 0.61) + wire_bytes / 100_000 seam_latency_ms = round( - model_ms + encode_ms + framing_ms + metadata_ms + copy_allocation_ms - + compression_ms + decompression_ms + connection_setup_ms + queue_wait_ms + transport_ms, + model_ms + encode_ms + activation_decode_ms + framing_ms + metadata_ms + copy_allocation_ms + + compression_ms + decompression_ms + connection_setup_ms + queue_wait_ms + transport_ms + + local_http_forwarding_ms, 4, ) return SeamSample( phase=phase, token_index=token_index, session_id=self.session_id, activation_id=f"benchmark-activation-{self._activation_count}", seam="head->tail", mode=self.mode, cache_mode=self.cache_mode, model_ms=model_ms, encode_ms=encode_ms, + activation_decode_ms=activation_decode_ms, framing_ms=framing_ms, metadata_ms=metadata_ms, copy_allocation_ms=copy_allocation_ms, copy_allocation_bytes=copy_allocation_bytes, compression_ms=compression_ms, decompression_ms=decompression_ms, connection_setup_ms=connection_setup_ms, queue_wait_ms=queue_wait_ms, + local_http_forwarding_ms=local_http_forwarding_ms, transport_ms=round(transport_ms, 4), seam_latency_ms=seam_latency_ms, payload_bytes=payload_bytes, wire_bytes=wire_bytes, compression_ratio=round(payload_bytes / wire_bytes, 4), connection_attempted=connection_attempted, @@ -329,9 +342,10 @@ def run_real_model_lan_benchmark(url: str, *, model: str, timeout: float = 120.0 sample = SeamSample( phase="decode", token_index=0, session_id=session_id, activation_id="lan-activation-1", seam="head->tail", mode="direct", cache_mode="cached", model_ms=0.0, encode_ms=0.0, + activation_decode_ms=0.0, framing_ms=0.0, metadata_ms=0.0, copy_allocation_ms=0.0, copy_allocation_bytes=0, compression_ms=0.0, decompression_ms=0.0, connection_setup_ms=elapsed_ms, - queue_wait_ms=0.0, transport_ms=elapsed_ms, seam_latency_ms=elapsed_ms, + queue_wait_ms=0.0, local_http_forwarding_ms=0.0, transport_ms=elapsed_ms, seam_latency_ms=elapsed_ms, payload_bytes=len(body), wire_bytes=len(body) + len(response_body), compression_ratio=1.0, connection_attempted=True, ) @@ -354,6 +368,10 @@ def format_summary(report: dict) -> str: f"{decode['tokens_per_sec']:.1f} tok/s; {decode['bytes_per_token']:.0f} B/tok; " f"seam {seam['payload_bytes']}/{seam['wire_bytes']} B " f"({seam['compression_ratio']:.2f}x); connections {run['connections']['attempts']}; " + f"model/encode/decode {decode['model_execution_ms']:.2f}/" + f"{decode['activation_encoding_ms']:.2f}/{decode['activation_decoding_ms']:.2f} ms; " + f"compression {decode['compression_cpu_ms']:.2f} ms; " + f"HTTP {decode['local_http_forwarding_ms']:.2f} ms; " f"queue p95 {decode['p95_queue_wait_ms']:.2f} ms" ) return "\n".join(lines) diff --git a/tests/test_route_session_benchmark.py b/tests/test_route_session_benchmark.py index c1a9294..603bd50 100644 --- a/tests/test_route_session_benchmark.py +++ b/tests/test_route_session_benchmark.py @@ -32,12 +32,18 @@ def test_matrix_reports_direct_relay_prefill_decode_and_machine_readable_metrics assert {"p50_latency_ms", "p95_latency_ms", "payload_bytes", "compression_ratio", "connection_attempts", "p95_queue_wait_ms"} <= set(run["phases"]["decode"]) sample = run["samples"][0] + assert sample["model_ms"] > 0 + assert sample["encode_ms"] > 0 + assert sample["activation_decode_ms"] > 0 assert sample["framing_ms"] > 0 assert sample["metadata_ms"] > 0 assert sample["copy_allocation_ms"] > 0 assert sample["copy_allocation_bytes"] >= sample["payload_bytes"] + assert sample["local_http_forwarding_ms"] > 0 assert len(run["samples"]) == 1 + len(run["output_tokens"]) - assert {"tokens_per_sec", "bytes_per_token", "compression_cpu_ms", "peak_buffered_bytes"} <= set(run["phases"]["decode"]) + assert {"tokens_per_sec", "bytes_per_token", "compression_cpu_ms", "peak_buffered_bytes", + "model_execution_ms", "activation_encoding_ms", "activation_decoding_ms", + "local_http_forwarding_ms"} <= set(run["phases"]["decode"]) def test_cached_sessions_reuse_one_connection_and_preserve_stub_tokens(): @@ -74,7 +80,10 @@ def test_cli_writes_json_artifact_and_human_summary(tmp_path, capsys): report = json.loads(output.read_text()) assert report["schema_version"] == 1 assert "Route Session benchmark" in capsys.readouterr().out - assert "relay" in format_summary(report) + summary = format_summary(report) + assert "relay" in summary + assert "model/encode/decode" in summary + assert "HTTP" in summary def test_performance_gate_checks_comparison_identity_session_and_cleanup():