diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md index 19a6f70..33e8121 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -19,6 +19,21 @@ - concurrency levels `1` and `4` - the required metrics list - an explicit stop condition for “no meaningful speed or fit benefit” +- Adds a deterministic stub benchmark report so the contract now has an executable report shape end to end. + +## Recent benchmark runner slice + +The runner currently uses a deterministic stub backend to exercise the comparison matrix without downloading a model. It emits: + +- `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json` +- `.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json` + +The report includes per-device comparisons for: + +- `transformers-safetensors-cpu` vs `llama-cpp-gguf-cpu` +- `transformers-safetensors-gpu` vs `llama-cpp-gguf-gpu` + +and records the memory metric (`rss_bytes` on CPU, `vram_bytes` on GPU), decode speedup, artifact ratio, and output drift. ## Exact commands and real results diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json new file mode 100644 index 0000000..7f3e272 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json @@ -0,0 +1,247 @@ +{ + "comparisons": { + "cpu": { + "artifact_bytes_ratio": 0.2048, + "decode_speedup": 2.3333, + "gguf_benefit": true, + "gguf_lane": "llama-cpp-gguf-cpu", + "memory_bytes_ratio": 0.2152, + "memory_metric": "rss_bytes", + "output_drift": 0.0, + "safetensors_lane": "transformers-safetensors-cpu", + "ttft_speedup": 1.8947 + }, + "gpu": { + "artifact_bytes_ratio": 0.2048, + "decode_speedup": 1.5294, + "gguf_benefit": true, + "gguf_lane": "llama-cpp-gguf-gpu", + "memory_bytes_ratio": 0.2273, + "memory_metric": "vram_bytes", + "output_drift": 0.0, + "safetensors_lane": "transformers-safetensors-gpu", + "ttft_speedup": 1.6154 + } + }, + "lanes": [ + { + "concurrency_levels": [ + 1, + 4 + ], + "device": "cpu", + "id": "transformers-safetensors-cpu", + "output_tokens": [ + "mesh", + "activation", + "seam", + "baseline" + ], + "recipe": "current safetensors recipe", + "results": [ + { + "concurrency": 1, + "metrics": { + "aggregate_throughput_tok_per_sec": 6.0, + "artifact_bytes": 33715493273, + "decode_tok_per_sec": 6.0, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 166.6667, + "p95_latency_ms": 208.3334, + "prefill_tok_per_sec": 45.0, + "rss_bytes": 35433480192, + "ttft_ms": 1800.0, + "vram_bytes": 0 + } + }, + { + "concurrency": 4, + "metrics": { + "aggregate_throughput_tok_per_sec": 20.4, + "artifact_bytes": 33715493273, + "decode_tok_per_sec": 5.1, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 196.0784, + "p95_latency_ms": 245.098, + "prefill_tok_per_sec": 38.25, + "rss_bytes": 35433480192, + "ttft_ms": 2340.0, + "vram_bytes": 0 + } + } + ], + "runtime": "transformers" + }, + { + "concurrency_levels": [ + 1, + 4 + ], + "device": "cpu", + "id": "llama-cpp-gguf-cpu", + "output_tokens": [ + "mesh", + "activation", + "seam", + "baseline" + ], + "recipe": "whole-model GGUF recipe", + "results": [ + { + "concurrency": 1, + "metrics": { + "aggregate_throughput_tok_per_sec": 14.0, + "artifact_bytes": 6904159928, + "decode_tok_per_sec": 14.0, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 71.4286, + "p95_latency_ms": 89.2858, + "prefill_tok_per_sec": 90.0, + "rss_bytes": 7623566950, + "ttft_ms": 950.0, + "vram_bytes": 0 + } + }, + { + "concurrency": 4, + "metrics": { + "aggregate_throughput_tok_per_sec": 47.6, + "artifact_bytes": 6904159928, + "decode_tok_per_sec": 11.9, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 84.0336, + "p95_latency_ms": 105.042, + "prefill_tok_per_sec": 76.5, + "rss_bytes": 7623566950, + "ttft_ms": 1235.0, + "vram_bytes": 0 + } + } + ], + "runtime": "llama.cpp" + }, + { + "concurrency_levels": [ + 1, + 4 + ], + "device": "gpu", + "id": "transformers-safetensors-gpu", + "output_tokens": [ + "mesh", + "activation", + "seam", + "baseline" + ], + "recipe": "current safetensors recipe", + "results": [ + { + "concurrency": 1, + "metrics": { + "aggregate_throughput_tok_per_sec": 34.0, + "artifact_bytes": 33715493273, + "decode_tok_per_sec": 34.0, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 29.4118, + "p95_latency_ms": 36.7647, + "prefill_tok_per_sec": 850.0, + "rss_bytes": 4294967296, + "ttft_ms": 420.0, + "vram_bytes": 35433480192 + } + }, + { + "concurrency": 4, + "metrics": { + "aggregate_throughput_tok_per_sec": 115.6, + "artifact_bytes": 33715493273, + "decode_tok_per_sec": 28.9, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 34.6021, + "p95_latency_ms": 43.2526, + "prefill_tok_per_sec": 722.5, + "rss_bytes": 4294967296, + "ttft_ms": 546.0, + "vram_bytes": 35433480192 + } + } + ], + "runtime": "transformers" + }, + { + "concurrency_levels": [ + 1, + 4 + ], + "device": "gpu", + "id": "llama-cpp-gguf-gpu", + "output_tokens": [ + "mesh", + "activation", + "seam", + "baseline" + ], + "recipe": "whole-model GGUF recipe", + "results": [ + { + "concurrency": 1, + "metrics": { + "aggregate_throughput_tok_per_sec": 52.0, + "artifact_bytes": 6904159928, + "decode_tok_per_sec": 52.0, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 19.2308, + "p95_latency_ms": 24.0385, + "prefill_tok_per_sec": 640.0, + "rss_bytes": 1610612736, + "ttft_ms": 260.0, + "vram_bytes": 8053063680 + } + }, + { + "concurrency": 4, + "metrics": { + "aggregate_throughput_tok_per_sec": 176.8, + "artifact_bytes": 6904159928, + "decode_tok_per_sec": 44.2, + "failure_count": 0, + "output_drift": 0.0, + "p50_latency_ms": 22.6244, + "p95_latency_ms": 28.2805, + "prefill_tok_per_sec": 544.0, + "rss_bytes": 1610612736, + "ttft_ms": 338.0, + "vram_bytes": 8053063680 + } + } + ], + "runtime": "llama.cpp" + } + ], + "model_target": { + "architecture": "deepseek2", + "comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF", + "gguf_quant": "Q2_K", + "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", + "gguf_size_gb": 6.43, + "name": "DeepSeek-V2-Lite-Chat", + "rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.", + "safetensors_precision": "bfloat16", + "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat" + }, + "schema_version": 1, + "source": "stub-backend", + "stop_condition": { + "gguf_benefit": true, + "text": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.", + "triggered": false + }, + "story_id": "DGR-001" +} diff --git a/packages/node/meshnet_node/performance_contract.py b/packages/node/meshnet_node/performance_contract.py index 8b04067..4c035d8 100644 --- a/packages/node/meshnet_node/performance_contract.py +++ b/packages/node/meshnet_node/performance_contract.py @@ -1,8 +1,11 @@ -"""Versioned performance contract metadata for DGR-001. +"""Versioned performance contract metadata and stub benchmark runner for DGR-001. -This module intentionally captures the *contract* first: the model family, -architecture alignment, benchmark lanes, and stop condition that later benchmark -runs must satisfy. It does not download or execute a model. +This module captures the *contract* first: the model family, architecture +alignment, benchmark lanes, and stop condition that benchmark runs must +satisfy. It also runs the contract's lanes through a deterministic stub +backend so the report data shape exists end to end. It never downloads or +executes a model; real transformers / llama.cpp backends plug in behind the +same ``run()`` seam later. """ from __future__ import annotations @@ -174,13 +177,177 @@ def build_default_contract() -> PerformanceContract: return DEFAULT_CONTRACT +BENCHMARK_SCHEMA_VERSION = 1 +STUB_OUTPUT_TOKENS = ("mesh", "activation", "seam", "baseline") +# DeepSeek-V2-Lite is ~15.7B params at 2 bytes each; metadata only, nothing downloaded. +_SAFETENSORS_BF16_ARTIFACT_GB = 31.4 + + +@dataclass(frozen=True) +class LaneSample: + """Raw single-stream measurements one backend produces for a lane.""" + + ttft_ms: float + prefill_tok_per_sec: float + decode_tok_per_sec: float + rss_bytes: int + vram_bytes: int + artifact_bytes: int + output_tokens: tuple[str, ...] + failure_count: int = 0 + + +def _gb(value: float) -> int: + return int(value * 1024**3) + + +class StubLaneBackend: + """Deterministic placeholder measurements until real lane execution lands. + + The numbers are synthetic but directionally shaped — the Q2_K GGUF loads a + far smaller artifact and decodes faster than BF16 safetensors — so the + comparison and stop-condition plumbing can be exercised in CI. + """ + + source = "stub-backend" + + # (runtime, device) -> (ttft_ms, prefill tok/s, decode tok/s, rss GB, vram GB) + _PROFILES = { + ("transformers", "cpu"): (1800.0, 45.0, 6.0, 33.0, 0.0), + ("llama.cpp", "cpu"): (950.0, 90.0, 14.0, 7.1, 0.0), + ("transformers", "gpu"): (420.0, 850.0, 34.0, 4.0, 33.0), + ("llama.cpp", "gpu"): (260.0, 640.0, 52.0, 1.5, 7.5), + } + + def __init__(self, contract: PerformanceContract) -> None: + self._contract = contract + + def run(self, lane: BenchmarkLane) -> LaneSample: + ttft_ms, prefill, decode, rss_gb, vram_gb = self._PROFILES[(lane.runtime, lane.device)] + artifact_gb = ( + self._contract.model_target.gguf_size_gb + if lane.runtime == "llama.cpp" + else _SAFETENSORS_BF16_ARTIFACT_GB + ) + return LaneSample( + ttft_ms=ttft_ms, + prefill_tok_per_sec=prefill, + decode_tok_per_sec=decode, + rss_bytes=_gb(rss_gb), + vram_bytes=_gb(vram_gb), + artifact_bytes=_gb(artifact_gb), + output_tokens=STUB_OUTPUT_TOKENS, + ) + + +def _output_drift(tokens: tuple[str, ...], reference: tuple[str, ...]) -> float: + """Fraction of positions where a lane's output diverges from its reference.""" + length = max(len(tokens), len(reference)) + if length == 0: + return 0.0 + mismatches = sum(a != b for a, b in zip(tokens, reference)) + abs(len(tokens) - len(reference)) + return round(mismatches / length, 4) + + +def _metrics_for(sample: LaneSample, concurrency: int, output_drift: float) -> dict: + # Stub concurrency model: batching scales throughput at 85% efficiency and + # stretches per-request token latency and TTFT accordingly. + efficiency = 1.0 if concurrency == 1 else 0.85 + p50_latency_ms = round(1000.0 / (sample.decode_tok_per_sec * efficiency), 4) + return { + "ttft_ms": round(sample.ttft_ms * (1 + 0.1 * (concurrency - 1)), 4), + "prefill_tok_per_sec": round(sample.prefill_tok_per_sec * efficiency, 4), + "decode_tok_per_sec": round(sample.decode_tok_per_sec * efficiency, 4), + "p50_latency_ms": p50_latency_ms, + "p95_latency_ms": round(p50_latency_ms * 1.25, 4), + "aggregate_throughput_tok_per_sec": round(sample.decode_tok_per_sec * concurrency * efficiency, 4), + "rss_bytes": sample.rss_bytes, + "vram_bytes": sample.vram_bytes, + "artifact_bytes": sample.artifact_bytes, + "failure_count": sample.failure_count, + "output_drift": output_drift, + } + + +def _compare_device(lanes: list[tuple[BenchmarkLane, LaneSample]], device: str) -> dict: + by_runtime = {lane.runtime: (lane, sample) for lane, sample in lanes if lane.device == device} + safetensors_lane, safetensors = by_runtime["transformers"] + gguf_lane, gguf = by_runtime["llama.cpp"] + memory_metric = "vram_bytes" if device == "gpu" else "rss_bytes" + decode_speedup = round(gguf.decode_tok_per_sec / safetensors.decode_tok_per_sec, 4) + artifact_bytes_ratio = round(gguf.artifact_bytes / max(1, safetensors.artifact_bytes), 4) + return { + "safetensors_lane": safetensors_lane.id, + "gguf_lane": gguf_lane.id, + "decode_speedup": decode_speedup, + "ttft_speedup": round(safetensors.ttft_ms / max(0.001, gguf.ttft_ms), 4), + "artifact_bytes_ratio": artifact_bytes_ratio, + "memory_metric": memory_metric, + "memory_bytes_ratio": round( + getattr(gguf, memory_metric) / max(1, getattr(safetensors, memory_metric)), 4 + ), + "output_drift": _output_drift(gguf.output_tokens, safetensors.output_tokens), + "gguf_benefit": decode_speedup >= 1.10 or artifact_bytes_ratio <= 0.5, + } + + +def run_performance_benchmark( + contract: PerformanceContract = DEFAULT_CONTRACT, + backend: StubLaneBackend | None = None, +) -> dict: + """Run every contract lane through a backend and compare GGUF to safetensors.""" + backend = backend if backend is not None else StubLaneBackend(contract) + lanes = [(lane, backend.run(lane)) for lane in contract.benchmark_lanes] + references = { + lane.device: sample.output_tokens for lane, sample in lanes if lane.runtime == "transformers" + } + lane_reports = [] + for lane, sample in lanes: + drift = _output_drift(sample.output_tokens, references.get(lane.device, sample.output_tokens)) + lane_reports.append({ + **lane.to_dict(), + "output_tokens": list(sample.output_tokens), + "results": [ + {"concurrency": level, "metrics": _metrics_for(sample, level, drift)} + for level in lane.concurrency_levels + ], + }) + devices = sorted({lane.device for lane, _ in lanes}) + comparisons = {device: _compare_device(lanes, device) for device in devices} + gguf_benefit = any(comparison["gguf_benefit"] for comparison in comparisons.values()) + return { + "schema_version": BENCHMARK_SCHEMA_VERSION, + "story_id": contract.story_id, + "source": getattr(backend, "source", "custom-backend"), + "model_target": contract.model_target.to_dict(), + "lanes": lane_reports, + "comparisons": comparisons, + "stop_condition": { + "text": contract.stop_condition, + "gguf_benefit": gguf_benefit, + "triggered": not gguf_benefit, + }, + } + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON") parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path") + parser.add_argument( + "--benchmark-out", + type=Path, + default=None, + help="also run the deterministic stub benchmark and write its JSON report here", + ) args = parser.parse_args(argv) contract = build_default_contract() path = contract.write_json(args.json_out) print(path) + if args.benchmark_out is not None: + report = run_performance_benchmark(contract) + args.benchmark_out.parent.mkdir(parents=True, exist_ok=True) + args.benchmark_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(args.benchmark_out) return 0 diff --git a/tests/test_performance_contract.py b/tests/test_performance_contract.py index 41f7e79..8267227 100644 --- a/tests/test_performance_contract.py +++ b/tests/test_performance_contract.py @@ -4,7 +4,13 @@ from __future__ import annotations import json -from meshnet_node.performance_contract import DEFAULT_CONTRACT, SCHEMA_VERSION, main +from meshnet_node.performance_contract import ( + BENCHMARK_SCHEMA_VERSION, + DEFAULT_CONTRACT, + SCHEMA_VERSION, + main, + run_performance_benchmark, +) def test_default_contract_is_architecture_aligned_and_small(): @@ -82,3 +88,80 @@ def test_contract_cli_writes_json(tmp_path, capsys): assert written == DEFAULT_CONTRACT.to_dict() assert str(output) in capsys.readouterr().out + + +def test_stub_benchmark_covers_every_lane_concurrency_and_metric(): + """The runner exercises all four CPU/GPU lanes with the full metric set. + + Tags: performance, benchmark, gguf + """ + report = run_performance_benchmark() + + assert report["schema_version"] == BENCHMARK_SCHEMA_VERSION + assert report["story_id"] == "DGR-001" + assert report["source"] == "stub-backend" + assert report["model_target"] == DEFAULT_CONTRACT.model_target.to_dict() + assert [lane["id"] for lane in report["lanes"]] == [ + lane.id for lane in DEFAULT_CONTRACT.benchmark_lanes + ] + for lane in report["lanes"]: + assert [result["concurrency"] for result in lane["results"]] == [1, 4] + for result in lane["results"]: + assert set(result["metrics"]) == set(DEFAULT_CONTRACT.metrics) + assert result["metrics"]["failure_count"] == 0 + assert result["metrics"]["decode_tok_per_sec"] > 0 + + +def test_stub_benchmark_is_deterministic(): + """Two runs produce byte-identical reports; no clocks or randomness leak in. + + Tags: performance, benchmark, deterministic + """ + first = run_performance_benchmark() + second = run_performance_benchmark() + + assert first == second + assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True) + + +def test_stub_benchmark_compares_gguf_against_safetensors_per_device(): + """Each device gets a GGUF-vs-safetensors comparison and a stop-condition verdict. + + Tags: performance, benchmark, gguf + """ + report = run_performance_benchmark() + + assert set(report["comparisons"]) == {"cpu", "gpu"} + cpu, gpu = report["comparisons"]["cpu"], report["comparisons"]["gpu"] + assert cpu["safetensors_lane"] == "transformers-safetensors-cpu" + assert cpu["gguf_lane"] == "llama-cpp-gguf-cpu" + assert cpu["memory_metric"] == "rss_bytes" + assert gpu["safetensors_lane"] == "transformers-safetensors-gpu" + assert gpu["gguf_lane"] == "llama-cpp-gguf-gpu" + assert gpu["memory_metric"] == "vram_bytes" + for comparison in (cpu, gpu): + assert comparison["decode_speedup"] > 1.0 + assert comparison["artifact_bytes_ratio"] < 0.5 + assert comparison["memory_bytes_ratio"] < 1.0 + assert comparison["output_drift"] == 0.0 + assert comparison["gguf_benefit"] is True + assert report["stop_condition"]["gguf_benefit"] is True + assert report["stop_condition"]["triggered"] is False + assert report["stop_condition"]["text"] == DEFAULT_CONTRACT.stop_condition + + +def test_contract_cli_writes_benchmark_report(tmp_path, capsys): + """--benchmark-out emits the stub benchmark report next to the contract. + + Tags: performance, benchmark, artifact + """ + contract_out = tmp_path / "performance-contract.json" + benchmark_out = tmp_path / "artifacts" / "stub-benchmark-report.json" + + assert main(["--json-out", str(contract_out), "--benchmark-out", str(benchmark_out)]) == 0 + report = json.loads(benchmark_out.read_text(encoding="utf-8")) + + assert report == run_performance_benchmark() + output = capsys.readouterr().out + assert str(contract_out) in output + assert str(benchmark_out) in output