From c7554ef7d836f1010f5b9d182fe37e3da511285c Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 18:13:54 +0300 Subject: [PATCH] feat: add DGR-001 performance contract --- .../evidence/DGR-001/README.md | 66 +++++++ .../DGR-001/performance-contract.json | 51 ++++++ ...ensors-versus-gguf-performance-contract.md | 4 + .../node/meshnet_node/performance_contract.py | 161 ++++++++++++++++++ tests/test_performance_contract.py | 63 +++++++ 5 files changed, 345 insertions(+) create mode 100644 .scratch/distributed-gguf-runtime/evidence/DGR-001/README.md create mode 100644 .scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json create mode 100644 packages/node/meshnet_node/performance_contract.py create mode 100644 tests/test_performance_contract.py diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md new file mode 100644 index 0000000..38f8689 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -0,0 +1,66 @@ +# DGR-001 — performance contract baseline + +## Files changed + +- `packages/node/meshnet_node/performance_contract.py` +- `tests/test_performance_contract.py` +- `.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md` +- `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json` + +## What this slice does + +- Locks the DGR-001 benchmark contract in code. +- Pins the architecture-aligned baseline to **DeepSeek-V2-Lite-Chat** (`deepseek2`). +- Uses the smallest DeepSeek-family GGUF target selected for this story: **Q2_K** via `second-state/DeepSeek-V2-Lite-Chat-GGUF`. +- Exposes a machine-readable JSON contract with: + - benchmark lanes for `transformers` safetensors and `llama.cpp` GGUF + - concurrency levels `1` and `4` + - the required metrics list + - an explicit stop condition for “no meaningful speed or fit benefit” + +## Exact commands and real results + +### Targeted tests + +```bash +pytest -q tests/test_performance_contract.py tests/test_route_session_benchmark.py +``` + +Result: `9 passed in 0.14s` + +### Contract artifact generation + +```bash +PYTHONPATH=packages/node python -m meshnet_node.performance_contract --json-out .scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json +``` + +Result: wrote `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json` + +### Python compile check + +```bash +python -m compileall packages/node/meshnet_node/performance_contract.py tests/test_performance_contract.py +``` + +Result: passed + +## Limitations + +- This slice captures the DGR-001 contract and baseline selection only. +- It does **not** download or run a real model yet. +- Real safetensors vs GGUF execution, TTFT/prefill/decode measurements, RSS/VRAM capture, and output-drift comparison are still to be implemented against the contract. + +## Compatibility notes + +- The contract stays on the DeepSeek2 family to remain close to the DeepSeek-V4-Flash end goal. +- A smaller non-DeepSeek model can still be used later for loader-plumbing smoke tests, but it does not replace this baseline. +- Model artifacts must stay on the mounted drive and not under `/home`. + +## Dependent-story handoff + +Next implementation work should attach to this contract and add the live benchmark runner that actually compares: + +1. current Transformers/safetensors recipe +2. whole-model llama.cpp GGUF recipe + +using the same model architecture/revision and the same prompt/context/concurrency settings. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json new file mode 100644 index 0000000..37f412e --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json @@ -0,0 +1,51 @@ +{ + "benchmark_lanes": [ + { + "concurrency_levels": [ + 1, + 4 + ], + "id": "transformers-safetensors", + "recipe": "current safetensors recipe", + "runtime": "transformers" + }, + { + "concurrency_levels": [ + 1, + 4 + ], + "id": "llama-cpp-gguf", + "recipe": "whole-model GGUF recipe", + "runtime": "llama.cpp" + } + ], + "metrics": [ + "ttft_ms", + "prefill_tok_per_sec", + "decode_tok_per_sec", + "p50_latency_ms", + "p95_latency_ms", + "aggregate_throughput_tok_per_sec", + "rss_bytes", + "vram_bytes", + "artifact_bytes", + "failure_count", + "output_drift" + ], + "model_target": { + "architecture": "deepseek2", + "gguf_quant": "Q2_K", + "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", + "gguf_size_gb": 6.43, + "name": "DeepSeek-V2-Lite-Chat", + "rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.", + "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat" + }, + "notes": [ + "Real model execution stays opt-in and must keep model artifacts on the mounted drive.", + "Use the tiny fallback only for loader plumbing smoke tests; it does not replace the architecture-aligned baseline." + ], + "schema_version": 1, + "stop_condition": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.", + "story_id": "DGR-001" +} diff --git a/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md b/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md index f191ccc..569f174 100644 --- a/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md +++ b/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md @@ -13,6 +13,10 @@ Status: ready-for-agent As a runtime engineer, I need a controlled baseline so that GGUF work proceeds from measured speed, memory, and quality rather than reputation. +## Baseline model target + +Use the smallest *DeepSeek-family* GGUF that still points toward DeepSeek-V4-Flash. Current choice: **DeepSeek-V2-Lite GGUF Q2_K** (~6.5GB, `deepseek2` architecture). Reserve smaller non-DeepSeek fallback models only for loader plumbing smoke tests if needed; they do not count as the DGR-001 architecture-aligned baseline. + ## Expected durable outputs - Benchmark harness and deterministic tests diff --git a/packages/node/meshnet_node/performance_contract.py b/packages/node/meshnet_node/performance_contract.py new file mode 100644 index 0000000..fa113a1 --- /dev/null +++ b/packages/node/meshnet_node/performance_contract.py @@ -0,0 +1,161 @@ +"""Versioned performance contract metadata for DGR-001. + +This module intentionally captures the *contract* first: the model family, +architecture alignment, benchmark lanes, and stop condition that later benchmark +runs must satisfy. It does not download or execute a model. +""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path + +SCHEMA_VERSION = 1 +CONTRACT_ID = "DGR-001" +DEFAULT_OUTPUT_PATH = Path(".scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json") + + +@dataclass(frozen=True) +class ModelTarget: + """Architecture-aligned model target for the DGR-001 benchmark contract.""" + + name: str + architecture: str + safetensors_repo: str + gguf_repo: str + gguf_quant: str + gguf_size_gb: float + rationale: str + + def to_dict(self) -> dict: + return { + "name": self.name, + "architecture": self.architecture, + "safetensors_repo": self.safetensors_repo, + "gguf_repo": self.gguf_repo, + "gguf_quant": self.gguf_quant, + "gguf_size_gb": self.gguf_size_gb, + "rationale": self.rationale, + } + + +@dataclass(frozen=True) +class BenchmarkLane: + """One side of the comparison the contract requires.""" + + id: str + runtime: str + recipe: str + concurrency_levels: tuple[int, ...] + + def to_dict(self) -> dict: + return { + "id": self.id, + "runtime": self.runtime, + "recipe": self.recipe, + "concurrency_levels": list(self.concurrency_levels), + } + + +@dataclass(frozen=True) +class PerformanceContract: + """Machine-readable contract for the DGR-001 benchmark story.""" + + schema_version: int + story_id: str + model_target: ModelTarget + benchmark_lanes: tuple[BenchmarkLane, ...] + metrics: tuple[str, ...] + stop_condition: str + notes: tuple[str, ...] = () + + def to_dict(self) -> dict: + return { + "schema_version": self.schema_version, + "story_id": self.story_id, + "model_target": self.model_target.to_dict(), + "benchmark_lanes": [lane.to_dict() for lane in self.benchmark_lanes], + "metrics": list(self.metrics), + "stop_condition": self.stop_condition, + "notes": list(self.notes), + } + + def write_json(self, path: str | Path) -> Path: + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n", encoding="utf-8") + return path + + +DEFAULT_CONTRACT = PerformanceContract( + schema_version=SCHEMA_VERSION, + story_id=CONTRACT_ID, + model_target=ModelTarget( + name="DeepSeek-V2-Lite-Chat", + architecture="deepseek2", + safetensors_repo="deepseek-ai/DeepSeek-V2-Lite-Chat", + gguf_repo="second-state/DeepSeek-V2-Lite-Chat-GGUF", + gguf_quant="Q2_K", + gguf_size_gb=6.43, + rationale=( + "Smallest DeepSeek-family benchmark anchor that still points toward " + "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead " + "of falling back to a tiny but architecture-mismatched smoke model." + ), + ), + benchmark_lanes=( + BenchmarkLane( + id="transformers-safetensors", + runtime="transformers", + recipe="current safetensors recipe", + concurrency_levels=(1, 4), + ), + BenchmarkLane( + id="llama-cpp-gguf", + runtime="llama.cpp", + recipe="whole-model GGUF recipe", + concurrency_levels=(1, 4), + ), + ), + metrics=( + "ttft_ms", + "prefill_tok_per_sec", + "decode_tok_per_sec", + "p50_latency_ms", + "p95_latency_ms", + "aggregate_throughput_tok_per_sec", + "rss_bytes", + "vram_bytes", + "artifact_bytes", + "failure_count", + "output_drift", + ), + stop_condition=( + "Stop if GGUF does not provide a meaningful speed or fit benefit over the " + "safetensors baseline for the chosen DeepSeek-family model target." + ), + notes=( + "Real model execution stays opt-in and must keep model artifacts on the mounted drive.", + "Use the tiny fallback only for loader plumbing smoke tests; it does not replace the architecture-aligned baseline.", + ), +) + + +def build_default_contract() -> PerformanceContract: + return DEFAULT_CONTRACT + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON") + parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path") + args = parser.parse_args(argv) + contract = build_default_contract() + path = contract.write_json(args.json_out) + print(path) + return 0 + + +if __name__ == "__main__": # pragma: no cover - CLI entry point + raise SystemExit(main()) diff --git a/tests/test_performance_contract.py b/tests/test_performance_contract.py new file mode 100644 index 0000000..8bd6e90 --- /dev/null +++ b/tests/test_performance_contract.py @@ -0,0 +1,63 @@ +"""Tests for the DGR-001 performance contract metadata.""" + +from __future__ import annotations + +import json + +from meshnet_node.performance_contract import DEFAULT_CONTRACT, SCHEMA_VERSION, main + + +def test_default_contract_is_architecture_aligned_and_small(): + """The baseline stays on DeepSeek2 and uses the smallest DeepSeek-family GGUF. + + Tags: performance, model, gguf + """ + payload = DEFAULT_CONTRACT.to_dict() + + assert payload["schema_version"] == SCHEMA_VERSION + assert payload["story_id"] == "DGR-001" + assert payload["model_target"] == { + "name": "DeepSeek-V2-Lite-Chat", + "architecture": "deepseek2", + "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat", + "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", + "gguf_quant": "Q2_K", + "gguf_size_gb": 6.43, + "rationale": ( + "Smallest DeepSeek-family benchmark anchor that still points toward " + "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead " + "of falling back to a tiny but architecture-mismatched smoke model." + ), + } + assert payload["benchmark_lanes"] == [ + { + "id": "transformers-safetensors", + "runtime": "transformers", + "recipe": "current safetensors recipe", + "concurrency_levels": [1, 4], + }, + { + "id": "llama-cpp-gguf", + "runtime": "llama.cpp", + "recipe": "whole-model GGUF recipe", + "concurrency_levels": [1, 4], + }, + ] + assert "ttft_ms" in payload["metrics"] + assert "output_drift" in payload["metrics"] + assert "meaningful speed or fit benefit" in payload["stop_condition"] + assert any("mounted drive" in note for note in payload["notes"]) + + +def test_contract_cli_writes_json(tmp_path, capsys): + """The contract can be emitted as a machine-readable artifact. + + Tags: performance, artifact + """ + output = tmp_path / "performance-contract.json" + + assert main(["--json-out", str(output)]) == 0 + written = json.loads(output.read_text(encoding="utf-8")) + + assert written == DEFAULT_CONTRACT.to_dict() + assert str(output) in capsys.readouterr().out