popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add deterministic CPU/GPU benchmark runner slice

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-14 21:39:13 +03:00
parent 5b33bf8b99
commit e6f6782995
4 changed files with 517 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md
View File

@@ -19,6 +19,21 @@
- concurrency levels `1` and `4`
- the required metrics list
- an explicit stop condition for “no meaningful speed or fit benefit”
- Adds a deterministic stub benchmark report so the contract now has an executable report shape end to end.
## Recent benchmark runner slice
The runner currently uses a deterministic stub backend to exercise the comparison matrix without downloading a model. It emits:
- `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json`
- `.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json`
The report includes per-device comparisons for:
- `transformers-safetensors-cpu` vs `llama-cpp-gguf-cpu`
- `transformers-safetensors-gpu` vs `llama-cpp-gguf-gpu`
and records the memory metric (`rss_bytes` on CPU, `vram_bytes` on GPU), decode speedup, artifact ratio, and output drift.
## Exact commands and real results

247
.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json Normal file
View File

@@ -0,0 +1,247 @@
{
"comparisons": {
"cpu": {
"artifact_bytes_ratio": 0.2048,
"decode_speedup": 2.3333,
"gguf_benefit": true,
"gguf_lane": "llama-cpp-gguf-cpu",
"memory_bytes_ratio": 0.2152,
"memory_metric": "rss_bytes",
"output_drift": 0.0,
"safetensors_lane": "transformers-safetensors-cpu",
"ttft_speedup": 1.8947
},
"gpu": {
"artifact_bytes_ratio": 0.2048,
"decode_speedup": 1.5294,
"gguf_benefit": true,
"gguf_lane": "llama-cpp-gguf-gpu",
"memory_bytes_ratio": 0.2273,
"memory_metric": "vram_bytes",
"output_drift": 0.0,
"safetensors_lane": "transformers-safetensors-gpu",
"ttft_speedup": 1.6154
}
},
"lanes": [
{
"concurrency_levels": [
1,
4
],
"device": "cpu",
"id": "transformers-safetensors-cpu",
"output_tokens": [
"mesh",
"activation",
"seam",
"baseline"
],
"recipe": "current safetensors recipe",
"results": [
{
"concurrency": 1,
"metrics": {
"aggregate_throughput_tok_per_sec": 6.0,
"artifact_bytes": 33715493273,
"decode_tok_per_sec": 6.0,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 166.6667,
"p95_latency_ms": 208.3334,
"prefill_tok_per_sec": 45.0,
"rss_bytes": 35433480192,
"ttft_ms": 1800.0,
"vram_bytes": 0
}
},
{
"concurrency": 4,
"metrics": {
"aggregate_throughput_tok_per_sec": 20.4,
"artifact_bytes": 33715493273,
"decode_tok_per_sec": 5.1,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 196.0784,
"p95_latency_ms": 245.098,
"prefill_tok_per_sec": 38.25,
"rss_bytes": 35433480192,
"ttft_ms": 2340.0,
"vram_bytes": 0
}
}
],
"runtime": "transformers"
},
{
"concurrency_levels": [
1,
4
],
"device": "cpu",
"id": "llama-cpp-gguf-cpu",
"output_tokens": [
"mesh",
"activation",
"seam",
"baseline"
],
"recipe": "whole-model GGUF recipe",
"results": [
{
"concurrency": 1,
"metrics": {
"aggregate_throughput_tok_per_sec": 14.0,
"artifact_bytes": 6904159928,
"decode_tok_per_sec": 14.0,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 71.4286,
"p95_latency_ms": 89.2858,
"prefill_tok_per_sec": 90.0,
"rss_bytes": 7623566950,
"ttft_ms": 950.0,
"vram_bytes": 0
}
},
{
"concurrency": 4,
"metrics": {
"aggregate_throughput_tok_per_sec": 47.6,
"artifact_bytes": 6904159928,
"decode_tok_per_sec": 11.9,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 84.0336,
"p95_latency_ms": 105.042,
"prefill_tok_per_sec": 76.5,
"rss_bytes": 7623566950,
"ttft_ms": 1235.0,
"vram_bytes": 0
}
}
],
"runtime": "llama.cpp"
},
{
"concurrency_levels": [
1,
4
],
"device": "gpu",
"id": "transformers-safetensors-gpu",
"output_tokens": [
"mesh",
"activation",
"seam",
"baseline"
],
"recipe": "current safetensors recipe",
"results": [
{
"concurrency": 1,
"metrics": {
"aggregate_throughput_tok_per_sec": 34.0,
"artifact_bytes": 33715493273,
"decode_tok_per_sec": 34.0,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 29.4118,
"p95_latency_ms": 36.7647,
"prefill_tok_per_sec": 850.0,
"rss_bytes": 4294967296,
"ttft_ms": 420.0,
"vram_bytes": 35433480192
}
},
{
"concurrency": 4,
"metrics": {
"aggregate_throughput_tok_per_sec": 115.6,
"artifact_bytes": 33715493273,
"decode_tok_per_sec": 28.9,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 34.6021,
"p95_latency_ms": 43.2526,
"prefill_tok_per_sec": 722.5,
"rss_bytes": 4294967296,
"ttft_ms": 546.0,
"vram_bytes": 35433480192
}
}
],
"runtime": "transformers"
},
{
"concurrency_levels": [
1,
4
],
"device": "gpu",
"id": "llama-cpp-gguf-gpu",
"output_tokens": [
"mesh",
"activation",
"seam",
"baseline"
],
"recipe": "whole-model GGUF recipe",
"results": [
{
"concurrency": 1,
"metrics": {
"aggregate_throughput_tok_per_sec": 52.0,
"artifact_bytes": 6904159928,
"decode_tok_per_sec": 52.0,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 19.2308,
"p95_latency_ms": 24.0385,
"prefill_tok_per_sec": 640.0,
"rss_bytes": 1610612736,
"ttft_ms": 260.0,
"vram_bytes": 8053063680
}
},
{
"concurrency": 4,
"metrics": {
"aggregate_throughput_tok_per_sec": 176.8,
"artifact_bytes": 6904159928,
"decode_tok_per_sec": 44.2,
"failure_count": 0,
"output_drift": 0.0,
"p50_latency_ms": 22.6244,
"p95_latency_ms": 28.2805,
"prefill_tok_per_sec": 544.0,
"rss_bytes": 1610612736,
"ttft_ms": 338.0,
"vram_bytes": 8053063680
}
}
],
"runtime": "llama.cpp"
}
],
"model_target": {
"architecture": "deepseek2",
"comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF",
"gguf_quant": "Q2_K",
"gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
"gguf_size_gb": 6.43,
"name": "DeepSeek-V2-Lite-Chat",
"rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.",
"safetensors_precision": "bfloat16",
"safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat"
},
"schema_version": 1,
"source": "stub-backend",
"stop_condition": {
"gguf_benefit": true,
"text": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.",
"triggered": false
},
"story_id": "DGR-001"
}

175
packages/node/meshnet_node/performance_contract.py
View File

@@ -1,8 +1,11 @@
"""Versioned performance contract metadata for DGR-001.
"""Versioned performance contract metadata and stub benchmark runner for DGR-001.
This module intentionally captures the *contract* first: the model family,
architecture alignment, benchmark lanes, and stop condition that later benchmark
runs must satisfy. It does not download or execute a model.
This module captures the *contract* first: the model family, architecture
alignment, benchmark lanes, and stop condition that benchmark runs must
satisfy. It also runs the contract's lanes through a deterministic stub
backend so the report data shape exists end to end. It never downloads or
executes a model; real transformers / llama.cpp backends plug in behind the
same ``run()`` seam later.
"""
from __future__ import annotations
@@ -174,13 +177,177 @@ def build_default_contract() -> PerformanceContract:
return DEFAULT_CONTRACT
BENCHMARK_SCHEMA_VERSION = 1
STUB_OUTPUT_TOKENS = ("mesh", "activation", "seam", "baseline")
# DeepSeek-V2-Lite is ~15.7B params at 2 bytes each; metadata only, nothing downloaded.
_SAFETENSORS_BF16_ARTIFACT_GB = 31.4
@dataclass(frozen=True)
class LaneSample:
"""Raw single-stream measurements one backend produces for a lane."""
ttft_ms: float
prefill_tok_per_sec: float
decode_tok_per_sec: float
rss_bytes: int
vram_bytes: int
artifact_bytes: int
output_tokens: tuple[str, ...]
failure_count: int = 0
def _gb(value: float) -> int:
return int(value * 1024**3)
class StubLaneBackend:
"""Deterministic placeholder measurements until real lane execution lands.
The numbers are synthetic but directionally shaped — the Q2_K GGUF loads a
far smaller artifact and decodes faster than BF16 safetensors — so the
comparison and stop-condition plumbing can be exercised in CI.
"""
source = "stub-backend"
# (runtime, device) -> (ttft_ms, prefill tok/s, decode tok/s, rss GB, vram GB)
_PROFILES = {
("transformers", "cpu"): (1800.0, 45.0, 6.0, 33.0, 0.0),
("llama.cpp", "cpu"): (950.0, 90.0, 14.0, 7.1, 0.0),
("transformers", "gpu"): (420.0, 850.0, 34.0, 4.0, 33.0),
("llama.cpp", "gpu"): (260.0, 640.0, 52.0, 1.5, 7.5),
}
def __init__(self, contract: PerformanceContract) -> None:
self._contract = contract
def run(self, lane: BenchmarkLane) -> LaneSample:
ttft_ms, prefill, decode, rss_gb, vram_gb = self._PROFILES[(lane.runtime, lane.device)]
artifact_gb = (
self._contract.model_target.gguf_size_gb
if lane.runtime == "llama.cpp"
else _SAFETENSORS_BF16_ARTIFACT_GB
)
return LaneSample(
ttft_ms=ttft_ms,
prefill_tok_per_sec=prefill,
decode_tok_per_sec=decode,
rss_bytes=_gb(rss_gb),
vram_bytes=_gb(vram_gb),
artifact_bytes=_gb(artifact_gb),
output_tokens=STUB_OUTPUT_TOKENS,
)
def _output_drift(tokens: tuple[str, ...], reference: tuple[str, ...]) -> float:
"""Fraction of positions where a lane's output diverges from its reference."""
length = max(len(tokens), len(reference))
if length == 0:
return 0.0
mismatches = sum(a != b for a, b in zip(tokens, reference)) + abs(len(tokens) - len(reference))
return round(mismatches / length, 4)
def _metrics_for(sample: LaneSample, concurrency: int, output_drift: float) -> dict:
# Stub concurrency model: batching scales throughput at 85% efficiency and
# stretches per-request token latency and TTFT accordingly.
efficiency = 1.0 if concurrency == 1 else 0.85
p50_latency_ms = round(1000.0 / (sample.decode_tok_per_sec * efficiency), 4)
return {
"ttft_ms": round(sample.ttft_ms * (1 + 0.1 * (concurrency - 1)), 4),
"prefill_tok_per_sec": round(sample.prefill_tok_per_sec * efficiency, 4),
"decode_tok_per_sec": round(sample.decode_tok_per_sec * efficiency, 4),
"p50_latency_ms": p50_latency_ms,
"p95_latency_ms": round(p50_latency_ms * 1.25, 4),
"aggregate_throughput_tok_per_sec": round(sample.decode_tok_per_sec * concurrency * efficiency, 4),
"rss_bytes": sample.rss_bytes,
"vram_bytes": sample.vram_bytes,
"artifact_bytes": sample.artifact_bytes,
"failure_count": sample.failure_count,
"output_drift": output_drift,
}
def _compare_device(lanes: list[tuple[BenchmarkLane, LaneSample]], device: str) -> dict:
by_runtime = {lane.runtime: (lane, sample) for lane, sample in lanes if lane.device == device}
safetensors_lane, safetensors = by_runtime["transformers"]
gguf_lane, gguf = by_runtime["llama.cpp"]
memory_metric = "vram_bytes" if device == "gpu" else "rss_bytes"
decode_speedup = round(gguf.decode_tok_per_sec / safetensors.decode_tok_per_sec, 4)
artifact_bytes_ratio = round(gguf.artifact_bytes / max(1, safetensors.artifact_bytes), 4)
return {
"safetensors_lane": safetensors_lane.id,
"gguf_lane": gguf_lane.id,
"decode_speedup": decode_speedup,
"ttft_speedup": round(safetensors.ttft_ms / max(0.001, gguf.ttft_ms), 4),
"artifact_bytes_ratio": artifact_bytes_ratio,
"memory_metric": memory_metric,
"memory_bytes_ratio": round(
getattr(gguf, memory_metric) / max(1, getattr(safetensors, memory_metric)), 4
),
"output_drift": _output_drift(gguf.output_tokens, safetensors.output_tokens),
"gguf_benefit": decode_speedup >= 1.10 or artifact_bytes_ratio <= 0.5,
}
def run_performance_benchmark(
contract: PerformanceContract = DEFAULT_CONTRACT,
backend: StubLaneBackend | None = None,
) -> dict:
"""Run every contract lane through a backend and compare GGUF to safetensors."""
backend = backend if backend is not None else StubLaneBackend(contract)
lanes = [(lane, backend.run(lane)) for lane in contract.benchmark_lanes]
references = {
lane.device: sample.output_tokens for lane, sample in lanes if lane.runtime == "transformers"
}
lane_reports = []
for lane, sample in lanes:
drift = _output_drift(sample.output_tokens, references.get(lane.device, sample.output_tokens))
lane_reports.append({
**lane.to_dict(),
"output_tokens": list(sample.output_tokens),
"results": [
{"concurrency": level, "metrics": _metrics_for(sample, level, drift)}
for level in lane.concurrency_levels
],
})
devices = sorted({lane.device for lane, _ in lanes})
comparisons = {device: _compare_device(lanes, device) for device in devices}
gguf_benefit = any(comparison["gguf_benefit"] for comparison in comparisons.values())
return {
"schema_version": BENCHMARK_SCHEMA_VERSION,
"story_id": contract.story_id,
"source": getattr(backend, "source", "custom-backend"),
"model_target": contract.model_target.to_dict(),
"lanes": lane_reports,
"comparisons": comparisons,
"stop_condition": {
"text": contract.stop_condition,
"gguf_benefit": gguf_benefit,
"triggered": not gguf_benefit,
},
}
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON")
parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path")
parser.add_argument(
"--benchmark-out",
type=Path,
default=None,
help="also run the deterministic stub benchmark and write its JSON report here",
)
args = parser.parse_args(argv)
contract = build_default_contract()
path = contract.write_json(args.json_out)
print(path)
if args.benchmark_out is not None:
report = run_performance_benchmark(contract)
args.benchmark_out.parent.mkdir(parents=True, exist_ok=True)
args.benchmark_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(args.benchmark_out)
return 0

85
tests/test_performance_contract.py
View File

@@ -4,7 +4,13 @@ from __future__ import annotations
import json
from meshnet_node.performance_contract import DEFAULT_CONTRACT, SCHEMA_VERSION, main
from meshnet_node.performance_contract import (
BENCHMARK_SCHEMA_VERSION,
DEFAULT_CONTRACT,
SCHEMA_VERSION,
main,
run_performance_benchmark,
)
def test_default_contract_is_architecture_aligned_and_small():
@@ -82,3 +88,80 @@ def test_contract_cli_writes_json(tmp_path, capsys):
assert written == DEFAULT_CONTRACT.to_dict()
assert str(output) in capsys.readouterr().out
def test_stub_benchmark_covers_every_lane_concurrency_and_metric():
"""The runner exercises all four CPU/GPU lanes with the full metric set.
Tags: performance, benchmark, gguf
"""
report = run_performance_benchmark()
assert report["schema_version"] == BENCHMARK_SCHEMA_VERSION
assert report["story_id"] == "DGR-001"
assert report["source"] == "stub-backend"
assert report["model_target"] == DEFAULT_CONTRACT.model_target.to_dict()
assert [lane["id"] for lane in report["lanes"]] == [
lane.id for lane in DEFAULT_CONTRACT.benchmark_lanes
]
for lane in report["lanes"]:
assert [result["concurrency"] for result in lane["results"]] == [1, 4]
for result in lane["results"]:
assert set(result["metrics"]) == set(DEFAULT_CONTRACT.metrics)
assert result["metrics"]["failure_count"] == 0
assert result["metrics"]["decode_tok_per_sec"] > 0
def test_stub_benchmark_is_deterministic():
"""Two runs produce byte-identical reports; no clocks or randomness leak in.
Tags: performance, benchmark, deterministic
"""
first = run_performance_benchmark()
second = run_performance_benchmark()
assert first == second
assert json.dumps(first, sort_keys=True) == json.dumps(second, sort_keys=True)
def test_stub_benchmark_compares_gguf_against_safetensors_per_device():
"""Each device gets a GGUF-vs-safetensors comparison and a stop-condition verdict.
Tags: performance, benchmark, gguf
"""
report = run_performance_benchmark()
assert set(report["comparisons"]) == {"cpu", "gpu"}
cpu, gpu = report["comparisons"]["cpu"], report["comparisons"]["gpu"]
assert cpu["safetensors_lane"] == "transformers-safetensors-cpu"
assert cpu["gguf_lane"] == "llama-cpp-gguf-cpu"
assert cpu["memory_metric"] == "rss_bytes"
assert gpu["safetensors_lane"] == "transformers-safetensors-gpu"
assert gpu["gguf_lane"] == "llama-cpp-gguf-gpu"
assert gpu["memory_metric"] == "vram_bytes"
for comparison in (cpu, gpu):
assert comparison["decode_speedup"] > 1.0
assert comparison["artifact_bytes_ratio"] < 0.5
assert comparison["memory_bytes_ratio"] < 1.0
assert comparison["output_drift"] == 0.0
assert comparison["gguf_benefit"] is True
assert report["stop_condition"]["gguf_benefit"] is True
assert report["stop_condition"]["triggered"] is False
assert report["stop_condition"]["text"] == DEFAULT_CONTRACT.stop_condition
def test_contract_cli_writes_benchmark_report(tmp_path, capsys):
"""--benchmark-out emits the stub benchmark report next to the contract.
Tags: performance, benchmark, artifact
"""
contract_out = tmp_path / "performance-contract.json"
benchmark_out = tmp_path / "artifacts" / "stub-benchmark-report.json"
assert main(["--json-out", str(contract_out), "--benchmark-out", str(benchmark_out)]) == 0
report = json.loads(benchmark_out.read_text(encoding="utf-8"))
assert report == run_performance_benchmark()
output = capsys.readouterr().out
assert str(contract_out) in output
assert str(benchmark_out) in output