feat: add deterministic CPU/GPU benchmark runner slice
This commit is contained in:
Dobromir Popov
@@ -19,6 +19,21 @@
|
||||
- concurrency levels `1` and `4`
|
||||
- the required metrics list
|
||||
- an explicit stop condition for “no meaningful speed or fit benefit”
|
||||
- Adds a deterministic stub benchmark report so the contract now has an executable report shape end to end.
|
||||
|
||||
## Recent benchmark runner slice
|
||||
|
||||
The runner currently uses a deterministic stub backend to exercise the comparison matrix without downloading a model. It emits:
|
||||
|
||||
- `.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json`
|
||||
- `.scratch/distributed-gguf-runtime/evidence/DGR-001/stub-benchmark-report.json`
|
||||
|
||||
The report includes per-device comparisons for:
|
||||
|
||||
- `transformers-safetensors-cpu` vs `llama-cpp-gguf-cpu`
|
||||
- `transformers-safetensors-gpu` vs `llama-cpp-gguf-gpu`
|
||||
|
||||
and records the memory metric (`rss_bytes` on CPU, `vram_bytes` on GPU), decode speedup, artifact ratio, and output drift.
|
||||
|
||||
## Exact commands and real results
|
||||
|
||||
|
||||
@@ -0,0 +1,247 @@
|
||||
{
|
||||
"comparisons": {
|
||||
"cpu": {
|
||||
"artifact_bytes_ratio": 0.2048,
|
||||
"decode_speedup": 2.3333,
|
||||
"gguf_benefit": true,
|
||||
"gguf_lane": "llama-cpp-gguf-cpu",
|
||||
"memory_bytes_ratio": 0.2152,
|
||||
"memory_metric": "rss_bytes",
|
||||
"output_drift": 0.0,
|
||||
"safetensors_lane": "transformers-safetensors-cpu",
|
||||
"ttft_speedup": 1.8947
|
||||
},
|
||||
"gpu": {
|
||||
"artifact_bytes_ratio": 0.2048,
|
||||
"decode_speedup": 1.5294,
|
||||
"gguf_benefit": true,
|
||||
"gguf_lane": "llama-cpp-gguf-gpu",
|
||||
"memory_bytes_ratio": 0.2273,
|
||||
"memory_metric": "vram_bytes",
|
||||
"output_drift": 0.0,
|
||||
"safetensors_lane": "transformers-safetensors-gpu",
|
||||
"ttft_speedup": 1.6154
|
||||
}
|
||||
},
|
||||
"lanes": [
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"device": "cpu",
|
||||
"id": "transformers-safetensors-cpu",
|
||||
"output_tokens": [
|
||||
"mesh",
|
||||
"activation",
|
||||
"seam",
|
||||
"baseline"
|
||||
],
|
||||
"recipe": "current safetensors recipe",
|
||||
"results": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 6.0,
|
||||
"artifact_bytes": 33715493273,
|
||||
"decode_tok_per_sec": 6.0,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 166.6667,
|
||||
"p95_latency_ms": 208.3334,
|
||||
"prefill_tok_per_sec": 45.0,
|
||||
"rss_bytes": 35433480192,
|
||||
"ttft_ms": 1800.0,
|
||||
"vram_bytes": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 20.4,
|
||||
"artifact_bytes": 33715493273,
|
||||
"decode_tok_per_sec": 5.1,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 196.0784,
|
||||
"p95_latency_ms": 245.098,
|
||||
"prefill_tok_per_sec": 38.25,
|
||||
"rss_bytes": 35433480192,
|
||||
"ttft_ms": 2340.0,
|
||||
"vram_bytes": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"runtime": "transformers"
|
||||
},
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"device": "cpu",
|
||||
"id": "llama-cpp-gguf-cpu",
|
||||
"output_tokens": [
|
||||
"mesh",
|
||||
"activation",
|
||||
"seam",
|
||||
"baseline"
|
||||
],
|
||||
"recipe": "whole-model GGUF recipe",
|
||||
"results": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 14.0,
|
||||
"artifact_bytes": 6904159928,
|
||||
"decode_tok_per_sec": 14.0,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 71.4286,
|
||||
"p95_latency_ms": 89.2858,
|
||||
"prefill_tok_per_sec": 90.0,
|
||||
"rss_bytes": 7623566950,
|
||||
"ttft_ms": 950.0,
|
||||
"vram_bytes": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 47.6,
|
||||
"artifact_bytes": 6904159928,
|
||||
"decode_tok_per_sec": 11.9,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 84.0336,
|
||||
"p95_latency_ms": 105.042,
|
||||
"prefill_tok_per_sec": 76.5,
|
||||
"rss_bytes": 7623566950,
|
||||
"ttft_ms": 1235.0,
|
||||
"vram_bytes": 0
|
||||
}
|
||||
}
|
||||
],
|
||||
"runtime": "llama.cpp"
|
||||
},
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"device": "gpu",
|
||||
"id": "transformers-safetensors-gpu",
|
||||
"output_tokens": [
|
||||
"mesh",
|
||||
"activation",
|
||||
"seam",
|
||||
"baseline"
|
||||
],
|
||||
"recipe": "current safetensors recipe",
|
||||
"results": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 34.0,
|
||||
"artifact_bytes": 33715493273,
|
||||
"decode_tok_per_sec": 34.0,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 29.4118,
|
||||
"p95_latency_ms": 36.7647,
|
||||
"prefill_tok_per_sec": 850.0,
|
||||
"rss_bytes": 4294967296,
|
||||
"ttft_ms": 420.0,
|
||||
"vram_bytes": 35433480192
|
||||
}
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 115.6,
|
||||
"artifact_bytes": 33715493273,
|
||||
"decode_tok_per_sec": 28.9,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 34.6021,
|
||||
"p95_latency_ms": 43.2526,
|
||||
"prefill_tok_per_sec": 722.5,
|
||||
"rss_bytes": 4294967296,
|
||||
"ttft_ms": 546.0,
|
||||
"vram_bytes": 35433480192
|
||||
}
|
||||
}
|
||||
],
|
||||
"runtime": "transformers"
|
||||
},
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"device": "gpu",
|
||||
"id": "llama-cpp-gguf-gpu",
|
||||
"output_tokens": [
|
||||
"mesh",
|
||||
"activation",
|
||||
"seam",
|
||||
"baseline"
|
||||
],
|
||||
"recipe": "whole-model GGUF recipe",
|
||||
"results": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 52.0,
|
||||
"artifact_bytes": 6904159928,
|
||||
"decode_tok_per_sec": 52.0,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 19.2308,
|
||||
"p95_latency_ms": 24.0385,
|
||||
"prefill_tok_per_sec": 640.0,
|
||||
"rss_bytes": 1610612736,
|
||||
"ttft_ms": 260.0,
|
||||
"vram_bytes": 8053063680
|
||||
}
|
||||
},
|
||||
{
|
||||
"concurrency": 4,
|
||||
"metrics": {
|
||||
"aggregate_throughput_tok_per_sec": 176.8,
|
||||
"artifact_bytes": 6904159928,
|
||||
"decode_tok_per_sec": 44.2,
|
||||
"failure_count": 0,
|
||||
"output_drift": 0.0,
|
||||
"p50_latency_ms": 22.6244,
|
||||
"p95_latency_ms": 28.2805,
|
||||
"prefill_tok_per_sec": 544.0,
|
||||
"rss_bytes": 1610612736,
|
||||
"ttft_ms": 338.0,
|
||||
"vram_bytes": 8053063680
|
||||
}
|
||||
}
|
||||
],
|
||||
"runtime": "llama.cpp"
|
||||
}
|
||||
],
|
||||
"model_target": {
|
||||
"architecture": "deepseek2",
|
||||
"comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF",
|
||||
"gguf_quant": "Q2_K",
|
||||
"gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
|
||||
"gguf_size_gb": 6.43,
|
||||
"name": "DeepSeek-V2-Lite-Chat",
|
||||
"rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.",
|
||||
"safetensors_precision": "bfloat16",
|
||||
"safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
},
|
||||
"schema_version": 1,
|
||||
"source": "stub-backend",
|
||||
"stop_condition": {
|
||||
"gguf_benefit": true,
|
||||
"text": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.",
|
||||
"triggered": false
|
||||
},
|
||||
"story_id": "DGR-001"
|
||||
}
|
||||
Reference in New Issue
Block a user