feat: add DGR-001 performance contract
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"benchmark_lanes": [
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"id": "transformers-safetensors",
|
||||
"recipe": "current safetensors recipe",
|
||||
"runtime": "transformers"
|
||||
},
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"id": "llama-cpp-gguf",
|
||||
"recipe": "whole-model GGUF recipe",
|
||||
"runtime": "llama.cpp"
|
||||
}
|
||||
],
|
||||
"metrics": [
|
||||
"ttft_ms",
|
||||
"prefill_tok_per_sec",
|
||||
"decode_tok_per_sec",
|
||||
"p50_latency_ms",
|
||||
"p95_latency_ms",
|
||||
"aggregate_throughput_tok_per_sec",
|
||||
"rss_bytes",
|
||||
"vram_bytes",
|
||||
"artifact_bytes",
|
||||
"failure_count",
|
||||
"output_drift"
|
||||
],
|
||||
"model_target": {
|
||||
"architecture": "deepseek2",
|
||||
"gguf_quant": "Q2_K",
|
||||
"gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
|
||||
"gguf_size_gb": 6.43,
|
||||
"name": "DeepSeek-V2-Lite-Chat",
|
||||
"rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.",
|
||||
"safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
},
|
||||
"notes": [
|
||||
"Real model execution stays opt-in and must keep model artifacts on the mounted drive.",
|
||||
"Use the tiny fallback only for loader plumbing smoke tests; it does not replace the architecture-aligned baseline."
|
||||
],
|
||||
"schema_version": 1,
|
||||
"stop_condition": "Stop if GGUF does not provide a meaningful speed or fit benefit over the safetensors baseline for the chosen DeepSeek-family model target.",
|
||||
"story_id": "DGR-001"
|
||||
}
|
||||
Reference in New Issue
Block a user