72 lines
2.3 KiB
JSON
72 lines
2.3 KiB
JSON
{
|
|
"contract_version": 1,
|
|
"fit_benefit": true,
|
|
"plan_id": "dgr-001-controlled-whole-model-baseline-v1",
|
|
"quality_lane_pass": false,
|
|
"rationale": [
|
|
"the near-lossless quality lane failed: the GGUF runtime disagrees with the safetensors reference beyond what near-lossless weights can explain",
|
|
"a meaningful speed benefit was measured",
|
|
"a meaningful fit benefit was measured"
|
|
],
|
|
"recipes": [
|
|
{
|
|
"comparable": true,
|
|
"failures": 0,
|
|
"fit_benefit": false,
|
|
"incomparable_reason": "",
|
|
"lane": "quality",
|
|
"measurements": {
|
|
"aggregate_concurrency": 4,
|
|
"aggregate_throughput_speedup": 4.7873,
|
|
"artifact_size_ratio": 0.9946,
|
|
"artifact_size_win": false,
|
|
"compared_prompts": 3,
|
|
"decode_speedup": 2.4173,
|
|
"exact_match_rate": 0.3333,
|
|
"expected_prompts": 3,
|
|
"failure_rate": 0.0,
|
|
"mean_similarity": 0.9471,
|
|
"resident_memory_ratio": 0.5721,
|
|
"ttft_ratio": 0.3767
|
|
},
|
|
"quality_pass": false,
|
|
"reasons": [
|
|
"single-request decode 2.42x reference (>= 1.25x) at TTFT ratio 0.38",
|
|
"aggregate throughput at concurrency 4 is 4.79x reference (>= 1.25x)",
|
|
"peak resident memory is 0.57x reference (<= 0.75x)",
|
|
"quality lane exact-match 0.33 / similarity 0.947 versus the reference (fail)"
|
|
],
|
|
"recipe_id": "llama-cpp-near-lossless-quality",
|
|
"speed_benefit": false
|
|
},
|
|
{
|
|
"comparable": true,
|
|
"failures": 0,
|
|
"fit_benefit": true,
|
|
"incomparable_reason": "",
|
|
"lane": "performance-fit",
|
|
"measurements": {
|
|
"aggregate_concurrency": 4,
|
|
"aggregate_throughput_speedup": 4.2048,
|
|
"artifact_size_ratio": 0.398,
|
|
"artifact_size_win": true,
|
|
"decode_speedup": 5.0967,
|
|
"failure_rate": 0.0,
|
|
"resident_memory_ratio": 0.2792,
|
|
"ttft_ratio": 0.5395
|
|
},
|
|
"quality_pass": null,
|
|
"reasons": [
|
|
"single-request decode 5.10x reference (>= 1.25x) at TTFT ratio 0.54",
|
|
"aggregate throughput at concurrency 4 is 4.20x reference (>= 1.25x)",
|
|
"peak resident memory is 0.28x reference (<= 0.75x)"
|
|
],
|
|
"recipe_id": "llama-cpp-quantized-performance-fit",
|
|
"speed_benefit": true
|
|
}
|
|
],
|
|
"speed_benefit": true,
|
|
"stop_condition_met": true,
|
|
"verdict": "stop"
|
|
}
|