72 lines
2.3 KiB
JSON
72 lines
2.3 KiB
JSON
{
|
|
"contract_version": 1,
|
|
"fit_benefit": true,
|
|
"plan_id": "dgr-001-controlled-whole-model-baseline-v1",
|
|
"quality_lane_pass": false,
|
|
"rationale": [
|
|
"the near-lossless quality lane failed: the GGUF runtime disagrees with the safetensors reference beyond what near-lossless weights can explain",
|
|
"a meaningful speed benefit was measured",
|
|
"a meaningful fit benefit was measured"
|
|
],
|
|
"recipes": [
|
|
{
|
|
"comparable": true,
|
|
"failures": 0,
|
|
"fit_benefit": false,
|
|
"incomparable_reason": "",
|
|
"lane": "quality",
|
|
"measurements": {
|
|
"aggregate_concurrency": 4,
|
|
"aggregate_throughput_speedup": 4.4882,
|
|
"artifact_size_ratio": 0.9946,
|
|
"artifact_size_win": false,
|
|
"compared_prompts": 3,
|
|
"decode_speedup": 1.9064,
|
|
"exact_match_rate": 0.3333,
|
|
"expected_prompts": 3,
|
|
"failure_rate": 0.0,
|
|
"mean_similarity": 0.9471,
|
|
"resident_memory_ratio": 0.5722,
|
|
"ttft_ratio": 1.1729
|
|
},
|
|
"quality_pass": false,
|
|
"reasons": [
|
|
"single-request decode 1.91x reference (>= 1.25x) at TTFT ratio 1.17",
|
|
"aggregate throughput at concurrency 4 is 4.49x reference (>= 1.25x)",
|
|
"peak resident memory is 0.57x reference (<= 0.75x)",
|
|
"quality lane exact-match 0.33 / similarity 0.947 versus the reference (fail)"
|
|
],
|
|
"recipe_id": "llama-cpp-near-lossless-quality",
|
|
"speed_benefit": false
|
|
},
|
|
{
|
|
"comparable": true,
|
|
"failures": 0,
|
|
"fit_benefit": true,
|
|
"incomparable_reason": "",
|
|
"lane": "performance-fit",
|
|
"measurements": {
|
|
"aggregate_concurrency": 4,
|
|
"aggregate_throughput_speedup": 4.3826,
|
|
"artifact_size_ratio": 0.398,
|
|
"artifact_size_win": true,
|
|
"decode_speedup": 3.6869,
|
|
"failure_rate": 0.0,
|
|
"resident_memory_ratio": 0.2793,
|
|
"ttft_ratio": 0.6367
|
|
},
|
|
"quality_pass": null,
|
|
"reasons": [
|
|
"single-request decode 3.69x reference (>= 1.25x) at TTFT ratio 0.64",
|
|
"aggregate throughput at concurrency 4 is 4.38x reference (>= 1.25x)",
|
|
"peak resident memory is 0.28x reference (<= 0.75x)"
|
|
],
|
|
"recipe_id": "llama-cpp-quantized-performance-fit",
|
|
"speed_benefit": true
|
|
}
|
|
],
|
|
"speed_benefit": true,
|
|
"stop_condition_met": true,
|
|
"verdict": "stop"
|
|
}
|