feat: add DGR-001 performance contract

This commit is contained in:
Dobromir Popov
2026-07-14 18:13:54 +03:00
parent 7b3399760e
commit c7554ef7d8
5 changed files with 345 additions and 0 deletions

View File

@@ -0,0 +1,63 @@
"""Tests for the DGR-001 performance contract metadata."""
from __future__ import annotations
import json
from meshnet_node.performance_contract import DEFAULT_CONTRACT, SCHEMA_VERSION, main
def test_default_contract_is_architecture_aligned_and_small():
"""The baseline stays on DeepSeek2 and uses the smallest DeepSeek-family GGUF.
Tags: performance, model, gguf
"""
payload = DEFAULT_CONTRACT.to_dict()
assert payload["schema_version"] == SCHEMA_VERSION
assert payload["story_id"] == "DGR-001"
assert payload["model_target"] == {
"name": "DeepSeek-V2-Lite-Chat",
"architecture": "deepseek2",
"safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat",
"gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
"gguf_quant": "Q2_K",
"gguf_size_gb": 6.43,
"rationale": (
"Smallest DeepSeek-family benchmark anchor that still points toward "
"DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead "
"of falling back to a tiny but architecture-mismatched smoke model."
),
}
assert payload["benchmark_lanes"] == [
{
"id": "transformers-safetensors",
"runtime": "transformers",
"recipe": "current safetensors recipe",
"concurrency_levels": [1, 4],
},
{
"id": "llama-cpp-gguf",
"runtime": "llama.cpp",
"recipe": "whole-model GGUF recipe",
"concurrency_levels": [1, 4],
},
]
assert "ttft_ms" in payload["metrics"]
assert "output_drift" in payload["metrics"]
assert "meaningful speed or fit benefit" in payload["stop_condition"]
assert any("mounted drive" in note for note in payload["notes"])
def test_contract_cli_writes_json(tmp_path, capsys):
"""The contract can be emitted as a machine-readable artifact.
Tags: performance, artifact
"""
output = tmp_path / "performance-contract.json"
assert main(["--json-out", str(output)]) == 0
written = json.loads(output.read_text(encoding="utf-8"))
assert written == DEFAULT_CONTRACT.to_dict()
assert str(output) in capsys.readouterr().out