85 lines
2.9 KiB
Python
85 lines
2.9 KiB
Python
"""Tests for the DGR-001 performance contract metadata."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from meshnet_node.performance_contract import DEFAULT_CONTRACT, SCHEMA_VERSION, main
|
|
|
|
|
|
def test_default_contract_is_architecture_aligned_and_small():
|
|
"""The baseline stays on DeepSeek2 and uses the smallest DeepSeek-family GGUF.
|
|
|
|
Tags: performance, model, gguf
|
|
"""
|
|
payload = DEFAULT_CONTRACT.to_dict()
|
|
|
|
assert payload["schema_version"] == SCHEMA_VERSION
|
|
assert payload["story_id"] == "DGR-001"
|
|
assert payload["model_target"] == {
|
|
"name": "DeepSeek-V2-Lite-Chat",
|
|
"architecture": "deepseek2",
|
|
"safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat",
|
|
"safetensors_precision": "bfloat16",
|
|
"gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
|
|
"gguf_quant": "Q2_K",
|
|
"gguf_size_gb": 6.43,
|
|
"comparison_policy": (
|
|
"same model/revision, closest practical low-footprint precision pair: "
|
|
"BF16 safetensors versus Q2_K GGUF"
|
|
),
|
|
"rationale": (
|
|
"Smallest DeepSeek-family benchmark anchor that still points toward "
|
|
"DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead "
|
|
"of falling back to a tiny but architecture-mismatched smoke model."
|
|
),
|
|
}
|
|
assert payload["benchmark_lanes"] == [
|
|
{
|
|
"id": "transformers-safetensors-cpu",
|
|
"runtime": "transformers",
|
|
"device": "cpu",
|
|
"recipe": "current safetensors recipe",
|
|
"concurrency_levels": [1, 4],
|
|
},
|
|
{
|
|
"id": "llama-cpp-gguf-cpu",
|
|
"runtime": "llama.cpp",
|
|
"device": "cpu",
|
|
"recipe": "whole-model GGUF recipe",
|
|
"concurrency_levels": [1, 4],
|
|
},
|
|
{
|
|
"id": "transformers-safetensors-gpu",
|
|
"runtime": "transformers",
|
|
"device": "gpu",
|
|
"recipe": "current safetensors recipe",
|
|
"concurrency_levels": [1, 4],
|
|
},
|
|
{
|
|
"id": "llama-cpp-gguf-gpu",
|
|
"runtime": "llama.cpp",
|
|
"device": "gpu",
|
|
"recipe": "whole-model GGUF recipe",
|
|
"concurrency_levels": [1, 4],
|
|
},
|
|
]
|
|
assert "ttft_ms" in payload["metrics"]
|
|
assert "output_drift" in payload["metrics"]
|
|
assert "meaningful speed or fit benefit" in payload["stop_condition"]
|
|
assert any("mounted drive" in note for note in payload["notes"])
|
|
|
|
|
|
def test_contract_cli_writes_json(tmp_path, capsys):
|
|
"""The contract can be emitted as a machine-readable artifact.
|
|
|
|
Tags: performance, artifact
|
|
"""
|
|
output = tmp_path / "performance-contract.json"
|
|
|
|
assert main(["--json-out", str(output)]) == 0
|
|
written = json.loads(output.read_text(encoding="utf-8"))
|
|
|
|
assert written == DEFAULT_CONTRACT.to_dict()
|
|
assert str(output) in capsys.readouterr().out
|