popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden DGR-001 performance contract evidence

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 19:10:24 +03:00
parent e24db7854f
commit 9e67b829e3
16 changed files with 3674 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
tests/test_recipe_benchmark.py
View File

@@ -8,12 +8,24 @@ report.
from __future__ import annotations
import pytest
import copy
import time
from pathlib import Path
import pytest
from meshnet_node.performance_contract import (
STOP_CONDITION,
ContractThresholds,
PerformanceContract,
PerformanceContractError,
_canonical_sha256,
evaluate_contract,
parse_contract,
)
from meshnet_node.recipe_drivers import (
_artifact_sha256,
_validate_config,
require_real_inference,
)
from meshnet_node.recipe_benchmark import (
BenchmarkError,
@@ -344,3 +356,209 @@ def test_contract_requires_a_quality_lane_then_allows_quantized_fit_benefit():
assert evaluation.quality_lane_pass is True
assert evaluation.fit_benefit is True
assert evaluation.verdict == "optimize"
def test_peak_memory_is_sampled_while_requests_are_in_flight():
class TransientMemoryDriver(FakeDriver):
def memory_probe(self) -> tuple[int, int]:
return (99_000_000 if self.in_flight else 1_000_000), 0
measurement = measure_recipe(
TransientMemoryDriver(generation_delay_s=0.05),
recipe("transient", Lane.QUALITY, reference=True),
plan(warmup_requests=0),
)
assert measurement.metrics[1].peak_rss_bytes == 99_000_000
assert measurement.metrics[4].peak_rss_bytes == 99_000_000
def test_real_inference_requires_explicit_opt_in(monkeypatch):
monkeypatch.delenv("MESHNET_ENABLE_REAL_INFERENCE_TESTS", raising=False)
with pytest.raises(BenchmarkError, match="opt-in"):
require_real_inference()
def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path):
artifact = tmp_path / "model.gguf"
artifact.write_bytes(b"real model bytes")
config = {
"artifact_storage_root": str(tmp_path),
"plan": {
"model_id": "test/model",
"model_revision": "revision-1",
"prompts": [
{"id": "p1", "text": "one"},
{"id": "p2", "text": "two"},
{"id": "p3", "text": "three"},
],
"concurrency_levels": [1, 4],
"repeats": 3,
"warmup_requests": 1,
"sampling": {
"temperature": 0.0,
"top_k": 1,
"top_p": 1.0,
"max_output_tokens": 32,
},
},
"recipes": [{
"id": "recipe",
"source_model_id": "test/model",
"source_model_revision": "revision-1",
"artifact_path": str(artifact),
"artifact_sha256": _artifact_sha256(artifact),
"device": "cpu",
"driver": {
"type": "llama-cpp-server",
"binary": str(artifact),
"binary_sha256": _artifact_sha256(artifact),
"gguf_path": str(artifact),
"device": "cpu",
"threads": 8,
"n_parallel": 4,
},
}],
}
_validate_config(config)
config["recipes"][0]["artifact_sha256"] = "0" * 64
with pytest.raises(BenchmarkError, match="digest mismatch"):
_validate_config(config)
config["recipes"][0]["artifact_sha256"] = _artifact_sha256(artifact)
config["recipes"][0]["driver"]["binary_sha256"] = "0" * 64
with pytest.raises(BenchmarkError, match="binary SHA-256 mismatch"):
_validate_config(config)
config["recipes"][0]["driver"]["binary_sha256"] = _artifact_sha256(artifact)
config["recipes"][0]["driver"]["n_gpu_layers"] = 1
with pytest.raises(BenchmarkError, match="CPU-only"):
_validate_config(config)
def _lock_real_report(report: dict) -> PerformanceContract:
report["evidence_class"] = "local-real"
report["host"] = {
"hostname": "test-host",
"platform": "test-platform",
"python": "3.12",
"cpu_count": 8,
}
for entry in report["recipes"]:
entry["recipe"]["source_model_id"] = report["plan"]["model_id"]
entry["recipe"]["source_model_revision"] = report["plan"]["model_revision"]
entry["recipe"]["artifact_sha256"] = "a" * 64
return PerformanceContract(
contract_version=1,
locked_at="2026-07-13T00:00:00Z",
locked_by="test",
plan_id=report["plan"]["plan_id"],
thresholds=ContractThresholds(),
baseline={
"required_evidence_class": "local-real",
"required_recipes": [entry["recipe"]["id"] for entry in report["recipes"]],
"required_concurrency_levels": [1, 4],
},
stop_condition="test",
)
def test_locked_contract_rejects_synthetic_evidence():
reference = measure_recipe(
FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan()
)
quality = measure_recipe(FakeDriver(), recipe("quality", Lane.QUALITY), plan())
q4 = measure_recipe(FakeDriver(), recipe("q4", Lane.PERFORMANCE_FIT), plan())
report = build_report(
plan(), [reference, quality, q4], host={}, evidence_class="synthetic"
)
contract = _lock_real_report(report)
report["evidence_class"] = "synthetic"
with pytest.raises(PerformanceContractError, match="evidence class"):
evaluate_contract(contract, report)
def test_quality_lane_requires_every_prompt_to_be_compared():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
reference = measure_recipe(
FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()
)
quality = measure_recipe(
FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()
)
q4 = measure_recipe(
FakeDriver(texts=texts, rss_bytes=1_000_000),
recipe("q4", Lane.PERFORMANCE_FIT),
plan(),
)
report = build_report(
plan(), [reference, quality, q4], host={}, evidence_class="synthetic"
)
contract = _lock_real_report(report)
incomplete = copy.deepcopy(report)
quality_drift = next(
drift for drift in incomplete["drift"] if drift["recipe_id"] == "quality"
)
quality_drift["compared_prompts"] = 1
quality_drift["per_prompt"] = quality_drift["per_prompt"][:1]
evaluation = evaluate_contract(contract, incomplete)
assert evaluation.quality_lane_pass is False
assert evaluation.verdict == "stop"
def test_locked_contract_rejects_changed_plan_and_token_counts():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
reference = measure_recipe(
FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()
)
quality = measure_recipe(
FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()
)
q4 = measure_recipe(
FakeDriver(texts=texts), recipe("q4", Lane.PERFORMANCE_FIT), plan()
)
report = build_report(
plan(), [reference, quality, q4], host={}, evidence_class="synthetic"
)
contract = _lock_real_report(report)
contract.baseline["required_plan_sha256"] = _canonical_sha256(report["plan"])
changed_plan = copy.deepcopy(report)
changed_plan["plan"]["prompts"][0]["text"] = "changed after locking"
with pytest.raises(PerformanceContractError, match="plan digest"):
evaluate_contract(contract, changed_plan)
changed_tokens = copy.deepcopy(report)
quality_entry = next(
entry for entry in changed_tokens["recipes"] if entry["recipe"]["id"] == "quality"
)
quality_entry["outcomes"][0]["decode_tokens"] += 1
with pytest.raises(PerformanceContractError, match="different prompt/decode token counts"):
evaluate_contract(contract, changed_tokens)
def test_v1_contract_thresholds_and_stop_condition_are_immutable():
raw = PerformanceContract(
contract_version=1,
locked_at="2026-07-13T00:00:00Z",
locked_by="test",
plan_id="test-plan",
thresholds=ContractThresholds(),
baseline={},
stop_condition=STOP_CONDITION,
).to_dict()
parse_contract(raw)
changed_threshold = copy.deepcopy(raw)
changed_threshold["thresholds"]["min_decode_speedup"] = 1.01
with pytest.raises(PerformanceContractError, match="immutable v1 thresholds"):
parse_contract(changed_threshold)
changed_stop = copy.deepcopy(raw)
changed_stop["stop_condition"] = "promote everything"
with pytest.raises(PerformanceContractError, match="stop condition differs"):
parse_contract(changed_stop)