feat: DGR-001 - Lock the safetensors-versus-GGUF performance contract

This commit is contained in:
Dobromir Popov
2026-07-13 17:55:55 +03:00
parent 59f2486bf2
commit e24db7854f
8 changed files with 248 additions and 4 deletions

View File

@@ -9,6 +9,12 @@ report.
from __future__ import annotations
import pytest
import time
from meshnet_node.performance_contract import (
ContractThresholds,
PerformanceContract,
evaluate_contract,
)
from meshnet_node.recipe_benchmark import (
BenchmarkError,
BenchmarkPlan,
@@ -60,6 +66,7 @@ class FakeDriver:
texts: dict[str, str] | None = None,
fail_at_concurrency: int | None = None,
decode_tokens: int = 8,
generation_delay_s: float = 0.0,
) -> None:
self.decode_ms_per_token = decode_ms_per_token
self.prefill_ms = prefill_ms
@@ -69,6 +76,7 @@ class FakeDriver:
self.texts = texts or {}
self.fail_at_concurrency = fail_at_concurrency
self.decode_tokens = decode_tokens
self.generation_delay_s = generation_delay_s
self.in_flight = 0
self.max_in_flight = 0
self.loads = 0
@@ -86,6 +94,8 @@ class FakeDriver:
self.in_flight += 1
self.max_in_flight = max(self.max_in_flight, self.in_flight)
try:
if self.generation_delay_s:
time.sleep(self.generation_delay_s)
if self.fail_at_concurrency and self.in_flight >= self.fail_at_concurrency:
raise RuntimeError("slot exhausted")
self.generations += 1
@@ -138,8 +148,8 @@ def test_measure_runs_every_prompt_at_every_concurrency_level():
def test_concurrency_level_actually_overlaps_requests():
driver = FakeDriver(decode_ms_per_token=5.0)
measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(4,)))
driver = FakeDriver(decode_ms_per_token=5.0, generation_delay_s=0.02)
measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(1, 4)))
assert driver.max_in_flight > 1, "concurrency 4 must run requests in parallel, not serially"
@@ -155,7 +165,7 @@ def test_driver_is_closed_even_when_every_request_fails():
def test_failed_requests_are_reported_not_raised():
driver = FakeDriver(fail_at_concurrency=4)
driver = FakeDriver(fail_at_concurrency=4, generation_delay_s=0.02)
measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan())
assert measurement.metrics[1].failures == 0
@@ -308,3 +318,29 @@ def test_unavailable_recipes_are_recorded_rather_than_dropped():
assert entry["available"] is False
assert "not found" in entry["unavailable_reason"]
assert report["drift"] == [], "an unmeasured recipe has no drift to report"
def test_contract_requires_a_quality_lane_then_allows_quantized_fit_benefit():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
reference = measure_recipe(
FakeDriver(texts=texts, rss_bytes=4_000_000), recipe("safetensors", Lane.QUALITY, reference=True), plan()
)
quality = measure_recipe(
FakeDriver(texts=texts), recipe("gguf-f16", Lane.QUALITY), plan()
)
q4 = measure_recipe(
FakeDriver(texts={prompt.text: "different quantized answer" for prompt in PROMPTS},
rss_bytes=1_000_000, decode_ms_per_token=20.0),
recipe("gguf-q4", Lane.PERFORMANCE_FIT), plan()
)
report = build_report(plan(), [reference, quality, q4], host={}, evidence_class="synthetic")
contract = PerformanceContract(
contract_version=1, locked_at="2026-07-13T00:00:00Z", locked_by="test",
plan_id="test-plan", thresholds=ContractThresholds(), baseline={}, stop_condition="test",
)
evaluation = evaluate_contract(contract, report)
assert evaluation.quality_lane_pass is True
assert evaluation.fit_benefit is True
assert evaluation.verdict == "optimize"