feat: DGR-001 - Lock the safetensors-versus-GGUF performance contract
This commit is contained in:
@@ -9,6 +9,12 @@ report.
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import time
|
||||
from meshnet_node.performance_contract import (
|
||||
ContractThresholds,
|
||||
PerformanceContract,
|
||||
evaluate_contract,
|
||||
)
|
||||
from meshnet_node.recipe_benchmark import (
|
||||
BenchmarkError,
|
||||
BenchmarkPlan,
|
||||
@@ -60,6 +66,7 @@ class FakeDriver:
|
||||
texts: dict[str, str] | None = None,
|
||||
fail_at_concurrency: int | None = None,
|
||||
decode_tokens: int = 8,
|
||||
generation_delay_s: float = 0.0,
|
||||
) -> None:
|
||||
self.decode_ms_per_token = decode_ms_per_token
|
||||
self.prefill_ms = prefill_ms
|
||||
@@ -69,6 +76,7 @@ class FakeDriver:
|
||||
self.texts = texts or {}
|
||||
self.fail_at_concurrency = fail_at_concurrency
|
||||
self.decode_tokens = decode_tokens
|
||||
self.generation_delay_s = generation_delay_s
|
||||
self.in_flight = 0
|
||||
self.max_in_flight = 0
|
||||
self.loads = 0
|
||||
@@ -86,6 +94,8 @@ class FakeDriver:
|
||||
self.in_flight += 1
|
||||
self.max_in_flight = max(self.max_in_flight, self.in_flight)
|
||||
try:
|
||||
if self.generation_delay_s:
|
||||
time.sleep(self.generation_delay_s)
|
||||
if self.fail_at_concurrency and self.in_flight >= self.fail_at_concurrency:
|
||||
raise RuntimeError("slot exhausted")
|
||||
self.generations += 1
|
||||
@@ -138,8 +148,8 @@ def test_measure_runs_every_prompt_at_every_concurrency_level():
|
||||
|
||||
|
||||
def test_concurrency_level_actually_overlaps_requests():
|
||||
driver = FakeDriver(decode_ms_per_token=5.0)
|
||||
measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(4,)))
|
||||
driver = FakeDriver(decode_ms_per_token=5.0, generation_delay_s=0.02)
|
||||
measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(1, 4)))
|
||||
|
||||
assert driver.max_in_flight > 1, "concurrency 4 must run requests in parallel, not serially"
|
||||
|
||||
@@ -155,7 +165,7 @@ def test_driver_is_closed_even_when_every_request_fails():
|
||||
|
||||
|
||||
def test_failed_requests_are_reported_not_raised():
|
||||
driver = FakeDriver(fail_at_concurrency=4)
|
||||
driver = FakeDriver(fail_at_concurrency=4, generation_delay_s=0.02)
|
||||
measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan())
|
||||
|
||||
assert measurement.metrics[1].failures == 0
|
||||
@@ -308,3 +318,29 @@ def test_unavailable_recipes_are_recorded_rather_than_dropped():
|
||||
assert entry["available"] is False
|
||||
assert "not found" in entry["unavailable_reason"]
|
||||
assert report["drift"] == [], "an unmeasured recipe has no drift to report"
|
||||
|
||||
|
||||
def test_contract_requires_a_quality_lane_then_allows_quantized_fit_benefit():
|
||||
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
|
||||
reference = measure_recipe(
|
||||
FakeDriver(texts=texts, rss_bytes=4_000_000), recipe("safetensors", Lane.QUALITY, reference=True), plan()
|
||||
)
|
||||
quality = measure_recipe(
|
||||
FakeDriver(texts=texts), recipe("gguf-f16", Lane.QUALITY), plan()
|
||||
)
|
||||
q4 = measure_recipe(
|
||||
FakeDriver(texts={prompt.text: "different quantized answer" for prompt in PROMPTS},
|
||||
rss_bytes=1_000_000, decode_ms_per_token=20.0),
|
||||
recipe("gguf-q4", Lane.PERFORMANCE_FIT), plan()
|
||||
)
|
||||
report = build_report(plan(), [reference, quality, q4], host={}, evidence_class="synthetic")
|
||||
contract = PerformanceContract(
|
||||
contract_version=1, locked_at="2026-07-13T00:00:00Z", locked_by="test",
|
||||
plan_id="test-plan", thresholds=ContractThresholds(), baseline={}, stop_condition="test",
|
||||
)
|
||||
|
||||
evaluation = evaluate_contract(contract, report)
|
||||
|
||||
assert evaluation.quality_lane_pass is True
|
||||
assert evaluation.fit_benefit is True
|
||||
assert evaluation.verdict == "optimize"
|
||||
|
||||
Reference in New Issue
Block a user