feat: DGR-001 - Lock the safetensors-versus-GGUF performance contract

tests/test_recipe_benchmark.py

@@ -0,0 +1,310 @@
+"""The recipe benchmark's measurement core, driven by a scripted fake runtime.
+
+These tests never load a model, touch a GPU, or open a socket: the core is
+deliberately runtime-free so the arithmetic and the lane rules can be pinned
+down exactly, and the real drivers only have to be honest about what they
+report.
+"""
+
+from __future__ import annotations
+
+import pytest
+from meshnet_node.recipe_benchmark import (
+    BenchmarkError,
+    BenchmarkPlan,
+    GenerationSample,
+    Lane,
+    LoadStats,
+    PromptSpec,
+    RecipeSpec,
+    SamplingPolicy,
+    build_report,
+    compute_drift,
+    measure_recipe,
+    summarize_concurrency,
+    RequestOutcome,
+)
+
+PROMPTS = (
+    PromptSpec(id="short", text="Say hello.", context_class="short"),
+    PromptSpec(id="long", text="Summarize the following. " * 40, context_class="long"),
+)
+
+
+def plan(**overrides) -> BenchmarkPlan:
+    defaults = dict(
+        plan_id="test-plan",
+        model_id="test/model",
+        model_revision="revision-1",
+        prompts=PROMPTS,
+        sampling=SamplingPolicy(max_output_tokens=8),
+        concurrency_levels=(1, 4),
+        repeats=1,
+        warmup_requests=0,
+    )
+    defaults.update(overrides)
+    return BenchmarkPlan(**defaults)
+
+
+class FakeDriver:
+    """A runtime with fixed timings, so every metric below has one right answer."""
+
+    def __init__(
+        self,
+        *,
+        decode_ms_per_token: float = 10.0,
+        prefill_ms: float = 100.0,
+        artifact_bytes: int = 1_000_000,
+        rss_bytes: int = 4_000_000,
+        vram_bytes: int = 0,
+        texts: dict[str, str] | None = None,
+        fail_at_concurrency: int | None = None,
+        decode_tokens: int = 8,
+    ) -> None:
+        self.decode_ms_per_token = decode_ms_per_token
+        self.prefill_ms = prefill_ms
+        self.artifact_bytes = artifact_bytes
+        self.rss_bytes = rss_bytes
+        self.vram_bytes = vram_bytes
+        self.texts = texts or {}
+        self.fail_at_concurrency = fail_at_concurrency
+        self.decode_tokens = decode_tokens
+        self.in_flight = 0
+        self.max_in_flight = 0
+        self.loads = 0
+        self.closes = 0
+        self.generations = 0
+
+    def load(self) -> LoadStats:
+        self.loads += 1
+        return LoadStats(
+            artifact_bytes=self.artifact_bytes, load_ms=50.0,
+            rss_bytes=self.rss_bytes, vram_bytes=self.vram_bytes,
+        )
+
+    def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
+        self.in_flight += 1
+        self.max_in_flight = max(self.max_in_flight, self.in_flight)
+        try:
+            if self.fail_at_concurrency and self.in_flight >= self.fail_at_concurrency:
+                raise RuntimeError("slot exhausted")
+            self.generations += 1
+            decode_ms = self.decode_ms_per_token * self.decode_tokens
+            return GenerationSample(
+                text=self.texts.get(prompt, "hello world"),
+                prompt_tokens=10,
+                decode_tokens=self.decode_tokens,
+                ttft_ms=self.prefill_ms,
+                prefill_ms=self.prefill_ms,
+                decode_ms=decode_ms,
+                total_ms=self.prefill_ms + decode_ms,
+            )
+        finally:
+            self.in_flight -= 1
+
+    def memory_probe(self) -> tuple[int, int]:
+        return self.rss_bytes, self.vram_bytes
+
+    def close(self) -> None:
+        self.closes += 1
+
+
+def recipe(recipe_id: str, lane: Lane, *, reference: bool = False, device: str = "cpu") -> RecipeSpec:
+    return RecipeSpec(
+        id=recipe_id, runtime="fake", weight_format="fake", weight_quantization="bf16",
+        lane=lane, device=device, is_reference=reference,
+    )
+
+
+def test_plan_rejects_an_experiment_it_cannot_run():
+    with pytest.raises(BenchmarkError):
+        plan(prompts=())
+    with pytest.raises(BenchmarkError):
+        plan(concurrency_levels=(0,))
+    with pytest.raises(BenchmarkError):
+        plan(repeats=0)
+
+
+def test_measure_runs_every_prompt_at_every_concurrency_level():
+    driver = FakeDriver()
+    measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan())
+
+    # 2 prompts x (1 + 4) requests-per-level.
+    assert len(measurement.outcomes) == 2 * 1 + 2 * 4
+    assert sorted(measurement.metrics) == [1, 4]
+    assert driver.loads == 1
+    assert driver.closes == 1
+    assert measurement.available
+
+
+def test_concurrency_level_actually_overlaps_requests():
+    driver = FakeDriver(decode_ms_per_token=5.0)
+    measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(4,)))
+
+    assert driver.max_in_flight > 1, "concurrency 4 must run requests in parallel, not serially"
+
+
+def test_driver_is_closed_even_when_every_request_fails():
+    driver = FakeDriver(fail_at_concurrency=1)
+    measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan())
+
+    assert driver.closes == 1
+    assert all(not outcome.ok for outcome in measurement.outcomes)
+    assert measurement.metrics[1].failures == 2
+    assert measurement.metrics[1].failure_reasons == ("RuntimeError: slot exhausted",)
+
+
+def test_failed_requests_are_reported_not_raised():
+    driver = FakeDriver(fail_at_concurrency=4)
+    measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan())
+
+    assert measurement.metrics[1].failures == 0
+    assert measurement.metrics[4].failures > 0
+    assert measurement.metrics[4].requests == 8
+
+
+def test_summary_arithmetic_is_exact():
+    outcomes = [
+        RequestOutcome(
+            recipe_id="r", concurrency=2, prompt_id="p", repeat=0, ok=True,
+            latency_ms=200.0, ttft_ms=100.0, prefill_ms=100.0, decode_ms=100.0,
+            prompt_tokens=10, decode_tokens=10,
+        ),
+        RequestOutcome(
+            recipe_id="r", concurrency=2, prompt_id="p", repeat=1, ok=True,
+            latency_ms=400.0, ttft_ms=200.0, prefill_ms=200.0, decode_ms=200.0,
+            prompt_tokens=10, decode_tokens=10,
+        ),
+    ]
+    metrics = summarize_concurrency(
+        outcomes, concurrency=2, wall_ms=1000.0, peak_rss_bytes=7, peak_vram_bytes=9
+    )
+
+    assert metrics.latency_p50_ms == 200.0
+    assert metrics.latency_p95_ms == 400.0
+    # 10 tok / 0.1 s = 100 tok/s and 10 tok / 0.2 s = 50 tok/s, averaged.
+    assert metrics.decode_tokens_per_sec == 75.0
+    # 20 decoded tokens over a 1 s wall clock, regardless of per-request rates.
+    assert metrics.aggregate_decode_tokens_per_sec == 20.0
+    assert (metrics.peak_rss_bytes, metrics.peak_vram_bytes) == (7, 9)
+
+
+def test_aggregate_throughput_credits_overlap_but_per_request_rate_does_not():
+    """Two runtimes with identical per-request speed must be told apart by overlap."""
+    serial = summarize_concurrency(
+        [
+            RequestOutcome(recipe_id="s", concurrency=4, prompt_id="p", repeat=i, ok=True,
+                           latency_ms=100.0, decode_ms=100.0, decode_tokens=10)
+            for i in range(4)
+        ],
+        concurrency=4, wall_ms=400.0, peak_rss_bytes=0, peak_vram_bytes=0,
+    )
+    batched = summarize_concurrency(
+        [
+            RequestOutcome(recipe_id="b", concurrency=4, prompt_id="p", repeat=i, ok=True,
+                           latency_ms=100.0, decode_ms=100.0, decode_tokens=10)
+            for i in range(4)
+        ],
+        concurrency=4, wall_ms=100.0, peak_rss_bytes=0, peak_vram_bytes=0,
+    )
+
+    assert serial.decode_tokens_per_sec == batched.decode_tokens_per_sec == 100.0
+    assert serial.aggregate_decode_tokens_per_sec == 100.0
+    assert batched.aggregate_decode_tokens_per_sec == 400.0
+
+
+def test_drift_against_the_reference_is_exact_for_an_identical_runtime():
+    texts = {prompt.text: f"answer for {prompt.id}" for prompt in PROMPTS}
+    reference = measure_recipe(
+        FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()
+    )
+    twin = measure_recipe(FakeDriver(texts=texts), recipe("twin", Lane.QUALITY), plan())
+
+    drift = compute_drift(twin, reference)
+    assert drift.compared_prompts == 2
+    assert drift.exact_match_rate == 1.0
+    assert drift.mean_similarity == 1.0
+    assert drift.advisory is False
+
+
+def test_quantized_drift_is_advisory_and_never_an_equivalence_claim():
+    reference = measure_recipe(
+        FakeDriver(texts={prompt.text: "the capital is Paris" for prompt in PROMPTS}),
+        recipe("ref", Lane.QUALITY, reference=True), plan(),
+    )
+    quantized = measure_recipe(
+        FakeDriver(texts={prompt.text: "the capital is Lyon" for prompt in PROMPTS}),
+        recipe("q4", Lane.PERFORMANCE_FIT), plan(),
+    )
+
+    drift = compute_drift(quantized, reference)
+    assert drift.advisory is True, "a quantized recipe's drift must be advisory"
+    assert drift.exact_match_rate == 0.0
+    assert 0.0 < drift.mean_similarity < 1.0
+    assert drift.per_prompt[0]["first_divergence_char"] > 0
+
+
+def test_report_needs_exactly_one_quality_lane_reference():
+    measurement = measure_recipe(FakeDriver(), recipe("a", Lane.QUALITY, reference=True), plan())
+    second = measure_recipe(FakeDriver(), recipe("b", Lane.QUALITY, reference=True), plan())
+    quantized = measure_recipe(FakeDriver(), recipe("q", Lane.PERFORMANCE_FIT), plan())
+
+    with pytest.raises(BenchmarkError, match="exactly one reference"):
+        build_report(plan(), [measurement, second], host={}, evidence_class="synthetic")
+    with pytest.raises(BenchmarkError, match="exactly one reference"):
+        build_report(plan(), [quantized], host={}, evidence_class="synthetic")
+
+
+def test_reference_recipe_may_not_be_quantized():
+    quantized_reference = measure_recipe(
+        FakeDriver(), recipe("q", Lane.PERFORMANCE_FIT, reference=True), plan()
+    )
+    with pytest.raises(BenchmarkError, match="quality lane"):
+        build_report(plan(), [quantized_reference], host={}, evidence_class="synthetic")
+
+
+def test_report_must_declare_how_it_was_produced():
+    measurement = measure_recipe(FakeDriver(), recipe("a", Lane.QUALITY, reference=True), plan())
+    with pytest.raises(BenchmarkError, match="evidence class"):
+        build_report(plan(), [measurement], host={}, evidence_class="probably-real")
+
+
+def test_report_carries_every_metric_the_contract_reads():
+    reference = measure_recipe(FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan())
+    quantized = measure_recipe(
+        FakeDriver(decode_ms_per_token=4.0, artifact_bytes=400_000, rss_bytes=1_000_000),
+        recipe("q4", Lane.PERFORMANCE_FIT), plan(),
+    )
+    report = build_report(
+        plan(), [reference, quantized], host={"cpu": "test"}, evidence_class="synthetic"
+    )
+
+    assert report["schema_version"] == 1
+    assert report["reference_recipe_id"] == "ref"
+    entry = next(e for e in report["recipes"] if e["recipe"]["id"] == "q4")
+    cell = entry["concurrency"]["1"]
+    for metric in (
+        "ttft_p50_ms", "ttft_p95_ms", "latency_p50_ms", "latency_p95_ms",
+        "prefill_tokens_per_sec", "decode_tokens_per_sec", "aggregate_decode_tokens_per_sec",
+        "peak_rss_bytes", "peak_vram_bytes", "failures",
+    ):
+        assert metric in cell, f"the contract reads {metric}, so the report must carry it"
+    assert entry["load"]["artifact_bytes"] == 400_000
+    assert [d["recipe_id"] for d in report["drift"]] == ["q4"]
+
+
+def test_unavailable_recipes_are_recorded_rather_than_dropped():
+    from meshnet_node.recipe_benchmark import RecipeMeasurement
+
+    reference = measure_recipe(FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan())
+    missing = RecipeMeasurement(
+        recipe=recipe("q4", Lane.PERFORMANCE_FIT),
+        load=LoadStats(artifact_bytes=0, load_ms=0.0),
+        unavailable_reason="BenchmarkError: GGUF artifact not found",
+    )
+    report = build_report(plan(), [reference, missing], host={}, evidence_class="synthetic")
+
+    entry = next(e for e in report["recipes"] if e["recipe"]["id"] == "q4")
+    assert entry["available"] is False
+    assert "not found" in entry["unavailable_reason"]
+    assert report["drift"] == [], "an unmeasured recipe has no drift to report"