"""The recipe benchmark's measurement core, driven by a scripted fake runtime. These tests never load a model, touch a GPU, or open a socket: the core is deliberately runtime-free so the arithmetic and the lane rules can be pinned down exactly, and the real drivers only have to be honest about what they report. """ from __future__ import annotations import pytest import time from meshnet_node.performance_contract import ( ContractThresholds, PerformanceContract, evaluate_contract, ) from meshnet_node.recipe_benchmark import ( BenchmarkError, BenchmarkPlan, GenerationSample, Lane, LoadStats, PromptSpec, RecipeSpec, SamplingPolicy, build_report, compute_drift, measure_recipe, summarize_concurrency, RequestOutcome, ) PROMPTS = ( PromptSpec(id="short", text="Say hello.", context_class="short"), PromptSpec(id="long", text="Summarize the following. " * 40, context_class="long"), ) def plan(**overrides) -> BenchmarkPlan: defaults = dict( plan_id="test-plan", model_id="test/model", model_revision="revision-1", prompts=PROMPTS, sampling=SamplingPolicy(max_output_tokens=8), concurrency_levels=(1, 4), repeats=1, warmup_requests=0, ) defaults.update(overrides) return BenchmarkPlan(**defaults) class FakeDriver: """A runtime with fixed timings, so every metric below has one right answer.""" def __init__( self, *, decode_ms_per_token: float = 10.0, prefill_ms: float = 100.0, artifact_bytes: int = 1_000_000, rss_bytes: int = 4_000_000, vram_bytes: int = 0, texts: dict[str, str] | None = None, fail_at_concurrency: int | None = None, decode_tokens: int = 8, generation_delay_s: float = 0.0, ) -> None: self.decode_ms_per_token = decode_ms_per_token self.prefill_ms = prefill_ms self.artifact_bytes = artifact_bytes self.rss_bytes = rss_bytes self.vram_bytes = vram_bytes self.texts = texts or {} self.fail_at_concurrency = fail_at_concurrency self.decode_tokens = decode_tokens self.generation_delay_s = generation_delay_s self.in_flight = 0 self.max_in_flight = 0 self.loads = 0 self.closes = 0 self.generations = 0 def load(self) -> LoadStats: self.loads += 1 return LoadStats( artifact_bytes=self.artifact_bytes, load_ms=50.0, rss_bytes=self.rss_bytes, vram_bytes=self.vram_bytes, ) def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample: self.in_flight += 1 self.max_in_flight = max(self.max_in_flight, self.in_flight) try: if self.generation_delay_s: time.sleep(self.generation_delay_s) if self.fail_at_concurrency and self.in_flight >= self.fail_at_concurrency: raise RuntimeError("slot exhausted") self.generations += 1 decode_ms = self.decode_ms_per_token * self.decode_tokens return GenerationSample( text=self.texts.get(prompt, "hello world"), prompt_tokens=10, decode_tokens=self.decode_tokens, ttft_ms=self.prefill_ms, prefill_ms=self.prefill_ms, decode_ms=decode_ms, total_ms=self.prefill_ms + decode_ms, ) finally: self.in_flight -= 1 def memory_probe(self) -> tuple[int, int]: return self.rss_bytes, self.vram_bytes def close(self) -> None: self.closes += 1 def recipe(recipe_id: str, lane: Lane, *, reference: bool = False, device: str = "cpu") -> RecipeSpec: return RecipeSpec( id=recipe_id, runtime="fake", weight_format="fake", weight_quantization="bf16", lane=lane, device=device, is_reference=reference, ) def test_plan_rejects_an_experiment_it_cannot_run(): with pytest.raises(BenchmarkError): plan(prompts=()) with pytest.raises(BenchmarkError): plan(concurrency_levels=(0,)) with pytest.raises(BenchmarkError): plan(repeats=0) def test_measure_runs_every_prompt_at_every_concurrency_level(): driver = FakeDriver() measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan()) # 2 prompts x (1 + 4) requests-per-level. assert len(measurement.outcomes) == 2 * 1 + 2 * 4 assert sorted(measurement.metrics) == [1, 4] assert driver.loads == 1 assert driver.closes == 1 assert measurement.available def test_concurrency_level_actually_overlaps_requests(): driver = FakeDriver(decode_ms_per_token=5.0, generation_delay_s=0.02) measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(1, 4))) assert driver.max_in_flight > 1, "concurrency 4 must run requests in parallel, not serially" def test_driver_is_closed_even_when_every_request_fails(): driver = FakeDriver(fail_at_concurrency=1) measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan()) assert driver.closes == 1 assert all(not outcome.ok for outcome in measurement.outcomes) assert measurement.metrics[1].failures == 2 assert measurement.metrics[1].failure_reasons == ("RuntimeError: slot exhausted",) def test_failed_requests_are_reported_not_raised(): driver = FakeDriver(fail_at_concurrency=4, generation_delay_s=0.02) measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan()) assert measurement.metrics[1].failures == 0 assert measurement.metrics[4].failures > 0 assert measurement.metrics[4].requests == 8 def test_summary_arithmetic_is_exact(): outcomes = [ RequestOutcome( recipe_id="r", concurrency=2, prompt_id="p", repeat=0, ok=True, latency_ms=200.0, ttft_ms=100.0, prefill_ms=100.0, decode_ms=100.0, prompt_tokens=10, decode_tokens=10, ), RequestOutcome( recipe_id="r", concurrency=2, prompt_id="p", repeat=1, ok=True, latency_ms=400.0, ttft_ms=200.0, prefill_ms=200.0, decode_ms=200.0, prompt_tokens=10, decode_tokens=10, ), ] metrics = summarize_concurrency( outcomes, concurrency=2, wall_ms=1000.0, peak_rss_bytes=7, peak_vram_bytes=9 ) assert metrics.latency_p50_ms == 200.0 assert metrics.latency_p95_ms == 400.0 # 10 tok / 0.1 s = 100 tok/s and 10 tok / 0.2 s = 50 tok/s, averaged. assert metrics.decode_tokens_per_sec == 75.0 # 20 decoded tokens over a 1 s wall clock, regardless of per-request rates. assert metrics.aggregate_decode_tokens_per_sec == 20.0 assert (metrics.peak_rss_bytes, metrics.peak_vram_bytes) == (7, 9) def test_aggregate_throughput_credits_overlap_but_per_request_rate_does_not(): """Two runtimes with identical per-request speed must be told apart by overlap.""" serial = summarize_concurrency( [ RequestOutcome(recipe_id="s", concurrency=4, prompt_id="p", repeat=i, ok=True, latency_ms=100.0, decode_ms=100.0, decode_tokens=10) for i in range(4) ], concurrency=4, wall_ms=400.0, peak_rss_bytes=0, peak_vram_bytes=0, ) batched = summarize_concurrency( [ RequestOutcome(recipe_id="b", concurrency=4, prompt_id="p", repeat=i, ok=True, latency_ms=100.0, decode_ms=100.0, decode_tokens=10) for i in range(4) ], concurrency=4, wall_ms=100.0, peak_rss_bytes=0, peak_vram_bytes=0, ) assert serial.decode_tokens_per_sec == batched.decode_tokens_per_sec == 100.0 assert serial.aggregate_decode_tokens_per_sec == 100.0 assert batched.aggregate_decode_tokens_per_sec == 400.0 def test_drift_against_the_reference_is_exact_for_an_identical_runtime(): texts = {prompt.text: f"answer for {prompt.id}" for prompt in PROMPTS} reference = measure_recipe( FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan() ) twin = measure_recipe(FakeDriver(texts=texts), recipe("twin", Lane.QUALITY), plan()) drift = compute_drift(twin, reference) assert drift.compared_prompts == 2 assert drift.exact_match_rate == 1.0 assert drift.mean_similarity == 1.0 assert drift.advisory is False def test_quantized_drift_is_advisory_and_never_an_equivalence_claim(): reference = measure_recipe( FakeDriver(texts={prompt.text: "the capital is Paris" for prompt in PROMPTS}), recipe("ref", Lane.QUALITY, reference=True), plan(), ) quantized = measure_recipe( FakeDriver(texts={prompt.text: "the capital is Lyon" for prompt in PROMPTS}), recipe("q4", Lane.PERFORMANCE_FIT), plan(), ) drift = compute_drift(quantized, reference) assert drift.advisory is True, "a quantized recipe's drift must be advisory" assert drift.exact_match_rate == 0.0 assert 0.0 < drift.mean_similarity < 1.0 assert drift.per_prompt[0]["first_divergence_char"] > 0 def test_report_needs_exactly_one_quality_lane_reference(): measurement = measure_recipe(FakeDriver(), recipe("a", Lane.QUALITY, reference=True), plan()) second = measure_recipe(FakeDriver(), recipe("b", Lane.QUALITY, reference=True), plan()) quantized = measure_recipe(FakeDriver(), recipe("q", Lane.PERFORMANCE_FIT), plan()) with pytest.raises(BenchmarkError, match="exactly one reference"): build_report(plan(), [measurement, second], host={}, evidence_class="synthetic") with pytest.raises(BenchmarkError, match="exactly one reference"): build_report(plan(), [quantized], host={}, evidence_class="synthetic") def test_reference_recipe_may_not_be_quantized(): quantized_reference = measure_recipe( FakeDriver(), recipe("q", Lane.PERFORMANCE_FIT, reference=True), plan() ) with pytest.raises(BenchmarkError, match="quality lane"): build_report(plan(), [quantized_reference], host={}, evidence_class="synthetic") def test_report_must_declare_how_it_was_produced(): measurement = measure_recipe(FakeDriver(), recipe("a", Lane.QUALITY, reference=True), plan()) with pytest.raises(BenchmarkError, match="evidence class"): build_report(plan(), [measurement], host={}, evidence_class="probably-real") def test_report_carries_every_metric_the_contract_reads(): reference = measure_recipe(FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan()) quantized = measure_recipe( FakeDriver(decode_ms_per_token=4.0, artifact_bytes=400_000, rss_bytes=1_000_000), recipe("q4", Lane.PERFORMANCE_FIT), plan(), ) report = build_report( plan(), [reference, quantized], host={"cpu": "test"}, evidence_class="synthetic" ) assert report["schema_version"] == 1 assert report["reference_recipe_id"] == "ref" entry = next(e for e in report["recipes"] if e["recipe"]["id"] == "q4") cell = entry["concurrency"]["1"] for metric in ( "ttft_p50_ms", "ttft_p95_ms", "latency_p50_ms", "latency_p95_ms", "prefill_tokens_per_sec", "decode_tokens_per_sec", "aggregate_decode_tokens_per_sec", "peak_rss_bytes", "peak_vram_bytes", "failures", ): assert metric in cell, f"the contract reads {metric}, so the report must carry it" assert entry["load"]["artifact_bytes"] == 400_000 assert [d["recipe_id"] for d in report["drift"]] == ["q4"] def test_unavailable_recipes_are_recorded_rather_than_dropped(): from meshnet_node.recipe_benchmark import RecipeMeasurement reference = measure_recipe(FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan()) missing = RecipeMeasurement( recipe=recipe("q4", Lane.PERFORMANCE_FIT), load=LoadStats(artifact_bytes=0, load_ms=0.0), unavailable_reason="BenchmarkError: GGUF artifact not found", ) report = build_report(plan(), [reference, missing], host={}, evidence_class="synthetic") entry = next(e for e in report["recipes"] if e["recipe"]["id"] == "q4") assert entry["available"] is False assert "not found" in entry["unavailable_reason"] assert report["drift"] == [], "an unmeasured recipe has no drift to report" def test_contract_requires_a_quality_lane_then_allows_quantized_fit_benefit(): texts = {prompt.text: "same greedy answer" for prompt in PROMPTS} reference = measure_recipe( FakeDriver(texts=texts, rss_bytes=4_000_000), recipe("safetensors", Lane.QUALITY, reference=True), plan() ) quality = measure_recipe( FakeDriver(texts=texts), recipe("gguf-f16", Lane.QUALITY), plan() ) q4 = measure_recipe( FakeDriver(texts={prompt.text: "different quantized answer" for prompt in PROMPTS}, rss_bytes=1_000_000, decode_ms_per_token=20.0), recipe("gguf-q4", Lane.PERFORMANCE_FIT), plan() ) report = build_report(plan(), [reference, quality, q4], host={}, evidence_class="synthetic") contract = PerformanceContract( contract_version=1, locked_at="2026-07-13T00:00:00Z", locked_by="test", plan_id="test-plan", thresholds=ContractThresholds(), baseline={}, stop_condition="test", ) evaluation = evaluate_contract(contract, report) assert evaluation.quality_lane_pass is True assert evaluation.fit_benefit is True assert evaluation.verdict == "optimize"