feat: DGR-001 - Lock the safetensors-versus-GGUF performance contract

packages/node/meshnet_node/recipe_benchmark.py

@@ -0,0 +1,648 @@
+"""Controlled safetensors-versus-GGUF recipe benchmark.
+
+This is a *model recipe* benchmark, unlike
+:mod:`meshnet_node.route_session_benchmark`, which is a transport harness.  It
+answers one question: on one machine, with one model revision and one fixed
+workload, what do the Transformers/safetensors recipe and the whole-model
+llama.cpp/GGUF recipes actually cost in speed, memory, fit, and output drift?
+
+Two ideas keep the answer honest.
+
+**Lanes.**  A recipe belongs to exactly one :class:`Lane`.  The quality lane
+holds near-lossless recipes (bf16/f16 weights) whose outputs may legitimately be
+compared for numerical agreement.  The performance-fit lane holds quantized
+recipes (Q8_0, Q4_K_M, ...).  Quantized recipes are judged on speed, memory, and
+artifact size only; their drift is *reported* but never read as evidence that
+Q4 and bf16 are numerically equivalent, because they are not.  The lane is a
+property of the recipe, so nothing downstream can quietly cross the boundary.
+
+**Drivers.**  The measurement core here is pure and runtime-free: it drives a
+:class:`RecipeDriver` and computes metrics.  Real runtimes live in
+:mod:`meshnet_node.recipe_drivers` and are imported only on demand, which keeps
+the default test suite deterministic, GPU-free and model-download-free while the
+same code path produces the real evidence.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import time
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict, dataclass, field
+from difflib import SequenceMatcher
+from enum import Enum
+from pathlib import Path
+from typing import Any, Protocol, Sequence
+
+# Layout of the report document produced by :func:`build_report`.
+REPORT_SCHEMA_VERSION = 1
+
+
+class Lane(str, Enum):
+    """Why a recipe is being measured at all.
+
+    ``QUALITY`` recipes carry near-lossless weights, so comparing their output
+    with the reference recipe is a meaningful correctness signal.
+    ``PERFORMANCE_FIT`` recipes carry quantized weights: they exist to be faster
+    or to fit, and their drift is descriptive, never a pass/fail equivalence
+    claim.
+    """
+
+    QUALITY = "quality"
+    PERFORMANCE_FIT = "performance-fit"
+
+
+class BenchmarkError(RuntimeError):
+    """Raised when a benchmark cannot be run as specified."""
+
+
+@dataclass(frozen=True)
+class SamplingPolicy:
+    """The sampling policy every recipe must be given, identically.
+
+    Greedy by default: sampling noise would otherwise be indistinguishable from
+    quantization drift, and the whole point of the quality lane is to tell those
+    two apart.
+    """
+
+    temperature: float = 0.0
+    top_p: float = 1.0
+    top_k: int = 1
+    seed: int = 1234
+    max_output_tokens: int = 64
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class PromptSpec:
+    """One fixed prompt, tagged with the context length it is meant to exercise."""
+
+    id: str
+    text: str
+    context_class: str = "short"
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class BenchmarkPlan:
+    """The controlled variables: identical for every recipe in a report.
+
+    A plan is the experiment.  If two recipes were measured under different
+    plans, their numbers are not comparable and the report must not pretend they
+    are, so the plan is recorded once at the top of the document rather than
+    per-recipe.
+    """
+
+    plan_id: str
+    model_id: str
+    model_revision: str
+    prompts: tuple[PromptSpec, ...]
+    sampling: SamplingPolicy = SamplingPolicy()
+    concurrency_levels: tuple[int, ...] = (1, 4)
+    repeats: int = 1
+    warmup_requests: int = 1
+
+    def __post_init__(self) -> None:
+        if not self.prompts:
+            raise BenchmarkError("a benchmark plan needs at least one prompt")
+        if not self.concurrency_levels or any(level < 1 for level in self.concurrency_levels):
+            raise BenchmarkError("concurrency levels must all be >= 1")
+        if self.repeats < 1:
+            raise BenchmarkError("repeats must be >= 1")
+
+    def to_dict(self) -> dict:
+        return {
+            "plan_id": self.plan_id,
+            "model_id": self.model_id,
+            "model_revision": self.model_revision,
+            "prompts": [prompt.to_dict() for prompt in self.prompts],
+            "sampling": self.sampling.to_dict(),
+            "concurrency_levels": list(self.concurrency_levels),
+            "repeats": self.repeats,
+            "warmup_requests": self.warmup_requests,
+        }
+
+
+@dataclass(frozen=True)
+class RecipeSpec:
+    """One runtime recipe under test.
+
+    ``is_reference`` marks the single recipe every other recipe's output drift is
+    measured against — the current Transformers/safetensors route, which
+    decision Gate 8 keeps as the correctness backend.
+    """
+
+    id: str
+    runtime: str
+    weight_format: str
+    weight_quantization: str
+    lane: Lane
+    device: str
+    artifact_path: str = ""
+    is_reference: bool = False
+    notes: str = ""
+
+    def to_dict(self) -> dict:
+        data = asdict(self)
+        data["lane"] = self.lane.value
+        return data
+
+
+@dataclass(frozen=True)
+class LoadStats:
+    """What loading the recipe cost, before any token is generated."""
+
+    artifact_bytes: int
+    load_ms: float
+    rss_bytes: int = 0
+    vram_bytes: int = 0
+    backend_detail: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class GenerationSample:
+    """One completed generation as reported by a driver.
+
+    ``prefill_ms``/``decode_ms`` are the runtime's own split where it exposes one
+    (llama.cpp does); drivers that cannot split honestly report ``prefill_ms`` as
+    the time to the first token.  ``queue_wait_ms`` separates time spent waiting
+    for a runtime slot from time spent computing, so a concurrency-4 TTFT is not
+    silently read as a slower prefill.
+    """
+
+    text: str
+    prompt_tokens: int
+    decode_tokens: int
+    ttft_ms: float
+    prefill_ms: float
+    decode_ms: float
+    total_ms: float
+    queue_wait_ms: float = 0.0
+
+
+class RecipeDriver(Protocol):
+    """The seam every runtime implements; the measurement core knows nothing else."""
+
+    def load(self) -> LoadStats:
+        """Load the artifact and return its cost."""
+
+    def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
+        """Run one complete generation under the given sampling policy."""
+
+    def memory_probe(self) -> tuple[int, int]:
+        """Return ``(rss_bytes, vram_bytes)`` observed right now."""
+
+    def close(self) -> None:
+        """Release the runtime."""
+
+
+@dataclass(frozen=True)
+class RequestOutcome:
+    """One request attempt, successful or not.
+
+    A failure is a first-class result, not an exception that aborts the run: a
+    recipe that cannot sustain concurrency 4 has told us something, and the
+    report must carry it rather than lose it.
+    """
+
+    recipe_id: str
+    concurrency: int
+    prompt_id: str
+    repeat: int
+    ok: bool
+    latency_ms: float
+    ttft_ms: float = 0.0
+    prefill_ms: float = 0.0
+    decode_ms: float = 0.0
+    queue_wait_ms: float = 0.0
+    prompt_tokens: int = 0
+    decode_tokens: int = 0
+    text: str = ""
+    error: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+def _percentile(values: Sequence[float], percentile: float) -> float:
+    """Nearest-rank percentile; 0.0 for an empty sample."""
+    ordered = sorted(values)
+    if not ordered:
+        return 0.0
+    rank = max(1, -(-len(ordered) * percentile // 100))
+    return round(ordered[int(rank) - 1], 4)
+
+
+def _mean(values: Sequence[float]) -> float:
+    return round(statistics.fmean(values), 4) if values else 0.0
+
+
+@dataclass(frozen=True)
+class ConcurrencyMetrics:
+    """Aggregate metrics for one recipe at one concurrency level."""
+
+    concurrency: int
+    requests: int
+    failures: int
+    wall_ms: float
+    ttft_p50_ms: float
+    ttft_p95_ms: float
+    latency_p50_ms: float
+    latency_p95_ms: float
+    prefill_tokens_per_sec: float
+    decode_tokens_per_sec: float
+    aggregate_decode_tokens_per_sec: float
+    peak_rss_bytes: int
+    peak_vram_bytes: int
+    failure_reasons: tuple[str, ...] = ()
+
+    def to_dict(self) -> dict:
+        data = asdict(self)
+        data["failure_reasons"] = list(self.failure_reasons)
+        return data
+
+
+def summarize_concurrency(
+    outcomes: Sequence[RequestOutcome],
+    *,
+    concurrency: int,
+    wall_ms: float,
+    peak_rss_bytes: int,
+    peak_vram_bytes: int,
+) -> ConcurrencyMetrics:
+    """Aggregate one recipe/concurrency cell.
+
+    Per-request rates are averaged over successful requests; aggregate
+    throughput is total decoded tokens over the wall clock of the whole cell,
+    which is the only figure that credits a runtime for overlapping work.
+    """
+    ok = [outcome for outcome in outcomes if outcome.ok]
+    failures = [outcome for outcome in outcomes if not outcome.ok]
+    decode_tokens = sum(outcome.decode_tokens for outcome in ok)
+
+    prefill_rates = [
+        outcome.prompt_tokens / (outcome.prefill_ms / 1000)
+        for outcome in ok
+        if outcome.prefill_ms > 0 and outcome.prompt_tokens
+    ]
+    decode_rates = [
+        outcome.decode_tokens / (outcome.decode_ms / 1000)
+        for outcome in ok
+        if outcome.decode_ms > 0 and outcome.decode_tokens
+    ]
+    return ConcurrencyMetrics(
+        concurrency=concurrency,
+        requests=len(outcomes),
+        failures=len(failures),
+        wall_ms=round(wall_ms, 4),
+        ttft_p50_ms=_percentile([outcome.ttft_ms for outcome in ok], 50),
+        ttft_p95_ms=_percentile([outcome.ttft_ms for outcome in ok], 95),
+        latency_p50_ms=_percentile([outcome.latency_ms for outcome in ok], 50),
+        latency_p95_ms=_percentile([outcome.latency_ms for outcome in ok], 95),
+        prefill_tokens_per_sec=_mean(prefill_rates),
+        decode_tokens_per_sec=_mean(decode_rates),
+        aggregate_decode_tokens_per_sec=round(decode_tokens / max(1e-6, wall_ms / 1000), 4),
+        peak_rss_bytes=peak_rss_bytes,
+        peak_vram_bytes=peak_vram_bytes,
+        failure_reasons=tuple(sorted({outcome.error for outcome in failures if outcome.error})),
+    )
+
+
+@dataclass
+class RecipeMeasurement:
+    """Everything measured for one recipe across every concurrency level."""
+
+    recipe: RecipeSpec
+    load: LoadStats
+    metrics: dict[int, ConcurrencyMetrics] = field(default_factory=dict)
+    outcomes: list[RequestOutcome] = field(default_factory=list)
+    unavailable_reason: str = ""
+
+    @property
+    def available(self) -> bool:
+        return not self.unavailable_reason
+
+    def outputs_by_prompt(self) -> dict[str, str]:
+        """First successful output per prompt, at the lowest concurrency measured.
+
+        Drift is a property of the recipe, not of load: concurrency must not
+        change greedy output, so the least-contended sample is the fair one.
+        """
+        best: dict[str, tuple[int, str]] = {}
+        for outcome in self.outcomes:
+            if not outcome.ok:
+                continue
+            seen = best.get(outcome.prompt_id)
+            if seen is None or outcome.concurrency < seen[0]:
+                best[outcome.prompt_id] = (outcome.concurrency, outcome.text)
+        return {prompt_id: text for prompt_id, (_, text) in best.items()}
+
+    def to_dict(self) -> dict:
+        return {
+            "recipe": self.recipe.to_dict(),
+            "available": self.available,
+            "unavailable_reason": self.unavailable_reason,
+            "load": self.load.to_dict(),
+            "concurrency": {
+                str(level): metrics.to_dict() for level, metrics in sorted(self.metrics.items())
+            },
+            "outcomes": [outcome.to_dict() for outcome in self.outcomes],
+        }
+
+
+@dataclass(frozen=True)
+class DriftReport:
+    """Output drift of one recipe against the reference recipe.
+
+    ``advisory`` is true for every performance-fit recipe: the number is
+    published, but a Q4 recipe disagreeing with bf16 is expected behaviour, not a
+    defect, and no gate may read it as one.
+    """
+
+    recipe_id: str
+    lane: Lane
+    reference_id: str
+    compared_prompts: int
+    exact_match_rate: float
+    mean_similarity: float
+    advisory: bool
+    per_prompt: tuple[dict[str, Any], ...] = ()
+
+    def to_dict(self) -> dict:
+        return {
+            "recipe_id": self.recipe_id,
+            "lane": self.lane.value,
+            "reference_id": self.reference_id,
+            "compared_prompts": self.compared_prompts,
+            "exact_match_rate": self.exact_match_rate,
+            "mean_similarity": self.mean_similarity,
+            "advisory": self.advisory,
+            "per_prompt": list(self.per_prompt),
+        }
+
+
+def _first_divergence(left: str, right: str) -> int:
+    """Index of the first differing character, or -1 when the strings agree."""
+    for index, (a, b) in enumerate(zip(left, right)):
+        if a != b:
+            return index
+    return -1 if len(left) == len(right) else min(len(left), len(right))
+
+
+def compute_drift(
+    measurement: RecipeMeasurement,
+    reference: RecipeMeasurement,
+) -> DriftReport:
+    """Compare one recipe's greedy outputs with the reference recipe's."""
+    reference_outputs = reference.outputs_by_prompt()
+    outputs = measurement.outputs_by_prompt()
+    shared = sorted(set(outputs) & set(reference_outputs))
+
+    per_prompt: list[dict[str, Any]] = []
+    exact = 0
+    similarities: list[float] = []
+    for prompt_id in shared:
+        got, want = outputs[prompt_id], reference_outputs[prompt_id]
+        matches = got == want
+        exact += matches
+        similarity = round(SequenceMatcher(None, want, got).ratio(), 4)
+        similarities.append(similarity)
+        per_prompt.append({
+            "prompt_id": prompt_id,
+            "exact_match": matches,
+            "similarity": similarity,
+            "first_divergence_char": _first_divergence(want, got),
+            "reference_text": want,
+            "recipe_text": got,
+        })
+
+    return DriftReport(
+        recipe_id=measurement.recipe.id,
+        lane=measurement.recipe.lane,
+        reference_id=reference.recipe.id,
+        compared_prompts=len(shared),
+        exact_match_rate=round(exact / len(shared), 4) if shared else 0.0,
+        mean_similarity=_mean(similarities),
+        advisory=measurement.recipe.lane is Lane.PERFORMANCE_FIT,
+        per_prompt=tuple(per_prompt),
+    )
+
+
+class _PeakMemory:
+    """Sample a driver's memory while requests are in flight."""
+
+    def __init__(self, driver: RecipeDriver) -> None:
+        self._driver = driver
+        self.peak_rss = 0
+        self.peak_vram = 0
+
+    def sample(self) -> None:
+        try:
+            rss, vram = self._driver.memory_probe()
+        except Exception:  # a probe must never fail a benchmark run
+            return
+        self.peak_rss = max(self.peak_rss, rss)
+        self.peak_vram = max(self.peak_vram, vram)
+
+
+def _run_request(
+    driver: RecipeDriver,
+    recipe: RecipeSpec,
+    prompt: PromptSpec,
+    sampling: SamplingPolicy,
+    concurrency: int,
+    repeat: int,
+    memory: _PeakMemory,
+) -> RequestOutcome:
+    started = time.monotonic()
+    try:
+        sample = driver.generate(prompt.text, sampling)
+    except Exception as exc:  # a failed request is data, not a crashed benchmark
+        return RequestOutcome(
+            recipe_id=recipe.id,
+            concurrency=concurrency,
+            prompt_id=prompt.id,
+            repeat=repeat,
+            ok=False,
+            latency_ms=round((time.monotonic() - started) * 1000, 4),
+            error=f"{type(exc).__name__}: {exc}",
+        )
+    finally:
+        memory.sample()
+
+    return RequestOutcome(
+        recipe_id=recipe.id,
+        concurrency=concurrency,
+        prompt_id=prompt.id,
+        repeat=repeat,
+        ok=True,
+        latency_ms=round(sample.total_ms, 4),
+        ttft_ms=round(sample.ttft_ms, 4),
+        prefill_ms=round(sample.prefill_ms, 4),
+        decode_ms=round(sample.decode_ms, 4),
+        queue_wait_ms=round(sample.queue_wait_ms, 4),
+        prompt_tokens=sample.prompt_tokens,
+        decode_tokens=sample.decode_tokens,
+        text=sample.text,
+    )
+
+
+def measure_recipe(
+    driver: RecipeDriver,
+    recipe: RecipeSpec,
+    plan: BenchmarkPlan,
+) -> RecipeMeasurement:
+    """Load one recipe and run the whole plan against it.
+
+    The driver is closed exactly once, whatever happens, so a recipe that dies at
+    concurrency 4 still releases its weights before the next recipe loads.
+    """
+    load = driver.load()
+    memory = _PeakMemory(driver)
+    memory.sample()
+    measurement = RecipeMeasurement(recipe=recipe, load=load)
+
+    try:
+        for _ in range(plan.warmup_requests):
+            try:
+                driver.generate(plan.prompts[0].text, plan.sampling)
+            except Exception:  # a failing warmup is reported by the real requests
+                break
+
+        for concurrency in plan.concurrency_levels:
+            requests = [
+                (prompt, repeat)
+                for repeat in range(plan.repeats)
+                for prompt in plan.prompts
+                for _ in range(concurrency)
+            ]
+            started = time.monotonic()
+            with ThreadPoolExecutor(max_workers=concurrency) as pool:
+                outcomes = list(pool.map(
+                    lambda item: _run_request(
+                        driver, recipe, item[0], plan.sampling, concurrency, item[1], memory
+                    ),
+                    requests,
+                ))
+            wall_ms = (time.monotonic() - started) * 1000
+
+            measurement.outcomes.extend(outcomes)
+            measurement.metrics[concurrency] = summarize_concurrency(
+                outcomes,
+                concurrency=concurrency,
+                wall_ms=wall_ms,
+                peak_rss_bytes=memory.peak_rss,
+                peak_vram_bytes=memory.peak_vram,
+            )
+    finally:
+        driver.close()
+
+    return measurement
+
+
+def build_report(
+    plan: BenchmarkPlan,
+    measurements: Sequence[RecipeMeasurement],
+    *,
+    host: dict[str, Any],
+    evidence_class: str,
+) -> dict:
+    """Assemble the machine-readable benchmark document.
+
+    ``evidence_class`` is one of ``synthetic``, ``local-real`` or
+    ``multi-machine-real`` and is never inferred: a report that cannot say how it
+    was produced cannot be trusted by a release gate.
+    """
+    if evidence_class not in {"synthetic", "local-real", "multi-machine-real"}:
+        raise BenchmarkError(f"unknown evidence class {evidence_class!r}")
+
+    references = [m for m in measurements if m.recipe.is_reference]
+    if len(references) != 1:
+        raise BenchmarkError(
+            f"exactly one reference recipe is required, got {len(references)}"
+        )
+    reference = references[0]
+    if reference.recipe.lane is not Lane.QUALITY:
+        raise BenchmarkError("the reference recipe must sit in the quality lane")
+
+    drift = [
+        compute_drift(measurement, reference).to_dict()
+        for measurement in measurements
+        if measurement is not reference and measurement.available
+    ]
+    return {
+        "schema_version": REPORT_SCHEMA_VERSION,
+        "evidence_class": evidence_class,
+        "plan": plan.to_dict(),
+        "host": host,
+        "reference_recipe_id": reference.recipe.id,
+        "recipes": [measurement.to_dict() for measurement in measurements],
+        "drift": drift,
+    }
+
+
+def format_summary(report: dict) -> str:
+    """Render the human-readable companion to the JSON artifact."""
+    plan = report["plan"]
+    lines = [
+        f"Recipe benchmark {plan['plan_id']} ({report['evidence_class']})",
+        f"model {plan['model_id']}@{plan['model_revision']}",
+    ]
+    for entry in report["recipes"]:
+        recipe = entry["recipe"]
+        if not entry["available"]:
+            lines.append(f"{recipe['id']:38} UNAVAILABLE: {entry['unavailable_reason']}")
+            continue
+        artifact_gb = entry["load"]["artifact_bytes"] / 1e9
+        for level, metrics in entry["concurrency"].items():
+            lines.append(
+                f"{recipe['id']:38} [{recipe['lane']:16}] c={level:>2} "
+                f"ttft p50/p95 {metrics['ttft_p50_ms']:8.1f}/{metrics['ttft_p95_ms']:8.1f} ms; "
+                f"prefill {metrics['prefill_tokens_per_sec']:7.1f} tok/s; "
+                f"decode {metrics['decode_tokens_per_sec']:6.1f} tok/s; "
+                f"aggregate {metrics['aggregate_decode_tokens_per_sec']:7.1f} tok/s; "
+                f"rss {metrics['peak_rss_bytes'] / 1e9:5.2f} GB; "
+                f"vram {metrics['peak_vram_bytes'] / 1e9:5.2f} GB; "
+                f"artifact {artifact_gb:5.2f} GB; failures {metrics['failures']}"
+            )
+    for entry in report["drift"]:
+        tag = "advisory" if entry["advisory"] else "gated"
+        lines.append(
+            f"drift {entry['recipe_id']:32} vs {entry['reference_id']:28} "
+            f"exact {entry['exact_match_rate']:.2f}; similarity {entry['mean_similarity']:.3f} ({tag})"
+        )
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        description="Run the controlled safetensors-versus-GGUF recipe benchmark"
+    )
+    parser.add_argument("--config", type=Path, required=True, help="benchmark configuration JSON")
+    parser.add_argument("--json-out", type=Path, help="write the JSON report to this path")
+    parser.add_argument("--summary-out", type=Path, help="write the text summary to this path")
+    args = parser.parse_args(argv)
+
+    from .recipe_drivers import run_configured_benchmark  # heavy runtimes: import on demand
+
+    report = run_configured_benchmark(json.loads(args.config.read_text(encoding="utf-8")))
+    summary = format_summary(report)
+    if args.json_out:
+        args.json_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    if args.summary_out:
+        args.summary_out.write_text(summary + "\n", encoding="utf-8")
+    print(summary)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover - CLI entry point
+    raise SystemExit(main())