fix: harden DGR-001 performance contract evidence

packages/node/meshnet_node/performance_contract.py

@@ -25,12 +25,13 @@ the release gate is allowed to reach.
 
 from __future__ import annotations
 
+import hashlib
 import json
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Any, Mapping
 
-from .recipe_benchmark import Lane
+from .recipe_benchmark import Lane, REPORT_SCHEMA_VERSION
 
 # Layout of the contract document understood by this reader.
 CONTRACT_SCHEMA_VERSION = 1
@@ -140,6 +141,150 @@ def _ratio(value: float, reference: float) -> float:
     return round(value / reference, 4)
 
 
+def _canonical_sha256(value: Any) -> str:
+    payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -> None:
+    """Fail closed when a report is not the experiment the contract locked."""
+    try:
+        schema_version = report["schema_version"]
+        evidence_class = report["evidence_class"]
+        plan = report["plan"]
+        recipes = report["recipes"]
+        reference_id = report["reference_recipe_id"]
+        host = report["host"]
+    except (KeyError, TypeError) as exc:
+        raise PerformanceContractError("benchmark report is missing required structure") from exc
+
+    if schema_version != REPORT_SCHEMA_VERSION:
+        raise PerformanceContractError(
+            f"report schema {schema_version!r} is not supported schema {REPORT_SCHEMA_VERSION}"
+        )
+    if plan.get("plan_id") != contract.plan_id:
+        raise PerformanceContractError(
+            f"report plan {plan.get('plan_id')!r} does not match locked plan {contract.plan_id!r}"
+        )
+    required_plan_sha256 = contract.baseline.get("required_plan_sha256")
+    measured_plan_sha256 = _canonical_sha256(plan)
+    if required_plan_sha256 and measured_plan_sha256 != required_plan_sha256:
+        raise PerformanceContractError(
+            f"report plan digest {measured_plan_sha256} does not match locked digest "
+            f"{required_plan_sha256}"
+        )
+    minimum_repeats = int(contract.baseline.get("minimum_repeats", 0))
+    minimum_prompts = int(contract.baseline.get("minimum_prompt_count", 0))
+    if int(plan.get("repeats", 0)) < minimum_repeats:
+        raise PerformanceContractError("report has too few repeats for the locked contract")
+    if len(plan.get("prompts", ())) < minimum_prompts:
+        raise PerformanceContractError("report has too few prompts for the locked contract")
+    minimum_output_tokens = int(contract.baseline.get("minimum_output_tokens", 0))
+    if int(plan.get("sampling", {}).get("max_output_tokens", 0)) < minimum_output_tokens:
+        raise PerformanceContractError("report output length is below the locked contract")
+    required_evidence = contract.baseline.get("required_evidence_class")
+    if required_evidence and evidence_class != required_evidence:
+        raise PerformanceContractError(
+            f"report evidence class {evidence_class!r} does not satisfy {required_evidence!r}"
+        )
+    if required_evidence and (
+        not isinstance(host, Mapping)
+        or any(key not in host for key in ("hostname", "platform", "python", "cpu_count"))
+    ):
+        raise PerformanceContractError("report lacks measured host provenance")
+    if not isinstance(recipes, list) or not recipes:
+        raise PerformanceContractError("report contains no recipes")
+
+    recipe_ids = [entry.get("recipe", {}).get("id") for entry in recipes]
+    if len(set(recipe_ids)) != len(recipe_ids) or None in recipe_ids:
+        raise PerformanceContractError("report recipe IDs must be present and unique")
+    required_recipes = set(contract.baseline.get("required_recipes", ()))
+    missing_recipes = required_recipes - set(recipe_ids)
+    if missing_recipes:
+        raise PerformanceContractError(
+            f"report is missing required recipes {sorted(missing_recipes)}"
+        )
+    if reference_id not in recipe_ids:
+        raise PerformanceContractError("report reference recipe is absent")
+
+    levels = {int(level) for level in plan.get("concurrency_levels", ())}
+    required_levels = {
+        int(level) for level in contract.baseline.get("required_concurrency_levels", ())
+    }
+    if not required_levels.issubset(levels):
+        raise PerformanceContractError(
+            f"report concurrency {sorted(levels)} lacks required levels {sorted(required_levels)}"
+        )
+    if not plan.get("prompts"):
+        raise PerformanceContractError("report plan contains no prompts")
+
+    model_id = plan.get("model_id")
+    model_revision = plan.get("model_revision")
+    required_device = contract.baseline.get("required_device")
+    for entry in recipes:
+        recipe = entry.get("recipe", {})
+        if required_device and recipe.get("device") != required_device:
+            raise PerformanceContractError(
+                f"recipe {recipe.get('id')!r} did not run on locked device {required_device!r}"
+            )
+        if required_evidence:
+            if recipe.get("source_model_id") != model_id:
+                raise PerformanceContractError("report mixes source model IDs")
+            if recipe.get("source_model_revision") != model_revision:
+                raise PerformanceContractError("report mixes source model revisions")
+            digest = recipe.get("artifact_sha256", "")
+            if not isinstance(digest, str) or len(digest) != 64:
+                raise PerformanceContractError("report lacks an artifact SHA-256 digest")
+        if entry.get("available"):
+            cells = entry.get("concurrency", {})
+            missing_cells = required_levels - {int(level) for level in cells}
+            if missing_cells:
+                raise PerformanceContractError(
+                    f"recipe {recipe.get('id')!r} lacks concurrency cells {sorted(missing_cells)}"
+                )
+
+    reference = next(entry for entry in recipes if entry["recipe"]["id"] == reference_id)
+    if not reference.get("available"):
+        raise PerformanceContractError("reference recipe is unavailable")
+    reference_failures = sum(
+        int(cell.get("failures", 0)) for cell in reference.get("concurrency", {}).values()
+    )
+    if reference_failures:
+        raise PerformanceContractError("reference recipe contains failed requests")
+
+    def token_counts(entry: Mapping[str, Any]) -> dict[tuple[str, int, int], list[tuple[int, int]]]:
+        counts: dict[tuple[str, int, int], list[tuple[int, int]]] = {}
+        for outcome in entry.get("outcomes", ()):
+            if not outcome.get("ok"):
+                continue
+            key = (
+                str(outcome.get("prompt_id")),
+                int(outcome.get("concurrency", 0)),
+                int(outcome.get("repeat", -1)),
+            )
+            counts.setdefault(key, []).append(
+                (int(outcome.get("prompt_tokens", 0)), int(outcome.get("decode_tokens", 0)))
+            )
+        return {key: sorted(values) for key, values in counts.items()}
+
+    reference_counts = token_counts(reference)
+    prompt_ids = {str(prompt["id"]) for prompt in plan["prompts"]}
+    repeats = int(plan["repeats"])
+    for prompt_id in prompt_ids:
+        for level in required_levels:
+            for repeat in range(repeats):
+                values = reference_counts.get((prompt_id, level, repeat), ())
+                if len(values) != level:
+                    raise PerformanceContractError(
+                        "reference recipe lacks complete prompt/repeat/concurrency coverage"
+                    )
+    for entry in recipes:
+        if entry.get("available") and token_counts(entry) != reference_counts:
+            raise PerformanceContractError(
+                f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
+            )
+
+
 @dataclass(frozen=True)
 class RecipeEvaluation:
     """How one GGUF recipe fared against the reference under the contract."""
@@ -196,6 +341,7 @@ def _evaluate_recipe(
     drift_by_recipe: Mapping[str, Mapping[str, Any]],
     thresholds: ContractThresholds,
     concurrency_levels: list[int],
+    expected_prompt_count: int,
 ) -> RecipeEvaluation:
     recipe = entry["recipe"]
     lane = recipe["lane"]
@@ -237,7 +383,7 @@ def _evaluate_recipe(
             and 0 < ttft_ratio <= thresholds.max_ttft_ratio
         )
         if single_request_win:
-            speed_benefit = True
+            speed_benefit = lane == Lane.PERFORMANCE_FIT.value
             reasons.append(
                 f"single-request decode {decode_speedup:.2f}x reference "
                 f"(>= {thresholds.min_decode_speedup:.2f}x) at TTFT ratio {ttft_ratio:.2f}"
@@ -259,7 +405,7 @@ def _evaluate_recipe(
             measurements["aggregate_throughput_speedup"] = aggregate_speedup
             measurements["aggregate_concurrency"] = top
             if aggregate_speedup >= thresholds.min_aggregate_throughput_speedup:
-                speed_benefit = True
+                speed_benefit = lane == Lane.PERFORMANCE_FIT.value
                 reasons.append(
                     f"aggregate throughput at concurrency {top} is {aggregate_speedup:.2f}x reference "
                     f"(>= {thresholds.min_aggregate_throughput_speedup:.2f}x)"
@@ -279,7 +425,7 @@ def _evaluate_recipe(
         measurements["resident_memory_ratio"] = resident_ratio
         measurements["artifact_size_ratio"] = artifact_ratio
         if 0 < resident_ratio <= thresholds.max_resident_memory_ratio:
-            fit_benefit = True
+            fit_benefit = lane == Lane.PERFORMANCE_FIT.value
             reasons.append(
                 f"peak resident memory is {resident_ratio:.2f}x reference "
                 f"(<= {thresholds.max_resident_memory_ratio:.2f}x)"
@@ -309,12 +455,24 @@ def _evaluate_recipe(
             quality_pass = False
             reasons.append("quality-lane recipe has no drift measurement against the reference")
         else:
+            complete_coverage = drift.get("compared_prompts") == expected_prompt_count
             quality_pass = (
-                drift["exact_match_rate"] >= thresholds.min_quality_exact_match_rate
+                complete_coverage
+                and failures == 0
+                and drift["exact_match_rate"] >= thresholds.min_quality_exact_match_rate
                 and drift["mean_similarity"] >= thresholds.min_quality_mean_similarity
             )
+            measurements["compared_prompts"] = drift.get("compared_prompts", 0)
+            measurements["expected_prompts"] = expected_prompt_count
             measurements["exact_match_rate"] = drift["exact_match_rate"]
             measurements["mean_similarity"] = drift["mean_similarity"]
+            if not complete_coverage:
+                reasons.append(
+                    f"quality lane compared {drift.get('compared_prompts', 0)} of "
+                    f"{expected_prompt_count} required prompts"
+                )
+            if failures:
+                reasons.append("quality lane contains failed requests")
             reasons.append(
                 f"quality lane exact-match {drift['exact_match_rate']:.2f} / similarity "
                 f"{drift['mean_similarity']:.3f} versus the reference "
@@ -339,6 +497,7 @@ def evaluate_contract(
     runtime that fails the quality lane is broken, and no amount of speed
     redeems it, so the verdict is ``stop`` regardless of the other numbers.
     """
+    _validate_report(contract, report)
     entries = _recipe_entries(report)
     reference_id = report["reference_recipe_id"]
     reference = entries.get(reference_id)
@@ -351,7 +510,14 @@ def evaluate_contract(
     concurrency_levels = list(report["plan"]["concurrency_levels"])
 
     evaluations = tuple(
-        _evaluate_recipe(entry, reference, drift_by_recipe, contract.thresholds, concurrency_levels)
+        _evaluate_recipe(
+            entry,
+            reference,
+            drift_by_recipe,
+            contract.thresholds,
+            concurrency_levels,
+            len(report["plan"]["prompts"]),
+        )
         for recipe_id, entry in entries.items()
         if recipe_id != reference_id
     )
@@ -439,18 +605,29 @@ def parse_contract(data: Any, source: str = "<memory>") -> PerformanceContract:
     raw_thresholds = data.get("thresholds")
     if not isinstance(raw_thresholds, Mapping):
         raise PerformanceContractError(f"'thresholds' in {source} must be a JSON object")
-    known = {field for field in ContractThresholds().to_dict()}
+    known = set(ContractThresholds().to_dict())
     unknown = set(raw_thresholds) - known
-    if unknown:
+    missing = known - set(raw_thresholds)
+    if unknown or missing:
         raise PerformanceContractError(
-            f"{source} carries unknown thresholds {sorted(unknown)}; this node enforces {sorted(known)}"
+            f"{source} threshold keys differ from v1; unknown={sorted(unknown)}, "
+            f"missing={sorted(missing)}"
         )
     thresholds = ContractThresholds(**{
         key: float(value) for key, value in raw_thresholds.items()
     })
+    contract_version = int(data["contract_version"])
+    if contract_version != 1 or thresholds != ContractThresholds():
+        raise PerformanceContractError(
+            f"{source} changes immutable v1 thresholds without a supported contract version"
+        )
+    if str(data["stop_condition"]) != STOP_CONDITION:
+        raise PerformanceContractError(
+            f"{source} stop condition differs from executable v1 semantics"
+        )
 
     return PerformanceContract(
-        contract_version=int(data["contract_version"]),
+        contract_version=contract_version,
         locked_at=str(data["locked_at"]),
         locked_by=str(data["locked_by"]),
         plan_id=str(data["plan_id"]),
@@ -486,6 +663,7 @@ def baseline_from_report(report: Mapping[str, Any]) -> dict[str, Any]:
         "evidence_class": report["evidence_class"],
         "model_id": report["plan"]["model_id"],
         "model_revision": report["plan"]["model_revision"],
+        "plan_sha256": _canonical_sha256(report["plan"]),
         "reference_recipe_id": report["reference_recipe_id"],
         "host": report["host"],
         "recipes": {},

packages/node/meshnet_node/recipe_benchmark.py

@@ -28,6 +28,7 @@ from __future__ import annotations
 import argparse
 import json
 import statistics
+import threading
 import time
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import asdict, dataclass, field
@@ -443,7 +444,7 @@ def compute_drift(
 
 
 class _PeakMemory:
-    """Sample a driver's memory while requests are in flight."""
+    """Continuously sample a driver's memory while requests are in flight."""
 
     def __init__(self, driver: RecipeDriver) -> None:
         self._driver = driver
@@ -458,6 +459,12 @@ class _PeakMemory:
         self.peak_rss = max(self.peak_rss, rss)
         self.peak_vram = max(self.peak_vram, vram)
 
+    def sample_until(self, stop: threading.Event, interval_s: float = 0.01) -> None:
+        """Sample until ``stop`` is set, including one final post-request probe."""
+        while not stop.wait(interval_s):
+            self.sample()
+        self.sample()
+
 
 def _run_request(
     driver: RecipeDriver,
@@ -512,8 +519,6 @@ def measure_recipe(
     concurrency 4 still releases its weights before the next recipe loads.
     """
     load = driver.load()
-    memory = _PeakMemory(driver)
-    memory.sample()
     measurement = RecipeMeasurement(recipe=recipe, load=load)
 
     try:
@@ -524,6 +529,17 @@ def measure_recipe(
                 break
 
         for concurrency in plan.concurrency_levels:
+            # Measure each cell independently and sample while requests are in
+            # flight; post-request probes alone miss transient KV/workspace peaks.
+            memory = _PeakMemory(driver)
+            memory.sample()
+            stop_sampling = threading.Event()
+            sampler = threading.Thread(
+                target=memory.sample_until,
+                args=(stop_sampling,),
+                name=f"recipe-memory-{recipe.id}-c{concurrency}",
+                daemon=True,
+            )
             requests = [
                 (prompt, repeat)
                 for repeat in range(plan.repeats)
@@ -531,13 +547,18 @@ def measure_recipe(
                 for _ in range(concurrency)
             ]
             started = time.monotonic()
-            with ThreadPoolExecutor(max_workers=concurrency) as pool:
-                outcomes = list(pool.map(
-                    lambda item: _run_request(
-                        driver, recipe, item[0], plan.sampling, concurrency, item[1], memory
-                    ),
-                    requests,
-                ))
+            sampler.start()
+            try:
+                with ThreadPoolExecutor(max_workers=concurrency) as pool:
+                    outcomes = list(pool.map(
+                        lambda item: _run_request(
+                            driver, recipe, item[0], plan.sampling, concurrency, item[1], memory
+                        ),
+                        requests,
+                    ))
+            finally:
+                stop_sampling.set()
+                sampler.join()
             wall_ms = (time.monotonic() - started) * 1000
 
             measurement.outcomes.extend(outcomes)

packages/node/meshnet_node/recipe_drivers.py

@@ -20,12 +20,16 @@ rules:
 
 from __future__ import annotations
 
+import hashlib
+import hmac
 import json
 import os
 import platform
+import re
 import socket
 import subprocess
 import sys
+import tempfile
 import time
 import urllib.error
 import urllib.request
@@ -85,6 +89,37 @@ def _directory_bytes(path: Path) -> int:
     return sum(entry.stat().st_size for entry in path.rglob("*") if entry.is_file())
 
 
+def _artifact_sha256(path: Path) -> str:
+    """Hash an artifact file or a deterministic directory content manifest.
+
+    A file uses the ordinary SHA-256 digest. A directory hashes each sorted
+    relative path, resolved file size, and file bytes, so tokenizer/config drift
+    cannot hide behind a weight-only digest.
+    """
+    digest = hashlib.sha256()
+    if path.is_file():
+        entries = [(None, path)]
+    else:
+        entries = [
+            (entry.relative_to(path).as_posix(), entry)
+            for entry in sorted(path.rglob("*"))
+            if entry.is_file()
+        ]
+        if not entries:
+            raise BenchmarkError(f"artifact directory is empty: {path}")
+
+    for relative, entry in entries:
+        if relative is not None:
+            encoded = relative.encode("utf-8")
+            digest.update(len(encoded).to_bytes(8, "big"))
+            digest.update(encoded)
+            digest.update(entry.stat().st_size.to_bytes(8, "big"))
+        with entry.open("rb") as stream:
+            while chunk := stream.read(8 * 1024 * 1024):
+                digest.update(chunk)
+    return digest.hexdigest()
+
+
 def _host_manifest() -> dict[str, Any]:
     """Capture non-secret host facts with the report rather than trusting prose."""
     manifest: dict[str, Any] = {
@@ -95,8 +130,10 @@ def _host_manifest() -> dict[str, Any]:
     }
     try:
         import torch
+        import transformers
 
         manifest["torch_version"] = torch.__version__
+        manifest["transformers_version"] = transformers.__version__
         manifest["cuda_available"] = bool(torch.cuda.is_available())
         if torch.cuda.is_available():
             manifest["accelerator_name"] = torch.cuda.get_device_name(0)
@@ -109,7 +146,7 @@ def _host_manifest() -> dict[str, Any]:
 
 
 def _validate_config(config: Mapping[str, Any]) -> None:
-    """Reject a comparison that could silently mix models or use home storage."""
+    """Reject comparisons that mix models, artifacts, devices, or budgets."""
     try:
         plan = config["plan"]
         root = Path(config["artifact_storage_root"]).resolve(strict=True)
@@ -122,17 +159,75 @@ def _validate_config(config: Mapping[str, Any]) -> None:
         raise BenchmarkError("model artifacts must use configured mounted-drive storage, never /home")
     if not isinstance(recipes, list) or not recipes:
         raise BenchmarkError("benchmark config needs at least one recipe")
+
+    sampling = plan.get("sampling", {})
+    if (
+        float(sampling.get("temperature", 0.0)) != 0.0
+        or int(sampling.get("top_k", 1)) != 1
+        or float(sampling.get("top_p", 1.0)) != 1.0
+    ):
+        raise BenchmarkError("the quality comparison requires greedy sampling")
+
+    if len(plan.get("prompts", ())) < 3 or int(plan.get("repeats", 0)) < 3:
+        raise BenchmarkError("contract-grade evidence requires at least 3 prompts and 3 repeats")
+    if int(plan.get("warmup_requests", 0)) < 1:
+        raise BenchmarkError("contract-grade evidence requires at least one warmup")
+    if int(sampling.get("max_output_tokens", 0)) < 32:
+        raise BenchmarkError("contract-grade evidence requires at least 32 output tokens")
+
+    thread_budgets: set[int] = set()
+    max_concurrency = max(int(level) for level in plan.get("concurrency_levels", (1, 4)))
     for spec in recipes:
         if spec.get("source_model_id") != plan.get("model_id"):
             raise BenchmarkError("every recipe must declare the plan's exact source_model_id")
         if spec.get("source_model_revision") != plan.get("model_revision"):
             raise BenchmarkError("every recipe must declare the plan's exact source_model_revision")
+
         digest = spec.get("artifact_sha256", "")
-        if not isinstance(digest, str) or len(digest) != 64:
-            raise BenchmarkError("every recipe must declare its exact 64-character artifact_sha256")
+        if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None:
+            raise BenchmarkError("every recipe must declare a lowercase SHA-256 artifact digest")
         artifact = Path(spec.get("artifact_path", "")).resolve(strict=True)
         if artifact != root and root not in artifact.parents:
             raise BenchmarkError("every model artifact must be beneath artifact_storage_root")
+        actual_digest = _artifact_sha256(artifact)
+        if not hmac.compare_digest(digest, actual_digest):
+            raise BenchmarkError(
+                f"artifact digest mismatch for {spec.get('id', '<unknown>')}: "
+                f"declared {digest}, measured {actual_digest}"
+            )
+
+        driver = spec.get("driver")
+        if not isinstance(driver, Mapping):
+            raise BenchmarkError("every recipe needs a driver object")
+        kind = driver.get("type")
+        if kind == "transformers":
+            driver_artifact = Path(driver.get("model_path", "")).resolve(strict=True)
+        elif kind == "llama-cpp-server":
+            driver_artifact = Path(driver.get("gguf_path", "")).resolve(strict=True)
+            binary = Path(driver.get("binary", "")).resolve(strict=True)
+            binary_digest = driver.get("binary_sha256", "")
+            if (
+                not isinstance(binary_digest, str)
+                or re.fullmatch(r"[0-9a-f]{64}", binary_digest) is None
+                or not hmac.compare_digest(binary_digest, _artifact_sha256(binary))
+            ):
+                raise BenchmarkError("llama.cpp binary SHA-256 mismatch")
+            if int(driver.get("n_parallel", max_concurrency)) < max_concurrency:
+                raise BenchmarkError("llama.cpp parallel slots must cover maximum concurrency")
+            if driver.get("device", "cpu") != "cpu" or int(driver.get("n_gpu_layers", 0)) != 0:
+                raise BenchmarkError(
+                    "v1 benchmark supports CPU-only llama.cpp until process VRAM is measurable"
+                )
+        else:
+            raise BenchmarkError(f"unknown driver type {kind!r}")
+        if driver_artifact != artifact:
+            raise BenchmarkError("driver artifact path must match the hashed recipe artifact")
+        if driver.get("device", "cpu") != spec.get("device"):
+            raise BenchmarkError("recipe and driver must declare the same device")
+        thread_budgets.add(int(driver.get("threads", 8)))
+
+    if len(thread_budgets) != 1:
+        raise BenchmarkError("every recipe must use the same CPU thread budget")
 
 
 class TransformersDriver:
@@ -160,8 +255,10 @@ class TransformersDriver:
         self._model: Any = None
         self._tokenizer: Any = None
         self._torch: Any = None
+        self._rss_baseline = 0
 
     def load(self) -> LoadStats:
+        self._rss_baseline = _process_rss()
         import torch
         from transformers import AutoModelForCausalLM, AutoTokenizer
 
@@ -185,7 +282,7 @@ class TransformersDriver:
         return LoadStats(
             artifact_bytes=_directory_bytes(self.model_path),
             load_ms=round(load_ms, 4),
-            rss_bytes=_process_rss(),
+            rss_bytes=max(0, _process_rss() - self._rss_baseline),
             vram_bytes=self._vram_bytes(),
             backend_detail=(
                 f"torch {torch.__version__}; dtype {self.dtype}; "
@@ -262,7 +359,7 @@ class TransformersDriver:
         return logits.argmax(dim=-1)
 
     def memory_probe(self) -> tuple[int, int]:
-        return _process_rss(), self._vram_bytes()
+        return max(0, _process_rss() - self._rss_baseline), self._vram_bytes()
 
     def close(self) -> None:
         self._model = None
@@ -295,6 +392,7 @@ class LlamaCppServerDriver:
         binary: str,
         gguf_path: str,
         *,
+        binary_sha256: str,
         device: str = "cpu",
         threads: int = 8,
         n_parallel: int = 4,
@@ -303,6 +401,7 @@ class LlamaCppServerDriver:
         startup_timeout_s: float = 120.0,
     ) -> None:
         self.binary = Path(binary)
+        self.binary_sha256 = binary_sha256
         self.gguf_path = Path(gguf_path)
         self.device = device
         self.threads = threads
@@ -318,11 +417,34 @@ class LlamaCppServerDriver:
     def _url(self) -> str:
         return f"http://127.0.0.1:{self._port}"
 
+    def _log_excerpt(self) -> str:
+        if self._log is None:
+            return ""
+        try:
+            self._log.flush()
+            self._log.seek(0)
+            return self._log.read()[-4096:].decode("utf-8", errors="replace").strip()
+        except Exception:
+            return ""
+
     def load(self) -> LoadStats:
         if not self.binary.exists():
             raise BenchmarkError(f"llama-server binary not found at {self.binary}")
         if not self.gguf_path.exists():
             raise BenchmarkError(f"GGUF artifact not found at {self.gguf_path}")
+        measured_binary_sha256 = _artifact_sha256(self.binary)
+        if not hmac.compare_digest(self.binary_sha256, measured_binary_sha256):
+            raise BenchmarkError("llama-server binary changed after config validation")
+        version = " | ".join(
+            subprocess.run(
+                [str(self.binary), "--version"],
+                check=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                timeout=10,
+            ).stdout.strip().splitlines()
+        )
 
         self._port = _free_port()
         command = [
@@ -339,9 +461,9 @@ class LlamaCppServerDriver:
             "--no-webui",
         ]
         started = time.monotonic()
-        self._log = subprocess.PIPE
+        self._log = tempfile.TemporaryFile(mode="w+b")
         self._process = subprocess.Popen(
-            command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+            command, stdout=self._log, stderr=subprocess.STDOUT
         )
         self._await_health(started)
         load_ms = (time.monotonic() - started) * 1000
@@ -352,7 +474,8 @@ class LlamaCppServerDriver:
             rss_bytes=_process_rss(self._process.pid),
             vram_bytes=0,
             backend_detail=(
-                f"llama-server; threads {self.threads}; parallel slots {self.n_parallel}; "
+                f"{version}; binary sha256 {measured_binary_sha256}; "
+                f"threads {self.threads}; parallel slots {self.n_parallel}; "
                 f"ctx/slot {self.context_per_slot}; gpu layers {self.n_gpu_layers}"
             ),
         )
@@ -361,7 +484,8 @@ class LlamaCppServerDriver:
         while time.monotonic() - started < self.startup_timeout_s:
             if self._process is not None and self._process.poll() is not None:
                 raise BenchmarkError(
-                    f"llama-server exited with code {self._process.returncode} during startup"
+                    f"llama-server exited with code {self._process.returncode} during startup; "
+                    f"log tail: {self._log_excerpt()}"
                 )
             try:
                 with urllib.request.urlopen(f"{self._url}/health", timeout=2) as response:
@@ -370,7 +494,8 @@ class LlamaCppServerDriver:
             except (urllib.error.URLError, OSError):
                 time.sleep(0.25)
         raise BenchmarkError(
-            f"llama-server did not become healthy within {self.startup_timeout_s:.0f}s"
+            f"llama-server did not become healthy within {self.startup_timeout_s:.0f}s; "
+            f"log tail: {self._log_excerpt()}"
         )
 
     def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
@@ -397,7 +522,6 @@ class LlamaCppServerDriver:
         )
 
         started = time.monotonic()
-        ttft_ms = 0.0
         chunks: list[str] = []
         timings: Mapping[str, Any] = {}
         with urllib.request.urlopen(request, timeout=600) as response:
@@ -407,8 +531,6 @@ class LlamaCppServerDriver:
                     continue
                 payload = json.loads(line[len("data:"):].strip())
                 content = payload.get("content", "")
-                if content and not ttft_ms:
-                    ttft_ms = (time.monotonic() - started) * 1000
                 chunks.append(content)
                 if payload.get("stop"):
                     timings = payload.get("timings") or {}
@@ -422,8 +544,14 @@ class LlamaCppServerDriver:
         return GenerationSample(
             text="".join(chunks),
             prompt_tokens=int(timings.get("prompt_n", 0)),
-            decode_tokens=int(timings.get("predicted_n", 0)),
-            ttft_ms=ttft_ms or total_ms,
+            # llama.cpp starts predicted_ms after sampling the first token while
+            # predicted_n includes it. Exclude that token to match the
+            # Transformers inter-token decode metric.
+            decode_tokens=max(0, int(timings.get("predicted_n", 0)) - 1),
+            # Use the runtime's prompt/first-token timing, matching the
+            # in-process Transformers boundary. HTTP/SSE and slot delay remain
+            # represented by total latency and queue_wait_ms.
+            ttft_ms=prefill_ms,
             prefill_ms=prefill_ms,
             decode_ms=decode_ms,
             total_ms=total_ms,
@@ -438,15 +566,18 @@ class LlamaCppServerDriver:
         return _process_rss(self._process.pid), 0
 
     def close(self) -> None:
-        if self._process is None:
-            return
-        self._process.terminate()
-        try:
-            self._process.wait(timeout=20)
-        except subprocess.TimeoutExpired:
-            self._process.kill()
-            self._process.wait(timeout=10)
-        self._process = None
+        if self._process is not None:
+            if self._process.poll() is None:
+                self._process.terminate()
+                try:
+                    self._process.wait(timeout=20)
+                except subprocess.TimeoutExpired:
+                    self._process.kill()
+                    self._process.wait(timeout=10)
+            self._process = None
+        if self._log is not None:
+            self._log.close()
+            self._log = None
 
 
 def build_driver(spec: Mapping[str, Any], plan: BenchmarkPlan) -> RecipeDriverBundle:
@@ -511,6 +642,7 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
     measurements = []
     for spec in config["recipes"]:
         recipe = _recipe_from_config(spec)
+        driver = None
         try:
             driver = build_driver(spec, plan)
             measurements.append(measure_recipe(driver, recipe, plan))
@@ -520,10 +652,13 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
                 load=LoadStats(artifact_bytes=0, load_ms=0.0),
                 unavailable_reason=f"{type(exc).__name__}: {exc}",
             ))
+        finally:
+            if driver is not None:
+                driver.close()
 
     return build_report(
         plan,
         measurements,
-        host={**_host_manifest(), **dict(config.get("host", {}))},
+        host={**dict(config.get("host", {})), **_host_manifest()},
         evidence_class=config.get("evidence_class", "local-real"),
     )