fix: harden DGR-001 performance contract evidence

packages/node/meshnet_node/performance_contract.py

@@ -25,12 +25,13 @@ the release gate is allowed to reach.
 
 from __future__ import annotations
 
+import hashlib
 import json
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Any, Mapping
 
-from .recipe_benchmark import Lane
+from .recipe_benchmark import Lane, REPORT_SCHEMA_VERSION
 
 # Layout of the contract document understood by this reader.
 CONTRACT_SCHEMA_VERSION = 1
@@ -140,6 +141,150 @@ def _ratio(value: float, reference: float) -> float:
     return round(value / reference, 4)
 
 
+def _canonical_sha256(value: Any) -> str:
+    payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -> None:
+    """Fail closed when a report is not the experiment the contract locked."""
+    try:
+        schema_version = report["schema_version"]
+        evidence_class = report["evidence_class"]
+        plan = report["plan"]
+        recipes = report["recipes"]
+        reference_id = report["reference_recipe_id"]
+        host = report["host"]
+    except (KeyError, TypeError) as exc:
+        raise PerformanceContractError("benchmark report is missing required structure") from exc
+
+    if schema_version != REPORT_SCHEMA_VERSION:
+        raise PerformanceContractError(
+            f"report schema {schema_version!r} is not supported schema {REPORT_SCHEMA_VERSION}"
+        )
+    if plan.get("plan_id") != contract.plan_id:
+        raise PerformanceContractError(
+            f"report plan {plan.get('plan_id')!r} does not match locked plan {contract.plan_id!r}"
+        )
+    required_plan_sha256 = contract.baseline.get("required_plan_sha256")
+    measured_plan_sha256 = _canonical_sha256(plan)
+    if required_plan_sha256 and measured_plan_sha256 != required_plan_sha256:
+        raise PerformanceContractError(
+            f"report plan digest {measured_plan_sha256} does not match locked digest "
+            f"{required_plan_sha256}"
+        )
+    minimum_repeats = int(contract.baseline.get("minimum_repeats", 0))
+    minimum_prompts = int(contract.baseline.get("minimum_prompt_count", 0))
+    if int(plan.get("repeats", 0)) < minimum_repeats:
+        raise PerformanceContractError("report has too few repeats for the locked contract")
+    if len(plan.get("prompts", ())) < minimum_prompts:
+        raise PerformanceContractError("report has too few prompts for the locked contract")
+    minimum_output_tokens = int(contract.baseline.get("minimum_output_tokens", 0))
+    if int(plan.get("sampling", {}).get("max_output_tokens", 0)) < minimum_output_tokens:
+        raise PerformanceContractError("report output length is below the locked contract")
+    required_evidence = contract.baseline.get("required_evidence_class")
+    if required_evidence and evidence_class != required_evidence:
+        raise PerformanceContractError(
+            f"report evidence class {evidence_class!r} does not satisfy {required_evidence!r}"
+        )
+    if required_evidence and (
+        not isinstance(host, Mapping)
+        or any(key not in host for key in ("hostname", "platform", "python", "cpu_count"))
+    ):
+        raise PerformanceContractError("report lacks measured host provenance")
+    if not isinstance(recipes, list) or not recipes:
+        raise PerformanceContractError("report contains no recipes")
+
+    recipe_ids = [entry.get("recipe", {}).get("id") for entry in recipes]
+    if len(set(recipe_ids)) != len(recipe_ids) or None in recipe_ids:
+        raise PerformanceContractError("report recipe IDs must be present and unique")
+    required_recipes = set(contract.baseline.get("required_recipes", ()))
+    missing_recipes = required_recipes - set(recipe_ids)
+    if missing_recipes:
+        raise PerformanceContractError(
+            f"report is missing required recipes {sorted(missing_recipes)}"
+        )
+    if reference_id not in recipe_ids:
+        raise PerformanceContractError("report reference recipe is absent")
+
+    levels = {int(level) for level in plan.get("concurrency_levels", ())}
+    required_levels = {
+        int(level) for level in contract.baseline.get("required_concurrency_levels", ())
+    }
+    if not required_levels.issubset(levels):
+        raise PerformanceContractError(
+            f"report concurrency {sorted(levels)} lacks required levels {sorted(required_levels)}"
+        )
+    if not plan.get("prompts"):
+        raise PerformanceContractError("report plan contains no prompts")
+
+    model_id = plan.get("model_id")
+    model_revision = plan.get("model_revision")
+    required_device = contract.baseline.get("required_device")
+    for entry in recipes:
+        recipe = entry.get("recipe", {})
+        if required_device and recipe.get("device") != required_device:
+            raise PerformanceContractError(
+                f"recipe {recipe.get('id')!r} did not run on locked device {required_device!r}"
+            )
+        if required_evidence:
+            if recipe.get("source_model_id") != model_id:
+                raise PerformanceContractError("report mixes source model IDs")
+            if recipe.get("source_model_revision") != model_revision:
+                raise PerformanceContractError("report mixes source model revisions")
+            digest = recipe.get("artifact_sha256", "")
+            if not isinstance(digest, str) or len(digest) != 64:
+                raise PerformanceContractError("report lacks an artifact SHA-256 digest")
+        if entry.get("available"):
+            cells = entry.get("concurrency", {})
+            missing_cells = required_levels - {int(level) for level in cells}
+            if missing_cells:
+                raise PerformanceContractError(
+                    f"recipe {recipe.get('id')!r} lacks concurrency cells {sorted(missing_cells)}"
+                )
+
+    reference = next(entry for entry in recipes if entry["recipe"]["id"] == reference_id)
+    if not reference.get("available"):
+        raise PerformanceContractError("reference recipe is unavailable")
+    reference_failures = sum(
+        int(cell.get("failures", 0)) for cell in reference.get("concurrency", {}).values()
+    )
+    if reference_failures:
+        raise PerformanceContractError("reference recipe contains failed requests")
+
+    def token_counts(entry: Mapping[str, Any]) -> dict[tuple[str, int, int], list[tuple[int, int]]]:
+        counts: dict[tuple[str, int, int], list[tuple[int, int]]] = {}
+        for outcome in entry.get("outcomes", ()):
+            if not outcome.get("ok"):
+                continue
+            key = (
+                str(outcome.get("prompt_id")),
+                int(outcome.get("concurrency", 0)),
+                int(outcome.get("repeat", -1)),
+            )
+            counts.setdefault(key, []).append(
+                (int(outcome.get("prompt_tokens", 0)), int(outcome.get("decode_tokens", 0)))
+            )
+        return {key: sorted(values) for key, values in counts.items()}
+
+    reference_counts = token_counts(reference)
+    prompt_ids = {str(prompt["id"]) for prompt in plan["prompts"]}
+    repeats = int(plan["repeats"])
+    for prompt_id in prompt_ids:
+        for level in required_levels:
+            for repeat in range(repeats):
+                values = reference_counts.get((prompt_id, level, repeat), ())
+                if len(values) != level:
+                    raise PerformanceContractError(
+                        "reference recipe lacks complete prompt/repeat/concurrency coverage"
+                    )
+    for entry in recipes:
+        if entry.get("available") and token_counts(entry) != reference_counts:
+            raise PerformanceContractError(
+                f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
+            )
+
+
 @dataclass(frozen=True)
 class RecipeEvaluation:
     """How one GGUF recipe fared against the reference under the contract."""
@@ -196,6 +341,7 @@ def _evaluate_recipe(
     drift_by_recipe: Mapping[str, Mapping[str, Any]],
     thresholds: ContractThresholds,
     concurrency_levels: list[int],
+    expected_prompt_count: int,
 ) -> RecipeEvaluation:
     recipe = entry["recipe"]
     lane = recipe["lane"]
@@ -237,7 +383,7 @@ def _evaluate_recipe(
             and 0 < ttft_ratio <= thresholds.max_ttft_ratio
         )
         if single_request_win:
-            speed_benefit = True
+            speed_benefit = lane == Lane.PERFORMANCE_FIT.value
             reasons.append(
                 f"single-request decode {decode_speedup:.2f}x reference "
                 f"(>= {thresholds.min_decode_speedup:.2f}x) at TTFT ratio {ttft_ratio:.2f}"
@@ -259,7 +405,7 @@ def _evaluate_recipe(
             measurements["aggregate_throughput_speedup"] = aggregate_speedup
             measurements["aggregate_concurrency"] = top
             if aggregate_speedup >= thresholds.min_aggregate_throughput_speedup:
-                speed_benefit = True
+                speed_benefit = lane == Lane.PERFORMANCE_FIT.value
                 reasons.append(
                     f"aggregate throughput at concurrency {top} is {aggregate_speedup:.2f}x reference "
                     f"(>= {thresholds.min_aggregate_throughput_speedup:.2f}x)"
@@ -279,7 +425,7 @@ def _evaluate_recipe(
         measurements["resident_memory_ratio"] = resident_ratio
         measurements["artifact_size_ratio"] = artifact_ratio
         if 0 < resident_ratio <= thresholds.max_resident_memory_ratio:
-            fit_benefit = True
+            fit_benefit = lane == Lane.PERFORMANCE_FIT.value
             reasons.append(
                 f"peak resident memory is {resident_ratio:.2f}x reference "
                 f"(<= {thresholds.max_resident_memory_ratio:.2f}x)"
@@ -309,12 +455,24 @@ def _evaluate_recipe(
             quality_pass = False
             reasons.append("quality-lane recipe has no drift measurement against the reference")
         else:
+            complete_coverage = drift.get("compared_prompts") == expected_prompt_count
             quality_pass = (
-                drift["exact_match_rate"] >= thresholds.min_quality_exact_match_rate
+                complete_coverage
+                and failures == 0
+                and drift["exact_match_rate"] >= thresholds.min_quality_exact_match_rate
                 and drift["mean_similarity"] >= thresholds.min_quality_mean_similarity
             )
+            measurements["compared_prompts"] = drift.get("compared_prompts", 0)
+            measurements["expected_prompts"] = expected_prompt_count
             measurements["exact_match_rate"] = drift["exact_match_rate"]
             measurements["mean_similarity"] = drift["mean_similarity"]
+            if not complete_coverage:
+                reasons.append(
+                    f"quality lane compared {drift.get('compared_prompts', 0)} of "
+                    f"{expected_prompt_count} required prompts"
+                )
+            if failures:
+                reasons.append("quality lane contains failed requests")
             reasons.append(
                 f"quality lane exact-match {drift['exact_match_rate']:.2f} / similarity "
                 f"{drift['mean_similarity']:.3f} versus the reference "
@@ -339,6 +497,7 @@ def evaluate_contract(
     runtime that fails the quality lane is broken, and no amount of speed
     redeems it, so the verdict is ``stop`` regardless of the other numbers.
     """
+    _validate_report(contract, report)
     entries = _recipe_entries(report)
     reference_id = report["reference_recipe_id"]
     reference = entries.get(reference_id)
@@ -351,7 +510,14 @@ def evaluate_contract(
     concurrency_levels = list(report["plan"]["concurrency_levels"])
 
     evaluations = tuple(
-        _evaluate_recipe(entry, reference, drift_by_recipe, contract.thresholds, concurrency_levels)
+        _evaluate_recipe(
+            entry,
+            reference,
+            drift_by_recipe,
+            contract.thresholds,
+            concurrency_levels,
+            len(report["plan"]["prompts"]),
+        )
         for recipe_id, entry in entries.items()
         if recipe_id != reference_id
     )
@@ -439,18 +605,29 @@ def parse_contract(data: Any, source: str = "<memory>") -> PerformanceContract:
     raw_thresholds = data.get("thresholds")
     if not isinstance(raw_thresholds, Mapping):
         raise PerformanceContractError(f"'thresholds' in {source} must be a JSON object")
-    known = {field for field in ContractThresholds().to_dict()}
+    known = set(ContractThresholds().to_dict())
     unknown = set(raw_thresholds) - known
-    if unknown:
+    missing = known - set(raw_thresholds)
+    if unknown or missing:
         raise PerformanceContractError(
-            f"{source} carries unknown thresholds {sorted(unknown)}; this node enforces {sorted(known)}"
+            f"{source} threshold keys differ from v1; unknown={sorted(unknown)}, "
+            f"missing={sorted(missing)}"
         )
     thresholds = ContractThresholds(**{
         key: float(value) for key, value in raw_thresholds.items()
     })
+    contract_version = int(data["contract_version"])
+    if contract_version != 1 or thresholds != ContractThresholds():
+        raise PerformanceContractError(
+            f"{source} changes immutable v1 thresholds without a supported contract version"
+        )
+    if str(data["stop_condition"]) != STOP_CONDITION:
+        raise PerformanceContractError(
+            f"{source} stop condition differs from executable v1 semantics"
+        )
 
     return PerformanceContract(
-        contract_version=int(data["contract_version"]),
+        contract_version=contract_version,
         locked_at=str(data["locked_at"]),
         locked_by=str(data["locked_by"]),
         plan_id=str(data["plan_id"]),
@@ -486,6 +663,7 @@ def baseline_from_report(report: Mapping[str, Any]) -> dict[str, Any]:
         "evidence_class": report["evidence_class"],
         "model_id": report["plan"]["model_id"],
         "model_revision": report["plan"]["model_revision"],
+        "plan_sha256": _canonical_sha256(report["plan"]),
         "reference_recipe_id": report["reference_recipe_id"],
         "host": report["host"],
         "recipes": {},