feat: add signed ROCm diagnostic lane

packages/node/meshnet_node/performance_contract.py

@@ -29,6 +29,7 @@ import base64
 import binascii
 import hashlib
 import json
+from collections import Counter
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Any, Mapping
@@ -370,22 +371,101 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -
             )
         return {key: sorted(values) for key, values in counts.items()}
 
-    reference_counts = token_counts(reference)
+    def outcome_counts(entry: Mapping[str, Any]) -> Counter[tuple[str, int, int]]:
+        return Counter(
+            (
+                str(outcome.get("prompt_id")),
+                int(outcome.get("concurrency", 0)),
+                int(outcome.get("repeat", -1)),
+            )
+            for outcome in entry.get("outcomes", ())
+        )
+
     prompt_ids = {str(prompt["id"]) for prompt in plan["prompts"]}
     repeats = int(plan["repeats"])
+    for entry in recipes:
+        if not entry.get("available"):
+            continue
+        recipe_id = str(entry["recipe"]["id"])
+        cells = entry.get("concurrency", {})
+        actual_levels = {int(level) for level in cells}
+        if actual_levels != levels:
+            raise PerformanceContractError(
+                f"recipe {recipe_id!r} has unexpected concurrency cells"
+            )
+        for outcome in entry.get("outcomes", ()):
+            try:
+                outcome_recipe_id = str(outcome["recipe_id"])
+                prompt_id = str(outcome["prompt_id"])
+                level = int(outcome["concurrency"])
+                repeat = int(outcome["repeat"])
+            except (KeyError, TypeError, ValueError) as exc:
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} contains a malformed raw outcome"
+                ) from exc
+            if outcome_recipe_id != recipe_id:
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} contains an outcome for {outcome_recipe_id!r}"
+                )
+            if prompt_id not in prompt_ids or level not in levels or not 0 <= repeat < repeats:
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} contains an out-of-domain raw outcome"
+                )
+            if not isinstance(outcome.get("ok"), bool):
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} contains an outcome without boolean ok status"
+                )
+
+        coverage = outcome_counts(entry)
+        for prompt_id in prompt_ids:
+            for level in levels:
+                for repeat in range(repeats):
+                    if coverage[(prompt_id, level, repeat)] != level:
+                        raise PerformanceContractError(
+                            f"recipe {recipe_id!r} lacks complete request coverage"
+                        )
+        for level in levels:
+            cell = cells.get(str(level), cells.get(level))
+            raw = [
+                outcome
+                for outcome in entry.get("outcomes", ())
+                if int(outcome["concurrency"]) == level
+            ]
+            if int(cell.get("concurrency", -1)) != level:
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} cell identity does not match concurrency {level}"
+                )
+            if int(cell.get("requests", -1)) != len(raw):
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} aggregate requests do not match raw outcomes"
+                )
+            raw_failures = sum(not outcome["ok"] for outcome in raw)
+            if int(cell.get("failures", -1)) != raw_failures:
+                raise PerformanceContractError(
+                    f"recipe {recipe_id!r} aggregate failures do not match raw outcomes"
+                )
+
+    reference_counts = token_counts(reference)
     for prompt_id in prompt_ids:
-        for level in required_levels:
+        for level in levels:
             for repeat in range(repeats):
                 values = reference_counts.get((prompt_id, level, repeat), ())
                 if len(values) != level:
                     raise PerformanceContractError(
                         "reference recipe lacks complete prompt/repeat/concurrency coverage"
                     )
+    if contract.thresholds.max_failure_rate != 0:
+        raise PerformanceContractError(
+            "nonzero failure tolerance requires an explicit failed-request token policy"
+        )
     for entry in recipes:
-        if entry.get("available") and token_counts(entry) != reference_counts:
-            raise PerformanceContractError(
-                f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
-            )
+        if not entry.get("available"):
+            continue
+        for key, values in token_counts(entry).items():
+            if Counter(values) - Counter(reference_counts.get(key, ())):
+                raise PerformanceContractError(
+                    f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
+                )
 
 
 @dataclass(frozen=True)
@@ -543,7 +623,12 @@ def _evaluate_recipe(
     requests = sum(cell["requests"] for cell in entry["concurrency"].values())
     failure_rate = _ratio(failures, requests) if requests else 0.0
     measurements["failure_rate"] = failure_rate
-    if failure_rate > thresholds.max_failure_rate:
+    failure_limit_exceeded = requests == 0 or (
+        failures > 0
+        if thresholds.max_failure_rate == 0
+        else failures / requests > thresholds.max_failure_rate
+    )
+    if failure_limit_exceeded:
         reasons.append(f"failure rate {failure_rate:.2%} exceeds the contract limit")
         speed_benefit = False
         fit_benefit = False