popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add signed ROCm diagnostic lane

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 21:24:43 +03:00
parent b1c9deeb01
commit ef2a9e67e8
16 changed files with 4443 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
packages/node/meshnet_node/performance_contract.py
View File

@@ -29,6 +29,7 @@ import base64
import binascii
import hashlib
import json
from collections import Counter
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Mapping
@@ -370,22 +371,101 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -
)
return {key: sorted(values) for key, values in counts.items()}
reference_counts = token_counts(reference)
def outcome_counts(entry: Mapping[str, Any]) -> Counter[tuple[str, int, int]]:
return Counter(
(
str(outcome.get("prompt_id")),
int(outcome.get("concurrency", 0)),
int(outcome.get("repeat", -1)),
)
for outcome in entry.get("outcomes", ())
)
prompt_ids = {str(prompt["id"]) for prompt in plan["prompts"]}
repeats = int(plan["repeats"])
for entry in recipes:
if not entry.get("available"):
continue
recipe_id = str(entry["recipe"]["id"])
cells = entry.get("concurrency", {})
actual_levels = {int(level) for level in cells}
if actual_levels != levels:
raise PerformanceContractError(
f"recipe {recipe_id!r} has unexpected concurrency cells"
)
for outcome in entry.get("outcomes", ()):
try:
outcome_recipe_id = str(outcome["recipe_id"])
prompt_id = str(outcome["prompt_id"])
level = int(outcome["concurrency"])
repeat = int(outcome["repeat"])
except (KeyError, TypeError, ValueError) as exc:
raise PerformanceContractError(
f"recipe {recipe_id!r} contains a malformed raw outcome"
) from exc
if outcome_recipe_id != recipe_id:
raise PerformanceContractError(
f"recipe {recipe_id!r} contains an outcome for {outcome_recipe_id!r}"
)
if prompt_id not in prompt_ids or level not in levels or not 0 <= repeat < repeats:
raise PerformanceContractError(
f"recipe {recipe_id!r} contains an out-of-domain raw outcome"
)
if not isinstance(outcome.get("ok"), bool):
raise PerformanceContractError(
f"recipe {recipe_id!r} contains an outcome without boolean ok status"
)
coverage = outcome_counts(entry)
for prompt_id in prompt_ids:
for level in levels:
for repeat in range(repeats):
if coverage[(prompt_id, level, repeat)] != level:
raise PerformanceContractError(
f"recipe {recipe_id!r} lacks complete request coverage"
)
for level in levels:
cell = cells.get(str(level), cells.get(level))
raw = [
outcome
for outcome in entry.get("outcomes", ())
if int(outcome["concurrency"]) == level
]
if int(cell.get("concurrency", -1)) != level:
raise PerformanceContractError(
f"recipe {recipe_id!r} cell identity does not match concurrency {level}"
)
if int(cell.get("requests", -1)) != len(raw):
raise PerformanceContractError(
f"recipe {recipe_id!r} aggregate requests do not match raw outcomes"
)
raw_failures = sum(not outcome["ok"] for outcome in raw)
if int(cell.get("failures", -1)) != raw_failures:
raise PerformanceContractError(
f"recipe {recipe_id!r} aggregate failures do not match raw outcomes"
)
reference_counts = token_counts(reference)
for prompt_id in prompt_ids:
for level in required_levels:
for level in levels:
for repeat in range(repeats):
values = reference_counts.get((prompt_id, level, repeat), ())
if len(values) != level:
raise PerformanceContractError(
"reference recipe lacks complete prompt/repeat/concurrency coverage"
)
if contract.thresholds.max_failure_rate != 0:
raise PerformanceContractError(
"nonzero failure tolerance requires an explicit failed-request token policy"
)
for entry in recipes:
if entry.get("available") and token_counts(entry) != reference_counts:
raise PerformanceContractError(
f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
)
if not entry.get("available"):
continue
for key, values in token_counts(entry).items():
if Counter(values) - Counter(reference_counts.get(key, ())):
raise PerformanceContractError(
f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
)
@dataclass(frozen=True)
@@ -543,7 +623,12 @@ def _evaluate_recipe(
requests = sum(cell["requests"] for cell in entry["concurrency"].values())
failure_rate = _ratio(failures, requests) if requests else 0.0
measurements["failure_rate"] = failure_rate
if failure_rate > thresholds.max_failure_rate:
failure_limit_exceeded = requests == 0 or (
failures > 0
if thresholds.max_failure_rate == 0
else failures / requests > thresholds.max_failure_rate
)
if failure_limit_exceeded:
reasons.append(f"failure rate {failure_rate:.2%} exceeds the contract limit")
speed_benefit = False
fit_benefit = False

18
packages/node/meshnet_node/recipe_benchmark.py
View File

@@ -660,13 +660,27 @@ def main(argv: list[str] | None = None) -> int:
description="Run the controlled safetensors-versus-GGUF recipe benchmark"
)
parser.add_argument("--config", type=Path, required=True, help="benchmark configuration JSON")
parser.add_argument(
"--profile",
choices=("contract-v1", "gpu-diagnostic"),
default="contract-v1",
help="validation and provenance profile (GPU diagnostics are not v1-eligible)",
)
parser.add_argument("--json-out", type=Path, help="write the JSON report to this path")
parser.add_argument("--summary-out", type=Path, help="write the text summary to this path")
args = parser.parse_args(argv)
from .recipe_drivers import run_configured_benchmark # heavy runtimes: import on demand
from .recipe_drivers import ( # heavy runtimes: import on demand
run_configured_benchmark,
run_configured_gpu_diagnostic,
)
report = run_configured_benchmark(json.loads(args.config.read_text(encoding="utf-8")))
runner = (
run_configured_gpu_diagnostic
if args.profile == "gpu-diagnostic"
else run_configured_benchmark
)
report = runner(json.loads(args.config.read_text(encoding="utf-8")))
summary = format_summary(report)
if args.json_out:
args.json_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")

140
packages/node/meshnet_node/recipe_drivers.py
View File

@@ -64,6 +64,9 @@ from .recipe_benchmark import (
REAL_INFERENCE_ENV = "MESHNET_ENABLE_REAL_INFERENCE_TESTS"
EVIDENCE_SIGNING_KEY_ENV = "MESHNET_EVIDENCE_SIGNING_KEY"
CONTRACT_V1_PROFILE = "contract-v1"
GPU_DIAGNOSTIC_PROFILE = "gpu-diagnostic"
GPU_DIAGNOSTIC_REPORT_PRODUCER = "meshnet_node.recipe_drivers.run_configured_gpu_diagnostic/v1"
def real_inference_enabled() -> bool:
@@ -214,8 +217,12 @@ def _host_manifest(config: Mapping[str, Any] | None = None) -> dict[str, Any]:
return manifest
def _validate_config(config: Mapping[str, Any]) -> None:
def _validate_config(
config: Mapping[str, Any], *, profile: str = CONTRACT_V1_PROFILE
) -> None:
"""Reject comparisons that mix models, artifacts, devices, or budgets."""
if profile not in {CONTRACT_V1_PROFILE, GPU_DIAGNOSTIC_PROFILE}:
raise BenchmarkError(f"unknown benchmark validation profile {profile!r}")
try:
plan = config["plan"]
root = Path(config["artifact_storage_root"]).resolve(strict=True)
@@ -267,7 +274,11 @@ def _validate_config(config: Mapping[str, Any]) -> None:
driver = spec.get("driver")
if not isinstance(driver, Mapping):
raise BenchmarkError("every recipe needs a driver object")
raise BenchmarkError("every recipe must declare a driver block")
if "artifact_sha256" in driver:
raise BenchmarkError(
"driver artifact_sha256 is forbidden; the validated recipe digest is authoritative"
)
kind = driver.get("type")
if kind == "transformers":
driver_artifact = Path(driver.get("model_path", "")).resolve(strict=True)
@@ -283,18 +294,40 @@ def _validate_config(config: Mapping[str, Any]) -> None:
raise BenchmarkError("llama.cpp binary SHA-256 mismatch")
if int(driver.get("n_parallel", max_concurrency)) < max_concurrency:
raise BenchmarkError("llama.cpp parallel slots must cover maximum concurrency")
if driver.get("device", "cpu") != "cpu" or int(driver.get("n_gpu_layers", 0)) != 0:
driver_device = driver.get("device", "cpu")
gpu_layers = int(driver.get("n_gpu_layers", 0))
if profile == CONTRACT_V1_PROFILE and (
driver_device != "cpu" or gpu_layers != 0
):
raise BenchmarkError(
"v1 benchmark supports CPU-only llama.cpp until process VRAM is measurable"
)
if profile == GPU_DIAGNOSTIC_PROFILE and (
driver_device != "cuda" or gpu_layers <= 0
):
raise BenchmarkError(
"GPU diagnostic requires CUDA/ROCm llama.cpp with positive n_gpu_layers"
)
else:
raise BenchmarkError(f"unknown driver type {kind!r}")
if driver_artifact != artifact:
raise BenchmarkError("driver artifact path must match the hashed recipe artifact")
if driver.get("device", "cpu") != spec.get("device"):
raise BenchmarkError("recipe and driver must declare the same device")
if profile == CONTRACT_V1_PROFILE and spec.get("device") != "cpu":
raise BenchmarkError("contract-v1 requires every recipe to run on CPU")
if profile == GPU_DIAGNOSTIC_PROFILE and spec.get("device") != "cuda":
raise BenchmarkError("GPU diagnostic requires every recipe to declare device 'cuda'")
thread_budgets.add(int(driver.get("threads", 8)))
host = config.get("host", {})
if not isinstance(host, Mapping):
raise BenchmarkError("benchmark host metadata must be an object")
if profile == GPU_DIAGNOSTIC_PROFILE and host.get(
"benchmark_lane"
) != "rocm-gpu-diagnostic":
raise BenchmarkError("GPU diagnostic requires the rocm-gpu-diagnostic host marker")
if len(thread_budgets) != 1:
raise BenchmarkError("every recipe must use the same CPU thread budget")
@@ -313,11 +346,13 @@ class TransformersDriver:
self,
model_path: str,
*,
artifact_sha256: str | None = None,
device: str = "cpu",
dtype: str = "bfloat16",
threads: int = 8,
) -> None:
self.model_path = Path(model_path)
self.artifact_sha256 = artifact_sha256
self.device = device
self.dtype = dtype
self.threads = threads
@@ -327,6 +362,10 @@ class TransformersDriver:
self._rss_baseline = 0
def load(self) -> LoadStats:
if self.artifact_sha256 is not None:
measured_artifact_sha256 = _artifact_sha256(self.model_path)
if not hmac.compare_digest(self.artifact_sha256, measured_artifact_sha256):
raise BenchmarkError("Transformers artifact changed after config validation")
self._rss_baseline = _process_rss()
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -445,6 +484,39 @@ def _free_port() -> int:
return int(probe.getsockname()[1])
def _gpu_offload_evidence(log_text: str, requested_layers: int) -> str:
"""Extract measured ROCm device and layer placement from llama-server logs."""
device_matches = re.findall(
r"-\s+(ROCm\d+)\s+:\s+(.+?)\s+\(\d+\s+MiB", log_text
)
offload_matches = re.findall(
r"offloaded\s+(\d+)/(\d+)\s+layers\s+to\s+GPU", log_text
)
if not device_matches:
raise BenchmarkError("GPU diagnostic found no measured ROCm device in llama-server logs")
if not offload_matches:
raise BenchmarkError("GPU diagnostic found no measured layer offload in llama-server logs")
backend, device_name = device_matches[-1]
offloaded, total = (int(value) for value in offload_matches[-1])
required = min(requested_layers, total)
if requested_layers <= 0 or offloaded < required:
raise BenchmarkError(
f"GPU diagnostic requested {requested_layers} layers but measured "
f"only {offloaded}/{total} offloaded"
)
return (
f"measured accelerator {backend}: {device_name}; "
f"measured offload {offloaded}/{total} layers"
)
def _gpu_layer_config_detail(device: str, layers: int) -> str:
# Immutable v1 pins the historical CPU wording exactly. GPU diagnostics use
# the clearer requested/measured distinction without changing v1 identity.
return f"gpu layers {layers}" if device == "cpu" else f"requested gpu layers {layers}"
class LlamaCppServerDriver:
"""The whole-model llama.cpp/GGUF recipe, driven through ``llama-server``.
@@ -462,6 +534,7 @@ class LlamaCppServerDriver:
gguf_path: str,
*,
binary_sha256: str,
artifact_sha256: str | None = None,
device: str = "cpu",
threads: int = 8,
n_parallel: int = 4,
@@ -472,6 +545,7 @@ class LlamaCppServerDriver:
self.binary = Path(binary)
self.binary_sha256 = binary_sha256
self.gguf_path = Path(gguf_path)
self.artifact_sha256 = artifact_sha256
self.device = device
self.threads = threads
self.n_parallel = n_parallel
@@ -486,21 +560,27 @@ class LlamaCppServerDriver:
def _url(self) -> str:
return f"http://127.0.0.1:{self._port}"
def _log_excerpt(self) -> str:
def _log_text(self) -> str:
if self._log is None:
return ""
try:
self._log.flush()
self._log.seek(0)
return self._log.read()[-4096:].decode("utf-8", errors="replace").strip()
size = min(os.fstat(self._log.fileno()).st_size, 8 * 1024 * 1024)
return os.pread(self._log.fileno(), size, 0).decode("utf-8", errors="replace")
except Exception:
return ""
def _log_excerpt(self) -> str:
return self._log_text()[-4096:].strip()
def load(self) -> LoadStats:
if not self.binary.exists():
raise BenchmarkError(f"llama-server binary not found at {self.binary}")
if not self.gguf_path.exists():
raise BenchmarkError(f"GGUF artifact not found at {self.gguf_path}")
if self.artifact_sha256 is not None:
measured_artifact_sha256 = _artifact_sha256(self.gguf_path)
if not hmac.compare_digest(self.artifact_sha256, measured_artifact_sha256):
raise BenchmarkError("GGUF artifact changed after config validation")
measured_binary_sha256 = _artifact_sha256(self.binary)
if not hmac.compare_digest(self.binary_sha256, measured_binary_sha256):
raise BenchmarkError("llama-server binary changed after config validation")
@@ -516,8 +596,12 @@ class LlamaCppServerDriver:
)
self._port = _free_port()
command = [
str(self.binary),
command = [str(self.binary)]
if self.device == "cuda":
# Debug verbosity is required to capture measured device placement
# and the offloaded/total layer count in signed diagnostics.
command.extend(["-lv", "5"])
command.extend([
"--model", str(self.gguf_path),
"--host", "127.0.0.1",
"--port", str(self._port),
@@ -528,7 +612,7 @@ class LlamaCppServerDriver:
"--ctx-size", str(self.context_per_slot * self.n_parallel),
"--n-gpu-layers", str(self.n_gpu_layers),
"--no-webui",
]
])
started = time.monotonic()
self._log = tempfile.TemporaryFile(mode="w+b")
self._process = subprocess.Popen(
@@ -536,6 +620,9 @@ class LlamaCppServerDriver:
)
self._await_health(started)
load_ms = (time.monotonic() - started) * 1000
gpu_evidence = ""
if self.device == "cuda":
gpu_evidence = _gpu_offload_evidence(self._log_text(), self.n_gpu_layers)
return LoadStats(
artifact_bytes=self.gguf_path.stat().st_size,
@@ -545,7 +632,9 @@ class LlamaCppServerDriver:
backend_detail=(
f"{version}; binary sha256 {measured_binary_sha256}; "
f"threads {self.threads}; parallel slots {self.n_parallel}; "
f"ctx/slot {self.context_per_slot}; gpu layers {self.n_gpu_layers}"
f"ctx/slot {self.context_per_slot}; "
f"{_gpu_layer_config_detail(self.device, self.n_gpu_layers)}"
+ (f"; {gpu_evidence}" if gpu_evidence else "")
),
)
@@ -653,6 +742,7 @@ def build_driver(spec: Mapping[str, Any], plan: BenchmarkPlan) -> RecipeDriverBu
"""Construct the driver named by a recipe's ``driver`` block."""
driver_spec = dict(spec["driver"])
kind = driver_spec.pop("type")
driver_spec["artifact_sha256"] = spec.get("artifact_sha256")
if kind == "transformers":
return TransformersDriver(**driver_spec)
if kind == "llama-cpp-server":
@@ -696,14 +786,28 @@ def _recipe_from_config(spec: Mapping[str, Any]) -> RecipeSpec:
def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
"""Run every recipe in ``config`` against one shared plan and return the report.
"""Run contract-v1 evidence through the CPU-only, fail-closed profile."""
return _run_profiled_benchmark(config, profile=CONTRACT_V1_PROFILE)
A recipe whose runtime cannot start is recorded as unavailable with the real
reason rather than dropped: a report that silently omits the recipe that
crashed would read as a clean result.
"""
def run_configured_gpu_diagnostic(config: Mapping[str, Any]) -> dict:
"""Run signed ROCm diagnostics that are intentionally ineligible for v1."""
return _run_profiled_benchmark(config, profile=GPU_DIAGNOSTIC_PROFILE)
def _producer_for_profile(profile: str) -> str:
if profile == CONTRACT_V1_PROFILE:
return REAL_REPORT_PRODUCER
if profile == GPU_DIAGNOSTIC_PROFILE:
return GPU_DIAGNOSTIC_REPORT_PRODUCER
raise BenchmarkError(f"unknown benchmark validation profile {profile!r}")
def _run_profiled_benchmark(config: Mapping[str, Any], *, profile: str) -> dict:
"""Validate a closed profile and derive its producer before measurement."""
require_real_inference()
_validate_config(config)
_validate_config(config, profile=profile)
producer = _producer_for_profile(profile)
evidence_class = config.get("evidence_class", "local-real")
if evidence_class not in {"local-real", "multi-machine-real"}:
raise BenchmarkError("canonical real runner cannot emit synthetic evidence")
@@ -739,7 +843,7 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
evidence_class=evidence_class,
provenance={
"schema_version": PROVENANCE_SCHEMA_VERSION,
"producer": REAL_REPORT_PRODUCER,
"producer": producer,
"run_id": run_id,
"started_at": started_at,
"completed_at": _utc_now(),