popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add signed ROCm diagnostic lane

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 21:24:43 +03:00
parent b1c9deeb01
commit ef2a9e67e8
16 changed files with 4443 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
packages/node/meshnet_node/recipe_drivers.py
View File

@@ -64,6 +64,9 @@ from .recipe_benchmark import (
REAL_INFERENCE_ENV = "MESHNET_ENABLE_REAL_INFERENCE_TESTS"
EVIDENCE_SIGNING_KEY_ENV = "MESHNET_EVIDENCE_SIGNING_KEY"
CONTRACT_V1_PROFILE = "contract-v1"
GPU_DIAGNOSTIC_PROFILE = "gpu-diagnostic"
GPU_DIAGNOSTIC_REPORT_PRODUCER = "meshnet_node.recipe_drivers.run_configured_gpu_diagnostic/v1"
def real_inference_enabled() -> bool:
@@ -214,8 +217,12 @@ def _host_manifest(config: Mapping[str, Any] | None = None) -> dict[str, Any]:
return manifest
def _validate_config(config: Mapping[str, Any]) -> None:
def _validate_config(
config: Mapping[str, Any], *, profile: str = CONTRACT_V1_PROFILE
) -> None:
"""Reject comparisons that mix models, artifacts, devices, or budgets."""
if profile not in {CONTRACT_V1_PROFILE, GPU_DIAGNOSTIC_PROFILE}:
raise BenchmarkError(f"unknown benchmark validation profile {profile!r}")
try:
plan = config["plan"]
root = Path(config["artifact_storage_root"]).resolve(strict=True)
@@ -267,7 +274,11 @@ def _validate_config(config: Mapping[str, Any]) -> None:
driver = spec.get("driver")
if not isinstance(driver, Mapping):
raise BenchmarkError("every recipe needs a driver object")
raise BenchmarkError("every recipe must declare a driver block")
if "artifact_sha256" in driver:
raise BenchmarkError(
"driver artifact_sha256 is forbidden; the validated recipe digest is authoritative"
)
kind = driver.get("type")
if kind == "transformers":
driver_artifact = Path(driver.get("model_path", "")).resolve(strict=True)
@@ -283,18 +294,40 @@ def _validate_config(config: Mapping[str, Any]) -> None:
raise BenchmarkError("llama.cpp binary SHA-256 mismatch")
if int(driver.get("n_parallel", max_concurrency)) < max_concurrency:
raise BenchmarkError("llama.cpp parallel slots must cover maximum concurrency")
if driver.get("device", "cpu") != "cpu" or int(driver.get("n_gpu_layers", 0)) != 0:
driver_device = driver.get("device", "cpu")
gpu_layers = int(driver.get("n_gpu_layers", 0))
if profile == CONTRACT_V1_PROFILE and (
driver_device != "cpu" or gpu_layers != 0
):
raise BenchmarkError(
"v1 benchmark supports CPU-only llama.cpp until process VRAM is measurable"
)
if profile == GPU_DIAGNOSTIC_PROFILE and (
driver_device != "cuda" or gpu_layers <= 0
):
raise BenchmarkError(
"GPU diagnostic requires CUDA/ROCm llama.cpp with positive n_gpu_layers"
)
else:
raise BenchmarkError(f"unknown driver type {kind!r}")
if driver_artifact != artifact:
raise BenchmarkError("driver artifact path must match the hashed recipe artifact")
if driver.get("device", "cpu") != spec.get("device"):
raise BenchmarkError("recipe and driver must declare the same device")
if profile == CONTRACT_V1_PROFILE and spec.get("device") != "cpu":
raise BenchmarkError("contract-v1 requires every recipe to run on CPU")
if profile == GPU_DIAGNOSTIC_PROFILE and spec.get("device") != "cuda":
raise BenchmarkError("GPU diagnostic requires every recipe to declare device 'cuda'")
thread_budgets.add(int(driver.get("threads", 8)))
host = config.get("host", {})
if not isinstance(host, Mapping):
raise BenchmarkError("benchmark host metadata must be an object")
if profile == GPU_DIAGNOSTIC_PROFILE and host.get(
"benchmark_lane"
) != "rocm-gpu-diagnostic":
raise BenchmarkError("GPU diagnostic requires the rocm-gpu-diagnostic host marker")
if len(thread_budgets) != 1:
raise BenchmarkError("every recipe must use the same CPU thread budget")
@@ -313,11 +346,13 @@ class TransformersDriver:
self,
model_path: str,
*,
artifact_sha256: str | None = None,
device: str = "cpu",
dtype: str = "bfloat16",
threads: int = 8,
) -> None:
self.model_path = Path(model_path)
self.artifact_sha256 = artifact_sha256
self.device = device
self.dtype = dtype
self.threads = threads
@@ -327,6 +362,10 @@ class TransformersDriver:
self._rss_baseline = 0
def load(self) -> LoadStats:
if self.artifact_sha256 is not None:
measured_artifact_sha256 = _artifact_sha256(self.model_path)
if not hmac.compare_digest(self.artifact_sha256, measured_artifact_sha256):
raise BenchmarkError("Transformers artifact changed after config validation")
self._rss_baseline = _process_rss()
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -445,6 +484,39 @@ def _free_port() -> int:
return int(probe.getsockname()[1])
def _gpu_offload_evidence(log_text: str, requested_layers: int) -> str:
"""Extract measured ROCm device and layer placement from llama-server logs."""
device_matches = re.findall(
r"-\s+(ROCm\d+)\s+:\s+(.+?)\s+\(\d+\s+MiB", log_text
)
offload_matches = re.findall(
r"offloaded\s+(\d+)/(\d+)\s+layers\s+to\s+GPU", log_text
)
if not device_matches:
raise BenchmarkError("GPU diagnostic found no measured ROCm device in llama-server logs")
if not offload_matches:
raise BenchmarkError("GPU diagnostic found no measured layer offload in llama-server logs")
backend, device_name = device_matches[-1]
offloaded, total = (int(value) for value in offload_matches[-1])
required = min(requested_layers, total)
if requested_layers <= 0 or offloaded < required:
raise BenchmarkError(
f"GPU diagnostic requested {requested_layers} layers but measured "
f"only {offloaded}/{total} offloaded"
)
return (
f"measured accelerator {backend}: {device_name}; "
f"measured offload {offloaded}/{total} layers"
)
def _gpu_layer_config_detail(device: str, layers: int) -> str:
# Immutable v1 pins the historical CPU wording exactly. GPU diagnostics use
# the clearer requested/measured distinction without changing v1 identity.
return f"gpu layers {layers}" if device == "cpu" else f"requested gpu layers {layers}"
class LlamaCppServerDriver:
"""The whole-model llama.cpp/GGUF recipe, driven through ``llama-server``.
@@ -462,6 +534,7 @@ class LlamaCppServerDriver:
gguf_path: str,
*,
binary_sha256: str,
artifact_sha256: str | None = None,
device: str = "cpu",
threads: int = 8,
n_parallel: int = 4,
@@ -472,6 +545,7 @@ class LlamaCppServerDriver:
self.binary = Path(binary)
self.binary_sha256 = binary_sha256
self.gguf_path = Path(gguf_path)
self.artifact_sha256 = artifact_sha256
self.device = device
self.threads = threads
self.n_parallel = n_parallel
@@ -486,21 +560,27 @@ class LlamaCppServerDriver:
def _url(self) -> str:
return f"http://127.0.0.1:{self._port}"
def _log_excerpt(self) -> str:
def _log_text(self) -> str:
if self._log is None:
return ""
try:
self._log.flush()
self._log.seek(0)
return self._log.read()[-4096:].decode("utf-8", errors="replace").strip()
size = min(os.fstat(self._log.fileno()).st_size, 8 * 1024 * 1024)
return os.pread(self._log.fileno(), size, 0).decode("utf-8", errors="replace")
except Exception:
return ""
def _log_excerpt(self) -> str:
return self._log_text()[-4096:].strip()
def load(self) -> LoadStats:
if not self.binary.exists():
raise BenchmarkError(f"llama-server binary not found at {self.binary}")
if not self.gguf_path.exists():
raise BenchmarkError(f"GGUF artifact not found at {self.gguf_path}")
if self.artifact_sha256 is not None:
measured_artifact_sha256 = _artifact_sha256(self.gguf_path)
if not hmac.compare_digest(self.artifact_sha256, measured_artifact_sha256):
raise BenchmarkError("GGUF artifact changed after config validation")
measured_binary_sha256 = _artifact_sha256(self.binary)
if not hmac.compare_digest(self.binary_sha256, measured_binary_sha256):
raise BenchmarkError("llama-server binary changed after config validation")
@@ -516,8 +596,12 @@ class LlamaCppServerDriver:
)
self._port = _free_port()
command = [
str(self.binary),
command = [str(self.binary)]
if self.device == "cuda":
# Debug verbosity is required to capture measured device placement
# and the offloaded/total layer count in signed diagnostics.
command.extend(["-lv", "5"])
command.extend([
"--model", str(self.gguf_path),
"--host", "127.0.0.1",
"--port", str(self._port),
@@ -528,7 +612,7 @@ class LlamaCppServerDriver:
"--ctx-size", str(self.context_per_slot * self.n_parallel),
"--n-gpu-layers", str(self.n_gpu_layers),
"--no-webui",
]
])
started = time.monotonic()
self._log = tempfile.TemporaryFile(mode="w+b")
self._process = subprocess.Popen(
@@ -536,6 +620,9 @@ class LlamaCppServerDriver:
)
self._await_health(started)
load_ms = (time.monotonic() - started) * 1000
gpu_evidence = ""
if self.device == "cuda":
gpu_evidence = _gpu_offload_evidence(self._log_text(), self.n_gpu_layers)
return LoadStats(
artifact_bytes=self.gguf_path.stat().st_size,
@@ -545,7 +632,9 @@ class LlamaCppServerDriver:
backend_detail=(
f"{version}; binary sha256 {measured_binary_sha256}; "
f"threads {self.threads}; parallel slots {self.n_parallel}; "
f"ctx/slot {self.context_per_slot}; gpu layers {self.n_gpu_layers}"
f"ctx/slot {self.context_per_slot}; "
f"{_gpu_layer_config_detail(self.device, self.n_gpu_layers)}"
+ (f"; {gpu_evidence}" if gpu_evidence else "")
),
)
@@ -653,6 +742,7 @@ def build_driver(spec: Mapping[str, Any], plan: BenchmarkPlan) -> RecipeDriverBu
"""Construct the driver named by a recipe's ``driver`` block."""
driver_spec = dict(spec["driver"])
kind = driver_spec.pop("type")
driver_spec["artifact_sha256"] = spec.get("artifact_sha256")
if kind == "transformers":
return TransformersDriver(**driver_spec)
if kind == "llama-cpp-server":
@@ -696,14 +786,28 @@ def _recipe_from_config(spec: Mapping[str, Any]) -> RecipeSpec:
def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
"""Run every recipe in ``config`` against one shared plan and return the report.
"""Run contract-v1 evidence through the CPU-only, fail-closed profile."""
return _run_profiled_benchmark(config, profile=CONTRACT_V1_PROFILE)
A recipe whose runtime cannot start is recorded as unavailable with the real
reason rather than dropped: a report that silently omits the recipe that
crashed would read as a clean result.
"""
def run_configured_gpu_diagnostic(config: Mapping[str, Any]) -> dict:
"""Run signed ROCm diagnostics that are intentionally ineligible for v1."""
return _run_profiled_benchmark(config, profile=GPU_DIAGNOSTIC_PROFILE)
def _producer_for_profile(profile: str) -> str:
if profile == CONTRACT_V1_PROFILE:
return REAL_REPORT_PRODUCER
if profile == GPU_DIAGNOSTIC_PROFILE:
return GPU_DIAGNOSTIC_REPORT_PRODUCER
raise BenchmarkError(f"unknown benchmark validation profile {profile!r}")
def _run_profiled_benchmark(config: Mapping[str, Any], *, profile: str) -> dict:
"""Validate a closed profile and derive its producer before measurement."""
require_real_inference()
_validate_config(config)
_validate_config(config, profile=profile)
producer = _producer_for_profile(profile)
evidence_class = config.get("evidence_class", "local-real")
if evidence_class not in {"local-real", "multi-machine-real"}:
raise BenchmarkError("canonical real runner cannot emit synthetic evidence")
@@ -739,7 +843,7 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
evidence_class=evidence_class,
provenance={
"schema_version": PROVENANCE_SCHEMA_VERSION,
"producer": REAL_REPORT_PRODUCER,
"producer": producer,
"run_id": run_id,
"started_at": started_at,
"completed_at": _utc_now(),