popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden DGR-001 performance contract evidence

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 19:10:24 +03:00
parent e24db7854f
commit 9e67b829e3
16 changed files with 3674 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

185
packages/node/meshnet_node/recipe_drivers.py
View File

@@ -20,12 +20,16 @@ rules:
from __future__ import annotations
import hashlib
import hmac
import json
import os
import platform
import re
import socket
import subprocess
import sys
import tempfile
import time
import urllib.error
import urllib.request
@@ -85,6 +89,37 @@ def _directory_bytes(path: Path) -> int:
return sum(entry.stat().st_size for entry in path.rglob("*") if entry.is_file())
def _artifact_sha256(path: Path) -> str:
"""Hash an artifact file or a deterministic directory content manifest.
A file uses the ordinary SHA-256 digest. A directory hashes each sorted
relative path, resolved file size, and file bytes, so tokenizer/config drift
cannot hide behind a weight-only digest.
"""
digest = hashlib.sha256()
if path.is_file():
entries = [(None, path)]
else:
entries = [
(entry.relative_to(path).as_posix(), entry)
for entry in sorted(path.rglob("*"))
if entry.is_file()
]
if not entries:
raise BenchmarkError(f"artifact directory is empty: {path}")
for relative, entry in entries:
if relative is not None:
encoded = relative.encode("utf-8")
digest.update(len(encoded).to_bytes(8, "big"))
digest.update(encoded)
digest.update(entry.stat().st_size.to_bytes(8, "big"))
with entry.open("rb") as stream:
while chunk := stream.read(8 * 1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _host_manifest() -> dict[str, Any]:
"""Capture non-secret host facts with the report rather than trusting prose."""
manifest: dict[str, Any] = {
@@ -95,8 +130,10 @@ def _host_manifest() -> dict[str, Any]:
}
try:
import torch
import transformers
manifest["torch_version"] = torch.__version__
manifest["transformers_version"] = transformers.__version__
manifest["cuda_available"] = bool(torch.cuda.is_available())
if torch.cuda.is_available():
manifest["accelerator_name"] = torch.cuda.get_device_name(0)
@@ -109,7 +146,7 @@ def _host_manifest() -> dict[str, Any]:
def _validate_config(config: Mapping[str, Any]) -> None:
"""Reject a comparison that could silently mix models or use home storage."""
"""Reject comparisons that mix models, artifacts, devices, or budgets."""
try:
plan = config["plan"]
root = Path(config["artifact_storage_root"]).resolve(strict=True)
@@ -122,17 +159,75 @@ def _validate_config(config: Mapping[str, Any]) -> None:
raise BenchmarkError("model artifacts must use configured mounted-drive storage, never /home")
if not isinstance(recipes, list) or not recipes:
raise BenchmarkError("benchmark config needs at least one recipe")
sampling = plan.get("sampling", {})
if (
float(sampling.get("temperature", 0.0)) != 0.0
or int(sampling.get("top_k", 1)) != 1
or float(sampling.get("top_p", 1.0)) != 1.0
):
raise BenchmarkError("the quality comparison requires greedy sampling")
if len(plan.get("prompts", ())) < 3 or int(plan.get("repeats", 0)) < 3:
raise BenchmarkError("contract-grade evidence requires at least 3 prompts and 3 repeats")
if int(plan.get("warmup_requests", 0)) < 1:
raise BenchmarkError("contract-grade evidence requires at least one warmup")
if int(sampling.get("max_output_tokens", 0)) < 32:
raise BenchmarkError("contract-grade evidence requires at least 32 output tokens")
thread_budgets: set[int] = set()
max_concurrency = max(int(level) for level in plan.get("concurrency_levels", (1, 4)))
for spec in recipes:
if spec.get("source_model_id") != plan.get("model_id"):
raise BenchmarkError("every recipe must declare the plan's exact source_model_id")
if spec.get("source_model_revision") != plan.get("model_revision"):
raise BenchmarkError("every recipe must declare the plan's exact source_model_revision")
digest = spec.get("artifact_sha256", "")
if not isinstance(digest, str) or len(digest) != 64:
raise BenchmarkError("every recipe must declare its exact 64-character artifact_sha256")
if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None:
raise BenchmarkError("every recipe must declare a lowercase SHA-256 artifact digest")
artifact = Path(spec.get("artifact_path", "")).resolve(strict=True)
if artifact != root and root not in artifact.parents:
raise BenchmarkError("every model artifact must be beneath artifact_storage_root")
actual_digest = _artifact_sha256(artifact)
if not hmac.compare_digest(digest, actual_digest):
raise BenchmarkError(
f"artifact digest mismatch for {spec.get('id', '<unknown>')}: "
f"declared {digest}, measured {actual_digest}"
)
driver = spec.get("driver")
if not isinstance(driver, Mapping):
raise BenchmarkError("every recipe needs a driver object")
kind = driver.get("type")
if kind == "transformers":
driver_artifact = Path(driver.get("model_path", "")).resolve(strict=True)
elif kind == "llama-cpp-server":
driver_artifact = Path(driver.get("gguf_path", "")).resolve(strict=True)
binary = Path(driver.get("binary", "")).resolve(strict=True)
binary_digest = driver.get("binary_sha256", "")
if (
not isinstance(binary_digest, str)
or re.fullmatch(r"[0-9a-f]{64}", binary_digest) is None
or not hmac.compare_digest(binary_digest, _artifact_sha256(binary))
):
raise BenchmarkError("llama.cpp binary SHA-256 mismatch")
if int(driver.get("n_parallel", max_concurrency)) < max_concurrency:
raise BenchmarkError("llama.cpp parallel slots must cover maximum concurrency")
if driver.get("device", "cpu") != "cpu" or int(driver.get("n_gpu_layers", 0)) != 0:
raise BenchmarkError(
"v1 benchmark supports CPU-only llama.cpp until process VRAM is measurable"
)
else:
raise BenchmarkError(f"unknown driver type {kind!r}")
if driver_artifact != artifact:
raise BenchmarkError("driver artifact path must match the hashed recipe artifact")
if driver.get("device", "cpu") != spec.get("device"):
raise BenchmarkError("recipe and driver must declare the same device")
thread_budgets.add(int(driver.get("threads", 8)))
if len(thread_budgets) != 1:
raise BenchmarkError("every recipe must use the same CPU thread budget")
class TransformersDriver:
@@ -160,8 +255,10 @@ class TransformersDriver:
self._model: Any = None
self._tokenizer: Any = None
self._torch: Any = None
self._rss_baseline = 0
def load(self) -> LoadStats:
self._rss_baseline = _process_rss()
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -185,7 +282,7 @@ class TransformersDriver:
return LoadStats(
artifact_bytes=_directory_bytes(self.model_path),
load_ms=round(load_ms, 4),
rss_bytes=_process_rss(),
rss_bytes=max(0, _process_rss() - self._rss_baseline),
vram_bytes=self._vram_bytes(),
backend_detail=(
f"torch {torch.__version__}; dtype {self.dtype}; "
@@ -262,7 +359,7 @@ class TransformersDriver:
return logits.argmax(dim=-1)
def memory_probe(self) -> tuple[int, int]:
return _process_rss(), self._vram_bytes()
return max(0, _process_rss() - self._rss_baseline), self._vram_bytes()
def close(self) -> None:
self._model = None
@@ -295,6 +392,7 @@ class LlamaCppServerDriver:
binary: str,
gguf_path: str,
*,
binary_sha256: str,
device: str = "cpu",
threads: int = 8,
n_parallel: int = 4,
@@ -303,6 +401,7 @@ class LlamaCppServerDriver:
startup_timeout_s: float = 120.0,
) -> None:
self.binary = Path(binary)
self.binary_sha256 = binary_sha256
self.gguf_path = Path(gguf_path)
self.device = device
self.threads = threads
@@ -318,11 +417,34 @@ class LlamaCppServerDriver:
def _url(self) -> str:
return f"http://127.0.0.1:{self._port}"
def _log_excerpt(self) -> str:
if self._log is None:
return ""
try:
self._log.flush()
self._log.seek(0)
return self._log.read()[-4096:].decode("utf-8", errors="replace").strip()
except Exception:
return ""
def load(self) -> LoadStats:
if not self.binary.exists():
raise BenchmarkError(f"llama-server binary not found at {self.binary}")
if not self.gguf_path.exists():
raise BenchmarkError(f"GGUF artifact not found at {self.gguf_path}")
measured_binary_sha256 = _artifact_sha256(self.binary)
if not hmac.compare_digest(self.binary_sha256, measured_binary_sha256):
raise BenchmarkError("llama-server binary changed after config validation")
version = " | ".join(
subprocess.run(
[str(self.binary), "--version"],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=10,
).stdout.strip().splitlines()
)
self._port = _free_port()
command = [
@@ -339,9 +461,9 @@ class LlamaCppServerDriver:
"--no-webui",
]
started = time.monotonic()
self._log = subprocess.PIPE
self._log = tempfile.TemporaryFile(mode="w+b")
self._process = subprocess.Popen(
command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
command, stdout=self._log, stderr=subprocess.STDOUT
)
self._await_health(started)
load_ms = (time.monotonic() - started) * 1000
@@ -352,7 +474,8 @@ class LlamaCppServerDriver:
rss_bytes=_process_rss(self._process.pid),
vram_bytes=0,
backend_detail=(
f"llama-server; threads {self.threads}; parallel slots {self.n_parallel}; "
f"{version}; binary sha256 {measured_binary_sha256}; "
f"threads {self.threads}; parallel slots {self.n_parallel}; "
f"ctx/slot {self.context_per_slot}; gpu layers {self.n_gpu_layers}"
),
)
@@ -361,7 +484,8 @@ class LlamaCppServerDriver:
while time.monotonic() - started < self.startup_timeout_s:
if self._process is not None and self._process.poll() is not None:
raise BenchmarkError(
f"llama-server exited with code {self._process.returncode} during startup"
f"llama-server exited with code {self._process.returncode} during startup; "
f"log tail: {self._log_excerpt()}"
)
try:
with urllib.request.urlopen(f"{self._url}/health", timeout=2) as response:
@@ -370,7 +494,8 @@ class LlamaCppServerDriver:
except (urllib.error.URLError, OSError):
time.sleep(0.25)
raise BenchmarkError(
f"llama-server did not become healthy within {self.startup_timeout_s:.0f}s"
f"llama-server did not become healthy within {self.startup_timeout_s:.0f}s; "
f"log tail: {self._log_excerpt()}"
)
def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
@@ -397,7 +522,6 @@ class LlamaCppServerDriver:
)
started = time.monotonic()
ttft_ms = 0.0
chunks: list[str] = []
timings: Mapping[str, Any] = {}
with urllib.request.urlopen(request, timeout=600) as response:
@@ -407,8 +531,6 @@ class LlamaCppServerDriver:
continue
payload = json.loads(line[len("data:"):].strip())
content = payload.get("content", "")
if content and not ttft_ms:
ttft_ms = (time.monotonic() - started) * 1000
chunks.append(content)
if payload.get("stop"):
timings = payload.get("timings") or {}
@@ -422,8 +544,14 @@ class LlamaCppServerDriver:
return GenerationSample(
text="".join(chunks),
prompt_tokens=int(timings.get("prompt_n", 0)),
decode_tokens=int(timings.get("predicted_n", 0)),
ttft_ms=ttft_ms or total_ms,
# llama.cpp starts predicted_ms after sampling the first token while
# predicted_n includes it. Exclude that token to match the
# Transformers inter-token decode metric.
decode_tokens=max(0, int(timings.get("predicted_n", 0)) - 1),
# Use the runtime's prompt/first-token timing, matching the
# in-process Transformers boundary. HTTP/SSE and slot delay remain
# represented by total latency and queue_wait_ms.
ttft_ms=prefill_ms,
prefill_ms=prefill_ms,
decode_ms=decode_ms,
total_ms=total_ms,
@@ -438,15 +566,18 @@ class LlamaCppServerDriver:
return _process_rss(self._process.pid), 0
def close(self) -> None:
if self._process is None:
return
self._process.terminate()
try:
self._process.wait(timeout=20)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait(timeout=10)
self._process = None
if self._process is not None:
if self._process.poll() is None:
self._process.terminate()
try:
self._process.wait(timeout=20)
except subprocess.TimeoutExpired:
self._process.kill()
self._process.wait(timeout=10)
self._process = None
if self._log is not None:
self._log.close()
self._log = None
def build_driver(spec: Mapping[str, Any], plan: BenchmarkPlan) -> RecipeDriverBundle:
@@ -511,6 +642,7 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
measurements = []
for spec in config["recipes"]:
recipe = _recipe_from_config(spec)
driver = None
try:
driver = build_driver(spec, plan)
measurements.append(measure_recipe(driver, recipe, plan))
@@ -520,10 +652,13 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
load=LoadStats(artifact_bytes=0, load_ms=0.0),
unavailable_reason=f"{type(exc).__name__}: {exc}",
))
finally:
if driver is not None:
driver.close()
return build_report(
plan,
measurements,
host={**_host_manifest(), **dict(config.get("host", {}))},
host={**dict(config.get("host", {})), **_host_manifest()},
evidence_class=config.get("evidence_class", "local-real"),
)