popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add signed ROCm diagnostic lane

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 21:24:43 +03:00
parent b1c9deeb01
commit ef2a9e67e8
16 changed files with 4443 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

250
tests/test_recipe_benchmark.py
View File

@@ -11,12 +11,16 @@ from __future__ import annotations
import base64
import copy
import hashlib
import json
import time
from dataclasses import replace
from pathlib import Path
import pytest
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from meshnet_node import recipe_benchmark as recipe_benchmark_module
from meshnet_node import recipe_drivers as recipe_drivers_module
from meshnet_node.performance_contract import (
PROVENANCE_SCHEMA_VERSION,
REAL_REPORT_PRODUCER,
@@ -30,8 +34,15 @@ from meshnet_node.performance_contract import (
report_signing_payload,
)
from meshnet_node.recipe_drivers import (
CONTRACT_V1_PROFILE,
GPU_DIAGNOSTIC_PROFILE,
GPU_DIAGNOSTIC_REPORT_PRODUCER,
_artifact_sha256,
_gpu_offload_evidence,
_gpu_layer_config_detail,
_producer_for_profile,
_validate_config,
build_driver,
require_real_inference,
)
from meshnet_node.recipe_benchmark import (
@@ -353,6 +364,19 @@ def test_contract_requires_a_quality_lane_then_allows_quantized_fit_benefit():
recipe("gguf-q4", Lane.PERFORMANCE_FIT), plan()
)
report = build_report(plan(), [reference, quality, q4], host={}, evidence_class="synthetic")
reference_entry = next(
entry for entry in report["recipes"] if entry["recipe"]["id"] == "safetensors"
)
q4_entry = next(
entry for entry in report["recipes"] if entry["recipe"]["id"] == "gguf-q4"
)
for level, q4_cell in q4_entry["concurrency"].items():
reference_cell = reference_entry["concurrency"][level]
q4_cell["decode_tokens_per_sec"] = reference_cell["decode_tokens_per_sec"] / 2
q4_cell["aggregate_decode_tokens_per_sec"] = (
reference_cell["aggregate_decode_tokens_per_sec"] / 2
)
q4_cell["ttft_p50_ms"] = reference_cell["ttft_p50_ms"] * 2
contract = PerformanceContract(
contract_version=1, locked_at="2026-07-13T00:00:00Z", locked_by="test",
plan_id="test-plan", thresholds=ContractThresholds(), baseline={}, stop_condition="test",
@@ -429,6 +453,24 @@ def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path):
}
_validate_config(config)
nested_digest = copy.deepcopy(config)
nested_digest["recipes"][0]["driver"]["artifact_sha256"] = "f" * 64
with pytest.raises(BenchmarkError, match="artifact_sha256 is forbidden"):
_validate_config(nested_digest)
driver = build_driver(nested_digest["recipes"][0], plan())
assert driver.artifact_sha256 == config["recipes"][0]["artifact_sha256"]
cuda_transformers = copy.deepcopy(config)
cuda_transformers["recipes"][0]["device"] = "cuda"
cuda_transformers["recipes"][0]["driver"] = {
"type": "transformers",
"model_path": str(artifact),
"device": "cuda",
"threads": 8,
}
with pytest.raises(BenchmarkError, match="every recipe to run on CPU"):
_validate_config(cuda_transformers)
config["recipes"][0]["artifact_sha256"] = "0" * 64
with pytest.raises(BenchmarkError, match="digest mismatch"):
_validate_config(config)
@@ -443,6 +485,131 @@ def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path):
with pytest.raises(BenchmarkError, match="CPU-only"):
_validate_config(config)
config["recipes"][0]["device"] = "cuda"
config["recipes"][0]["driver"]["device"] = "cuda"
with pytest.raises(BenchmarkError, match="host marker"):
_validate_config(config, profile=GPU_DIAGNOSTIC_PROFILE)
config["host"] = []
with pytest.raises(BenchmarkError, match="host metadata must be an object"):
_validate_config(config, profile=GPU_DIAGNOSTIC_PROFILE)
config["host"] = {"benchmark_lane": "rocm-gpu-diagnostic"}
_validate_config(config, profile=GPU_DIAGNOSTIC_PROFILE)
config["recipes"][0]["driver"]["n_gpu_layers"] = 0
with pytest.raises(BenchmarkError, match="positive n_gpu_layers"):
_validate_config(config, profile=GPU_DIAGNOSTIC_PROFILE)
def test_gpu_offload_evidence_requires_measured_rocm_placement():
assert _gpu_layer_config_detail("cpu", 0) == "gpu layers 0"
assert _gpu_layer_config_detail("cuda", 99) == "requested gpu layers 99"
measured = """
common_param: - ROCm0 : Radeon 8060S Graphics (63963 MiB, 33058 MiB free)
load_tensors: offloaded 25/25 layers to GPU
"""
assert _gpu_offload_evidence(measured, 99) == (
"measured accelerator ROCm0: Radeon 8060S Graphics; "
"measured offload 25/25 layers"
)
with pytest.raises(BenchmarkError, match="no measured ROCm device"):
_gpu_offload_evidence("offloaded 25/25 layers to GPU", 99)
with pytest.raises(BenchmarkError, match="no measured layer offload"):
_gpu_offload_evidence(
"- ROCm0 : Radeon 8060S Graphics (63963 MiB, 33058 MiB free)", 99
)
with pytest.raises(BenchmarkError, match="measured only 4/25"):
_gpu_offload_evidence(measured.replace("25/25", "4/25"), 99)
def test_profiles_derive_fixed_producers_and_cli_dispatch(monkeypatch, tmp_path: Path):
assert _producer_for_profile(CONTRACT_V1_PROFILE) == REAL_REPORT_PRODUCER
assert _producer_for_profile(GPU_DIAGNOSTIC_PROFILE) == GPU_DIAGNOSTIC_REPORT_PRODUCER
with pytest.raises(BenchmarkError, match="unknown benchmark validation profile"):
_producer_for_profile("caller-selected-producer")
profiled_calls = []
def profiled_runner(config, *, profile):
profiled_calls.append((config, profile))
return {"profile": profile}
monkeypatch.setattr(recipe_drivers_module, "_run_profiled_benchmark", profiled_runner)
assert recipe_drivers_module.run_configured_benchmark({"source": "v1"}) == {
"profile": CONTRACT_V1_PROFILE
}
assert recipe_drivers_module.run_configured_gpu_diagnostic({"source": "gpu"}) == {
"profile": GPU_DIAGNOSTIC_PROFILE
}
assert profiled_calls == [
({"source": "v1"}, CONTRACT_V1_PROFILE),
({"source": "gpu"}, GPU_DIAGNOSTIC_PROFILE),
]
config_path = tmp_path / "config.json"
config_path.write_text(json.dumps({"test": "config"}), encoding="utf-8")
calls = []
def contract_runner(config):
calls.append((CONTRACT_V1_PROFILE, config))
return {"profile": CONTRACT_V1_PROFILE}
def gpu_runner(config):
calls.append((GPU_DIAGNOSTIC_PROFILE, config))
return {"profile": GPU_DIAGNOSTIC_PROFILE}
monkeypatch.setattr(recipe_drivers_module, "run_configured_benchmark", contract_runner)
monkeypatch.setattr(recipe_drivers_module, "run_configured_gpu_diagnostic", gpu_runner)
monkeypatch.setattr(
recipe_benchmark_module, "format_summary", lambda report: report["profile"]
)
recipe_benchmark_module.main(["--config", str(config_path)])
recipe_benchmark_module.main(
["--profile", GPU_DIAGNOSTIC_PROFILE, "--config", str(config_path)]
)
assert calls == [
(CONTRACT_V1_PROFILE, {"test": "config"}),
(GPU_DIAGNOSTIC_PROFILE, {"test": "config"}),
]
def test_committed_config_digest_matches_immutable_contract():
evidence = (
Path(__file__).parents[1]
/ ".scratch"
/ "distributed-gguf-runtime"
/ "evidence"
/ "DGR-001"
)
contract = parse_contract(
json.loads((evidence / "performance-contract.json").read_text())
)
config = json.loads((evidence / "benchmark-config.json").read_text())
assert _canonical_sha256(config) == contract.baseline["required_config_sha256"]
trusted = json.loads(
(evidence.parents[1] / "trusted-evidence-signers.json").read_text()
)
public_key = base64.b64decode(contract.baseline["required_signer_public_key"])
fingerprint = hashlib.sha256(public_key).hexdigest()
assert any(
signer["algorithm"] == "ed25519"
and signer["fingerprint_sha256"] == fingerprint
and signer["status"] == "active"
for signer in trusted["signers"]
)
expected_cpu_gpu_detail = _gpu_layer_config_detail("cpu", 0)
llama_backends = [
detail
for recipe_id, detail in contract.baseline["required_backend_detail"].items()
if recipe_id.startswith("llama-cpp-")
]
assert llama_backends
assert all(expected_cpu_gpu_detail in detail for detail in llama_backends)
assert all("requested gpu layers" not in detail for detail in llama_backends)
_TEST_SIGNING_KEY = Ed25519PrivateKey.generate()
@@ -559,6 +726,12 @@ def test_signed_real_report_rejects_tampering_and_rebinding():
with pytest.raises(PerformanceContractError, match="signature"):
evaluate_contract(contract, unsigned)
gpu_diagnostic = copy.deepcopy(report)
gpu_diagnostic["provenance"]["producer"] = GPU_DIAGNOSTIC_REPORT_PRODUCER
_resign_test_report(gpu_diagnostic)
with pytest.raises(PerformanceContractError, match="canonical real runner"):
evaluate_contract(contract, gpu_diagnostic)
tampered = copy.deepcopy(report)
tampered["recipes"][0]["recipe"]["artifact_sha256"] = "b" * 64
with pytest.raises(PerformanceContractError, match="signature verification failed"):
@@ -625,6 +798,83 @@ def test_quality_lane_requires_every_prompt_to_be_compared():
assert evaluation.verdict == "stop"
def test_failed_request_reaches_zero_tolerance_gate_instead_of_validation_error():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
measurements = [
measure_recipe(
FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()
),
measure_recipe(FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()),
measure_recipe(FakeDriver(texts=texts), recipe("q4", Lane.PERFORMANCE_FIT), plan()),
]
report = build_report(plan(), measurements, host={}, evidence_class="synthetic")
contract = _lock_real_report(report)
failed = copy.deepcopy(report)
quality_entry = next(
entry for entry in failed["recipes"] if entry["recipe"]["id"] == "quality"
)
outcome = quality_entry["outcomes"][0]
outcome["ok"] = False
outcome["error"] = "simulated runtime failure"
cell_key = next(
key for key in quality_entry["concurrency"] if int(key) == outcome["concurrency"]
)
quality_entry["concurrency"][cell_key]["failures"] = 1
_resign_test_report(failed)
evaluation = evaluate_contract(contract, failed)
quality_evaluation = next(
item for item in evaluation.recipes if item.recipe_id == "quality"
)
assert quality_evaluation.failures == 1
assert quality_evaluation.quality_pass is False
assert evaluation.verdict == "stop"
permissive_contract = replace(
contract,
thresholds=replace(contract.thresholds, max_failure_rate=0.01),
)
with pytest.raises(PerformanceContractError, match="explicit failed-request token policy"):
evaluate_contract(permissive_contract, failed)
inconsistent = copy.deepcopy(failed)
quality_entry = next(
entry for entry in inconsistent["recipes"] if entry["recipe"]["id"] == "quality"
)
quality_entry["concurrency"][cell_key]["failures"] = 0
_resign_test_report(inconsistent)
with pytest.raises(PerformanceContractError, match="failures do not match raw outcomes"):
evaluate_contract(contract, inconsistent)
wrong_requests = copy.deepcopy(report)
quality_entry = next(
entry for entry in wrong_requests["recipes"] if entry["recipe"]["id"] == "quality"
)
quality_entry["concurrency"][cell_key]["requests"] += 1
_resign_test_report(wrong_requests)
with pytest.raises(PerformanceContractError, match="requests do not match raw outcomes"):
evaluate_contract(contract, wrong_requests)
misidentified = copy.deepcopy(report)
quality_entry = next(
entry for entry in misidentified["recipes"] if entry["recipe"]["id"] == "quality"
)
quality_entry["outcomes"][0]["recipe_id"] = "other-recipe"
_resign_test_report(misidentified)
with pytest.raises(PerformanceContractError, match="contains an outcome for"):
evaluate_contract(contract, misidentified)
missing = copy.deepcopy(failed)
quality_entry = next(
entry for entry in missing["recipes"] if entry["recipe"]["id"] == "quality"
)
quality_entry["outcomes"].pop()
_resign_test_report(missing)
with pytest.raises(PerformanceContractError, match="complete request coverage"):
evaluate_contract(contract, missing)
def test_locked_contract_rejects_changed_plan_and_token_counts():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
reference = measure_recipe(