#!/usr/bin/env python3 """Build the DGR-001 parity summary from cryptographically verified reports.""" from __future__ import annotations import base64 import hashlib import json from pathlib import Path from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey from meshnet_node.performance_contract import ( _canonical_sha256, evaluate_contract, load_contract, report_signing_payload, ) ROOT = Path(__file__).resolve().parent def _read(name: str) -> dict: return json.loads((ROOT / name).read_text(encoding="utf-8")) def _file_sha256(name: str) -> str: return hashlib.sha256((ROOT / name).read_bytes()).hexdigest() def _drift(report: dict, recipe_id: str) -> dict: return next(item for item in report["drift"] if item["recipe_id"] == recipe_id) def _recipe(report: dict, recipe_id: str) -> dict: return next(item for item in report["recipes"] if item["recipe"]["id"] == recipe_id) def main() -> None: contract = load_contract(ROOT / "performance-contract.json") cpu_report = _read("results.json") gpu_config = _read("gpu-diagnostic-config.json") gpu_report = _read("gpu-diagnostic-results.json") cpu_evaluation = evaluate_contract(contract, cpu_report) if cpu_evaluation.verdict != "stop": raise RuntimeError("immutable CPU v1 evidence no longer evaluates to stop") public_key_bytes = base64.b64decode(contract.baseline["required_signer_public_key"]) public_key = Ed25519PublicKey.from_public_bytes(public_key_bytes) public_key.verify( base64.b64decode(gpu_report["provenance"]["signature"]), report_signing_payload(gpu_report), ) signer_fingerprint = hashlib.sha256(public_key_bytes).hexdigest() if gpu_report["provenance"]["signer_public_key_sha256"] != signer_fingerprint: raise RuntimeError("GPU report signer fingerprint does not match the contract trust key") if gpu_report["provenance"]["config_sha256"] != _canonical_sha256(gpu_config): raise RuntimeError("GPU report is not bound to gpu-diagnostic-config.json") if gpu_report.get("schema_version") != 1 or gpu_report.get("evidence_class") != "local-real": raise RuntimeError("GPU report must be schema-v1 local-real evidence") expected_producer = "meshnet_node.recipe_drivers.run_configured_gpu_diagnostic/v1" if gpu_report["provenance"].get("producer") != expected_producer: raise RuntimeError("GPU report was not emitted by the canonical diagnostic producer") if gpu_report.get("reference_recipe_id") != "transformers-fp32-rocm-quality-oracle": raise RuntimeError("GPU report uses the wrong quality reference") if gpu_report.get("host", {}).get("benchmark_lane") != "rocm-gpu-diagnostic": raise RuntimeError("GPU report lacks the diagnostic host marker") trusted = json.loads( (ROOT.parents[1] / "trusted-evidence-signers.json").read_text(encoding="utf-8") ) if not any( signer.get("algorithm") == "ed25519" and signer.get("fingerprint_sha256") == signer_fingerprint and signer.get("status") == "active" for signer in trusted.get("signers", ()) ): raise RuntimeError("GPU signer is not active in the trusted-signers registry") for field in ("model_id", "model_revision"): if gpu_report["plan"].get(field) != cpu_report["plan"].get(field): raise RuntimeError(f"CPU and GPU reports do not share {field}") if gpu_config["plan"].get(field) != gpu_report["plan"].get(field): raise RuntimeError(f"GPU config and report do not share {field}") expected_recipes = { "transformers-fp32-rocm-quality-oracle": ("quality", "cuda"), "llama-cpp-bf16-rocm-quality": ("quality", "cuda"), "transformers-bf16-rocm-throughput": ("performance-fit", "cuda"), "llama-cpp-q4-rocm-throughput": ("performance-fit", "cuda"), } actual_recipes = { entry["recipe"]["id"]: (entry["recipe"]["lane"], entry["recipe"]["device"]) for entry in gpu_report["recipes"] } if actual_recipes != expected_recipes: raise RuntimeError("GPU report recipe identities, lanes, or devices changed") gpu_prompt_ids = {prompt["id"] for prompt in gpu_report["plan"]["prompts"]} levels = {int(level) for level in gpu_report["plan"]["concurrency_levels"]} repeats = int(gpu_report["plan"]["repeats"]) expected_outcomes = len(gpu_prompt_ids) * repeats * sum(levels) for entry in gpu_report["recipes"]: recipe_id = entry["recipe"]["id"] if not entry.get("available") or len(entry.get("outcomes", ())) != expected_outcomes: raise RuntimeError(f"GPU recipe {recipe_id!r} lacks complete outcomes") if any( not outcome.get("ok") or outcome.get("recipe_id") != recipe_id or outcome.get("prompt_id") not in gpu_prompt_ids or int(outcome.get("concurrency", 0)) not in levels or not 0 <= int(outcome.get("repeat", -1)) < repeats for outcome in entry["outcomes"] ): raise RuntimeError(f"GPU recipe {recipe_id!r} contains failed or invalid outcomes") if {int(level) for level in entry["concurrency"]} != levels: raise RuntimeError(f"GPU recipe {recipe_id!r} has wrong concurrency cells") for prompt_id in gpu_prompt_ids: for level in levels: for repeat in range(repeats): count = sum( outcome["prompt_id"] == prompt_id and int(outcome["concurrency"]) == level and int(outcome["repeat"]) == repeat for outcome in entry["outcomes"] ) if count != level: raise RuntimeError( f"GPU recipe {recipe_id!r} lacks complete request coverage" ) if any( int(cell.get("failures", -1)) != 0 or int(cell.get("requests", -1)) != len( [ outcome for outcome in entry["outcomes"] if int(outcome["concurrency"]) == int(level) ] ) for level, cell in entry["concurrency"].items() ): raise RuntimeError(f"GPU recipe {recipe_id!r} aggregates do not match outcomes") cpu_quality = _drift(cpu_report, "llama-cpp-near-lossless-quality") gpu_quality = _drift(gpu_report, "llama-cpp-bf16-rocm-quality") cpu_recipe = _recipe(cpu_report, "llama-cpp-near-lossless-quality") gpu_recipe = _recipe(gpu_report, "llama-cpp-bf16-rocm-quality") gpu_backend = gpu_recipe["load"]["backend_detail"] if "measured accelerator ROCm0: Radeon 8060S Graphics" not in gpu_backend: raise RuntimeError("GPU report lacks measured ROCm device evidence") if "measured offload 25/25 layers" not in gpu_backend: raise RuntimeError("GPU report lacks measured layer-offload evidence") if cpu_recipe["recipe"]["artifact_sha256"] != gpu_recipe["recipe"]["artifact_sha256"]: raise RuntimeError("CPU and GPU diagnostics use different BF16 GGUF artifacts") if gpu_quality.get("compared_prompts") != len(gpu_prompt_ids): raise RuntimeError("GPU quality drift lacks complete prompt coverage") if {item["prompt_id"] for item in gpu_quality.get("per_prompt", ())} != gpu_prompt_ids: raise RuntimeError("GPU quality drift prompt identities do not match the plan") summary = { "schema_version": 2, "model_id": cpu_report["plan"]["model_id"], "model_revision": cpu_report["plan"]["model_revision"], "cpu_v1": { "report": "results.json", "report_sha256": _file_sha256("results.json"), "run_id": cpu_report["provenance"]["run_id"], "plan_id": cpu_report["plan"]["plan_id"], "plan_sha256": _canonical_sha256(cpu_report["plan"]), "config_sha256": cpu_report["provenance"]["config_sha256"], "device": "cpu", "quality_oracle": "Transformers BF16 safetensors", "candidate": "llama.cpp BF16 GGUF", "candidate_artifact_sha256": cpu_recipe["recipe"]["artifact_sha256"], "exact_match_rate": cpu_quality["exact_match_rate"], "mean_similarity": cpu_quality["mean_similarity"], "contract_verdict": cpu_evaluation.verdict, "root_cause": "undetermined; no logit-tie claim is acceptance evidence", }, "rocm_diagnostic": { "report": "gpu-diagnostic-results.json", "report_sha256": _file_sha256("gpu-diagnostic-results.json"), "run_id": gpu_report["provenance"]["run_id"], "producer": gpu_report["provenance"]["producer"], "signer_fingerprint": signer_fingerprint, "plan_id": gpu_report["plan"]["plan_id"], "plan_sha256": _canonical_sha256(gpu_report["plan"]), "config_sha256": gpu_report["provenance"]["config_sha256"], "device": "cuda (ROCm)", "quality_oracle": "Transformers float32 safetensors", "candidate": "llama.cpp BF16 GGUF", "candidate_artifact_sha256": gpu_recipe["recipe"]["artifact_sha256"], "measured_backend_detail": gpu_backend, "exact_match_rate": gpu_quality["exact_match_rate"], "mean_similarity": gpu_quality["mean_similarity"], "failures": sum( metrics["failures"] for entry in gpu_report["recipes"] for metrics in entry["concurrency"].values() ), "v1_eligible": False, }, "conclusion": { "v1_verdict_changed": False, "cpu_bf16_divergence_explained": False, "conversion_corruption_observed_in_rocm_sample": False, "scope": ( "The ROCm diagnostic establishes only that the same BF16 GGUF artifact " "matched the float32 oracle for three GPU sequences; it does not explain " "the CPU BF16 divergence or prove global conversion correctness." ), "recommended_v2_design": ( "Predeclare a float32 quality oracle separately from the BF16 performance " "reference, with a larger prompt corpus and immutable thresholds." ), }, } (ROOT / "quality-parity-diagnosis.json").write_text( json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) md = f"""# DGR-001 quality-parity evidence summary This summary is generated by `summarize-quality-parity.py` from signed reports. It contains no independent logit measurements or self-asserted verification flag. | Source | Device | Quality oracle | BF16 GGUF candidate | Exact | Similarity | Status | |---|---|---|---|---:|---:|---| | CPU v1 (`{summary['cpu_v1']['run_id']}`) | CPU | Transformers BF16 | llama.cpp BF16 | {summary['cpu_v1']['exact_match_rate']:.4f} | {summary['cpu_v1']['mean_similarity']:.4f} | immutable `stop` | | ROCm diagnostic (`{summary['rocm_diagnostic']['run_id']}`) | ROCm0 / Radeon 8060S | Transformers float32 | llama.cpp BF16 | {summary['rocm_diagnostic']['exact_match_rate']:.4f} | {summary['rocm_diagnostic']['mean_similarity']:.4f} | diagnostic only | ## Interpretation The CPU and ROCm rows use different plans, devices, kernels, and quality oracles. The CPU BF16 divergence remains unexplained and v1 remains `stop`. The signed ROCm report establishes the narrower fact that the same BF16 GGUF artifact matched the float32 oracle for all three GPU sequences with zero failures. Its signed backend detail records `ROCm0: Radeon 8060S Graphics` and measured `25/25` layer offload. No conversion corruption was observed in that three-sequence ROCm sample. This does not prove global conversion correctness and does not retroactively change or explain the CPU result. A future v2 should predeclare a float32 quality oracle separately from its BF16 performance reference and use a larger corpus. ## Reproduction and bindings - CPU report SHA-256: `{summary['cpu_v1']['report_sha256']}` - GPU report SHA-256: `{summary['rocm_diagnostic']['report_sha256']}` - BF16 GGUF SHA-256: `{summary['rocm_diagnostic']['candidate_artifact_sha256']}` - Signer fingerprint: `{signer_fingerprint}` - Exact verification command: see `commands.txt`. """ (ROOT / "quality-parity-diagnosis.md").write_text(md, encoding="utf-8") if __name__ == "__main__": main()