262 lines
12 KiB
Python
262 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Build the DGR-001 parity summary from cryptographically verified reports."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
|
|
|
from meshnet_node.performance_contract import (
|
|
_canonical_sha256,
|
|
evaluate_contract,
|
|
load_contract,
|
|
report_signing_payload,
|
|
)
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
|
|
|
|
def _read(name: str) -> dict:
|
|
return json.loads((ROOT / name).read_text(encoding="utf-8"))
|
|
|
|
|
|
def _file_sha256(name: str) -> str:
|
|
return hashlib.sha256((ROOT / name).read_bytes()).hexdigest()
|
|
|
|
|
|
def _drift(report: dict, recipe_id: str) -> dict:
|
|
return next(item for item in report["drift"] if item["recipe_id"] == recipe_id)
|
|
|
|
|
|
def _recipe(report: dict, recipe_id: str) -> dict:
|
|
return next(item for item in report["recipes"] if item["recipe"]["id"] == recipe_id)
|
|
|
|
|
|
def main() -> None:
|
|
contract = load_contract(ROOT / "performance-contract.json")
|
|
cpu_report = _read("results.json")
|
|
gpu_config = _read("gpu-diagnostic-config.json")
|
|
gpu_report = _read("gpu-diagnostic-results.json")
|
|
|
|
cpu_evaluation = evaluate_contract(contract, cpu_report)
|
|
if cpu_evaluation.verdict != "stop":
|
|
raise RuntimeError("immutable CPU v1 evidence no longer evaluates to stop")
|
|
|
|
public_key_bytes = base64.b64decode(contract.baseline["required_signer_public_key"])
|
|
public_key = Ed25519PublicKey.from_public_bytes(public_key_bytes)
|
|
public_key.verify(
|
|
base64.b64decode(gpu_report["provenance"]["signature"]),
|
|
report_signing_payload(gpu_report),
|
|
)
|
|
signer_fingerprint = hashlib.sha256(public_key_bytes).hexdigest()
|
|
if gpu_report["provenance"]["signer_public_key_sha256"] != signer_fingerprint:
|
|
raise RuntimeError("GPU report signer fingerprint does not match the contract trust key")
|
|
if gpu_report["provenance"]["config_sha256"] != _canonical_sha256(gpu_config):
|
|
raise RuntimeError("GPU report is not bound to gpu-diagnostic-config.json")
|
|
|
|
if gpu_report.get("schema_version") != 1 or gpu_report.get("evidence_class") != "local-real":
|
|
raise RuntimeError("GPU report must be schema-v1 local-real evidence")
|
|
expected_producer = "meshnet_node.recipe_drivers.run_configured_gpu_diagnostic/v1"
|
|
if gpu_report["provenance"].get("producer") != expected_producer:
|
|
raise RuntimeError("GPU report was not emitted by the canonical diagnostic producer")
|
|
if gpu_report.get("reference_recipe_id") != "transformers-fp32-rocm-quality-oracle":
|
|
raise RuntimeError("GPU report uses the wrong quality reference")
|
|
if gpu_report.get("host", {}).get("benchmark_lane") != "rocm-gpu-diagnostic":
|
|
raise RuntimeError("GPU report lacks the diagnostic host marker")
|
|
|
|
trusted = json.loads(
|
|
(ROOT.parents[1] / "trusted-evidence-signers.json").read_text(encoding="utf-8")
|
|
)
|
|
if not any(
|
|
signer.get("algorithm") == "ed25519"
|
|
and signer.get("fingerprint_sha256") == signer_fingerprint
|
|
and signer.get("status") == "active"
|
|
for signer in trusted.get("signers", ())
|
|
):
|
|
raise RuntimeError("GPU signer is not active in the trusted-signers registry")
|
|
|
|
for field in ("model_id", "model_revision"):
|
|
if gpu_report["plan"].get(field) != cpu_report["plan"].get(field):
|
|
raise RuntimeError(f"CPU and GPU reports do not share {field}")
|
|
if gpu_config["plan"].get(field) != gpu_report["plan"].get(field):
|
|
raise RuntimeError(f"GPU config and report do not share {field}")
|
|
|
|
expected_recipes = {
|
|
"transformers-fp32-rocm-quality-oracle": ("quality", "cuda"),
|
|
"llama-cpp-bf16-rocm-quality": ("quality", "cuda"),
|
|
"transformers-bf16-rocm-throughput": ("performance-fit", "cuda"),
|
|
"llama-cpp-q4-rocm-throughput": ("performance-fit", "cuda"),
|
|
}
|
|
actual_recipes = {
|
|
entry["recipe"]["id"]: (entry["recipe"]["lane"], entry["recipe"]["device"])
|
|
for entry in gpu_report["recipes"]
|
|
}
|
|
if actual_recipes != expected_recipes:
|
|
raise RuntimeError("GPU report recipe identities, lanes, or devices changed")
|
|
|
|
gpu_prompt_ids = {prompt["id"] for prompt in gpu_report["plan"]["prompts"]}
|
|
levels = {int(level) for level in gpu_report["plan"]["concurrency_levels"]}
|
|
repeats = int(gpu_report["plan"]["repeats"])
|
|
expected_outcomes = len(gpu_prompt_ids) * repeats * sum(levels)
|
|
for entry in gpu_report["recipes"]:
|
|
recipe_id = entry["recipe"]["id"]
|
|
if not entry.get("available") or len(entry.get("outcomes", ())) != expected_outcomes:
|
|
raise RuntimeError(f"GPU recipe {recipe_id!r} lacks complete outcomes")
|
|
if any(
|
|
not outcome.get("ok")
|
|
or outcome.get("recipe_id") != recipe_id
|
|
or outcome.get("prompt_id") not in gpu_prompt_ids
|
|
or int(outcome.get("concurrency", 0)) not in levels
|
|
or not 0 <= int(outcome.get("repeat", -1)) < repeats
|
|
for outcome in entry["outcomes"]
|
|
):
|
|
raise RuntimeError(f"GPU recipe {recipe_id!r} contains failed or invalid outcomes")
|
|
if {int(level) for level in entry["concurrency"]} != levels:
|
|
raise RuntimeError(f"GPU recipe {recipe_id!r} has wrong concurrency cells")
|
|
for prompt_id in gpu_prompt_ids:
|
|
for level in levels:
|
|
for repeat in range(repeats):
|
|
count = sum(
|
|
outcome["prompt_id"] == prompt_id
|
|
and int(outcome["concurrency"]) == level
|
|
and int(outcome["repeat"]) == repeat
|
|
for outcome in entry["outcomes"]
|
|
)
|
|
if count != level:
|
|
raise RuntimeError(
|
|
f"GPU recipe {recipe_id!r} lacks complete request coverage"
|
|
)
|
|
if any(
|
|
int(cell.get("failures", -1)) != 0
|
|
or int(cell.get("requests", -1))
|
|
!= len(
|
|
[
|
|
outcome
|
|
for outcome in entry["outcomes"]
|
|
if int(outcome["concurrency"]) == int(level)
|
|
]
|
|
)
|
|
for level, cell in entry["concurrency"].items()
|
|
):
|
|
raise RuntimeError(f"GPU recipe {recipe_id!r} aggregates do not match outcomes")
|
|
|
|
cpu_quality = _drift(cpu_report, "llama-cpp-near-lossless-quality")
|
|
gpu_quality = _drift(gpu_report, "llama-cpp-bf16-rocm-quality")
|
|
cpu_recipe = _recipe(cpu_report, "llama-cpp-near-lossless-quality")
|
|
gpu_recipe = _recipe(gpu_report, "llama-cpp-bf16-rocm-quality")
|
|
gpu_backend = gpu_recipe["load"]["backend_detail"]
|
|
if "measured accelerator ROCm0: Radeon 8060S Graphics" not in gpu_backend:
|
|
raise RuntimeError("GPU report lacks measured ROCm device evidence")
|
|
if "measured offload 25/25 layers" not in gpu_backend:
|
|
raise RuntimeError("GPU report lacks measured layer-offload evidence")
|
|
if cpu_recipe["recipe"]["artifact_sha256"] != gpu_recipe["recipe"]["artifact_sha256"]:
|
|
raise RuntimeError("CPU and GPU diagnostics use different BF16 GGUF artifacts")
|
|
if gpu_quality.get("compared_prompts") != len(gpu_prompt_ids):
|
|
raise RuntimeError("GPU quality drift lacks complete prompt coverage")
|
|
if {item["prompt_id"] for item in gpu_quality.get("per_prompt", ())} != gpu_prompt_ids:
|
|
raise RuntimeError("GPU quality drift prompt identities do not match the plan")
|
|
|
|
summary = {
|
|
"schema_version": 2,
|
|
"model_id": cpu_report["plan"]["model_id"],
|
|
"model_revision": cpu_report["plan"]["model_revision"],
|
|
"cpu_v1": {
|
|
"report": "results.json",
|
|
"report_sha256": _file_sha256("results.json"),
|
|
"run_id": cpu_report["provenance"]["run_id"],
|
|
"plan_id": cpu_report["plan"]["plan_id"],
|
|
"plan_sha256": _canonical_sha256(cpu_report["plan"]),
|
|
"config_sha256": cpu_report["provenance"]["config_sha256"],
|
|
"device": "cpu",
|
|
"quality_oracle": "Transformers BF16 safetensors",
|
|
"candidate": "llama.cpp BF16 GGUF",
|
|
"candidate_artifact_sha256": cpu_recipe["recipe"]["artifact_sha256"],
|
|
"exact_match_rate": cpu_quality["exact_match_rate"],
|
|
"mean_similarity": cpu_quality["mean_similarity"],
|
|
"contract_verdict": cpu_evaluation.verdict,
|
|
"root_cause": "undetermined; no logit-tie claim is acceptance evidence",
|
|
},
|
|
"rocm_diagnostic": {
|
|
"report": "gpu-diagnostic-results.json",
|
|
"report_sha256": _file_sha256("gpu-diagnostic-results.json"),
|
|
"run_id": gpu_report["provenance"]["run_id"],
|
|
"producer": gpu_report["provenance"]["producer"],
|
|
"signer_fingerprint": signer_fingerprint,
|
|
"plan_id": gpu_report["plan"]["plan_id"],
|
|
"plan_sha256": _canonical_sha256(gpu_report["plan"]),
|
|
"config_sha256": gpu_report["provenance"]["config_sha256"],
|
|
"device": "cuda (ROCm)",
|
|
"quality_oracle": "Transformers float32 safetensors",
|
|
"candidate": "llama.cpp BF16 GGUF",
|
|
"candidate_artifact_sha256": gpu_recipe["recipe"]["artifact_sha256"],
|
|
"measured_backend_detail": gpu_backend,
|
|
"exact_match_rate": gpu_quality["exact_match_rate"],
|
|
"mean_similarity": gpu_quality["mean_similarity"],
|
|
"failures": sum(
|
|
metrics["failures"]
|
|
for entry in gpu_report["recipes"]
|
|
for metrics in entry["concurrency"].values()
|
|
),
|
|
"v1_eligible": False,
|
|
},
|
|
"conclusion": {
|
|
"v1_verdict_changed": False,
|
|
"cpu_bf16_divergence_explained": False,
|
|
"conversion_corruption_observed_in_rocm_sample": False,
|
|
"scope": (
|
|
"The ROCm diagnostic establishes only that the same BF16 GGUF artifact "
|
|
"matched the float32 oracle for three GPU sequences; it does not explain "
|
|
"the CPU BF16 divergence or prove global conversion correctness."
|
|
),
|
|
"recommended_v2_design": (
|
|
"Predeclare a float32 quality oracle separately from the BF16 performance "
|
|
"reference, with a larger prompt corpus and immutable thresholds."
|
|
),
|
|
},
|
|
}
|
|
|
|
(ROOT / "quality-parity-diagnosis.json").write_text(
|
|
json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
|
)
|
|
md = f"""# DGR-001 quality-parity evidence summary
|
|
|
|
This summary is generated by `summarize-quality-parity.py` from signed reports.
|
|
It contains no independent logit measurements or self-asserted verification flag.
|
|
|
|
| Source | Device | Quality oracle | BF16 GGUF candidate | Exact | Similarity | Status |
|
|
|---|---|---|---|---:|---:|---|
|
|
| CPU v1 (`{summary['cpu_v1']['run_id']}`) | CPU | Transformers BF16 | llama.cpp BF16 | {summary['cpu_v1']['exact_match_rate']:.4f} | {summary['cpu_v1']['mean_similarity']:.4f} | immutable `stop` |
|
|
| ROCm diagnostic (`{summary['rocm_diagnostic']['run_id']}`) | ROCm0 / Radeon 8060S | Transformers float32 | llama.cpp BF16 | {summary['rocm_diagnostic']['exact_match_rate']:.4f} | {summary['rocm_diagnostic']['mean_similarity']:.4f} | diagnostic only |
|
|
|
|
## Interpretation
|
|
|
|
The CPU and ROCm rows use different plans, devices, kernels, and quality oracles.
|
|
The CPU BF16 divergence remains unexplained and v1 remains `stop`. The signed
|
|
ROCm report establishes the narrower fact that the same BF16 GGUF artifact
|
|
matched the float32 oracle for all three GPU sequences with zero failures.
|
|
Its signed backend detail records `ROCm0: Radeon 8060S Graphics` and measured
|
|
`25/25` layer offload.
|
|
|
|
No conversion corruption was observed in that three-sequence ROCm sample. This
|
|
does not prove global conversion correctness and does not retroactively change
|
|
or explain the CPU result. A future v2 should predeclare a float32 quality oracle
|
|
separately from its BF16 performance reference and use a larger corpus.
|
|
|
|
## Reproduction and bindings
|
|
|
|
- CPU report SHA-256: `{summary['cpu_v1']['report_sha256']}`
|
|
- GPU report SHA-256: `{summary['rocm_diagnostic']['report_sha256']}`
|
|
- BF16 GGUF SHA-256: `{summary['rocm_diagnostic']['candidate_artifact_sha256']}`
|
|
- Signer fingerprint: `{signer_fingerprint}`
|
|
- Exact verification command: see `commands.txt`.
|
|
"""
|
|
(ROOT / "quality-parity-diagnosis.md").write_text(md, encoding="utf-8")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|