fix: cryptographically bind DGR-001 evidence

This commit is contained in:
Dobromir Popov
2026-07-13 19:38:14 +03:00
parent 9e67b829e3
commit b1c9deeb01
11 changed files with 1349 additions and 903 deletions

View File

@@ -8,12 +8,18 @@ report.
from __future__ import annotations
import base64
import copy
import hashlib
import time
from pathlib import Path
import pytest
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
from meshnet_node.performance_contract import (
PROVENANCE_SCHEMA_VERSION,
REAL_REPORT_PRODUCER,
STOP_CONDITION,
ContractThresholds,
PerformanceContract,
@@ -21,6 +27,7 @@ from meshnet_node.performance_contract import (
_canonical_sha256,
evaluate_contract,
parse_contract,
report_signing_payload,
)
from meshnet_node.recipe_drivers import (
_artifact_sha256,
@@ -437,6 +444,16 @@ def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path):
_validate_config(config)
_TEST_SIGNING_KEY = Ed25519PrivateKey.generate()
def _resign_test_report(report: dict) -> None:
report["provenance"].pop("signature", None)
report["provenance"]["signature"] = base64.b64encode(
_TEST_SIGNING_KEY.sign(report_signing_payload(report))
).decode("ascii")
def _lock_real_report(report: dict) -> PerformanceContract:
report["evidence_class"] = "local-real"
report["host"] = {
@@ -449,7 +466,23 @@ def _lock_real_report(report: dict) -> PerformanceContract:
entry["recipe"]["source_model_id"] = report["plan"]["model_id"]
entry["recipe"]["source_model_revision"] = report["plan"]["model_revision"]
entry["recipe"]["artifact_sha256"] = "a" * 64
return PerformanceContract(
entry["load"]["backend_detail"] = f"fake-backend-{entry['recipe']['id']}"
public_key = _TEST_SIGNING_KEY.public_key().public_bytes(
serialization.Encoding.Raw, serialization.PublicFormat.Raw
)
config_sha256 = "c" * 64
report["provenance"] = {
"schema_version": PROVENANCE_SCHEMA_VERSION,
"producer": REAL_REPORT_PRODUCER,
"run_id": "test-run-id",
"started_at": "2026-07-13T00:00:00Z",
"completed_at": "2026-07-13T00:01:00Z",
"config_sha256": config_sha256,
"signature_algorithm": "ed25519",
"signer_public_key_sha256": hashlib.sha256(public_key).hexdigest(),
}
contract = PerformanceContract(
contract_version=1,
locked_at="2026-07-13T00:00:00Z",
locked_by="test",
@@ -459,9 +492,29 @@ def _lock_real_report(report: dict) -> PerformanceContract:
"required_evidence_class": "local-real",
"required_recipes": [entry["recipe"]["id"] for entry in report["recipes"]],
"required_concurrency_levels": [1, 4],
"required_config_sha256": config_sha256,
"required_signer_public_key": base64.b64encode(public_key).decode("ascii"),
"required_artifact_sha256": {
entry["recipe"]["id"]: entry["recipe"]["artifact_sha256"]
for entry in report["recipes"]
},
"required_recipe_runtime": {
entry["recipe"]["id"]: {
field: entry["recipe"].get(field)
for field in ("runtime", "weight_format", "weight_quantization", "device")
}
for entry in report["recipes"]
},
"required_backend_detail": {
entry["recipe"]["id"]: entry["load"]["backend_detail"]
for entry in report["recipes"]
},
"required_host_identity": {"python": "3.12"},
},
stop_condition="test",
)
_resign_test_report(report)
return contract
def test_locked_contract_rejects_synthetic_evidence():
@@ -480,6 +533,67 @@ def test_locked_contract_rejects_synthetic_evidence():
evaluate_contract(contract, report)
def test_non_synthetic_report_requires_canonical_provenance():
reference = measure_recipe(
FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan()
)
with pytest.raises(BenchmarkError, match="signed provenance"):
build_report(
plan(), [reference], host={}, evidence_class="local-real"
)
def test_signed_real_report_rejects_tampering_and_rebinding():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
measurements = [
measure_recipe(FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()),
measure_recipe(FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()),
measure_recipe(FakeDriver(texts=texts), recipe("q4", Lane.PERFORMANCE_FIT), plan()),
]
report = build_report(plan(), measurements, host={}, evidence_class="synthetic")
contract = _lock_real_report(report)
assert evaluate_contract(contract, report).verdict in {"promote", "optimize", "stop"}
unsigned = copy.deepcopy(report)
unsigned["provenance"].pop("signature")
with pytest.raises(PerformanceContractError, match="signature"):
evaluate_contract(contract, unsigned)
tampered = copy.deepcopy(report)
tampered["recipes"][0]["recipe"]["artifact_sha256"] = "b" * 64
with pytest.raises(PerformanceContractError, match="signature verification failed"):
evaluate_contract(contract, tampered)
rebound_artifact = copy.deepcopy(tampered)
_resign_test_report(rebound_artifact)
with pytest.raises(PerformanceContractError, match="artifact digest"):
evaluate_contract(contract, rebound_artifact)
rebound_config = copy.deepcopy(report)
rebound_config["provenance"]["config_sha256"] = "d" * 64
_resign_test_report(rebound_config)
with pytest.raises(PerformanceContractError, match="config digest"):
evaluate_contract(contract, rebound_config)
rebound_runtime = copy.deepcopy(report)
rebound_runtime["recipes"][1]["recipe"]["runtime"] = "other-runtime"
_resign_test_report(rebound_runtime)
with pytest.raises(PerformanceContractError, match="runtime identity"):
evaluate_contract(contract, rebound_runtime)
rebound_backend = copy.deepcopy(report)
rebound_backend["recipes"][1]["load"]["backend_detail"] = "other-backend"
_resign_test_report(rebound_backend)
with pytest.raises(PerformanceContractError, match="backend identity"):
evaluate_contract(contract, rebound_backend)
rebound_host = copy.deepcopy(report)
rebound_host["host"]["python"] = "9.9"
_resign_test_report(rebound_host)
with pytest.raises(PerformanceContractError, match="host/runtime field"):
evaluate_contract(contract, rebound_host)
def test_quality_lane_requires_every_prompt_to_be_compared():
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
reference = measure_recipe(
@@ -503,6 +617,7 @@ def test_quality_lane_requires_every_prompt_to_be_compared():
)
quality_drift["compared_prompts"] = 1
quality_drift["per_prompt"] = quality_drift["per_prompt"][:1]
_resign_test_report(incomplete)
evaluation = evaluate_contract(contract, incomplete)
@@ -529,6 +644,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts():
changed_plan = copy.deepcopy(report)
changed_plan["plan"]["prompts"][0]["text"] = "changed after locking"
_resign_test_report(changed_plan)
with pytest.raises(PerformanceContractError, match="plan digest"):
evaluate_contract(contract, changed_plan)
@@ -537,6 +653,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts():
entry for entry in changed_tokens["recipes"] if entry["recipe"]["id"] == "quality"
)
quality_entry["outcomes"][0]["decode_tokens"] += 1
_resign_test_report(changed_tokens)
with pytest.raises(PerformanceContractError, match="different prompt/decode token counts"):
evaluate_contract(contract, changed_tokens)