fix: cryptographically bind DGR-001 evidence
This commit is contained in:
@@ -8,12 +8,18 @@ report.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
from meshnet_node.performance_contract import (
|
||||
PROVENANCE_SCHEMA_VERSION,
|
||||
REAL_REPORT_PRODUCER,
|
||||
STOP_CONDITION,
|
||||
ContractThresholds,
|
||||
PerformanceContract,
|
||||
@@ -21,6 +27,7 @@ from meshnet_node.performance_contract import (
|
||||
_canonical_sha256,
|
||||
evaluate_contract,
|
||||
parse_contract,
|
||||
report_signing_payload,
|
||||
)
|
||||
from meshnet_node.recipe_drivers import (
|
||||
_artifact_sha256,
|
||||
@@ -437,6 +444,16 @@ def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path):
|
||||
_validate_config(config)
|
||||
|
||||
|
||||
_TEST_SIGNING_KEY = Ed25519PrivateKey.generate()
|
||||
|
||||
|
||||
def _resign_test_report(report: dict) -> None:
|
||||
report["provenance"].pop("signature", None)
|
||||
report["provenance"]["signature"] = base64.b64encode(
|
||||
_TEST_SIGNING_KEY.sign(report_signing_payload(report))
|
||||
).decode("ascii")
|
||||
|
||||
|
||||
def _lock_real_report(report: dict) -> PerformanceContract:
|
||||
report["evidence_class"] = "local-real"
|
||||
report["host"] = {
|
||||
@@ -449,7 +466,23 @@ def _lock_real_report(report: dict) -> PerformanceContract:
|
||||
entry["recipe"]["source_model_id"] = report["plan"]["model_id"]
|
||||
entry["recipe"]["source_model_revision"] = report["plan"]["model_revision"]
|
||||
entry["recipe"]["artifact_sha256"] = "a" * 64
|
||||
return PerformanceContract(
|
||||
entry["load"]["backend_detail"] = f"fake-backend-{entry['recipe']['id']}"
|
||||
|
||||
public_key = _TEST_SIGNING_KEY.public_key().public_bytes(
|
||||
serialization.Encoding.Raw, serialization.PublicFormat.Raw
|
||||
)
|
||||
config_sha256 = "c" * 64
|
||||
report["provenance"] = {
|
||||
"schema_version": PROVENANCE_SCHEMA_VERSION,
|
||||
"producer": REAL_REPORT_PRODUCER,
|
||||
"run_id": "test-run-id",
|
||||
"started_at": "2026-07-13T00:00:00Z",
|
||||
"completed_at": "2026-07-13T00:01:00Z",
|
||||
"config_sha256": config_sha256,
|
||||
"signature_algorithm": "ed25519",
|
||||
"signer_public_key_sha256": hashlib.sha256(public_key).hexdigest(),
|
||||
}
|
||||
contract = PerformanceContract(
|
||||
contract_version=1,
|
||||
locked_at="2026-07-13T00:00:00Z",
|
||||
locked_by="test",
|
||||
@@ -459,9 +492,29 @@ def _lock_real_report(report: dict) -> PerformanceContract:
|
||||
"required_evidence_class": "local-real",
|
||||
"required_recipes": [entry["recipe"]["id"] for entry in report["recipes"]],
|
||||
"required_concurrency_levels": [1, 4],
|
||||
"required_config_sha256": config_sha256,
|
||||
"required_signer_public_key": base64.b64encode(public_key).decode("ascii"),
|
||||
"required_artifact_sha256": {
|
||||
entry["recipe"]["id"]: entry["recipe"]["artifact_sha256"]
|
||||
for entry in report["recipes"]
|
||||
},
|
||||
"required_recipe_runtime": {
|
||||
entry["recipe"]["id"]: {
|
||||
field: entry["recipe"].get(field)
|
||||
for field in ("runtime", "weight_format", "weight_quantization", "device")
|
||||
}
|
||||
for entry in report["recipes"]
|
||||
},
|
||||
"required_backend_detail": {
|
||||
entry["recipe"]["id"]: entry["load"]["backend_detail"]
|
||||
for entry in report["recipes"]
|
||||
},
|
||||
"required_host_identity": {"python": "3.12"},
|
||||
},
|
||||
stop_condition="test",
|
||||
)
|
||||
_resign_test_report(report)
|
||||
return contract
|
||||
|
||||
|
||||
def test_locked_contract_rejects_synthetic_evidence():
|
||||
@@ -480,6 +533,67 @@ def test_locked_contract_rejects_synthetic_evidence():
|
||||
evaluate_contract(contract, report)
|
||||
|
||||
|
||||
def test_non_synthetic_report_requires_canonical_provenance():
|
||||
reference = measure_recipe(
|
||||
FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan()
|
||||
)
|
||||
with pytest.raises(BenchmarkError, match="signed provenance"):
|
||||
build_report(
|
||||
plan(), [reference], host={}, evidence_class="local-real"
|
||||
)
|
||||
|
||||
|
||||
def test_signed_real_report_rejects_tampering_and_rebinding():
|
||||
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
|
||||
measurements = [
|
||||
measure_recipe(FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()),
|
||||
measure_recipe(FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()),
|
||||
measure_recipe(FakeDriver(texts=texts), recipe("q4", Lane.PERFORMANCE_FIT), plan()),
|
||||
]
|
||||
report = build_report(plan(), measurements, host={}, evidence_class="synthetic")
|
||||
contract = _lock_real_report(report)
|
||||
assert evaluate_contract(contract, report).verdict in {"promote", "optimize", "stop"}
|
||||
|
||||
unsigned = copy.deepcopy(report)
|
||||
unsigned["provenance"].pop("signature")
|
||||
with pytest.raises(PerformanceContractError, match="signature"):
|
||||
evaluate_contract(contract, unsigned)
|
||||
|
||||
tampered = copy.deepcopy(report)
|
||||
tampered["recipes"][0]["recipe"]["artifact_sha256"] = "b" * 64
|
||||
with pytest.raises(PerformanceContractError, match="signature verification failed"):
|
||||
evaluate_contract(contract, tampered)
|
||||
|
||||
rebound_artifact = copy.deepcopy(tampered)
|
||||
_resign_test_report(rebound_artifact)
|
||||
with pytest.raises(PerformanceContractError, match="artifact digest"):
|
||||
evaluate_contract(contract, rebound_artifact)
|
||||
|
||||
rebound_config = copy.deepcopy(report)
|
||||
rebound_config["provenance"]["config_sha256"] = "d" * 64
|
||||
_resign_test_report(rebound_config)
|
||||
with pytest.raises(PerformanceContractError, match="config digest"):
|
||||
evaluate_contract(contract, rebound_config)
|
||||
|
||||
rebound_runtime = copy.deepcopy(report)
|
||||
rebound_runtime["recipes"][1]["recipe"]["runtime"] = "other-runtime"
|
||||
_resign_test_report(rebound_runtime)
|
||||
with pytest.raises(PerformanceContractError, match="runtime identity"):
|
||||
evaluate_contract(contract, rebound_runtime)
|
||||
|
||||
rebound_backend = copy.deepcopy(report)
|
||||
rebound_backend["recipes"][1]["load"]["backend_detail"] = "other-backend"
|
||||
_resign_test_report(rebound_backend)
|
||||
with pytest.raises(PerformanceContractError, match="backend identity"):
|
||||
evaluate_contract(contract, rebound_backend)
|
||||
|
||||
rebound_host = copy.deepcopy(report)
|
||||
rebound_host["host"]["python"] = "9.9"
|
||||
_resign_test_report(rebound_host)
|
||||
with pytest.raises(PerformanceContractError, match="host/runtime field"):
|
||||
evaluate_contract(contract, rebound_host)
|
||||
|
||||
|
||||
def test_quality_lane_requires_every_prompt_to_be_compared():
|
||||
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
|
||||
reference = measure_recipe(
|
||||
@@ -503,6 +617,7 @@ def test_quality_lane_requires_every_prompt_to_be_compared():
|
||||
)
|
||||
quality_drift["compared_prompts"] = 1
|
||||
quality_drift["per_prompt"] = quality_drift["per_prompt"][:1]
|
||||
_resign_test_report(incomplete)
|
||||
|
||||
evaluation = evaluate_contract(contract, incomplete)
|
||||
|
||||
@@ -529,6 +644,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts():
|
||||
|
||||
changed_plan = copy.deepcopy(report)
|
||||
changed_plan["plan"]["prompts"][0]["text"] = "changed after locking"
|
||||
_resign_test_report(changed_plan)
|
||||
with pytest.raises(PerformanceContractError, match="plan digest"):
|
||||
evaluate_contract(contract, changed_plan)
|
||||
|
||||
@@ -537,6 +653,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts():
|
||||
entry for entry in changed_tokens["recipes"] if entry["recipe"]["id"] == "quality"
|
||||
)
|
||||
quality_entry["outcomes"][0]["decode_tokens"] += 1
|
||||
_resign_test_report(changed_tokens)
|
||||
with pytest.raises(PerformanceContractError, match="different prompt/decode token counts"):
|
||||
evaluate_contract(contract, changed_tokens)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user