fix: cryptographically bind DGR-001 evidence
This commit is contained in:
Dobromir Popov
@@ -45,17 +45,17 @@ All recipes completed every request with zero failures.
|
||||
|
||||
| Metric | Transformers BF16 | llama.cpp BF16 | llama.cpp Q4_K_M |
|
||||
|---|---:|---:|---:|
|
||||
| Decode tok/s, c=1 | 46.1 | 88.0 | 170.1 |
|
||||
| Aggregate decode tok/s, c=4 | 47.1 | 211.4 | 206.4 |
|
||||
| TTFT p50, c=1 | 37.5 ms | 43.9 ms | 23.8 ms |
|
||||
| Decode tok/s, c=1 | 40.8 | 98.5 | 207.7 |
|
||||
| Aggregate decode tok/s, c=4 | 46.5 | 222.8 | 195.7 |
|
||||
| TTFT p50, c=1 | 40.0 ms | 15.1 ms | 21.6 ms |
|
||||
| Peak resident memory, c=1 | 1.94 GB | 1.11 GB | 0.54 GB |
|
||||
| Artifact size | 1.00 GB | 0.99 GB | 0.40 GB |
|
||||
| Failures | 0 | 0 | 0 |
|
||||
|
||||
Against the reference, the eligible Q4_K_M lane measured:
|
||||
|
||||
- single-request decode speedup: **3.69×**;
|
||||
- concurrency-4 aggregate throughput speedup: **4.38×**;
|
||||
- single-request decode speedup: **5.10×**;
|
||||
- concurrency-4 aggregate throughput speedup: **4.20×**;
|
||||
- resident-memory ratio: **0.279×**;
|
||||
- artifact-size ratio: **0.398×**.
|
||||
|
||||
@@ -99,9 +99,13 @@ Thresholds were not changed after observing these results.
|
||||
- llama.cpp's `predicted_n` includes the first token while `predicted_ms` begins
|
||||
after it; the driver subtracts that token so decode throughput matches the
|
||||
Transformers inter-token convention.
|
||||
- `performance_contract.py` rejects wrong plans, synthetic evidence, missing
|
||||
recipes/concurrency, mixed model revisions, incomplete quality coverage,
|
||||
failed references, and missing artifact/host provenance.
|
||||
- `performance_contract.py` rejects wrong plans, unsigned or incorrectly signed
|
||||
real evidence, wrong config/artifact/runtime/backend/host bindings, missing
|
||||
recipes/concurrency, mixed model revisions, incomplete quality coverage, and
|
||||
failed references.
|
||||
- Every non-synthetic report is Ed25519-signed over the complete canonical JSON,
|
||||
including raw outcomes and metrics. The contract pins the public key and exact
|
||||
config SHA-256; the private key remains outside Git at mode `0600`.
|
||||
- Quantized drift remains advisory. Only the near-lossless lane can satisfy the
|
||||
quality gate, and only performance-fit recipes can earn speed/fit benefits.
|
||||
|
||||
@@ -120,8 +124,8 @@ Thresholds were not changed after observing these results.
|
||||
## Verification
|
||||
|
||||
```text
|
||||
Targeted: 22 passed
|
||||
Full suite: 749 passed, 13 skipped
|
||||
Targeted: 24 passed
|
||||
Full suite: 751 passed, 13 skipped
|
||||
Earlier cancellation retry matrix, DGR-001: 4/5 passed
|
||||
Earlier cancellation retry matrix, clean d904c40: 4/5 passed
|
||||
compileall: passed
|
||||
@@ -129,9 +133,9 @@ git diff --check: passed
|
||||
Evidence JSON parse/integrity checks: passed
|
||||
```
|
||||
|
||||
The full-suite exception is documented in `known-unrelated-failure.md` and
|
||||
satisfies the issue's explicit clean-tree reproduction clause. DGR-001 changes
|
||||
no tracker/proxy files.
|
||||
An earlier intermittent tracker cancellation race reproduced at the same rate on
|
||||
the clean base and is retained in `known-unrelated-failure.md`; the final suite
|
||||
completed green. DGR-001 changes no tracker/proxy files.
|
||||
|
||||
The earlier Ralph claim that the full suite was blocked by Protobuf 6.33.6 was
|
||||
invalid: it used Hermes Agent's internal venv. Verification above used the
|
||||
|
||||
@@ -1,4 +1,14 @@
|
||||
{
|
||||
"artifact_sha256": {
|
||||
"llama-cpp-near-lossless-quality": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862",
|
||||
"llama-cpp-quantized-performance-fit": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5",
|
||||
"transformers-safetensors-reference": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6"
|
||||
},
|
||||
"backend_detail": {
|
||||
"llama-cpp-near-lossless-quality": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0",
|
||||
"llama-cpp-quantized-performance-fit": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0",
|
||||
"transformers-safetensors-reference": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cpu; intra-op threads 16"
|
||||
},
|
||||
"evidence_class": "local-real",
|
||||
"host": {
|
||||
"accelerator_name": "Radeon 8060S Graphics",
|
||||
@@ -10,6 +20,12 @@
|
||||
"hostname": "fedora",
|
||||
"llama_cpp_commit": "e920c523e3b8a0163fe498af5bf90df35ff51d25",
|
||||
"llama_cpp_version": "9991",
|
||||
"llama_server_identities": {
|
||||
"/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server": {
|
||||
"sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
|
||||
"version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64"
|
||||
}
|
||||
},
|
||||
"llama_server_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
|
||||
"platform": "Linux-7.0.14-101.fc43.x86_64-x86_64-with-glibc2.42",
|
||||
"python": "3.12.13",
|
||||
@@ -20,34 +36,65 @@
|
||||
"model_id": "Qwen/Qwen2.5-0.5B-Instruct",
|
||||
"model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
|
||||
"plan_sha256": "efe24690a9a7164bac6ab3fd0a6b22f078fc08aaefcfb96210ddf154e6050570",
|
||||
"provenance": {
|
||||
"completed_at": "2026-07-13T16:27:19.647692Z",
|
||||
"config_sha256": "00b2cce3e2f281bdf92fc5304ba5cac915a178ffccd3b9a25995ce39c00b90d3",
|
||||
"producer": "meshnet_node.recipe_drivers.run_configured_benchmark/v1",
|
||||
"run_id": "e4eedadf-22f6-4907-8990-985456961099",
|
||||
"schema_version": 1,
|
||||
"signature": "owev+/ToswP20C923G6E+srOCUBV5vrjmndVatr9CbTXakiFGqlHrTiEo+aymA4BcSwmG6KJTxlxO6WpLnpcAg==",
|
||||
"signature_algorithm": "ed25519",
|
||||
"signer_public_key_sha256": "8baca8742d9b3ed0c3fc54929c23f75ec8c1c739900aaf5334780d598ffa84de",
|
||||
"started_at": "2026-07-13T16:26:22.361501Z"
|
||||
},
|
||||
"recipe_runtime": {
|
||||
"llama-cpp-near-lossless-quality": {
|
||||
"device": "cpu",
|
||||
"runtime": "llama.cpp-9991-e920c523",
|
||||
"weight_format": "gguf",
|
||||
"weight_quantization": "bfloat16"
|
||||
},
|
||||
"llama-cpp-quantized-performance-fit": {
|
||||
"device": "cpu",
|
||||
"runtime": "llama.cpp-9991-e920c523",
|
||||
"weight_format": "gguf",
|
||||
"weight_quantization": "Q4_K_M"
|
||||
},
|
||||
"transformers-safetensors-reference": {
|
||||
"device": "cpu",
|
||||
"runtime": "transformers-5.13.0",
|
||||
"weight_format": "safetensors",
|
||||
"weight_quantization": "bfloat16"
|
||||
}
|
||||
},
|
||||
"recipes": {
|
||||
"llama-cpp-near-lossless-quality": {
|
||||
"artifact_bytes": 994156448,
|
||||
"available": true,
|
||||
"concurrency": {
|
||||
"1": {
|
||||
"aggregate_decode_tokens_per_sec": 73.7861,
|
||||
"decode_tokens_per_sec": 87.9728,
|
||||
"aggregate_decode_tokens_per_sec": 86.7339,
|
||||
"decode_tokens_per_sec": 98.5178,
|
||||
"failures": 0,
|
||||
"latency_p50_ms": 385.2049,
|
||||
"latency_p95_ms": 560.2939,
|
||||
"peak_rss_bytes": 1110708224,
|
||||
"latency_p50_ms": 333.023,
|
||||
"latency_p95_ms": 383.0597,
|
||||
"peak_rss_bytes": 1110728704,
|
||||
"peak_vram_bytes": 0,
|
||||
"prefill_tokens_per_sec": 1427.2072,
|
||||
"ttft_p50_ms": 43.929,
|
||||
"ttft_p95_ms": 107.003
|
||||
"prefill_tokens_per_sec": 1717.9451,
|
||||
"ttft_p50_ms": 15.069,
|
||||
"ttft_p95_ms": 63.766
|
||||
},
|
||||
"4": {
|
||||
"aggregate_decode_tokens_per_sec": 211.3515,
|
||||
"decode_tokens_per_sec": 73.8932,
|
||||
"aggregate_decode_tokens_per_sec": 222.788,
|
||||
"decode_tokens_per_sec": 76.6297,
|
||||
"failures": 0,
|
||||
"latency_p50_ms": 467.5094,
|
||||
"latency_p95_ms": 790.862,
|
||||
"peak_rss_bytes": 1129578496,
|
||||
"latency_p50_ms": 490.8738,
|
||||
"latency_p95_ms": 646.26,
|
||||
"peak_rss_bytes": 1139466240,
|
||||
"peak_vram_bytes": 0,
|
||||
"prefill_tokens_per_sec": 1077.8162,
|
||||
"ttft_p50_ms": 33.612,
|
||||
"ttft_p95_ms": 128.501
|
||||
"prefill_tokens_per_sec": 859.8985,
|
||||
"ttft_p50_ms": 32.445,
|
||||
"ttft_p95_ms": 218.387
|
||||
}
|
||||
},
|
||||
"device": "cpu",
|
||||
@@ -58,28 +105,28 @@
|
||||
"available": true,
|
||||
"concurrency": {
|
||||
"1": {
|
||||
"aggregate_decode_tokens_per_sec": 110.0458,
|
||||
"decode_tokens_per_sec": 170.131,
|
||||
"aggregate_decode_tokens_per_sec": 139.2693,
|
||||
"decode_tokens_per_sec": 207.712,
|
||||
"failures": 0,
|
||||
"latency_p50_ms": 258.0681,
|
||||
"latency_p95_ms": 465.8523,
|
||||
"peak_rss_bytes": 542167040,
|
||||
"latency_p50_ms": 168.3307,
|
||||
"latency_p95_ms": 305.1338,
|
||||
"peak_rss_bytes": 542081024,
|
||||
"peak_vram_bytes": 0,
|
||||
"prefill_tokens_per_sec": 783.3775,
|
||||
"ttft_p50_ms": 23.847,
|
||||
"ttft_p95_ms": 237.696
|
||||
"prefill_tokens_per_sec": 967.0195,
|
||||
"ttft_p50_ms": 21.582,
|
||||
"ttft_p95_ms": 147.859
|
||||
},
|
||||
"4": {
|
||||
"aggregate_decode_tokens_per_sec": 206.377,
|
||||
"decode_tokens_per_sec": 83.543,
|
||||
"aggregate_decode_tokens_per_sec": 195.6789,
|
||||
"decode_tokens_per_sec": 76.9497,
|
||||
"failures": 0,
|
||||
"latency_p50_ms": 413.3897,
|
||||
"latency_p95_ms": 910.253,
|
||||
"peak_rss_bytes": 572788736,
|
||||
"latency_p50_ms": 437.9196,
|
||||
"latency_p95_ms": 885.5355,
|
||||
"peak_rss_bytes": 573259776,
|
||||
"peak_vram_bytes": 0,
|
||||
"prefill_tokens_per_sec": 474.3116,
|
||||
"ttft_p50_ms": 67.945,
|
||||
"ttft_p95_ms": 431.804
|
||||
"prefill_tokens_per_sec": 572.4424,
|
||||
"ttft_p50_ms": 48.127,
|
||||
"ttft_p95_ms": 416.531
|
||||
}
|
||||
},
|
||||
"device": "cpu",
|
||||
@@ -90,28 +137,28 @@
|
||||
"available": true,
|
||||
"concurrency": {
|
||||
"1": {
|
||||
"aggregate_decode_tokens_per_sec": 40.3425,
|
||||
"decode_tokens_per_sec": 46.1451,
|
||||
"aggregate_decode_tokens_per_sec": 35.4722,
|
||||
"decode_tokens_per_sec": 40.7545,
|
||||
"failures": 0,
|
||||
"latency_p50_ms": 795.4807,
|
||||
"latency_p95_ms": 930.9725,
|
||||
"peak_rss_bytes": 1941213184,
|
||||
"latency_p50_ms": 818.3864,
|
||||
"latency_p95_ms": 1258.0673,
|
||||
"peak_rss_bytes": 1941458944,
|
||||
"peak_vram_bytes": 0,
|
||||
"prefill_tokens_per_sec": 671.8016,
|
||||
"ttft_p50_ms": 37.4548,
|
||||
"ttft_p95_ms": 193.4633
|
||||
"prefill_tokens_per_sec": 625.6467,
|
||||
"ttft_p50_ms": 40.0018,
|
||||
"ttft_p95_ms": 195.2551
|
||||
},
|
||||
"4": {
|
||||
"aggregate_decode_tokens_per_sec": 47.0903,
|
||||
"decode_tokens_per_sec": 13.1337,
|
||||
"aggregate_decode_tokens_per_sec": 46.5375,
|
||||
"decode_tokens_per_sec": 12.9506,
|
||||
"failures": 0,
|
||||
"latency_p50_ms": 2631.0031,
|
||||
"latency_p95_ms": 3073.7389,
|
||||
"peak_rss_bytes": 2177265664,
|
||||
"latency_p50_ms": 2481.8662,
|
||||
"latency_p95_ms": 3365.8395,
|
||||
"peak_rss_bytes": 2104832000,
|
||||
"peak_vram_bytes": 0,
|
||||
"prefill_tokens_per_sec": 247.5617,
|
||||
"ttft_p50_ms": 94.3995,
|
||||
"ttft_p95_ms": 444.6749
|
||||
"prefill_tokens_per_sec": 264.0101,
|
||||
"ttft_p50_ms": 97.0403,
|
||||
"ttft_p95_ms": 429.0665
|
||||
}
|
||||
},
|
||||
"device": "cpu",
|
||||
|
||||
@@ -4,6 +4,10 @@ LLAMA=/run/media/popov/d/DEV/llamacpp/llama.cpp
|
||||
ROCM_PY=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv-rocm/bin/python
|
||||
PROJECT_PY=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python
|
||||
OUT=/run/media/popov/DATA/llm/dgr-001
|
||||
SIGNING_KEY=/home/popov/.config/neuron-tai/keys/dgr-001-evidence-ed25519.pem
|
||||
|
||||
# Private signing key is outside Git and must remain owner-only
|
||||
stat -c '%a %n' "$SIGNING_KEY" # expected: 600
|
||||
|
||||
# Converter support check (no writes)
|
||||
$ROCM_PY $LLAMA/convert_hf_to_gguf.py "$SOURCE" --outtype f16 --outfile "$OUT/Qwen2.5-0.5B-Instruct-7ae5576-F16.gguf" --dry-run
|
||||
@@ -27,7 +31,8 @@ print(_artifact_sha256(Path('/run/media/popov/DATA/llm/safetensor/models/models-
|
||||
PY
|
||||
|
||||
# Canonical opt-in local-real benchmark
|
||||
MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 PYTHONPATH=packages/node $ROCM_PY -m meshnet_node.recipe_benchmark \
|
||||
MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 MESHNET_EVIDENCE_SIGNING_KEY="$SIGNING_KEY" \
|
||||
PYTHONPATH=packages/node $ROCM_PY -m meshnet_node.recipe_benchmark \
|
||||
--config .scratch/distributed-gguf-runtime/evidence/DGR-001/benchmark-config.json \
|
||||
--json-out .scratch/distributed-gguf-runtime/evidence/DGR-001/results.json \
|
||||
--summary-out .scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt
|
||||
|
||||
@@ -17,22 +17,22 @@
|
||||
"lane": "quality",
|
||||
"measurements": {
|
||||
"aggregate_concurrency": 4,
|
||||
"aggregate_throughput_speedup": 4.4882,
|
||||
"aggregate_throughput_speedup": 4.7873,
|
||||
"artifact_size_ratio": 0.9946,
|
||||
"artifact_size_win": false,
|
||||
"compared_prompts": 3,
|
||||
"decode_speedup": 1.9064,
|
||||
"decode_speedup": 2.4173,
|
||||
"exact_match_rate": 0.3333,
|
||||
"expected_prompts": 3,
|
||||
"failure_rate": 0.0,
|
||||
"mean_similarity": 0.9471,
|
||||
"resident_memory_ratio": 0.5722,
|
||||
"ttft_ratio": 1.1729
|
||||
"resident_memory_ratio": 0.5721,
|
||||
"ttft_ratio": 0.3767
|
||||
},
|
||||
"quality_pass": false,
|
||||
"reasons": [
|
||||
"single-request decode 1.91x reference (>= 1.25x) at TTFT ratio 1.17",
|
||||
"aggregate throughput at concurrency 4 is 4.49x reference (>= 1.25x)",
|
||||
"single-request decode 2.42x reference (>= 1.25x) at TTFT ratio 0.38",
|
||||
"aggregate throughput at concurrency 4 is 4.79x reference (>= 1.25x)",
|
||||
"peak resident memory is 0.57x reference (<= 0.75x)",
|
||||
"quality lane exact-match 0.33 / similarity 0.947 versus the reference (fail)"
|
||||
],
|
||||
@@ -47,18 +47,18 @@
|
||||
"lane": "performance-fit",
|
||||
"measurements": {
|
||||
"aggregate_concurrency": 4,
|
||||
"aggregate_throughput_speedup": 4.3826,
|
||||
"aggregate_throughput_speedup": 4.2048,
|
||||
"artifact_size_ratio": 0.398,
|
||||
"artifact_size_win": true,
|
||||
"decode_speedup": 3.6869,
|
||||
"decode_speedup": 5.0967,
|
||||
"failure_rate": 0.0,
|
||||
"resident_memory_ratio": 0.2793,
|
||||
"ttft_ratio": 0.6367
|
||||
"resident_memory_ratio": 0.2792,
|
||||
"ttft_ratio": 0.5395
|
||||
},
|
||||
"quality_pass": null,
|
||||
"reasons": [
|
||||
"single-request decode 3.69x reference (>= 1.25x) at TTFT ratio 0.64",
|
||||
"aggregate throughput at concurrency 4 is 4.38x reference (>= 1.25x)",
|
||||
"single-request decode 5.10x reference (>= 1.25x) at TTFT ratio 0.54",
|
||||
"aggregate throughput at concurrency 4 is 4.20x reference (>= 1.25x)",
|
||||
"peak resident memory is 0.28x reference (<= 0.75x)"
|
||||
],
|
||||
"recipe_id": "llama-cpp-quantized-performance-fit",
|
||||
|
||||
@@ -37,8 +37,51 @@
|
||||
"minimum_prompt_count": 3,
|
||||
"minimum_repeats": 3,
|
||||
"minimum_output_tokens": 32,
|
||||
"required_device": "cpu"
|
||||
"required_device": "cpu",
|
||||
"required_config_sha256": "00b2cce3e2f281bdf92fc5304ba5cac915a178ffccd3b9a25995ce39c00b90d3",
|
||||
"required_signer_public_key": "zQ/qRMwF/ydazzaxEI24Xvnrl5bZxzw16JYpP0bfRuI=",
|
||||
"required_artifact_sha256": {
|
||||
"transformers-safetensors-reference": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6",
|
||||
"llama-cpp-near-lossless-quality": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862",
|
||||
"llama-cpp-quantized-performance-fit": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5"
|
||||
},
|
||||
"required_recipe_runtime": {
|
||||
"transformers-safetensors-reference": {
|
||||
"runtime": "transformers-5.13.0",
|
||||
"weight_format": "safetensors",
|
||||
"weight_quantization": "bfloat16",
|
||||
"device": "cpu"
|
||||
},
|
||||
"llama-cpp-near-lossless-quality": {
|
||||
"runtime": "llama.cpp-9991-e920c523",
|
||||
"weight_format": "gguf",
|
||||
"weight_quantization": "bfloat16",
|
||||
"device": "cpu"
|
||||
},
|
||||
"llama-cpp-quantized-performance-fit": {
|
||||
"runtime": "llama.cpp-9991-e920c523",
|
||||
"weight_format": "gguf",
|
||||
"weight_quantization": "Q4_K_M",
|
||||
"device": "cpu"
|
||||
}
|
||||
},
|
||||
"required_backend_detail": {
|
||||
"transformers-safetensors-reference": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cpu; intra-op threads 16",
|
||||
"llama-cpp-near-lossless-quality": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0",
|
||||
"llama-cpp-quantized-performance-fit": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0"
|
||||
},
|
||||
"required_host_identity": {
|
||||
"python": "3.12.13",
|
||||
"torch_version": "2.10.0+rocm7.13.0a20260513",
|
||||
"transformers_version": "5.13.0",
|
||||
"llama_server_identities": {
|
||||
"/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server": {
|
||||
"sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
|
||||
"version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"stop_condition": "Stop the native llama.cpp/GGUF track when, on the same machine and device as the Transformers/safetensors reference and under this plan, no performance-fit GGUF recipe delivers either a meaningful speed benefit (>=25% higher single-request decode tokens/sec without a >25% worse TTFT, or >=25% higher aggregate throughput under concurrency) or a meaningful fit benefit (>=25% lower peak resident memory), or when the near-lossless quality lane fails, which indicates a broken runtime rather than a quantization trade-off.",
|
||||
"notes": "Quantized performance-fit output drift is reported as advisory only. It is not numerical-equivalence evidence. DGR-014 consumes this immutable v1 contract."
|
||||
"notes": "Quantized performance-fit output drift is reported as advisory only. It is not numerical-equivalence evidence. DGR-014 consumes this immutable v1 contract. Non-synthetic evidence must be Ed25519-signed by the pinned key and match the exact locked config, artifacts, runtimes, backends, and host runtime identity."
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,10 +1,10 @@
|
||||
Recipe benchmark dgr-001-controlled-whole-model-baseline-v1 (local-real)
|
||||
model Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf67be50417f59c2c2f167def9a775
|
||||
transformers-safetensors-reference [quality ] c= 1 ttft p50/p95 37.5/ 193.5 ms; prefill 671.8 tok/s; decode 46.1 tok/s; aggregate 40.3 tok/s; rss 1.94 GB; vram 0.00 GB; artifact 1.00 GB; failures 0
|
||||
transformers-safetensors-reference [quality ] c= 4 ttft p50/p95 94.4/ 444.7 ms; prefill 247.6 tok/s; decode 13.1 tok/s; aggregate 47.1 tok/s; rss 2.18 GB; vram 0.00 GB; artifact 1.00 GB; failures 0
|
||||
llama-cpp-near-lossless-quality [quality ] c= 1 ttft p50/p95 43.9/ 107.0 ms; prefill 1427.2 tok/s; decode 88.0 tok/s; aggregate 73.8 tok/s; rss 1.11 GB; vram 0.00 GB; artifact 0.99 GB; failures 0
|
||||
llama-cpp-near-lossless-quality [quality ] c= 4 ttft p50/p95 33.6/ 128.5 ms; prefill 1077.8 tok/s; decode 73.9 tok/s; aggregate 211.4 tok/s; rss 1.13 GB; vram 0.00 GB; artifact 0.99 GB; failures 0
|
||||
llama-cpp-quantized-performance-fit [performance-fit ] c= 1 ttft p50/p95 23.8/ 237.7 ms; prefill 783.4 tok/s; decode 170.1 tok/s; aggregate 110.0 tok/s; rss 0.54 GB; vram 0.00 GB; artifact 0.40 GB; failures 0
|
||||
llama-cpp-quantized-performance-fit [performance-fit ] c= 4 ttft p50/p95 67.9/ 431.8 ms; prefill 474.3 tok/s; decode 83.5 tok/s; aggregate 206.4 tok/s; rss 0.57 GB; vram 0.00 GB; artifact 0.40 GB; failures 0
|
||||
transformers-safetensors-reference [quality ] c= 1 ttft p50/p95 40.0/ 195.3 ms; prefill 625.6 tok/s; decode 40.8 tok/s; aggregate 35.5 tok/s; rss 1.94 GB; vram 0.00 GB; artifact 1.00 GB; failures 0
|
||||
transformers-safetensors-reference [quality ] c= 4 ttft p50/p95 97.0/ 429.1 ms; prefill 264.0 tok/s; decode 13.0 tok/s; aggregate 46.5 tok/s; rss 2.10 GB; vram 0.00 GB; artifact 1.00 GB; failures 0
|
||||
llama-cpp-near-lossless-quality [quality ] c= 1 ttft p50/p95 15.1/ 63.8 ms; prefill 1717.9 tok/s; decode 98.5 tok/s; aggregate 86.7 tok/s; rss 1.11 GB; vram 0.00 GB; artifact 0.99 GB; failures 0
|
||||
llama-cpp-near-lossless-quality [quality ] c= 4 ttft p50/p95 32.4/ 218.4 ms; prefill 859.9 tok/s; decode 76.6 tok/s; aggregate 222.8 tok/s; rss 1.14 GB; vram 0.00 GB; artifact 0.99 GB; failures 0
|
||||
llama-cpp-quantized-performance-fit [performance-fit ] c= 1 ttft p50/p95 21.6/ 147.9 ms; prefill 967.0 tok/s; decode 207.7 tok/s; aggregate 139.3 tok/s; rss 0.54 GB; vram 0.00 GB; artifact 0.40 GB; failures 0
|
||||
llama-cpp-quantized-performance-fit [performance-fit ] c= 4 ttft p50/p95 48.1/ 416.5 ms; prefill 572.4 tok/s; decode 76.9 tok/s; aggregate 195.7 tok/s; rss 0.57 GB; vram 0.00 GB; artifact 0.40 GB; failures 0
|
||||
drift llama-cpp-near-lossless-quality vs transformers-safetensors-reference exact 0.33; similarity 0.947 (gated)
|
||||
drift llama-cpp-quantized-performance-fit vs transformers-safetensors-reference exact 0.00; similarity 0.456 (advisory)
|
||||
|
||||
@@ -25,16 +25,23 @@ the release gate is allowed to reach.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
|
||||
from .recipe_benchmark import Lane, REPORT_SCHEMA_VERSION
|
||||
|
||||
# Layout of the contract document understood by this reader.
|
||||
CONTRACT_SCHEMA_VERSION = 1
|
||||
PROVENANCE_SCHEMA_VERSION = 1
|
||||
REAL_REPORT_PRODUCER = "meshnet_node.recipe_drivers.run_configured_benchmark/v1"
|
||||
|
||||
VERDICT_PROMOTE = "promote"
|
||||
VERDICT_OPTIMIZE = "optimize"
|
||||
@@ -146,6 +153,66 @@ def _canonical_sha256(value: Any) -> str:
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def report_signing_payload(report: Mapping[str, Any]) -> bytes:
|
||||
"""Canonical report bytes covered by the Ed25519 signature."""
|
||||
provenance = report.get("provenance")
|
||||
if not isinstance(provenance, Mapping):
|
||||
raise PerformanceContractError("real benchmark report lacks signed provenance")
|
||||
unsigned = dict(report)
|
||||
unsigned_provenance = dict(provenance)
|
||||
unsigned_provenance.pop("signature", None)
|
||||
unsigned["provenance"] = unsigned_provenance
|
||||
return json.dumps(
|
||||
unsigned, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _decode_base64(value: Any, label: str) -> bytes:
|
||||
if not isinstance(value, str):
|
||||
raise PerformanceContractError(f"report lacks {label}")
|
||||
try:
|
||||
return base64.b64decode(value, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise PerformanceContractError(f"report carries invalid {label}") from exc
|
||||
|
||||
|
||||
def _verify_real_provenance(
|
||||
contract: PerformanceContract, report: Mapping[str, Any]
|
||||
) -> None:
|
||||
provenance = report.get("provenance")
|
||||
if not isinstance(provenance, Mapping):
|
||||
raise PerformanceContractError("real benchmark report lacks signed provenance")
|
||||
if provenance.get("schema_version") != PROVENANCE_SCHEMA_VERSION:
|
||||
raise PerformanceContractError("report provenance schema is unsupported")
|
||||
if provenance.get("producer") != REAL_REPORT_PRODUCER:
|
||||
raise PerformanceContractError("report was not emitted by the canonical real runner")
|
||||
if provenance.get("signature_algorithm") != "ed25519":
|
||||
raise PerformanceContractError("report provenance is not Ed25519 signed")
|
||||
for field in ("run_id", "started_at", "completed_at"):
|
||||
if not provenance.get(field):
|
||||
raise PerformanceContractError(f"report provenance lacks {field}")
|
||||
|
||||
required_config = contract.baseline.get("required_config_sha256")
|
||||
if not required_config or provenance.get("config_sha256") != required_config:
|
||||
raise PerformanceContractError("report config digest does not match the locked config")
|
||||
|
||||
encoded_public_key = contract.baseline.get("required_signer_public_key")
|
||||
public_key_bytes = _decode_base64(encoded_public_key, "locked signer public key")
|
||||
if len(public_key_bytes) != 32:
|
||||
raise PerformanceContractError("locked Ed25519 public key must be 32 bytes")
|
||||
expected_fingerprint = hashlib.sha256(public_key_bytes).hexdigest()
|
||||
if provenance.get("signer_public_key_sha256") != expected_fingerprint:
|
||||
raise PerformanceContractError("report signer fingerprint does not match the contract")
|
||||
|
||||
signature = _decode_base64(provenance.get("signature"), "Ed25519 signature")
|
||||
try:
|
||||
Ed25519PublicKey.from_public_bytes(public_key_bytes).verify(
|
||||
signature, report_signing_payload(report)
|
||||
)
|
||||
except (InvalidSignature, ValueError) as exc:
|
||||
raise PerformanceContractError("report Ed25519 signature verification failed") from exc
|
||||
|
||||
|
||||
def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -> None:
|
||||
"""Fail closed when a report is not the experiment the contract locked."""
|
||||
try:
|
||||
@@ -187,6 +254,8 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -
|
||||
raise PerformanceContractError(
|
||||
f"report evidence class {evidence_class!r} does not satisfy {required_evidence!r}"
|
||||
)
|
||||
if evidence_class in {"local-real", "multi-machine-real"}:
|
||||
_verify_real_provenance(contract, report)
|
||||
if required_evidence and (
|
||||
not isinstance(host, Mapping)
|
||||
or any(key not in host for key in ("hostname", "platform", "python", "cpu_count"))
|
||||
@@ -221,8 +290,18 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -
|
||||
model_id = plan.get("model_id")
|
||||
model_revision = plan.get("model_revision")
|
||||
required_device = contract.baseline.get("required_device")
|
||||
required_artifacts = dict(contract.baseline.get("required_artifact_sha256") or {})
|
||||
required_runtimes = dict(contract.baseline.get("required_recipe_runtime") or {})
|
||||
required_backends = dict(contract.baseline.get("required_backend_detail") or {})
|
||||
required_host = dict(contract.baseline.get("required_host_identity") or {})
|
||||
for field, expected in required_host.items():
|
||||
if host.get(field) != expected:
|
||||
raise PerformanceContractError(
|
||||
f"report host/runtime field {field!r} does not match the locked identity"
|
||||
)
|
||||
for entry in recipes:
|
||||
recipe = entry.get("recipe", {})
|
||||
recipe_id = recipe.get("id")
|
||||
if required_device and recipe.get("device") != required_device:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe.get('id')!r} did not run on locked device {required_device!r}"
|
||||
@@ -235,6 +314,30 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -
|
||||
digest = recipe.get("artifact_sha256", "")
|
||||
if not isinstance(digest, str) or len(digest) != 64:
|
||||
raise PerformanceContractError("report lacks an artifact SHA-256 digest")
|
||||
expected_digest = required_artifacts.get(recipe_id)
|
||||
if not expected_digest or digest != expected_digest:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} artifact digest does not match the contract"
|
||||
)
|
||||
expected_runtime = required_runtimes.get(recipe_id)
|
||||
if not isinstance(expected_runtime, Mapping):
|
||||
raise PerformanceContractError(
|
||||
f"contract lacks runtime identity for recipe {recipe_id!r}"
|
||||
)
|
||||
actual_runtime = {
|
||||
field: recipe.get(field) for field in expected_runtime
|
||||
}
|
||||
if actual_runtime != dict(expected_runtime):
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} runtime identity does not match the contract"
|
||||
)
|
||||
if entry.get("available"):
|
||||
expected_backend = required_backends.get(recipe_id)
|
||||
actual_backend = entry.get("load", {}).get("backend_detail")
|
||||
if not expected_backend or actual_backend != expected_backend:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} backend identity does not match the contract"
|
||||
)
|
||||
if entry.get("available"):
|
||||
cells = entry.get("concurrency", {})
|
||||
missing_cells = required_levels - {int(level) for level in cells}
|
||||
@@ -666,6 +769,23 @@ def baseline_from_report(report: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"plan_sha256": _canonical_sha256(report["plan"]),
|
||||
"reference_recipe_id": report["reference_recipe_id"],
|
||||
"host": report["host"],
|
||||
"provenance": dict(report.get("provenance") or {}),
|
||||
"artifact_sha256": {
|
||||
recipe_id: entry["recipe"]["artifact_sha256"]
|
||||
for recipe_id, entry in entries.items()
|
||||
},
|
||||
"recipe_runtime": {
|
||||
recipe_id: {
|
||||
field: entry["recipe"].get(field)
|
||||
for field in ("runtime", "weight_format", "weight_quantization", "device")
|
||||
}
|
||||
for recipe_id, entry in entries.items()
|
||||
},
|
||||
"backend_detail": {
|
||||
recipe_id: entry.get("load", {}).get("backend_detail")
|
||||
for recipe_id, entry in entries.items()
|
||||
if entry.get("available")
|
||||
},
|
||||
"recipes": {},
|
||||
}
|
||||
for recipe_id, entry in entries.items():
|
||||
|
||||
@@ -35,7 +35,7 @@ from dataclasses import asdict, dataclass, field
|
||||
from difflib import SequenceMatcher
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol, Sequence
|
||||
from typing import Any, Mapping, Protocol, Sequence
|
||||
|
||||
# Layout of the report document produced by :func:`build_report`.
|
||||
REPORT_SCHEMA_VERSION = 1
|
||||
@@ -581,6 +581,7 @@ def build_report(
|
||||
*,
|
||||
host: dict[str, Any],
|
||||
evidence_class: str,
|
||||
provenance: Mapping[str, Any] | None = None,
|
||||
) -> dict:
|
||||
"""Assemble the machine-readable benchmark document.
|
||||
|
||||
@@ -590,6 +591,8 @@ def build_report(
|
||||
"""
|
||||
if evidence_class not in {"synthetic", "local-real", "multi-machine-real"}:
|
||||
raise BenchmarkError(f"unknown evidence class {evidence_class!r}")
|
||||
if evidence_class != "synthetic" and not isinstance(provenance, Mapping):
|
||||
raise BenchmarkError("non-synthetic reports require canonical signed provenance")
|
||||
|
||||
references = [m for m in measurements if m.recipe.is_reference]
|
||||
if len(references) != 1:
|
||||
@@ -605,7 +608,7 @@ def build_report(
|
||||
for measurement in measurements
|
||||
if measurement is not reference and measurement.available
|
||||
]
|
||||
return {
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA_VERSION,
|
||||
"evidence_class": evidence_class,
|
||||
"plan": plan.to_dict(),
|
||||
@@ -614,6 +617,9 @@ def build_report(
|
||||
"recipes": [measurement.to_dict() for measurement in measurements],
|
||||
"drift": drift,
|
||||
}
|
||||
if provenance is not None:
|
||||
report["provenance"] = dict(provenance)
|
||||
return report
|
||||
|
||||
|
||||
def format_summary(report: dict) -> str:
|
||||
|
||||
@@ -20,6 +20,7 @@ rules:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
@@ -27,15 +28,27 @@ import os
|
||||
import platform
|
||||
import re
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
from .performance_contract import (
|
||||
PROVENANCE_SCHEMA_VERSION,
|
||||
REAL_REPORT_PRODUCER,
|
||||
_canonical_sha256,
|
||||
report_signing_payload,
|
||||
)
|
||||
from .recipe_benchmark import (
|
||||
BenchmarkError,
|
||||
BenchmarkPlan,
|
||||
@@ -50,6 +63,7 @@ from .recipe_benchmark import (
|
||||
)
|
||||
|
||||
REAL_INFERENCE_ENV = "MESHNET_ENABLE_REAL_INFERENCE_TESTS"
|
||||
EVIDENCE_SIGNING_KEY_ENV = "MESHNET_EVIDENCE_SIGNING_KEY"
|
||||
|
||||
|
||||
def real_inference_enabled() -> bool:
|
||||
@@ -64,6 +78,37 @@ def require_real_inference() -> None:
|
||||
)
|
||||
|
||||
|
||||
def _load_evidence_signing_key() -> Ed25519PrivateKey:
|
||||
raw_path = os.environ.get(EVIDENCE_SIGNING_KEY_ENV)
|
||||
if not raw_path:
|
||||
raise BenchmarkError(
|
||||
f"real evidence requires {EVIDENCE_SIGNING_KEY_ENV} to name an Ed25519 private key"
|
||||
)
|
||||
path = Path(raw_path).expanduser().resolve(strict=True)
|
||||
if os.name != "nt" and stat.S_IMODE(path.stat().st_mode) != 0o600:
|
||||
raise BenchmarkError("evidence signing key must have mode 0600")
|
||||
key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||
if not isinstance(key, Ed25519PrivateKey):
|
||||
raise BenchmarkError("evidence signing key must be Ed25519")
|
||||
return key
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _sign_report(report: dict[str, Any], key: Ed25519PrivateKey) -> None:
|
||||
public_key = key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw, serialization.PublicFormat.Raw
|
||||
)
|
||||
report["provenance"]["signer_public_key_sha256"] = hashlib.sha256(
|
||||
public_key
|
||||
).hexdigest()
|
||||
report["provenance"]["signature"] = base64.b64encode(
|
||||
key.sign(report_signing_payload(report))
|
||||
).decode("ascii")
|
||||
|
||||
|
||||
def _process_rss(pid: int | None = None) -> int:
|
||||
"""Resident bytes for a process and its children, or 0 when unobservable."""
|
||||
try:
|
||||
@@ -120,7 +165,7 @@ def _artifact_sha256(path: Path) -> str:
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _host_manifest() -> dict[str, Any]:
|
||||
def _host_manifest(config: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Capture non-secret host facts with the report rather than trusting prose."""
|
||||
manifest: dict[str, Any] = {
|
||||
"hostname": socket.gethostname(),
|
||||
@@ -142,6 +187,30 @@ def _host_manifest() -> dict[str, Any]:
|
||||
)
|
||||
except ImportError:
|
||||
manifest["torch_version"] = None
|
||||
|
||||
llama_identities: dict[str, dict[str, str]] = {}
|
||||
for spec in (config or {}).get("recipes", ()):
|
||||
driver = spec.get("driver", {})
|
||||
if driver.get("type") != "llama-cpp-server":
|
||||
continue
|
||||
binary = Path(driver["binary"]).resolve(strict=True)
|
||||
key = str(binary)
|
||||
if key in llama_identities:
|
||||
continue
|
||||
version_result = subprocess.run(
|
||||
[str(binary), "--version"],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
llama_identities[key] = {
|
||||
"sha256": _artifact_sha256(binary),
|
||||
"version": " | ".join(version_result.stdout.strip().splitlines()),
|
||||
}
|
||||
if llama_identities:
|
||||
manifest["llama_server_identities"] = llama_identities
|
||||
return manifest
|
||||
|
||||
|
||||
@@ -635,6 +704,13 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
|
||||
"""
|
||||
require_real_inference()
|
||||
_validate_config(config)
|
||||
evidence_class = config.get("evidence_class", "local-real")
|
||||
if evidence_class not in {"local-real", "multi-machine-real"}:
|
||||
raise BenchmarkError("canonical real runner cannot emit synthetic evidence")
|
||||
signing_key = _load_evidence_signing_key()
|
||||
started_at = _utc_now()
|
||||
run_id = str(uuid.uuid4())
|
||||
config_sha256 = _canonical_sha256(config)
|
||||
plan = _plan_from_config(config)
|
||||
|
||||
from .recipe_benchmark import RecipeMeasurement # local import keeps the seam obvious
|
||||
@@ -656,9 +732,20 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
|
||||
if driver is not None:
|
||||
driver.close()
|
||||
|
||||
return build_report(
|
||||
report = build_report(
|
||||
plan,
|
||||
measurements,
|
||||
host={**dict(config.get("host", {})), **_host_manifest()},
|
||||
evidence_class=config.get("evidence_class", "local-real"),
|
||||
host={**dict(config.get("host", {})), **_host_manifest(config)},
|
||||
evidence_class=evidence_class,
|
||||
provenance={
|
||||
"schema_version": PROVENANCE_SCHEMA_VERSION,
|
||||
"producer": REAL_REPORT_PRODUCER,
|
||||
"run_id": run_id,
|
||||
"started_at": started_at,
|
||||
"completed_at": _utc_now(),
|
||||
"config_sha256": config_sha256,
|
||||
"signature_algorithm": "ed25519",
|
||||
},
|
||||
)
|
||||
_sign_report(report, signing_key)
|
||||
return report
|
||||
|
||||
@@ -8,12 +8,18 @@ report.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import copy
|
||||
import hashlib
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
from meshnet_node.performance_contract import (
|
||||
PROVENANCE_SCHEMA_VERSION,
|
||||
REAL_REPORT_PRODUCER,
|
||||
STOP_CONDITION,
|
||||
ContractThresholds,
|
||||
PerformanceContract,
|
||||
@@ -21,6 +27,7 @@ from meshnet_node.performance_contract import (
|
||||
_canonical_sha256,
|
||||
evaluate_contract,
|
||||
parse_contract,
|
||||
report_signing_payload,
|
||||
)
|
||||
from meshnet_node.recipe_drivers import (
|
||||
_artifact_sha256,
|
||||
@@ -437,6 +444,16 @@ def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path):
|
||||
_validate_config(config)
|
||||
|
||||
|
||||
_TEST_SIGNING_KEY = Ed25519PrivateKey.generate()
|
||||
|
||||
|
||||
def _resign_test_report(report: dict) -> None:
|
||||
report["provenance"].pop("signature", None)
|
||||
report["provenance"]["signature"] = base64.b64encode(
|
||||
_TEST_SIGNING_KEY.sign(report_signing_payload(report))
|
||||
).decode("ascii")
|
||||
|
||||
|
||||
def _lock_real_report(report: dict) -> PerformanceContract:
|
||||
report["evidence_class"] = "local-real"
|
||||
report["host"] = {
|
||||
@@ -449,7 +466,23 @@ def _lock_real_report(report: dict) -> PerformanceContract:
|
||||
entry["recipe"]["source_model_id"] = report["plan"]["model_id"]
|
||||
entry["recipe"]["source_model_revision"] = report["plan"]["model_revision"]
|
||||
entry["recipe"]["artifact_sha256"] = "a" * 64
|
||||
return PerformanceContract(
|
||||
entry["load"]["backend_detail"] = f"fake-backend-{entry['recipe']['id']}"
|
||||
|
||||
public_key = _TEST_SIGNING_KEY.public_key().public_bytes(
|
||||
serialization.Encoding.Raw, serialization.PublicFormat.Raw
|
||||
)
|
||||
config_sha256 = "c" * 64
|
||||
report["provenance"] = {
|
||||
"schema_version": PROVENANCE_SCHEMA_VERSION,
|
||||
"producer": REAL_REPORT_PRODUCER,
|
||||
"run_id": "test-run-id",
|
||||
"started_at": "2026-07-13T00:00:00Z",
|
||||
"completed_at": "2026-07-13T00:01:00Z",
|
||||
"config_sha256": config_sha256,
|
||||
"signature_algorithm": "ed25519",
|
||||
"signer_public_key_sha256": hashlib.sha256(public_key).hexdigest(),
|
||||
}
|
||||
contract = PerformanceContract(
|
||||
contract_version=1,
|
||||
locked_at="2026-07-13T00:00:00Z",
|
||||
locked_by="test",
|
||||
@@ -459,9 +492,29 @@ def _lock_real_report(report: dict) -> PerformanceContract:
|
||||
"required_evidence_class": "local-real",
|
||||
"required_recipes": [entry["recipe"]["id"] for entry in report["recipes"]],
|
||||
"required_concurrency_levels": [1, 4],
|
||||
"required_config_sha256": config_sha256,
|
||||
"required_signer_public_key": base64.b64encode(public_key).decode("ascii"),
|
||||
"required_artifact_sha256": {
|
||||
entry["recipe"]["id"]: entry["recipe"]["artifact_sha256"]
|
||||
for entry in report["recipes"]
|
||||
},
|
||||
"required_recipe_runtime": {
|
||||
entry["recipe"]["id"]: {
|
||||
field: entry["recipe"].get(field)
|
||||
for field in ("runtime", "weight_format", "weight_quantization", "device")
|
||||
}
|
||||
for entry in report["recipes"]
|
||||
},
|
||||
"required_backend_detail": {
|
||||
entry["recipe"]["id"]: entry["load"]["backend_detail"]
|
||||
for entry in report["recipes"]
|
||||
},
|
||||
"required_host_identity": {"python": "3.12"},
|
||||
},
|
||||
stop_condition="test",
|
||||
)
|
||||
_resign_test_report(report)
|
||||
return contract
|
||||
|
||||
|
||||
def test_locked_contract_rejects_synthetic_evidence():
|
||||
@@ -480,6 +533,67 @@ def test_locked_contract_rejects_synthetic_evidence():
|
||||
evaluate_contract(contract, report)
|
||||
|
||||
|
||||
def test_non_synthetic_report_requires_canonical_provenance():
|
||||
reference = measure_recipe(
|
||||
FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan()
|
||||
)
|
||||
with pytest.raises(BenchmarkError, match="signed provenance"):
|
||||
build_report(
|
||||
plan(), [reference], host={}, evidence_class="local-real"
|
||||
)
|
||||
|
||||
|
||||
def test_signed_real_report_rejects_tampering_and_rebinding():
|
||||
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
|
||||
measurements = [
|
||||
measure_recipe(FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()),
|
||||
measure_recipe(FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()),
|
||||
measure_recipe(FakeDriver(texts=texts), recipe("q4", Lane.PERFORMANCE_FIT), plan()),
|
||||
]
|
||||
report = build_report(plan(), measurements, host={}, evidence_class="synthetic")
|
||||
contract = _lock_real_report(report)
|
||||
assert evaluate_contract(contract, report).verdict in {"promote", "optimize", "stop"}
|
||||
|
||||
unsigned = copy.deepcopy(report)
|
||||
unsigned["provenance"].pop("signature")
|
||||
with pytest.raises(PerformanceContractError, match="signature"):
|
||||
evaluate_contract(contract, unsigned)
|
||||
|
||||
tampered = copy.deepcopy(report)
|
||||
tampered["recipes"][0]["recipe"]["artifact_sha256"] = "b" * 64
|
||||
with pytest.raises(PerformanceContractError, match="signature verification failed"):
|
||||
evaluate_contract(contract, tampered)
|
||||
|
||||
rebound_artifact = copy.deepcopy(tampered)
|
||||
_resign_test_report(rebound_artifact)
|
||||
with pytest.raises(PerformanceContractError, match="artifact digest"):
|
||||
evaluate_contract(contract, rebound_artifact)
|
||||
|
||||
rebound_config = copy.deepcopy(report)
|
||||
rebound_config["provenance"]["config_sha256"] = "d" * 64
|
||||
_resign_test_report(rebound_config)
|
||||
with pytest.raises(PerformanceContractError, match="config digest"):
|
||||
evaluate_contract(contract, rebound_config)
|
||||
|
||||
rebound_runtime = copy.deepcopy(report)
|
||||
rebound_runtime["recipes"][1]["recipe"]["runtime"] = "other-runtime"
|
||||
_resign_test_report(rebound_runtime)
|
||||
with pytest.raises(PerformanceContractError, match="runtime identity"):
|
||||
evaluate_contract(contract, rebound_runtime)
|
||||
|
||||
rebound_backend = copy.deepcopy(report)
|
||||
rebound_backend["recipes"][1]["load"]["backend_detail"] = "other-backend"
|
||||
_resign_test_report(rebound_backend)
|
||||
with pytest.raises(PerformanceContractError, match="backend identity"):
|
||||
evaluate_contract(contract, rebound_backend)
|
||||
|
||||
rebound_host = copy.deepcopy(report)
|
||||
rebound_host["host"]["python"] = "9.9"
|
||||
_resign_test_report(rebound_host)
|
||||
with pytest.raises(PerformanceContractError, match="host/runtime field"):
|
||||
evaluate_contract(contract, rebound_host)
|
||||
|
||||
|
||||
def test_quality_lane_requires_every_prompt_to_be_compared():
|
||||
texts = {prompt.text: "same greedy answer" for prompt in PROMPTS}
|
||||
reference = measure_recipe(
|
||||
@@ -503,6 +617,7 @@ def test_quality_lane_requires_every_prompt_to_be_compared():
|
||||
)
|
||||
quality_drift["compared_prompts"] = 1
|
||||
quality_drift["per_prompt"] = quality_drift["per_prompt"][:1]
|
||||
_resign_test_report(incomplete)
|
||||
|
||||
evaluation = evaluate_contract(contract, incomplete)
|
||||
|
||||
@@ -529,6 +644,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts():
|
||||
|
||||
changed_plan = copy.deepcopy(report)
|
||||
changed_plan["plan"]["prompts"][0]["text"] = "changed after locking"
|
||||
_resign_test_report(changed_plan)
|
||||
with pytest.raises(PerformanceContractError, match="plan digest"):
|
||||
evaluate_contract(contract, changed_plan)
|
||||
|
||||
@@ -537,6 +653,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts():
|
||||
entry for entry in changed_tokens["recipes"] if entry["recipe"]["id"] == "quality"
|
||||
)
|
||||
quality_entry["outcomes"][0]["decode_tokens"] += 1
|
||||
_resign_test_report(changed_tokens)
|
||||
with pytest.raises(PerformanceContractError, match="different prompt/decode token counts"):
|
||||
evaluate_contract(contract, changed_tokens)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user