From b1c9deeb013c94d5d4afb32bea825ff902d34cd5 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Mon, 13 Jul 2026 19:38:14 +0300 Subject: [PATCH] fix: cryptographically bind DGR-001 evidence --- .../evidence/DGR-001/README.md | 30 +- .../evidence/DGR-001/baseline.json | 143 +- .../evidence/DGR-001/commands.txt | 7 +- .../evidence/DGR-001/contract-evaluation.json | 24 +- .../DGR-001/performance-contract.json | 47 +- .../evidence/DGR-001/results.json | 1645 +++++++++-------- .../evidence/DGR-001/results.txt | 12 +- .../node/meshnet_node/performance_contract.py | 120 ++ .../node/meshnet_node/recipe_benchmark.py | 10 +- packages/node/meshnet_node/recipe_drivers.py | 95 +- tests/test_recipe_benchmark.py | 119 +- 11 files changed, 1349 insertions(+), 903 deletions(-) diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md index 8f2907c..8117974 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -45,17 +45,17 @@ All recipes completed every request with zero failures. | Metric | Transformers BF16 | llama.cpp BF16 | llama.cpp Q4_K_M | |---|---:|---:|---:| -| Decode tok/s, c=1 | 46.1 | 88.0 | 170.1 | -| Aggregate decode tok/s, c=4 | 47.1 | 211.4 | 206.4 | -| TTFT p50, c=1 | 37.5 ms | 43.9 ms | 23.8 ms | +| Decode tok/s, c=1 | 40.8 | 98.5 | 207.7 | +| Aggregate decode tok/s, c=4 | 46.5 | 222.8 | 195.7 | +| TTFT p50, c=1 | 40.0 ms | 15.1 ms | 21.6 ms | | Peak resident memory, c=1 | 1.94 GB | 1.11 GB | 0.54 GB | | Artifact size | 1.00 GB | 0.99 GB | 0.40 GB | | Failures | 0 | 0 | 0 | Against the reference, the eligible Q4_K_M lane measured: -- single-request decode speedup: **3.69×**; -- concurrency-4 aggregate throughput speedup: **4.38×**; +- single-request decode speedup: **5.10×**; +- concurrency-4 aggregate throughput speedup: **4.20×**; - resident-memory ratio: **0.279×**; - artifact-size ratio: **0.398×**. @@ -99,9 +99,13 @@ Thresholds were not changed after observing these results. - llama.cpp's `predicted_n` includes the first token while `predicted_ms` begins after it; the driver subtracts that token so decode throughput matches the Transformers inter-token convention. -- `performance_contract.py` rejects wrong plans, synthetic evidence, missing - recipes/concurrency, mixed model revisions, incomplete quality coverage, - failed references, and missing artifact/host provenance. +- `performance_contract.py` rejects wrong plans, unsigned or incorrectly signed + real evidence, wrong config/artifact/runtime/backend/host bindings, missing + recipes/concurrency, mixed model revisions, incomplete quality coverage, and + failed references. +- Every non-synthetic report is Ed25519-signed over the complete canonical JSON, + including raw outcomes and metrics. The contract pins the public key and exact + config SHA-256; the private key remains outside Git at mode `0600`. - Quantized drift remains advisory. Only the near-lossless lane can satisfy the quality gate, and only performance-fit recipes can earn speed/fit benefits. @@ -120,8 +124,8 @@ Thresholds were not changed after observing these results. ## Verification ```text -Targeted: 22 passed -Full suite: 749 passed, 13 skipped +Targeted: 24 passed +Full suite: 751 passed, 13 skipped Earlier cancellation retry matrix, DGR-001: 4/5 passed Earlier cancellation retry matrix, clean d904c40: 4/5 passed compileall: passed @@ -129,9 +133,9 @@ git diff --check: passed Evidence JSON parse/integrity checks: passed ``` -The full-suite exception is documented in `known-unrelated-failure.md` and -satisfies the issue's explicit clean-tree reproduction clause. DGR-001 changes -no tracker/proxy files. +An earlier intermittent tracker cancellation race reproduced at the same rate on +the clean base and is retained in `known-unrelated-failure.md`; the final suite +completed green. DGR-001 changes no tracker/proxy files. The earlier Ralph claim that the full suite was blocked by Protobuf 6.33.6 was invalid: it used Hermes Agent's internal venv. Verification above used the diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/baseline.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/baseline.json index 1a16ac1..43ae76b 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/baseline.json +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/baseline.json @@ -1,4 +1,14 @@ { + "artifact_sha256": { + "llama-cpp-near-lossless-quality": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862", + "llama-cpp-quantized-performance-fit": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5", + "transformers-safetensors-reference": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6" + }, + "backend_detail": { + "llama-cpp-near-lossless-quality": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0", + "llama-cpp-quantized-performance-fit": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0", + "transformers-safetensors-reference": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cpu; intra-op threads 16" + }, "evidence_class": "local-real", "host": { "accelerator_name": "Radeon 8060S Graphics", @@ -10,6 +20,12 @@ "hostname": "fedora", "llama_cpp_commit": "e920c523e3b8a0163fe498af5bf90df35ff51d25", "llama_cpp_version": "9991", + "llama_server_identities": { + "/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server": { + "sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd", + "version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64" + } + }, "llama_server_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd", "platform": "Linux-7.0.14-101.fc43.x86_64-x86_64-with-glibc2.42", "python": "3.12.13", @@ -20,34 +36,65 @@ "model_id": "Qwen/Qwen2.5-0.5B-Instruct", "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", "plan_sha256": "efe24690a9a7164bac6ab3fd0a6b22f078fc08aaefcfb96210ddf154e6050570", + "provenance": { + "completed_at": "2026-07-13T16:27:19.647692Z", + "config_sha256": "00b2cce3e2f281bdf92fc5304ba5cac915a178ffccd3b9a25995ce39c00b90d3", + "producer": "meshnet_node.recipe_drivers.run_configured_benchmark/v1", + "run_id": "e4eedadf-22f6-4907-8990-985456961099", + "schema_version": 1, + "signature": "owev+/ToswP20C923G6E+srOCUBV5vrjmndVatr9CbTXakiFGqlHrTiEo+aymA4BcSwmG6KJTxlxO6WpLnpcAg==", + "signature_algorithm": "ed25519", + "signer_public_key_sha256": "8baca8742d9b3ed0c3fc54929c23f75ec8c1c739900aaf5334780d598ffa84de", + "started_at": "2026-07-13T16:26:22.361501Z" + }, + "recipe_runtime": { + "llama-cpp-near-lossless-quality": { + "device": "cpu", + "runtime": "llama.cpp-9991-e920c523", + "weight_format": "gguf", + "weight_quantization": "bfloat16" + }, + "llama-cpp-quantized-performance-fit": { + "device": "cpu", + "runtime": "llama.cpp-9991-e920c523", + "weight_format": "gguf", + "weight_quantization": "Q4_K_M" + }, + "transformers-safetensors-reference": { + "device": "cpu", + "runtime": "transformers-5.13.0", + "weight_format": "safetensors", + "weight_quantization": "bfloat16" + } + }, "recipes": { "llama-cpp-near-lossless-quality": { "artifact_bytes": 994156448, "available": true, "concurrency": { "1": { - "aggregate_decode_tokens_per_sec": 73.7861, - "decode_tokens_per_sec": 87.9728, + "aggregate_decode_tokens_per_sec": 86.7339, + "decode_tokens_per_sec": 98.5178, "failures": 0, - "latency_p50_ms": 385.2049, - "latency_p95_ms": 560.2939, - "peak_rss_bytes": 1110708224, + "latency_p50_ms": 333.023, + "latency_p95_ms": 383.0597, + "peak_rss_bytes": 1110728704, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 1427.2072, - "ttft_p50_ms": 43.929, - "ttft_p95_ms": 107.003 + "prefill_tokens_per_sec": 1717.9451, + "ttft_p50_ms": 15.069, + "ttft_p95_ms": 63.766 }, "4": { - "aggregate_decode_tokens_per_sec": 211.3515, - "decode_tokens_per_sec": 73.8932, + "aggregate_decode_tokens_per_sec": 222.788, + "decode_tokens_per_sec": 76.6297, "failures": 0, - "latency_p50_ms": 467.5094, - "latency_p95_ms": 790.862, - "peak_rss_bytes": 1129578496, + "latency_p50_ms": 490.8738, + "latency_p95_ms": 646.26, + "peak_rss_bytes": 1139466240, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 1077.8162, - "ttft_p50_ms": 33.612, - "ttft_p95_ms": 128.501 + "prefill_tokens_per_sec": 859.8985, + "ttft_p50_ms": 32.445, + "ttft_p95_ms": 218.387 } }, "device": "cpu", @@ -58,28 +105,28 @@ "available": true, "concurrency": { "1": { - "aggregate_decode_tokens_per_sec": 110.0458, - "decode_tokens_per_sec": 170.131, + "aggregate_decode_tokens_per_sec": 139.2693, + "decode_tokens_per_sec": 207.712, "failures": 0, - "latency_p50_ms": 258.0681, - "latency_p95_ms": 465.8523, - "peak_rss_bytes": 542167040, + "latency_p50_ms": 168.3307, + "latency_p95_ms": 305.1338, + "peak_rss_bytes": 542081024, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 783.3775, - "ttft_p50_ms": 23.847, - "ttft_p95_ms": 237.696 + "prefill_tokens_per_sec": 967.0195, + "ttft_p50_ms": 21.582, + "ttft_p95_ms": 147.859 }, "4": { - "aggregate_decode_tokens_per_sec": 206.377, - "decode_tokens_per_sec": 83.543, + "aggregate_decode_tokens_per_sec": 195.6789, + "decode_tokens_per_sec": 76.9497, "failures": 0, - "latency_p50_ms": 413.3897, - "latency_p95_ms": 910.253, - "peak_rss_bytes": 572788736, + "latency_p50_ms": 437.9196, + "latency_p95_ms": 885.5355, + "peak_rss_bytes": 573259776, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 474.3116, - "ttft_p50_ms": 67.945, - "ttft_p95_ms": 431.804 + "prefill_tokens_per_sec": 572.4424, + "ttft_p50_ms": 48.127, + "ttft_p95_ms": 416.531 } }, "device": "cpu", @@ -90,28 +137,28 @@ "available": true, "concurrency": { "1": { - "aggregate_decode_tokens_per_sec": 40.3425, - "decode_tokens_per_sec": 46.1451, + "aggregate_decode_tokens_per_sec": 35.4722, + "decode_tokens_per_sec": 40.7545, "failures": 0, - "latency_p50_ms": 795.4807, - "latency_p95_ms": 930.9725, - "peak_rss_bytes": 1941213184, + "latency_p50_ms": 818.3864, + "latency_p95_ms": 1258.0673, + "peak_rss_bytes": 1941458944, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 671.8016, - "ttft_p50_ms": 37.4548, - "ttft_p95_ms": 193.4633 + "prefill_tokens_per_sec": 625.6467, + "ttft_p50_ms": 40.0018, + "ttft_p95_ms": 195.2551 }, "4": { - "aggregate_decode_tokens_per_sec": 47.0903, - "decode_tokens_per_sec": 13.1337, + "aggregate_decode_tokens_per_sec": 46.5375, + "decode_tokens_per_sec": 12.9506, "failures": 0, - "latency_p50_ms": 2631.0031, - "latency_p95_ms": 3073.7389, - "peak_rss_bytes": 2177265664, + "latency_p50_ms": 2481.8662, + "latency_p95_ms": 3365.8395, + "peak_rss_bytes": 2104832000, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 247.5617, - "ttft_p50_ms": 94.3995, - "ttft_p95_ms": 444.6749 + "prefill_tokens_per_sec": 264.0101, + "ttft_p50_ms": 97.0403, + "ttft_p95_ms": 429.0665 } }, "device": "cpu", diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt index d1020eb..0775d2a 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt @@ -4,6 +4,10 @@ LLAMA=/run/media/popov/d/DEV/llamacpp/llama.cpp ROCM_PY=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv-rocm/bin/python PROJECT_PY=/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python OUT=/run/media/popov/DATA/llm/dgr-001 +SIGNING_KEY=/home/popov/.config/neuron-tai/keys/dgr-001-evidence-ed25519.pem + +# Private signing key is outside Git and must remain owner-only +stat -c '%a %n' "$SIGNING_KEY" # expected: 600 # Converter support check (no writes) $ROCM_PY $LLAMA/convert_hf_to_gguf.py "$SOURCE" --outtype f16 --outfile "$OUT/Qwen2.5-0.5B-Instruct-7ae5576-F16.gguf" --dry-run @@ -27,7 +31,8 @@ print(_artifact_sha256(Path('/run/media/popov/DATA/llm/safetensor/models/models- PY # Canonical opt-in local-real benchmark -MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 PYTHONPATH=packages/node $ROCM_PY -m meshnet_node.recipe_benchmark \ +MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 MESHNET_EVIDENCE_SIGNING_KEY="$SIGNING_KEY" \ + PYTHONPATH=packages/node $ROCM_PY -m meshnet_node.recipe_benchmark \ --config .scratch/distributed-gguf-runtime/evidence/DGR-001/benchmark-config.json \ --json-out .scratch/distributed-gguf-runtime/evidence/DGR-001/results.json \ --summary-out .scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/contract-evaluation.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/contract-evaluation.json index 3b1ac24..2e02801 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/contract-evaluation.json +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/contract-evaluation.json @@ -17,22 +17,22 @@ "lane": "quality", "measurements": { "aggregate_concurrency": 4, - "aggregate_throughput_speedup": 4.4882, + "aggregate_throughput_speedup": 4.7873, "artifact_size_ratio": 0.9946, "artifact_size_win": false, "compared_prompts": 3, - "decode_speedup": 1.9064, + "decode_speedup": 2.4173, "exact_match_rate": 0.3333, "expected_prompts": 3, "failure_rate": 0.0, "mean_similarity": 0.9471, - "resident_memory_ratio": 0.5722, - "ttft_ratio": 1.1729 + "resident_memory_ratio": 0.5721, + "ttft_ratio": 0.3767 }, "quality_pass": false, "reasons": [ - "single-request decode 1.91x reference (>= 1.25x) at TTFT ratio 1.17", - "aggregate throughput at concurrency 4 is 4.49x reference (>= 1.25x)", + "single-request decode 2.42x reference (>= 1.25x) at TTFT ratio 0.38", + "aggregate throughput at concurrency 4 is 4.79x reference (>= 1.25x)", "peak resident memory is 0.57x reference (<= 0.75x)", "quality lane exact-match 0.33 / similarity 0.947 versus the reference (fail)" ], @@ -47,18 +47,18 @@ "lane": "performance-fit", "measurements": { "aggregate_concurrency": 4, - "aggregate_throughput_speedup": 4.3826, + "aggregate_throughput_speedup": 4.2048, "artifact_size_ratio": 0.398, "artifact_size_win": true, - "decode_speedup": 3.6869, + "decode_speedup": 5.0967, "failure_rate": 0.0, - "resident_memory_ratio": 0.2793, - "ttft_ratio": 0.6367 + "resident_memory_ratio": 0.2792, + "ttft_ratio": 0.5395 }, "quality_pass": null, "reasons": [ - "single-request decode 3.69x reference (>= 1.25x) at TTFT ratio 0.64", - "aggregate throughput at concurrency 4 is 4.38x reference (>= 1.25x)", + "single-request decode 5.10x reference (>= 1.25x) at TTFT ratio 0.54", + "aggregate throughput at concurrency 4 is 4.20x reference (>= 1.25x)", "peak resident memory is 0.28x reference (<= 0.75x)" ], "recipe_id": "llama-cpp-quantized-performance-fit", diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json index 7fdd79c..f6b6440 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json @@ -37,8 +37,51 @@ "minimum_prompt_count": 3, "minimum_repeats": 3, "minimum_output_tokens": 32, - "required_device": "cpu" + "required_device": "cpu", + "required_config_sha256": "00b2cce3e2f281bdf92fc5304ba5cac915a178ffccd3b9a25995ce39c00b90d3", + "required_signer_public_key": "zQ/qRMwF/ydazzaxEI24Xvnrl5bZxzw16JYpP0bfRuI=", + "required_artifact_sha256": { + "transformers-safetensors-reference": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6", + "llama-cpp-near-lossless-quality": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862", + "llama-cpp-quantized-performance-fit": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5" + }, + "required_recipe_runtime": { + "transformers-safetensors-reference": { + "runtime": "transformers-5.13.0", + "weight_format": "safetensors", + "weight_quantization": "bfloat16", + "device": "cpu" + }, + "llama-cpp-near-lossless-quality": { + "runtime": "llama.cpp-9991-e920c523", + "weight_format": "gguf", + "weight_quantization": "bfloat16", + "device": "cpu" + }, + "llama-cpp-quantized-performance-fit": { + "runtime": "llama.cpp-9991-e920c523", + "weight_format": "gguf", + "weight_quantization": "Q4_K_M", + "device": "cpu" + } + }, + "required_backend_detail": { + "transformers-safetensors-reference": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cpu; intra-op threads 16", + "llama-cpp-near-lossless-quality": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0", + "llama-cpp-quantized-performance-fit": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0" + }, + "required_host_identity": { + "python": "3.12.13", + "torch_version": "2.10.0+rocm7.13.0a20260513", + "transformers_version": "5.13.0", + "llama_server_identities": { + "/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server": { + "sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd", + "version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64" + } + } + } }, "stop_condition": "Stop the native llama.cpp/GGUF track when, on the same machine and device as the Transformers/safetensors reference and under this plan, no performance-fit GGUF recipe delivers either a meaningful speed benefit (>=25% higher single-request decode tokens/sec without a >25% worse TTFT, or >=25% higher aggregate throughput under concurrency) or a meaningful fit benefit (>=25% lower peak resident memory), or when the near-lossless quality lane fails, which indicates a broken runtime rather than a quantization trade-off.", - "notes": "Quantized performance-fit output drift is reported as advisory only. It is not numerical-equivalence evidence. DGR-014 consumes this immutable v1 contract." + "notes": "Quantized performance-fit output drift is reported as advisory only. It is not numerical-equivalence evidence. DGR-014 consumes this immutable v1 contract. Non-synthetic evidence must be Ed25519-signed by the pinned key and match the exact locked config, artifacts, runtimes, backends, and host runtime identity." } diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json index d2f355d..0f03695 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json @@ -82,6 +82,12 @@ "hostname": "fedora", "llama_cpp_commit": "e920c523e3b8a0163fe498af5bf90df35ff51d25", "llama_cpp_version": "9991", + "llama_server_identities": { + "/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server": { + "sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd", + "version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64" + } + }, "llama_server_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd", "platform": "Linux-7.0.14-101.fc43.x86_64-x86_64-with-glibc2.42", "python": "3.12.13", @@ -124,770 +130,781 @@ }, "warmup_requests": 2 }, + "provenance": { + "completed_at": "2026-07-13T16:27:19.647692Z", + "config_sha256": "00b2cce3e2f281bdf92fc5304ba5cac915a178ffccd3b9a25995ce39c00b90d3", + "producer": "meshnet_node.recipe_drivers.run_configured_benchmark/v1", + "run_id": "e4eedadf-22f6-4907-8990-985456961099", + "schema_version": 1, + "signature": "owev+/ToswP20C923G6E+srOCUBV5vrjmndVatr9CbTXakiFGqlHrTiEo+aymA4BcSwmG6KJTxlxO6WpLnpcAg==", + "signature_algorithm": "ed25519", + "signer_public_key_sha256": "8baca8742d9b3ed0c3fc54929c23f75ec8c1c739900aaf5334780d598ffa84de", + "started_at": "2026-07-13T16:26:22.361501Z" + }, "recipes": [ { "available": true, "concurrency": { "1": { - "aggregate_decode_tokens_per_sec": 40.3425, + "aggregate_decode_tokens_per_sec": 35.4722, "concurrency": 1, - "decode_tokens_per_sec": 46.1451, + "decode_tokens_per_sec": 40.7545, "failure_reasons": [], "failures": 0, - "latency_p50_ms": 795.4807, - "latency_p95_ms": 930.9725, - "peak_rss_bytes": 1941213184, + "latency_p50_ms": 818.3864, + "latency_p95_ms": 1258.0673, + "peak_rss_bytes": 1941458944, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 671.8016, + "prefill_tokens_per_sec": 625.6467, "requests": 9, - "ttft_p50_ms": 37.4548, - "ttft_p95_ms": 193.4633, - "wall_ms": 6915.7847 + "ttft_p50_ms": 40.0018, + "ttft_p95_ms": 195.2551, + "wall_ms": 7865.3174 }, "4": { - "aggregate_decode_tokens_per_sec": 47.0903, + "aggregate_decode_tokens_per_sec": 46.5375, "concurrency": 4, - "decode_tokens_per_sec": 13.1337, + "decode_tokens_per_sec": 12.9506, "failure_reasons": [], "failures": 0, - "latency_p50_ms": 2631.0031, - "latency_p95_ms": 3073.7389, - "peak_rss_bytes": 2177265664, + "latency_p50_ms": 2481.8662, + "latency_p95_ms": 3365.8395, + "peak_rss_bytes": 2104832000, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 247.5617, + "prefill_tokens_per_sec": 264.0101, "requests": 36, - "ttft_p50_ms": 94.3995, - "ttft_p95_ms": 444.6749, - "wall_ms": 23699.1528 + "ttft_p50_ms": 97.0403, + "ttft_p95_ms": 429.0665, + "wall_ms": 23980.6838 } }, "load": { "artifact_bytes": 999586347, "backend_detail": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cpu; intra-op threads 16", - "load_ms": 971.9102, - "rss_bytes": 877498368, + "load_ms": 979.2172, + "rss_bytes": 877441024, "vram_bytes": 0 }, "outcomes": [ { "concurrency": 1, - "decode_ms": 766.5448, + "decode_ms": 789.1253, "decode_tokens": 31, "error": "", - "latency_ms": 795.4807, + "latency_ms": 818.3864, "ok": true, - "prefill_ms": 28.9063, + "prefill_ms": 29.2306, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 28.9063 + "ttft_ms": 29.2306 }, { "concurrency": 1, - "decode_ms": 652.3796, + "decode_ms": 783.4, "decode_tokens": 31, "error": "", - "latency_ms": 692.2214, + "latency_ms": 831.0575, "ok": true, - "prefill_ms": 39.8146, + "prefill_ms": 47.6305, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 39.8146 + "ttft_ms": 47.6305 }, { "concurrency": 1, - "decode_ms": 737.481, + "decode_ms": 729.0472, "decode_tokens": 31, "error": "", - "latency_ms": 930.9725, + "latency_ms": 924.3293, "ok": true, - "prefill_ms": 193.4633, + "prefill_ms": 195.2551, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 193.4633 + "ttft_ms": 195.2551 }, { "concurrency": 1, - "decode_ms": 766.7118, + "decode_ms": 724.2915, "decode_tokens": 31, "error": "", - "latency_ms": 796.5856, + "latency_ms": 753.4753, "ok": true, - "prefill_ms": 29.8464, + "prefill_ms": 29.1565, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 29.8464 + "ttft_ms": 29.1565 }, { "concurrency": 1, - "decode_ms": 666.169, + "decode_ms": 711.203, "decode_tokens": 31, "error": "", - "latency_ms": 703.6503, + "latency_ms": 745.9763, "ok": true, - "prefill_ms": 37.4548, + "prefill_ms": 34.747, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 37.4548 + "ttft_ms": 34.747 }, { "concurrency": 1, - "decode_ms": 657.7801, + "decode_ms": 776.5576, "decode_tokens": 31, "error": "", - "latency_ms": 831.2304, + "latency_ms": 960.6125, "ok": true, - "prefill_ms": 173.4228, + "prefill_ms": 184.0279, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 173.4228 + "ttft_ms": 184.0279 }, { "concurrency": 1, - "decode_ms": 596.4379, + "decode_ms": 642.2951, "decode_tokens": 31, "error": "", - "latency_ms": 624.6906, + "latency_ms": 668.9401, "ok": true, - "prefill_ms": 28.2262, + "prefill_ms": 26.6176, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 28.2262 + "ttft_ms": 26.6176 }, { "concurrency": 1, - "decode_ms": 597.4485, + "decode_ms": 722.4096, "decode_tokens": 31, "error": "", - "latency_ms": 624.9351, + "latency_ms": 762.4393, "ok": true, - "prefill_ms": 27.4631, + "prefill_ms": 40.0018, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 27.4631 + "ttft_ms": 40.0018 }, { "concurrency": 1, - "decode_ms": 654.6493, + "decode_ms": 1109.9513, "decode_tokens": 31, "error": "", - "latency_ms": 799.0997, + "latency_ms": 1258.0673, "ok": true, - "prefill_ms": 144.4247, + "prefill_ms": 148.0872, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 144.4247 + "ttft_ms": 148.0872 }, { "concurrency": 4, - "decode_ms": 2870.971, + "decode_ms": 2337.8807, "decode_tokens": 31, "error": "", - "latency_ms": 2938.7696, + "latency_ms": 2500.639, "ok": true, - "prefill_ms": 67.7608, + "prefill_ms": 162.7268, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 67.7608 + "ttft_ms": 162.7268 }, { "concurrency": 4, - "decode_ms": 2878.6509, + "decode_ms": 2303.5448, "decode_tokens": 31, "error": "", - "latency_ms": 2951.2079, + "latency_ms": 2460.0828, "ok": true, - "prefill_ms": 72.5332, + "prefill_ms": 156.4924, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 72.5332 + "ttft_ms": 156.4924 }, { "concurrency": 4, - "decode_ms": 2876.61, + "decode_ms": 2316.9477, "decode_tokens": 31, "error": "", - "latency_ms": 2945.3565, + "latency_ms": 2481.8662, "ok": true, - "prefill_ms": 68.6974, + "prefill_ms": 164.89, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 68.6974 + "ttft_ms": 164.89 }, { "concurrency": 4, - "decode_ms": 2861.1328, + "decode_ms": 2306.1901, "decode_tokens": 31, "error": "", - "latency_ms": 2929.2516, + "latency_ms": 2482.8428, "ok": true, - "prefill_ms": 68.0086, + "prefill_ms": 176.6204, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 68.0086 + "ttft_ms": 176.6204 }, { "concurrency": 4, - "decode_ms": 2283.0476, + "decode_ms": 2294.0499, "decode_tokens": 31, "error": "", - "latency_ms": 2360.6441, + "latency_ms": 2367.6722, "ok": true, - "prefill_ms": 77.5755, + "prefill_ms": 73.5665, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 77.5755 + "ttft_ms": 73.5665 }, { "concurrency": 4, - "decode_ms": 2250.2644, + "decode_ms": 2310.4442, "decode_tokens": 31, "error": "", - "latency_ms": 2345.5795, + "latency_ms": 2407.1392, "ok": true, - "prefill_ms": 95.2694, + "prefill_ms": 96.6606, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 95.2694 + "ttft_ms": 96.6606 }, { "concurrency": 4, - "decode_ms": 2240.7462, + "decode_ms": 2311.9951, "decode_tokens": 31, "error": "", - "latency_ms": 2332.8119, + "latency_ms": 2401.913, "ok": true, - "prefill_ms": 92.0319, + "prefill_ms": 89.876, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 92.0319 + "ttft_ms": 89.876 }, { "concurrency": 4, - "decode_ms": 2259.3903, + "decode_ms": 2321.37, "decode_tokens": 31, "error": "", - "latency_ms": 2358.6972, + "latency_ms": 2424.403, "ok": true, - "prefill_ms": 99.2793, + "prefill_ms": 103.0054, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 99.2793 + "ttft_ms": 103.0054 }, { "concurrency": 4, - "decode_ms": 2350.3356, + "decode_ms": 2322.0048, "decode_tokens": 31, "error": "", - "latency_ms": 2756.1862, + "latency_ms": 2691.1307, "ok": true, - "prefill_ms": 405.831, + "prefill_ms": 369.0982, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 405.831 + "ttft_ms": 369.0982 }, { "concurrency": 4, - "decode_ms": 2296.8995, + "decode_ms": 2205.5395, "decode_tokens": 31, "error": "", - "latency_ms": 2729.1847, + "latency_ms": 2634.641, "ok": true, - "prefill_ms": 432.2451, + "prefill_ms": 429.0665, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 432.2451 + "ttft_ms": 429.0665 }, { "concurrency": 4, - "decode_ms": 2282.5935, + "decode_ms": 2196.7617, "decode_tokens": 31, "error": "", - "latency_ms": 2725.1834, + "latency_ms": 2615.7086, "ok": true, - "prefill_ms": 442.553, + "prefill_ms": 418.917, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 442.553 + "ttft_ms": 418.917 }, { "concurrency": 4, - "decode_ms": 2313.7306, + "decode_ms": 2247.9121, "decode_tokens": 31, "error": "", - "latency_ms": 2750.0004, + "latency_ms": 2680.2786, "ok": true, - "prefill_ms": 436.2347, + "prefill_ms": 432.3199, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 436.2347 + "ttft_ms": 432.3199 }, { "concurrency": 4, - "decode_ms": 2235.2735, + "decode_ms": 2203.407, "decode_tokens": 31, "error": "", - "latency_ms": 2303.1697, + "latency_ms": 2267.7641, "ok": true, - "prefill_ms": 67.8745, + "prefill_ms": 64.3272, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 67.8745 + "ttft_ms": 64.3272 }, { "concurrency": 4, - "decode_ms": 2197.0096, + "decode_ms": 2184.3202, "decode_tokens": 31, "error": "", - "latency_ms": 2265.9332, + "latency_ms": 2248.9142, "ok": true, - "prefill_ms": 68.8497, + "prefill_ms": 64.5731, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 68.8497 + "ttft_ms": 64.5731 }, { "concurrency": 4, - "decode_ms": 2214.0624, + "decode_ms": 2187.0349, "decode_tokens": 31, "error": "", - "latency_ms": 2291.0791, + "latency_ms": 2257.8805, "ok": true, - "prefill_ms": 76.9785, + "prefill_ms": 70.8191, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 76.9785 + "ttft_ms": 70.8191 }, { "concurrency": 4, - "decode_ms": 2181.2955, + "decode_ms": 2197.6307, "decode_tokens": 31, "error": "", - "latency_ms": 2254.5162, + "latency_ms": 2268.1916, "ok": true, - "prefill_ms": 73.1872, + "prefill_ms": 70.5298, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 73.1872 + "ttft_ms": 70.5298 }, { "concurrency": 4, - "decode_ms": 2925.6869, + "decode_ms": 2219.3714, "decode_tokens": 31, "error": "", - "latency_ms": 3063.8863, + "latency_ms": 2299.2901, "ok": true, - "prefill_ms": 138.1565, + "prefill_ms": 79.8752, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 138.1565 + "ttft_ms": 79.8752 }, { "concurrency": 4, - "decode_ms": 2902.3747, + "decode_ms": 2194.9725, "decode_tokens": 31, "error": "", - "latency_ms": 3084.7823, + "latency_ms": 2283.2617, "ok": true, - "prefill_ms": 182.3376, + "prefill_ms": 88.2587, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 182.3376 + "ttft_ms": 88.2587 }, { "concurrency": 4, - "decode_ms": 2875.2204, + "decode_ms": 2205.6012, "decode_tokens": 31, "error": "", - "latency_ms": 3073.7389, + "latency_ms": 2294.631, "ok": true, - "prefill_ms": 198.4802, + "prefill_ms": 89.0024, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 198.4802 + "ttft_ms": 89.0024 }, { "concurrency": 4, - "decode_ms": 2840.4175, + "decode_ms": 2299.5909, "decode_tokens": 31, "error": "", - "latency_ms": 3041.2979, + "latency_ms": 2396.7511, "ok": true, - "prefill_ms": 200.8536, + "prefill_ms": 97.0403, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 200.8536 + "ttft_ms": 97.0403 }, { "concurrency": 4, - "decode_ms": 2324.5449, + "decode_ms": 2269.6039, "decode_tokens": 31, "error": "", - "latency_ms": 2736.0303, + "latency_ms": 2665.8279, "ok": true, - "prefill_ms": 411.4573, + "prefill_ms": 396.1888, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 411.4573 + "ttft_ms": 396.1888 }, { "concurrency": 4, - "decode_ms": 2316.7447, + "decode_ms": 2223.5483, "decode_tokens": 31, "error": "", - "latency_ms": 2735.5942, + "latency_ms": 2639.5593, "ok": true, - "prefill_ms": 418.8192, + "prefill_ms": 415.9796, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 418.8192 + "ttft_ms": 415.9796 }, { "concurrency": 4, - "decode_ms": 2282.3126, + "decode_ms": 2230.8398, "decode_tokens": 31, "error": "", - "latency_ms": 2727.0413, + "latency_ms": 2657.5963, "ok": true, - "prefill_ms": 444.6749, + "prefill_ms": 426.6935, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 444.6749 + "ttft_ms": 426.6935 }, { "concurrency": 4, - "decode_ms": 2291.1708, + "decode_ms": 2221.9724, "decode_tokens": 31, "error": "", - "latency_ms": 2743.3143, + "latency_ms": 2608.1412, "ok": true, - "prefill_ms": 452.1187, + "prefill_ms": 386.1249, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 452.1187 + "ttft_ms": 386.1249 }, { "concurrency": 4, - "decode_ms": 2232.3363, + "decode_ms": 3091.2788, "decode_tokens": 31, "error": "", - "latency_ms": 2295.6625, + "latency_ms": 3159.6612, "ok": true, - "prefill_ms": 63.2855, + "prefill_ms": 68.3579, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 63.2855 + "ttft_ms": 68.3579 }, { "concurrency": 4, - "decode_ms": 2269.3066, + "decode_ms": 3053.4716, "decode_tokens": 31, "error": "", - "latency_ms": 2337.6075, + "latency_ms": 3128.5477, "ok": true, - "prefill_ms": 68.2744, + "prefill_ms": 75.0442, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 68.2744 + "ttft_ms": 75.0442 }, { "concurrency": 4, - "decode_ms": 2219.7869, + "decode_ms": 3080.1085, "decode_tokens": 31, "error": "", - "latency_ms": 2292.6208, + "latency_ms": 3154.079, "ok": true, - "prefill_ms": 72.7952, + "prefill_ms": 73.9445, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 72.7952 + "ttft_ms": 73.9445 }, { "concurrency": 4, - "decode_ms": 2226.0384, + "decode_ms": 3067.5886, "decode_tokens": 31, "error": "", - "latency_ms": 2303.4441, + "latency_ms": 3148.8735, "ok": true, - "prefill_ms": 77.3776, + "prefill_ms": 81.2505, "prompt_id": "short-fact", "prompt_tokens": 5, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se", - "ttft_ms": 77.3776 + "ttft_ms": 81.2505 }, { "concurrency": 4, - "decode_ms": 2202.8446, + "decode_ms": 2323.7094, "decode_tokens": 31, "error": "", - "latency_ms": 2291.3462, + "latency_ms": 2406.7763, "ok": true, - "prefill_ms": 88.441, + "prefill_ms": 82.9929, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 88.441 + "ttft_ms": 82.9929 }, { "concurrency": 4, - "decode_ms": 2168.871, + "decode_ms": 2314.5197, "decode_tokens": 31, "error": "", - "latency_ms": 2253.8977, + "latency_ms": 2408.2229, "ok": true, - "prefill_ms": 84.9928, + "prefill_ms": 93.668, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 84.9928 + "ttft_ms": 93.668 }, { "concurrency": 4, - "decode_ms": 2176.5775, + "decode_ms": 2304.5741, "decode_tokens": 31, "error": "", - "latency_ms": 2269.48, + "latency_ms": 2394.9373, "ok": true, - "prefill_ms": 92.8707, + "prefill_ms": 90.3384, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 92.8707 + "ttft_ms": 90.3384 }, { "concurrency": 4, - "decode_ms": 2193.5394, + "decode_ms": 2328.6365, "decode_tokens": 31, "error": "", - "latency_ms": 2287.9679, + "latency_ms": 2432.765, "ok": true, - "prefill_ms": 94.3995, + "prefill_ms": 104.0982, "prompt_id": "medium-code", "prompt_tokens": 24, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 94.3995 + "ttft_ms": 104.0982 }, { "concurrency": 4, - "decode_ms": 2264.7258, + "decode_ms": 2959.1949, "decode_tokens": 31, "error": "", - "latency_ms": 2656.3603, + "latency_ms": 3365.8395, "ok": true, - "prefill_ms": 391.5941, + "prefill_ms": 406.5841, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 391.5941 + "ttft_ms": 406.5841 }, { "concurrency": 4, - "decode_ms": 2201.3131, + "decode_ms": 2970.6675, "decode_tokens": 31, "error": "", - "latency_ms": 2631.0031, + "latency_ms": 3397.0527, "ok": true, - "prefill_ms": 429.6629, + "prefill_ms": 426.3508, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 429.6629 + "ttft_ms": 426.3508 }, { "concurrency": 4, - "decode_ms": 2209.0335, + "decode_ms": 2913.6743, "decode_tokens": 31, "error": "", - "latency_ms": 2639.3245, + "latency_ms": 3326.1048, "ok": true, - "prefill_ms": 430.2421, + "prefill_ms": 412.3929, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 430.2421 + "ttft_ms": 412.3929 }, { "concurrency": 4, - "decode_ms": 2201.2124, + "decode_ms": 2907.1259, "decode_tokens": 31, "error": "", - "latency_ms": 2624.0923, + "latency_ms": 3299.2804, "ok": true, - "prefill_ms": 422.8195, + "prefill_ms": 392.114, "prompt_id": "long-summary", "prompt_tokens": 191, "queue_wait_ms": 0.0, "recipe_id": "transformers-safetensors-reference", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed", - "ttft_ms": 422.8195 + "ttft_ms": 392.114 } ], "recipe": { @@ -910,765 +927,765 @@ "available": true, "concurrency": { "1": { - "aggregate_decode_tokens_per_sec": 73.7861, + "aggregate_decode_tokens_per_sec": 86.7339, "concurrency": 1, - "decode_tokens_per_sec": 87.9728, + "decode_tokens_per_sec": 98.5178, "failure_reasons": [], "failures": 0, - "latency_p50_ms": 385.2049, - "latency_p95_ms": 560.2939, - "peak_rss_bytes": 1110708224, + "latency_p50_ms": 333.023, + "latency_p95_ms": 383.0597, + "peak_rss_bytes": 1110728704, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 1427.2072, + "prefill_tokens_per_sec": 1717.9451, "requests": 9, - "ttft_p50_ms": 43.929, - "ttft_p95_ms": 107.003, - "wall_ms": 3781.2019 + "ttft_p50_ms": 15.069, + "ttft_p95_ms": 63.766, + "wall_ms": 3216.7354 }, "4": { - "aggregate_decode_tokens_per_sec": 211.3515, + "aggregate_decode_tokens_per_sec": 222.788, "concurrency": 4, - "decode_tokens_per_sec": 73.8932, + "decode_tokens_per_sec": 76.6297, "failure_reasons": [], "failures": 0, - "latency_p50_ms": 467.5094, - "latency_p95_ms": 790.862, - "peak_rss_bytes": 1129578496, + "latency_p50_ms": 490.8738, + "latency_p95_ms": 646.26, + "peak_rss_bytes": 1139466240, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 1077.8162, + "prefill_tokens_per_sec": 859.8985, "requests": 36, - "ttft_p50_ms": 33.612, - "ttft_p95_ms": 128.501, - "wall_ms": 5280.3028 + "ttft_p50_ms": 32.445, + "ttft_p95_ms": 218.387, + "wall_ms": 5009.2465 } }, "load": { "artifact_bytes": 994156448, "backend_detail": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0", - "load_ms": 509.9895, - "rss_bytes": 1086750720, + "load_ms": 513.098, + "rss_bytes": 1086771200, "vram_bytes": 0 }, "outcomes": [ { "concurrency": 1, - "decode_ms": 388.956, + "decode_ms": 326.476, "decode_tokens": 31, "error": "", - "latency_ms": 406.2852, + "latency_ms": 339.6427, "ok": true, - "prefill_ms": 15.533, + "prefill_ms": 11.875, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.7962, + "queue_wait_ms": 1.2917, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 15.533 + "ttft_ms": 11.875 }, { "concurrency": 1, - "decode_ms": 514.396, + "decode_ms": 311.923, "decode_tokens": 31, "error": "", - "latency_ms": 560.2939, + "latency_ms": 329.4616, "ok": true, - "prefill_ms": 43.929, + "prefill_ms": 16.03, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.9689, + "queue_wait_ms": 1.5086, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 43.929 + "ttft_ms": 16.03 }, { "concurrency": 1, - "decode_ms": 433.092, + "decode_ms": 313.864, "decode_tokens": 31, "error": "", - "latency_ms": 543.2542, + "latency_ms": 380.2495, "ok": true, - "prefill_ms": 107.003, + "prefill_ms": 63.766, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 3.1592, + "queue_wait_ms": 2.6195, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 107.003 + "ttft_ms": 63.766 }, { "concurrency": 1, - "decode_ms": 345.902, + "decode_ms": 310.351, "decode_tokens": 31, "error": "", - "latency_ms": 396.0584, + "latency_ms": 324.069, "ok": true, - "prefill_ms": 47.903, + "prefill_ms": 11.225, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 2.2534, + "queue_wait_ms": 2.493, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 47.903 + "ttft_ms": 11.225 }, { "concurrency": 1, - "decode_ms": 311.041, + "decode_ms": 315.083, "decode_tokens": 31, "error": "", - "latency_ms": 327.4633, + "latency_ms": 331.4476, "ok": true, - "prefill_ms": 14.864, + "prefill_ms": 15.069, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.5583, + "queue_wait_ms": 1.2956, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 14.864 + "ttft_ms": 15.069 }, { "concurrency": 1, - "decode_ms": 320.904, + "decode_ms": 320.067, "decode_tokens": 31, "error": "", - "latency_ms": 380.5965, + "latency_ms": 383.0597, "ok": true, - "prefill_ms": 57.209, + "prefill_ms": 60.269, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 2.4835, + "queue_wait_ms": 2.7237, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 57.209 + "ttft_ms": 60.269 }, { "concurrency": 1, - "decode_ms": 312.288, + "decode_ms": 320.389, "decode_tokens": 31, "error": "", - "latency_ms": 325.1266, + "latency_ms": 333.023, "ok": true, - "prefill_ms": 11.487, + "prefill_ms": 11.507, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.3516, + "queue_wait_ms": 1.127, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 11.487 + "ttft_ms": 11.507 }, { "concurrency": 1, - "decode_ms": 314.501, + "decode_ms": 305.305, "decode_tokens": 31, "error": "", - "latency_ms": 330.9473, + "latency_ms": 321.8616, "ok": true, - "prefill_ms": 15.082, + "prefill_ms": 15.038, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.3643, + "queue_wait_ms": 1.5186, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 15.082 + "ttft_ms": 15.038 }, { "concurrency": 1, - "decode_ms": 321.287, + "decode_ms": 309.595, "decode_tokens": 31, "error": "", - "latency_ms": 385.2049, + "latency_ms": 369.7193, "ok": true, - "prefill_ms": 61.466, + "prefill_ms": 57.7, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 2.4519, + "queue_wait_ms": 2.4243, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 61.466 + "ttft_ms": 57.7 }, { "concurrency": 4, - "decode_ms": 363.766, + "decode_ms": 346.864, "decode_tokens": 31, "error": "", - "latency_ms": 377.6431, + "latency_ms": 363.9806, "ok": true, - "prefill_ms": 11.719, + "prefill_ms": 14.475, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 2.1581, + "queue_wait_ms": 2.6416, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 11.719 + "ttft_ms": 14.475 }, { "concurrency": 4, - "decode_ms": 351.14, + "decode_ms": 346.856, "decode_tokens": 31, "error": "", - "latency_ms": 388.4667, + "latency_ms": 364.0486, "ok": true, - "prefill_ms": 23.562, + "prefill_ms": 14.752, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 13.7647, + "queue_wait_ms": 2.4406, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 23.562 + "ttft_ms": 14.752 }, { "concurrency": 4, - "decode_ms": 350.563, + "decode_ms": 346.497, "decode_tokens": 31, "error": "", - "latency_ms": 388.6212, + "latency_ms": 363.7474, "ok": true, - "prefill_ms": 24.498, + "prefill_ms": 15.223, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 13.5602, + "queue_wait_ms": 2.0274, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 24.498 + "ttft_ms": 15.223 }, { "concurrency": 4, - "decode_ms": 351.096, + "decode_ms": 346.877, "decode_tokens": 31, "error": "", - "latency_ms": 388.0489, + "latency_ms": 363.1133, "ok": true, - "prefill_ms": 23.815, + "prefill_ms": 14.63, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 13.1379, + "queue_wait_ms": 1.6063, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 23.815 + "ttft_ms": 14.63 }, { "concurrency": 4, - "decode_ms": 383.359, + "decode_ms": 352.14, "decode_tokens": 31, "error": "", - "latency_ms": 400.0232, + "latency_ms": 387.0363, "ok": true, - "prefill_ms": 15.127, + "prefill_ms": 32.445, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.5372, + "queue_wait_ms": 2.4513, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 15.127 + "ttft_ms": 32.445 }, { "concurrency": 4, - "decode_ms": 439.638, + "decode_ms": 352.137, "decode_tokens": 31, "error": "", - "latency_ms": 467.5094, + "latency_ms": 386.453, "ok": true, - "prefill_ms": 24.388, + "prefill_ms": 32.612, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 3.4834, + "queue_wait_ms": 1.704, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 24.388 + "ttft_ms": 32.612 }, { "concurrency": 4, - "decode_ms": 409.78, + "decode_ms": 352.121, "decode_tokens": 31, "error": "", - "latency_ms": 474.5689, + "latency_ms": 386.3724, "ok": true, - "prefill_ms": 41.109, + "prefill_ms": 32.771, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 23.6799, + "queue_wait_ms": 1.4804, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 41.109 + "ttft_ms": 32.771 }, { "concurrency": 4, - "decode_ms": 409.751, + "decode_ms": 352.101, "decode_tokens": 31, "error": "", - "latency_ms": 472.4775, + "latency_ms": 386.3013, "ok": true, - "prefill_ms": 41.27, + "prefill_ms": 32.919, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 21.4565, + "queue_wait_ms": 1.2813, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 41.27 + "ttft_ms": 32.919 }, { "concurrency": 4, - "decode_ms": 529.071, + "decode_ms": 424.326, "decode_tokens": 31, "error": "", - "latency_ms": 604.9495, + "latency_ms": 647.1063, "ok": true, - "prefill_ms": 69.549, + "prefill_ms": 218.058, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 6.3295, + "queue_wait_ms": 4.7223, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 69.549 + "ttft_ms": 218.058 }, { "concurrency": 4, - "decode_ms": 492.015, + "decode_ms": 424.156, "decode_tokens": 31, "error": "", - "latency_ms": 565.0987, + "latency_ms": 646.26, "ok": true, - "prefill_ms": 66.65, + "prefill_ms": 218.631, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 6.4337, + "queue_wait_ms": 3.473, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 66.65 + "ttft_ms": 218.631 }, { "concurrency": 4, - "decode_ms": 375.591, + "decode_ms": 424.365, "decode_tokens": 31, "error": "", - "latency_ms": 553.1371, + "latency_ms": 645.6859, "ok": true, - "prefill_ms": 128.501, + "prefill_ms": 218.224, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 49.0451, + "queue_wait_ms": 3.0969, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 128.501 + "ttft_ms": 218.224 }, { "concurrency": 4, - "decode_ms": 375.663, + "decode_ms": 424.305, "decode_tokens": 31, "error": "", - "latency_ms": 552.2825, + "latency_ms": 645.6746, "ok": true, - "prefill_ms": 128.598, + "prefill_ms": 218.387, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 48.0215, + "queue_wait_ms": 2.9826, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 0, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 128.598 + "ttft_ms": 218.387 }, { "concurrency": 4, - "decode_ms": 395.56, + "decode_ms": 460.812, "decode_tokens": 31, "error": "", - "latency_ms": 421.9957, + "latency_ms": 489.7045, "ok": true, - "prefill_ms": 22.876, + "prefill_ms": 26.141, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 3.5597, + "queue_wait_ms": 2.7515, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 22.876 + "ttft_ms": 26.141 }, { "concurrency": 4, - "decode_ms": 408.691, + "decode_ms": 447.676, "decode_tokens": 31, "error": "", - "latency_ms": 440.9788, + "latency_ms": 500.4384, "ok": true, - "prefill_ms": 21.361, + "prefill_ms": 28.144, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 10.9268, + "queue_wait_ms": 24.6184, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 21.361 + "ttft_ms": 28.144 }, { "concurrency": 4, - "decode_ms": 376.876, + "decode_ms": 447.81, "decode_tokens": 31, "error": "", - "latency_ms": 431.4856, + "latency_ms": 496.1159, "ok": true, - "prefill_ms": 33.454, + "prefill_ms": 28.233, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 21.1556, + "queue_wait_ms": 20.0729, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 33.454 + "ttft_ms": 28.233 }, { "concurrency": 4, - "decode_ms": 376.849, + "decode_ms": 447.846, "decode_tokens": 31, "error": "", - "latency_ms": 430.7598, + "latency_ms": 490.8738, "ok": true, - "prefill_ms": 33.612, + "prefill_ms": 28.369, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 20.2988, + "queue_wait_ms": 14.6588, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 33.612 + "ttft_ms": 28.369 }, { "concurrency": 4, - "decode_ms": 407.157, + "decode_ms": 550.9, "decode_tokens": 31, "error": "", - "latency_ms": 439.679, + "latency_ms": 584.8535, "ok": true, - "prefill_ms": 26.307, + "prefill_ms": 29.515, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 6.215, + "queue_wait_ms": 4.4385, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 26.307 + "ttft_ms": 29.515 }, { "concurrency": 4, - "decode_ms": 452.7, + "decode_ms": 515.045, "decode_tokens": 31, "error": "", - "latency_ms": 486.6358, + "latency_ms": 573.8089, "ok": true, - "prefill_ms": 23.791, + "prefill_ms": 47.181, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 10.1448, + "queue_wait_ms": 11.5829, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 23.791 + "ttft_ms": 47.181 }, { "concurrency": 4, - "decode_ms": 406.65, + "decode_ms": 480.138, "decode_tokens": 31, "error": "", - "latency_ms": 464.9895, + "latency_ms": 569.6531, "ok": true, - "prefill_ms": 39.807, + "prefill_ms": 45.676, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 18.5325, + "queue_wait_ms": 43.8391, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 39.807 + "ttft_ms": 45.676 }, { "concurrency": 4, - "decode_ms": 406.65, + "decode_ms": 480.022, "decode_tokens": 31, "error": "", - "latency_ms": 463.8505, + "latency_ms": 566.2952, "ok": true, - "prefill_ms": 39.662, + "prefill_ms": 45.891, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 17.5385, + "queue_wait_ms": 40.3822, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 39.662 + "ttft_ms": 45.891 }, { "concurrency": 4, - "decode_ms": 518.864, + "decode_ms": 532.421, "decode_tokens": 31, "error": "", - "latency_ms": 598.7536, + "latency_ms": 594.8715, "ok": true, - "prefill_ms": 67.908, + "prefill_ms": 59.373, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 11.9816, + "queue_wait_ms": 3.0775, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 67.908 + "ttft_ms": 59.373 }, { "concurrency": 4, - "decode_ms": 475.05, + "decode_ms": 365.659, "decode_tokens": 31, "error": "", - "latency_ms": 550.1841, + "latency_ms": 575.7742, "ok": true, - "prefill_ms": 66.691, + "prefill_ms": 178.428, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 8.4431, + "queue_wait_ms": 31.6872, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 66.691 + "ttft_ms": 178.428 }, { "concurrency": 4, - "decode_ms": 366.102, + "decode_ms": 365.658, "decode_tokens": 31, "error": "", - "latency_ms": 504.9402, + "latency_ms": 571.4574, "ok": true, - "prefill_ms": 120.353, + "prefill_ms": 178.58, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 18.4852, + "queue_wait_ms": 27.2194, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 120.353 + "ttft_ms": 178.58 }, { "concurrency": 4, - "decode_ms": 366.189, + "decode_ms": 365.623, "decode_tokens": 31, "error": "", - "latency_ms": 504.5586, + "latency_ms": 570.8456, "ok": true, - "prefill_ms": 120.462, + "prefill_ms": 178.746, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 17.9076, + "queue_wait_ms": 26.4766, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 1, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 120.462 + "ttft_ms": 178.746 }, { "concurrency": 4, - "decode_ms": 378.436, + "decode_ms": 370.443, "decode_tokens": 31, "error": "", - "latency_ms": 411.7135, + "latency_ms": 383.5684, "ok": true, - "prefill_ms": 22.159, + "prefill_ms": 11.612, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 11.1185, + "queue_wait_ms": 1.5134, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 22.159 + "ttft_ms": 11.612 }, { "concurrency": 4, - "decode_ms": 393.111, + "decode_ms": 362.394, "decode_tokens": 31, "error": "", - "latency_ms": 423.2137, + "latency_ms": 386.4942, "ok": true, - "prefill_ms": 20.398, + "prefill_ms": 21.305, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 9.7047, + "queue_wait_ms": 2.7952, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 20.398 + "ttft_ms": 21.305 }, { "concurrency": 4, - "decode_ms": 363.546, + "decode_ms": 350.516, "decode_tokens": 31, "error": "", - "latency_ms": 418.0173, + "latency_ms": 395.4336, "ok": true, - "prefill_ms": 32.05, + "prefill_ms": 23.0, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 22.4213, + "queue_wait_ms": 21.9176, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 32.05 + "ttft_ms": 23.0 }, { "concurrency": 4, - "decode_ms": 363.511, + "decode_ms": 350.53, "decode_tokens": 31, "error": "", - "latency_ms": 417.298, + "latency_ms": 393.0663, "ok": true, - "prefill_ms": 32.207, + "prefill_ms": 23.11, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 21.58, + "queue_wait_ms": 19.4263, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the", - "ttft_ms": 32.207 + "ttft_ms": 23.11 }, { "concurrency": 4, - "decode_ms": 416.716, + "decode_ms": 374.767, "decode_tokens": 31, "error": "", - "latency_ms": 445.7282, + "latency_ms": 391.7625, "ok": true, - "prefill_ms": 25.649, + "prefill_ms": 14.813, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 3.3632, + "queue_wait_ms": 2.1825, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 25.649 + "ttft_ms": 14.813 }, { "concurrency": 4, - "decode_ms": 463.304, + "decode_ms": 431.621, "decode_tokens": 31, "error": "", - "latency_ms": 498.9606, + "latency_ms": 465.0976, "ok": true, - "prefill_ms": 24.263, + "prefill_ms": 25.343, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 11.3936, + "queue_wait_ms": 8.1336, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 24.263 + "ttft_ms": 25.343 }, { "concurrency": 4, - "decode_ms": 463.28, + "decode_ms": 501.279, "decode_tokens": 31, "error": "", - "latency_ms": 523.3473, + "latency_ms": 543.2184, "ok": true, - "prefill_ms": 40.138, + "prefill_ms": 31.046, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 19.9293, + "queue_wait_ms": 10.8934, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 40.138 + "ttft_ms": 31.046 }, { "concurrency": 4, - "decode_ms": 463.324, + "decode_ms": 501.273, "decode_tokens": 31, "error": "", - "latency_ms": 522.9262, + "latency_ms": 542.1594, "ok": true, - "prefill_ms": 40.272, + "prefill_ms": 31.193, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 19.3302, + "queue_wait_ms": 9.6934, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >", - "ttft_ms": 40.272 + "ttft_ms": 31.193 }, { "concurrency": 4, - "decode_ms": 462.041, + "decode_ms": 549.266, "decode_tokens": 31, "error": "", - "latency_ms": 535.5183, + "latency_ms": 628.2905, "ok": true, - "prefill_ms": 67.981, + "prefill_ms": 69.487, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 5.4963, + "queue_wait_ms": 9.5375, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 67.981 + "ttft_ms": 69.487 }, { "concurrency": 4, - "decode_ms": 751.567, + "decode_ms": 459.836, "decode_tokens": 31, "error": "", - "latency_ms": 826.8821, + "latency_ms": 550.1822, "ok": true, - "prefill_ms": 67.036, + "prefill_ms": 79.286, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 8.2791, + "queue_wait_ms": 11.0602, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 67.036 + "ttft_ms": 79.286 }, { "concurrency": 4, - "decode_ms": 648.215, + "decode_ms": 353.562, "decode_tokens": 31, "error": "", - "latency_ms": 790.862, + "latency_ms": 478.789, "ok": true, - "prefill_ms": 121.592, + "prefill_ms": 116.484, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 21.055, + "queue_wait_ms": 8.743, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 121.592 + "ttft_ms": 116.484 }, { "concurrency": 4, - "decode_ms": 648.245, + "decode_ms": 353.539, "decode_tokens": 31, "error": "", - "latency_ms": 790.7863, + "latency_ms": 478.1728, "ok": true, - "prefill_ms": 121.708, + "prefill_ms": 116.378, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 20.8333, + "queue_wait_ms": 8.2558, "recipe_id": "llama-cpp-near-lossless-quality", "repeat": 2, "text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to", - "ttft_ms": 121.708 + "ttft_ms": 116.378 } ], "recipe": { @@ -1691,765 +1708,765 @@ "available": true, "concurrency": { "1": { - "aggregate_decode_tokens_per_sec": 110.0458, + "aggregate_decode_tokens_per_sec": 139.2693, "concurrency": 1, - "decode_tokens_per_sec": 170.131, + "decode_tokens_per_sec": 207.712, "failure_reasons": [], "failures": 0, - "latency_p50_ms": 258.0681, - "latency_p95_ms": 465.8523, - "peak_rss_bytes": 542167040, + "latency_p50_ms": 168.3307, + "latency_p95_ms": 305.1338, + "peak_rss_bytes": 542081024, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 783.3775, + "prefill_tokens_per_sec": 967.0195, "requests": 9, - "ttft_p50_ms": 23.847, - "ttft_p95_ms": 237.696, - "wall_ms": 2535.3081 + "ttft_p50_ms": 21.582, + "ttft_p95_ms": 147.859, + "wall_ms": 2003.3134 }, "4": { - "aggregate_decode_tokens_per_sec": 206.377, + "aggregate_decode_tokens_per_sec": 195.6789, "concurrency": 4, - "decode_tokens_per_sec": 83.543, + "decode_tokens_per_sec": 76.9497, "failure_reasons": [], "failures": 0, - "latency_p50_ms": 413.3897, - "latency_p95_ms": 910.253, - "peak_rss_bytes": 572788736, + "latency_p50_ms": 437.9196, + "latency_p95_ms": 885.5355, + "peak_rss_bytes": 573259776, "peak_vram_bytes": 0, - "prefill_tokens_per_sec": 474.3116, + "prefill_tokens_per_sec": 572.4424, "requests": 36, - "ttft_p50_ms": 67.945, - "ttft_p95_ms": 431.804, - "wall_ms": 5407.5795 + "ttft_p50_ms": 48.127, + "ttft_p95_ms": 416.531, + "wall_ms": 5703.2207 } }, "load": { "artifact_bytes": 397807520, "backend_detail": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0", - "load_ms": 504.2373, - "rss_bytes": 520003584, + "load_ms": 504.9034, + "rss_bytes": 519925760, "vram_bytes": 0 }, "outcomes": [ { "concurrency": 1, - "decode_ms": 155.567, + "decode_ms": 154.112, "decode_tokens": 31, "error": "", - "latency_ms": 168.2902, + "latency_ms": 166.3913, "ok": true, - "prefill_ms": 11.286, + "prefill_ms": 11.143, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.4372, + "queue_wait_ms": 1.1363, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of", - "ttft_ms": 11.286 + "ttft_ms": 11.143 }, { "concurrency": 1, - "decode_ms": 176.69, + "decode_ms": 160.212, "decode_tokens": 31, "error": "", - "latency_ms": 201.9608, + "latency_ms": 185.7731, "ok": true, - "prefill_ms": 23.847, + "prefill_ms": 24.1, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.4238, + "queue_wait_ms": 1.4611, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 23.847 + "ttft_ms": 24.1 }, { "concurrency": 1, - "decode_ms": 218.292, + "decode_ms": 154.87, "decode_tokens": 31, "error": "", - "latency_ms": 378.9965, + "latency_ms": 305.1338, "ok": true, - "prefill_ms": 157.91, + "prefill_ms": 147.859, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 2.7945, + "queue_wait_ms": 2.4048, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 157.91 + "ttft_ms": 147.859 }, { "concurrency": 1, - "decode_ms": 230.358, + "decode_ms": 141.334, "decode_tokens": 31, "error": "", - "latency_ms": 258.0681, + "latency_ms": 152.5336, "ok": true, - "prefill_ms": 23.289, + "prefill_ms": 9.418, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 4.4211, + "queue_wait_ms": 1.7816, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of", - "ttft_ms": 23.289 + "ttft_ms": 9.418 }, { "concurrency": 1, - "decode_ms": 257.889, + "decode_ms": 141.596, "decode_tokens": 31, "error": "", - "latency_ms": 317.6954, + "latency_ms": 164.0665, "ok": true, - "prefill_ms": 57.574, + "prefill_ms": 21.016, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 2.2324, + "queue_wait_ms": 1.4545, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 1, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 57.574 + "ttft_ms": 21.016 }, { "concurrency": 1, - "decode_ms": 224.541, + "decode_ms": 152.059, "decode_tokens": 31, "error": "", - "latency_ms": 465.8523, + "latency_ms": 296.5754, "ok": true, - "prefill_ms": 237.696, + "prefill_ms": 142.622, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 3.6153, + "queue_wait_ms": 1.8944, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 1, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 237.696 + "ttft_ms": 142.622 }, { "concurrency": 1, - "decode_ms": 146.754, + "decode_ms": 143.413, "decode_tokens": 31, "error": "", - "latency_ms": 158.7311, + "latency_ms": 155.4718, "ok": true, - "prefill_ms": 10.957, + "prefill_ms": 10.774, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.0201, + "queue_wait_ms": 1.2848, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of", - "ttft_ms": 10.957 + "ttft_ms": 10.774 }, { "concurrency": 1, - "decode_ms": 149.767, + "decode_ms": 145.251, "decode_tokens": 31, "error": "", - "latency_ms": 171.9368, + "latency_ms": 168.3307, "ok": true, - "prefill_ms": 20.755, + "prefill_ms": 21.582, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.4148, + "queue_wait_ms": 1.4977, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 20.755 + "ttft_ms": 21.582 }, { "concurrency": 1, - "decode_ms": 152.422, + "decode_ms": 152.806, "decode_tokens": 31, "error": "", - "latency_ms": 296.5435, + "latency_ms": 293.6305, "ok": true, - "prefill_ms": 142.152, + "prefill_ms": 138.543, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 1.9695, + "queue_wait_ms": 2.2815, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 142.152 + "ttft_ms": 138.543 }, { "concurrency": 4, - "decode_ms": 320.867, + "decode_ms": 376.127, "decode_tokens": 31, "error": "", - "latency_ms": 342.034, + "latency_ms": 401.7532, "ok": true, - "prefill_ms": 19.581, + "prefill_ms": 23.322, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.586, + "queue_wait_ms": 2.3042, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 19.581 + "ttft_ms": 23.322 }, { "concurrency": 4, - "decode_ms": 320.861, + "decode_ms": 375.793, "decode_tokens": 31, "error": "", - "latency_ms": 342.1077, + "latency_ms": 401.5063, "ok": true, - "prefill_ms": 19.743, + "prefill_ms": 23.77, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.5037, + "queue_wait_ms": 1.9433, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 19.743 + "ttft_ms": 23.77 }, { "concurrency": 4, - "decode_ms": 320.563, + "decode_ms": 354.788, "decode_tokens": 31, "error": "", - "latency_ms": 342.1894, + "latency_ms": 406.3609, "ok": true, - "prefill_ms": 20.201, + "prefill_ms": 26.2, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.4254, + "queue_wait_ms": 25.3729, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 0, + "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the", + "ttft_ms": 26.2 + }, + { + "concurrency": 4, + "decode_ms": 376.001, + "decode_tokens": 31, + "error": "", + "latency_ms": 400.5544, + "ok": true, + "prefill_ms": 23.17, + "prompt_id": "short-fact", + "prompt_tokens": 5, + "queue_wait_ms": 1.3834, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 20.201 + "ttft_ms": 23.17 }, { "concurrency": 4, - "decode_ms": 320.857, + "decode_ms": 405.905, "decode_tokens": 31, "error": "", - "latency_ms": 342.1242, + "latency_ms": 428.1187, "ok": true, - "prefill_ms": 19.827, - "prompt_id": "short-fact", - "prompt_tokens": 5, - "queue_wait_ms": 1.4402, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 0, - "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 19.827 - }, - { - "concurrency": 4, - "decode_ms": 337.081, - "decode_tokens": 31, - "error": "", - "latency_ms": 413.3897, - "ok": true, - "prefill_ms": 73.553, + "prefill_ms": 20.392, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 2.7557, + "queue_wait_ms": 1.8217, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 73.553 + "ttft_ms": 20.392 }, { "concurrency": 4, - "decode_ms": 337.059, + "decode_ms": 354.584, "decode_tokens": 31, "error": "", - "latency_ms": 413.1597, + "latency_ms": 437.9196, "ok": true, - "prefill_ms": 73.693, + "prefill_ms": 61.802, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 2.4077, + "queue_wait_ms": 21.5336, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 73.693 + "ttft_ms": 61.802 }, { "concurrency": 4, - "decode_ms": 337.097, + "decode_ms": 354.558, "decode_tokens": 31, "error": "", - "latency_ms": 412.6975, + "latency_ms": 437.755, "ok": true, - "prefill_ms": 73.271, + "prefill_ms": 61.931, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 2.3295, + "queue_wait_ms": 21.266, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 73.271 + "ttft_ms": 61.931 }, { "concurrency": 4, - "decode_ms": 337.095, + "decode_ms": 354.566, "decode_tokens": 31, "error": "", - "latency_ms": 412.3863, + "latency_ms": 435.8597, "ok": true, - "prefill_ms": 73.417, + "prefill_ms": 61.676, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 1.8743, + "queue_wait_ms": 19.6177, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 73.417 + "ttft_ms": 61.676 }, { "concurrency": 4, - "decode_ms": 343.399, + "decode_ms": 741.39, "decode_tokens": 31, "error": "", - "latency_ms": 778.9733, + "latency_ms": 885.5355, "ok": true, - "prefill_ms": 432.216, + "prefill_ms": 141.568, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 3.3583, + "queue_wait_ms": 2.5775, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 432.216 + "ttft_ms": 141.568 }, { "concurrency": 4, - "decode_ms": 343.728, + "decode_ms": 334.866, "decode_tokens": 31, "error": "", - "latency_ms": 778.3749, + "latency_ms": 868.3416, "ok": true, - "prefill_ms": 431.804, + "prefill_ms": 416.421, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 2.8429, + "queue_wait_ms": 117.0546, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 431.804 + "ttft_ms": 416.421 }, { "concurrency": 4, - "decode_ms": 343.766, + "decode_ms": 334.558, "decode_tokens": 31, "error": "", - "latency_ms": 777.9607, + "latency_ms": 867.4189, "ok": true, - "prefill_ms": 431.685, + "prefill_ms": 416.972, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 2.5097, + "queue_wait_ms": 115.8889, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 431.685 + "ttft_ms": 416.972 }, { "concurrency": 4, - "decode_ms": 343.778, + "decode_ms": 334.875, "decode_tokens": 31, "error": "", - "latency_ms": 777.2286, + "latency_ms": 866.8735, "ok": true, - "prefill_ms": 431.555, + "prefill_ms": 416.531, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 1.8956, + "queue_wait_ms": 115.4675, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 0, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 431.555 + "ttft_ms": 416.531 }, { "concurrency": 4, - "decode_ms": 321.54, + "decode_ms": 360.293, "decode_tokens": 31, "error": "", - "latency_ms": 341.2678, + "latency_ms": 373.9246, "ok": true, - "prefill_ms": 17.183, + "prefill_ms": 11.202, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 2.5448, + "queue_wait_ms": 2.4296, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 17.183 + "ttft_ms": 11.202 }, { "concurrency": 4, - "decode_ms": 321.55, + "decode_ms": 387.093, "decode_tokens": 31, "error": "", - "latency_ms": 340.9384, + "latency_ms": 412.5291, "ok": true, - "prefill_ms": 17.28, + "prefill_ms": 21.333, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 2.1084, + "queue_wait_ms": 4.1031, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the", + "ttft_ms": 21.333 + }, + { + "concurrency": 4, + "decode_ms": 387.107, + "decode_tokens": 31, + "error": "", + "latency_ms": 411.9945, + "ok": true, + "prefill_ms": 21.211, + "prompt_id": "short-fact", + "prompt_tokens": 5, + "queue_wait_ms": 3.6765, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the", + "ttft_ms": 21.211 + }, + { + "concurrency": 4, + "decode_ms": 376.091, + "decode_tokens": 31, + "error": "", + "latency_ms": 417.7572, + "ok": true, + "prefill_ms": 21.212, + "prompt_id": "short-fact", + "prompt_tokens": 5, + "queue_wait_ms": 20.4542, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 1, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 17.28 + "ttft_ms": 21.212 }, { "concurrency": 4, - "decode_ms": 321.541, + "decode_ms": 389.488, "decode_tokens": 31, "error": "", - "latency_ms": 340.9985, + "latency_ms": 431.0462, "ok": true, - "prefill_ms": 17.386, + "prefill_ms": 30.89, + "prompt_id": "medium-code", + "prompt_tokens": 24, + "queue_wait_ms": 10.6682, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", + "ttft_ms": 30.89 + }, + { + "concurrency": 4, + "decode_ms": 522.548, + "decode_tokens": 31, + "error": "", + "latency_ms": 573.2773, + "ok": true, + "prefill_ms": 48.127, + "prompt_id": "medium-code", + "prompt_tokens": 24, + "queue_wait_ms": 2.6023, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", + "ttft_ms": 48.127 + }, + { + "concurrency": 4, + "decode_ms": 522.512, + "decode_tokens": 31, + "error": "", + "latency_ms": 572.773, + "ok": true, + "prefill_ms": 48.289, + "prompt_id": "medium-code", + "prompt_tokens": 24, + "queue_wait_ms": 1.972, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", + "ttft_ms": 48.289 + }, + { + "concurrency": 4, + "decode_ms": 500.617, + "decode_tokens": 31, + "error": "", + "latency_ms": 576.4582, + "ok": true, + "prefill_ms": 32.565, + "prompt_id": "medium-code", + "prompt_tokens": 24, + "queue_wait_ms": 43.2762, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", + "ttft_ms": 32.565 + }, + { + "concurrency": 4, + "decode_ms": 774.96, + "decode_tokens": 31, + "error": "", + "latency_ms": 938.0479, + "ok": true, + "prefill_ms": 155.553, + "prompt_id": "long-summary", + "prompt_tokens": 191, + "queue_wait_ms": 7.5349, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", + "ttft_ms": 155.553 + }, + { + "concurrency": 4, + "decode_ms": 524.937, + "decode_tokens": 31, + "error": "", + "latency_ms": 826.886, + "ok": true, + "prefill_ms": 294.908, + "prompt_id": "long-summary", + "prompt_tokens": 191, + "queue_wait_ms": 7.041, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", + "ttft_ms": 294.908 + }, + { + "concurrency": 4, + "decode_ms": 524.939, + "decode_tokens": 31, + "error": "", + "latency_ms": 826.2408, + "ok": true, + "prefill_ms": 294.79, + "prompt_id": "long-summary", + "prompt_tokens": 191, + "queue_wait_ms": 6.5118, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", + "ttft_ms": 294.79 + }, + { + "concurrency": 4, + "decode_ms": 379.586, + "decode_tokens": 31, + "error": "", + "latency_ms": 827.0821, + "ok": true, + "prefill_ms": 155.842, + "prompt_id": "long-summary", + "prompt_tokens": 191, + "queue_wait_ms": 291.6541, + "recipe_id": "llama-cpp-quantized-performance-fit", + "repeat": 1, + "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", + "ttft_ms": 155.842 + }, + { + "concurrency": 4, + "decode_ms": 306.002, + "decode_tokens": 31, + "error": "", + "latency_ms": 330.1262, + "ok": true, + "prefill_ms": 20.307, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 2.0715, + "queue_wait_ms": 3.8172, "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 17.386 + "repeat": 2, + "text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of", + "ttft_ms": 20.307 }, { "concurrency": 4, - "decode_ms": 321.531, + "decode_ms": 388.127, "decode_tokens": 31, "error": "", - "latency_ms": 340.3135, + "latency_ms": 414.4426, "ok": true, - "prefill_ms": 17.493, + "prefill_ms": 20.412, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 1.2895, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 17.493 - }, - { - "concurrency": 4, - "decode_ms": 352.481, - "decode_tokens": 31, - "error": "", - "latency_ms": 399.8068, - "ok": true, - "prefill_ms": 45.386, - "prompt_id": "medium-code", - "prompt_tokens": 24, - "queue_wait_ms": 1.9398, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 45.386 - }, - { - "concurrency": 4, - "decode_ms": 352.45, - "decode_tokens": 31, - "error": "", - "latency_ms": 399.8142, - "ok": true, - "prefill_ms": 45.687, - "prompt_id": "medium-code", - "prompt_tokens": 24, - "queue_wait_ms": 1.6772, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 45.687 - }, - { - "concurrency": 4, - "decode_ms": 352.474, - "decode_tokens": 31, - "error": "", - "latency_ms": 399.46, - "ok": true, - "prefill_ms": 45.546, - "prompt_id": "medium-code", - "prompt_tokens": 24, - "queue_wait_ms": 1.44, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 45.546 - }, - { - "concurrency": 4, - "decode_ms": 329.9, - "decode_tokens": 31, - "error": "", - "latency_ms": 404.9024, - "ok": true, - "prefill_ms": 27.996, - "prompt_id": "medium-code", - "prompt_tokens": 24, - "queue_wait_ms": 47.0064, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 27.996 - }, - { - "concurrency": 4, - "decode_ms": 764.535, - "decode_tokens": 31, - "error": "", - "latency_ms": 900.1417, - "ok": true, - "prefill_ms": 132.886, - "prompt_id": "long-summary", - "prompt_tokens": 191, - "queue_wait_ms": 2.7207, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 132.886 - }, - { - "concurrency": 4, - "decode_ms": 374.002, - "decode_tokens": 31, - "error": "", - "latency_ms": 910.253, - "ok": true, - "prefill_ms": 401.437, - "prompt_id": "long-summary", - "prompt_tokens": 191, - "queue_wait_ms": 134.814, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 401.437 - }, - { - "concurrency": 4, - "decode_ms": 374.027, - "decode_tokens": 31, - "error": "", - "latency_ms": 910.5091, - "ok": true, - "prefill_ms": 401.538, - "prompt_id": "long-summary", - "prompt_tokens": 191, - "queue_wait_ms": 134.9441, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 401.538 - }, - { - "concurrency": 4, - "decode_ms": 373.991, - "decode_tokens": 31, - "error": "", - "latency_ms": 909.1499, - "ok": true, - "prefill_ms": 401.647, - "prompt_id": "long-summary", - "prompt_tokens": 191, - "queue_wait_ms": 133.5119, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 1, - "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 401.647 - }, - { - "concurrency": 4, - "decode_ms": 352.719, - "decode_tokens": 31, - "error": "", - "latency_ms": 364.9475, - "ok": true, - "prefill_ms": 10.966, - "prompt_id": "short-fact", - "prompt_tokens": 5, - "queue_wait_ms": 1.2625, + "queue_wait_ms": 5.9036, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 10.966 + "ttft_ms": 20.412 }, { "concurrency": 4, - "decode_ms": 390.304, + "decode_ms": 388.131, "decode_tokens": 31, "error": "", - "latency_ms": 422.7162, + "latency_ms": 413.676, "ok": true, - "prefill_ms": 25.97, + "prefill_ms": 20.524, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 6.4422, + "queue_wait_ms": 5.021, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 25.97 + "ttft_ms": 20.524 }, { "concurrency": 4, - "decode_ms": 390.314, + "decode_ms": 377.834, "decode_tokens": 31, "error": "", - "latency_ms": 422.0191, + "latency_ms": 417.606, "ok": true, - "prefill_ms": 25.841, + "prefill_ms": 20.665, "prompt_id": "short-fact", "prompt_tokens": 5, - "queue_wait_ms": 5.8641, + "queue_wait_ms": 19.107, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, - "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 25.841 + "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the Seine River, which flows through the center of", + "ttft_ms": 20.665 }, { "concurrency": 4, - "decode_ms": 390.314, + "decode_ms": 337.163, "decode_tokens": 31, "error": "", - "latency_ms": 421.728, + "latency_ms": 379.6094, "ok": true, - "prefill_ms": 26.086, - "prompt_id": "short-fact", - "prompt_tokens": 5, - "queue_wait_ms": 5.328, - "recipe_id": "llama-cpp-quantized-performance-fit", - "repeat": 2, - "text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the", - "ttft_ms": 26.086 - }, - { - "concurrency": 4, - "decode_ms": 366.352, - "decode_tokens": 31, - "error": "", - "latency_ms": 409.4103, - "ok": true, - "prefill_ms": 31.602, + "prefill_ms": 31.614, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 11.4563, + "queue_wait_ms": 10.8324, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 31.602 + "ttft_ms": 31.614 }, { "concurrency": 4, - "decode_ms": 515.907, + "decode_ms": 530.866, "decode_tokens": 31, "error": "", - "latency_ms": 587.374, + "latency_ms": 583.4067, "ok": true, - "prefill_ms": 67.945, + "prefill_ms": 48.478, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 3.522, + "queue_wait_ms": 4.0627, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 67.945 + "ttft_ms": 48.478 }, { "concurrency": 4, - "decode_ms": 515.867, + "decode_ms": 530.849, "decode_tokens": 31, "error": "", - "latency_ms": 587.0382, + "latency_ms": 582.1522, "ok": true, - "prefill_ms": 68.255, + "prefill_ms": 48.634, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 2.9162, + "queue_wait_ms": 2.6692, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 68.255 + "ttft_ms": 48.634 }, { "concurrency": 4, - "decode_ms": 515.918, + "decode_ms": 509.052, "decode_tokens": 31, "error": "", - "latency_ms": 586.3544, + "latency_ms": 589.1578, "ok": true, - "prefill_ms": 68.094, + "prefill_ms": 32.537, "prompt_id": "medium-code", "prompt_tokens": 24, - "queue_wait_ms": 2.3424, + "queue_wait_ms": 47.5688, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-", - "ttft_ms": 68.094 + "ttft_ms": 32.537 }, { "concurrency": 4, - "decode_ms": 707.553, + "decode_ms": 340.616, "decode_tokens": 31, "error": "", - "latency_ms": 875.8595, + "latency_ms": 502.6207, "ok": true, - "prefill_ms": 157.452, + "prefill_ms": 155.5, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 10.8545, + "queue_wait_ms": 6.5047, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 157.452 + "ttft_ms": 155.5 }, { "concurrency": 4, - "decode_ms": 478.62, + "decode_ms": 602.847, "decode_tokens": 31, "error": "", - "latency_ms": 784.2201, + "latency_ms": 752.0484, "ok": true, - "prefill_ms": 301.345, + "prefill_ms": 146.657, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 4.2551, + "queue_wait_ms": 2.5444, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 301.345 + "ttft_ms": 146.657 }, { "concurrency": 4, - "decode_ms": 478.621, + "decode_ms": 320.521, "decode_tokens": 31, "error": "", - "latency_ms": 783.3942, + "latency_ms": 761.1155, "ok": true, - "prefill_ms": 301.493, + "prefill_ms": 292.311, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 3.2802, + "queue_wait_ms": 148.2835, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 301.493 + "ttft_ms": 292.311 }, { "concurrency": 4, - "decode_ms": 325.451, + "decode_ms": 320.539, "decode_tokens": 31, "error": "", - "latency_ms": 786.8324, + "latency_ms": 759.5872, "ok": true, - "prefill_ms": 158.635, + "prefill_ms": 292.422, "prompt_id": "long-summary", "prompt_tokens": 191, - "queue_wait_ms": 302.7464, + "queue_wait_ms": 146.6262, "recipe_id": "llama-cpp-quantized-performance-fit", "repeat": 2, "text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions", - "ttft_ms": 158.635 + "ttft_ms": 292.422 } ], "recipe": { diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt index 406bfdd..1ff8c23 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt @@ -1,10 +1,10 @@ Recipe benchmark dgr-001-controlled-whole-model-baseline-v1 (local-real) model Qwen/Qwen2.5-0.5B-Instruct@7ae557604adf67be50417f59c2c2f167def9a775 -transformers-safetensors-reference [quality ] c= 1 ttft p50/p95 37.5/ 193.5 ms; prefill 671.8 tok/s; decode 46.1 tok/s; aggregate 40.3 tok/s; rss 1.94 GB; vram 0.00 GB; artifact 1.00 GB; failures 0 -transformers-safetensors-reference [quality ] c= 4 ttft p50/p95 94.4/ 444.7 ms; prefill 247.6 tok/s; decode 13.1 tok/s; aggregate 47.1 tok/s; rss 2.18 GB; vram 0.00 GB; artifact 1.00 GB; failures 0 -llama-cpp-near-lossless-quality [quality ] c= 1 ttft p50/p95 43.9/ 107.0 ms; prefill 1427.2 tok/s; decode 88.0 tok/s; aggregate 73.8 tok/s; rss 1.11 GB; vram 0.00 GB; artifact 0.99 GB; failures 0 -llama-cpp-near-lossless-quality [quality ] c= 4 ttft p50/p95 33.6/ 128.5 ms; prefill 1077.8 tok/s; decode 73.9 tok/s; aggregate 211.4 tok/s; rss 1.13 GB; vram 0.00 GB; artifact 0.99 GB; failures 0 -llama-cpp-quantized-performance-fit [performance-fit ] c= 1 ttft p50/p95 23.8/ 237.7 ms; prefill 783.4 tok/s; decode 170.1 tok/s; aggregate 110.0 tok/s; rss 0.54 GB; vram 0.00 GB; artifact 0.40 GB; failures 0 -llama-cpp-quantized-performance-fit [performance-fit ] c= 4 ttft p50/p95 67.9/ 431.8 ms; prefill 474.3 tok/s; decode 83.5 tok/s; aggregate 206.4 tok/s; rss 0.57 GB; vram 0.00 GB; artifact 0.40 GB; failures 0 +transformers-safetensors-reference [quality ] c= 1 ttft p50/p95 40.0/ 195.3 ms; prefill 625.6 tok/s; decode 40.8 tok/s; aggregate 35.5 tok/s; rss 1.94 GB; vram 0.00 GB; artifact 1.00 GB; failures 0 +transformers-safetensors-reference [quality ] c= 4 ttft p50/p95 97.0/ 429.1 ms; prefill 264.0 tok/s; decode 13.0 tok/s; aggregate 46.5 tok/s; rss 2.10 GB; vram 0.00 GB; artifact 1.00 GB; failures 0 +llama-cpp-near-lossless-quality [quality ] c= 1 ttft p50/p95 15.1/ 63.8 ms; prefill 1717.9 tok/s; decode 98.5 tok/s; aggregate 86.7 tok/s; rss 1.11 GB; vram 0.00 GB; artifact 0.99 GB; failures 0 +llama-cpp-near-lossless-quality [quality ] c= 4 ttft p50/p95 32.4/ 218.4 ms; prefill 859.9 tok/s; decode 76.6 tok/s; aggregate 222.8 tok/s; rss 1.14 GB; vram 0.00 GB; artifact 0.99 GB; failures 0 +llama-cpp-quantized-performance-fit [performance-fit ] c= 1 ttft p50/p95 21.6/ 147.9 ms; prefill 967.0 tok/s; decode 207.7 tok/s; aggregate 139.3 tok/s; rss 0.54 GB; vram 0.00 GB; artifact 0.40 GB; failures 0 +llama-cpp-quantized-performance-fit [performance-fit ] c= 4 ttft p50/p95 48.1/ 416.5 ms; prefill 572.4 tok/s; decode 76.9 tok/s; aggregate 195.7 tok/s; rss 0.57 GB; vram 0.00 GB; artifact 0.40 GB; failures 0 drift llama-cpp-near-lossless-quality vs transformers-safetensors-reference exact 0.33; similarity 0.947 (gated) drift llama-cpp-quantized-performance-fit vs transformers-safetensors-reference exact 0.00; similarity 0.456 (advisory) diff --git a/packages/node/meshnet_node/performance_contract.py b/packages/node/meshnet_node/performance_contract.py index efa1b41..4caabc8 100644 --- a/packages/node/meshnet_node/performance_contract.py +++ b/packages/node/meshnet_node/performance_contract.py @@ -25,16 +25,23 @@ the release gate is allowed to reach. from __future__ import annotations +import base64 +import binascii import hashlib import json from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Mapping +from cryptography.exceptions import InvalidSignature +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + from .recipe_benchmark import Lane, REPORT_SCHEMA_VERSION # Layout of the contract document understood by this reader. CONTRACT_SCHEMA_VERSION = 1 +PROVENANCE_SCHEMA_VERSION = 1 +REAL_REPORT_PRODUCER = "meshnet_node.recipe_drivers.run_configured_benchmark/v1" VERDICT_PROMOTE = "promote" VERDICT_OPTIMIZE = "optimize" @@ -146,6 +153,66 @@ def _canonical_sha256(value: Any) -> str: return hashlib.sha256(payload.encode("utf-8")).hexdigest() +def report_signing_payload(report: Mapping[str, Any]) -> bytes: + """Canonical report bytes covered by the Ed25519 signature.""" + provenance = report.get("provenance") + if not isinstance(provenance, Mapping): + raise PerformanceContractError("real benchmark report lacks signed provenance") + unsigned = dict(report) + unsigned_provenance = dict(provenance) + unsigned_provenance.pop("signature", None) + unsigned["provenance"] = unsigned_provenance + return json.dumps( + unsigned, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + + +def _decode_base64(value: Any, label: str) -> bytes: + if not isinstance(value, str): + raise PerformanceContractError(f"report lacks {label}") + try: + return base64.b64decode(value, validate=True) + except (binascii.Error, ValueError) as exc: + raise PerformanceContractError(f"report carries invalid {label}") from exc + + +def _verify_real_provenance( + contract: PerformanceContract, report: Mapping[str, Any] +) -> None: + provenance = report.get("provenance") + if not isinstance(provenance, Mapping): + raise PerformanceContractError("real benchmark report lacks signed provenance") + if provenance.get("schema_version") != PROVENANCE_SCHEMA_VERSION: + raise PerformanceContractError("report provenance schema is unsupported") + if provenance.get("producer") != REAL_REPORT_PRODUCER: + raise PerformanceContractError("report was not emitted by the canonical real runner") + if provenance.get("signature_algorithm") != "ed25519": + raise PerformanceContractError("report provenance is not Ed25519 signed") + for field in ("run_id", "started_at", "completed_at"): + if not provenance.get(field): + raise PerformanceContractError(f"report provenance lacks {field}") + + required_config = contract.baseline.get("required_config_sha256") + if not required_config or provenance.get("config_sha256") != required_config: + raise PerformanceContractError("report config digest does not match the locked config") + + encoded_public_key = contract.baseline.get("required_signer_public_key") + public_key_bytes = _decode_base64(encoded_public_key, "locked signer public key") + if len(public_key_bytes) != 32: + raise PerformanceContractError("locked Ed25519 public key must be 32 bytes") + expected_fingerprint = hashlib.sha256(public_key_bytes).hexdigest() + if provenance.get("signer_public_key_sha256") != expected_fingerprint: + raise PerformanceContractError("report signer fingerprint does not match the contract") + + signature = _decode_base64(provenance.get("signature"), "Ed25519 signature") + try: + Ed25519PublicKey.from_public_bytes(public_key_bytes).verify( + signature, report_signing_payload(report) + ) + except (InvalidSignature, ValueError) as exc: + raise PerformanceContractError("report Ed25519 signature verification failed") from exc + + def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -> None: """Fail closed when a report is not the experiment the contract locked.""" try: @@ -187,6 +254,8 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) - raise PerformanceContractError( f"report evidence class {evidence_class!r} does not satisfy {required_evidence!r}" ) + if evidence_class in {"local-real", "multi-machine-real"}: + _verify_real_provenance(contract, report) if required_evidence and ( not isinstance(host, Mapping) or any(key not in host for key in ("hostname", "platform", "python", "cpu_count")) @@ -221,8 +290,18 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) - model_id = plan.get("model_id") model_revision = plan.get("model_revision") required_device = contract.baseline.get("required_device") + required_artifacts = dict(contract.baseline.get("required_artifact_sha256") or {}) + required_runtimes = dict(contract.baseline.get("required_recipe_runtime") or {}) + required_backends = dict(contract.baseline.get("required_backend_detail") or {}) + required_host = dict(contract.baseline.get("required_host_identity") or {}) + for field, expected in required_host.items(): + if host.get(field) != expected: + raise PerformanceContractError( + f"report host/runtime field {field!r} does not match the locked identity" + ) for entry in recipes: recipe = entry.get("recipe", {}) + recipe_id = recipe.get("id") if required_device and recipe.get("device") != required_device: raise PerformanceContractError( f"recipe {recipe.get('id')!r} did not run on locked device {required_device!r}" @@ -235,6 +314,30 @@ def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) - digest = recipe.get("artifact_sha256", "") if not isinstance(digest, str) or len(digest) != 64: raise PerformanceContractError("report lacks an artifact SHA-256 digest") + expected_digest = required_artifacts.get(recipe_id) + if not expected_digest or digest != expected_digest: + raise PerformanceContractError( + f"recipe {recipe_id!r} artifact digest does not match the contract" + ) + expected_runtime = required_runtimes.get(recipe_id) + if not isinstance(expected_runtime, Mapping): + raise PerformanceContractError( + f"contract lacks runtime identity for recipe {recipe_id!r}" + ) + actual_runtime = { + field: recipe.get(field) for field in expected_runtime + } + if actual_runtime != dict(expected_runtime): + raise PerformanceContractError( + f"recipe {recipe_id!r} runtime identity does not match the contract" + ) + if entry.get("available"): + expected_backend = required_backends.get(recipe_id) + actual_backend = entry.get("load", {}).get("backend_detail") + if not expected_backend or actual_backend != expected_backend: + raise PerformanceContractError( + f"recipe {recipe_id!r} backend identity does not match the contract" + ) if entry.get("available"): cells = entry.get("concurrency", {}) missing_cells = required_levels - {int(level) for level in cells} @@ -666,6 +769,23 @@ def baseline_from_report(report: Mapping[str, Any]) -> dict[str, Any]: "plan_sha256": _canonical_sha256(report["plan"]), "reference_recipe_id": report["reference_recipe_id"], "host": report["host"], + "provenance": dict(report.get("provenance") or {}), + "artifact_sha256": { + recipe_id: entry["recipe"]["artifact_sha256"] + for recipe_id, entry in entries.items() + }, + "recipe_runtime": { + recipe_id: { + field: entry["recipe"].get(field) + for field in ("runtime", "weight_format", "weight_quantization", "device") + } + for recipe_id, entry in entries.items() + }, + "backend_detail": { + recipe_id: entry.get("load", {}).get("backend_detail") + for recipe_id, entry in entries.items() + if entry.get("available") + }, "recipes": {}, } for recipe_id, entry in entries.items(): diff --git a/packages/node/meshnet_node/recipe_benchmark.py b/packages/node/meshnet_node/recipe_benchmark.py index b2bbbc9..4a1c287 100644 --- a/packages/node/meshnet_node/recipe_benchmark.py +++ b/packages/node/meshnet_node/recipe_benchmark.py @@ -35,7 +35,7 @@ from dataclasses import asdict, dataclass, field from difflib import SequenceMatcher from enum import Enum from pathlib import Path -from typing import Any, Protocol, Sequence +from typing import Any, Mapping, Protocol, Sequence # Layout of the report document produced by :func:`build_report`. REPORT_SCHEMA_VERSION = 1 @@ -581,6 +581,7 @@ def build_report( *, host: dict[str, Any], evidence_class: str, + provenance: Mapping[str, Any] | None = None, ) -> dict: """Assemble the machine-readable benchmark document. @@ -590,6 +591,8 @@ def build_report( """ if evidence_class not in {"synthetic", "local-real", "multi-machine-real"}: raise BenchmarkError(f"unknown evidence class {evidence_class!r}") + if evidence_class != "synthetic" and not isinstance(provenance, Mapping): + raise BenchmarkError("non-synthetic reports require canonical signed provenance") references = [m for m in measurements if m.recipe.is_reference] if len(references) != 1: @@ -605,7 +608,7 @@ def build_report( for measurement in measurements if measurement is not reference and measurement.available ] - return { + report = { "schema_version": REPORT_SCHEMA_VERSION, "evidence_class": evidence_class, "plan": plan.to_dict(), @@ -614,6 +617,9 @@ def build_report( "recipes": [measurement.to_dict() for measurement in measurements], "drift": drift, } + if provenance is not None: + report["provenance"] = dict(provenance) + return report def format_summary(report: dict) -> str: diff --git a/packages/node/meshnet_node/recipe_drivers.py b/packages/node/meshnet_node/recipe_drivers.py index 9265bb7..94fe9cb 100644 --- a/packages/node/meshnet_node/recipe_drivers.py +++ b/packages/node/meshnet_node/recipe_drivers.py @@ -20,6 +20,7 @@ rules: from __future__ import annotations +import base64 import hashlib import hmac import json @@ -27,15 +28,27 @@ import os import platform import re import socket +import stat import subprocess import sys import tempfile import time import urllib.error import urllib.request +import uuid +from datetime import datetime, timezone from pathlib import Path from typing import Any, Mapping +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from .performance_contract import ( + PROVENANCE_SCHEMA_VERSION, + REAL_REPORT_PRODUCER, + _canonical_sha256, + report_signing_payload, +) from .recipe_benchmark import ( BenchmarkError, BenchmarkPlan, @@ -50,6 +63,7 @@ from .recipe_benchmark import ( ) REAL_INFERENCE_ENV = "MESHNET_ENABLE_REAL_INFERENCE_TESTS" +EVIDENCE_SIGNING_KEY_ENV = "MESHNET_EVIDENCE_SIGNING_KEY" def real_inference_enabled() -> bool: @@ -64,6 +78,37 @@ def require_real_inference() -> None: ) +def _load_evidence_signing_key() -> Ed25519PrivateKey: + raw_path = os.environ.get(EVIDENCE_SIGNING_KEY_ENV) + if not raw_path: + raise BenchmarkError( + f"real evidence requires {EVIDENCE_SIGNING_KEY_ENV} to name an Ed25519 private key" + ) + path = Path(raw_path).expanduser().resolve(strict=True) + if os.name != "nt" and stat.S_IMODE(path.stat().st_mode) != 0o600: + raise BenchmarkError("evidence signing key must have mode 0600") + key = serialization.load_pem_private_key(path.read_bytes(), password=None) + if not isinstance(key, Ed25519PrivateKey): + raise BenchmarkError("evidence signing key must be Ed25519") + return key + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _sign_report(report: dict[str, Any], key: Ed25519PrivateKey) -> None: + public_key = key.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + report["provenance"]["signer_public_key_sha256"] = hashlib.sha256( + public_key + ).hexdigest() + report["provenance"]["signature"] = base64.b64encode( + key.sign(report_signing_payload(report)) + ).decode("ascii") + + def _process_rss(pid: int | None = None) -> int: """Resident bytes for a process and its children, or 0 when unobservable.""" try: @@ -120,7 +165,7 @@ def _artifact_sha256(path: Path) -> str: return digest.hexdigest() -def _host_manifest() -> dict[str, Any]: +def _host_manifest(config: Mapping[str, Any] | None = None) -> dict[str, Any]: """Capture non-secret host facts with the report rather than trusting prose.""" manifest: dict[str, Any] = { "hostname": socket.gethostname(), @@ -142,6 +187,30 @@ def _host_manifest() -> dict[str, Any]: ) except ImportError: manifest["torch_version"] = None + + llama_identities: dict[str, dict[str, str]] = {} + for spec in (config or {}).get("recipes", ()): + driver = spec.get("driver", {}) + if driver.get("type") != "llama-cpp-server": + continue + binary = Path(driver["binary"]).resolve(strict=True) + key = str(binary) + if key in llama_identities: + continue + version_result = subprocess.run( + [str(binary), "--version"], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=10, + ) + llama_identities[key] = { + "sha256": _artifact_sha256(binary), + "version": " | ".join(version_result.stdout.strip().splitlines()), + } + if llama_identities: + manifest["llama_server_identities"] = llama_identities return manifest @@ -635,6 +704,13 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict: """ require_real_inference() _validate_config(config) + evidence_class = config.get("evidence_class", "local-real") + if evidence_class not in {"local-real", "multi-machine-real"}: + raise BenchmarkError("canonical real runner cannot emit synthetic evidence") + signing_key = _load_evidence_signing_key() + started_at = _utc_now() + run_id = str(uuid.uuid4()) + config_sha256 = _canonical_sha256(config) plan = _plan_from_config(config) from .recipe_benchmark import RecipeMeasurement # local import keeps the seam obvious @@ -656,9 +732,20 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict: if driver is not None: driver.close() - return build_report( + report = build_report( plan, measurements, - host={**dict(config.get("host", {})), **_host_manifest()}, - evidence_class=config.get("evidence_class", "local-real"), + host={**dict(config.get("host", {})), **_host_manifest(config)}, + evidence_class=evidence_class, + provenance={ + "schema_version": PROVENANCE_SCHEMA_VERSION, + "producer": REAL_REPORT_PRODUCER, + "run_id": run_id, + "started_at": started_at, + "completed_at": _utc_now(), + "config_sha256": config_sha256, + "signature_algorithm": "ed25519", + }, ) + _sign_report(report, signing_key) + return report diff --git a/tests/test_recipe_benchmark.py b/tests/test_recipe_benchmark.py index d5e2a52..9dc6ded 100644 --- a/tests/test_recipe_benchmark.py +++ b/tests/test_recipe_benchmark.py @@ -8,12 +8,18 @@ report. from __future__ import annotations +import base64 import copy +import hashlib import time from pathlib import Path import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey from meshnet_node.performance_contract import ( + PROVENANCE_SCHEMA_VERSION, + REAL_REPORT_PRODUCER, STOP_CONDITION, ContractThresholds, PerformanceContract, @@ -21,6 +27,7 @@ from meshnet_node.performance_contract import ( _canonical_sha256, evaluate_contract, parse_contract, + report_signing_payload, ) from meshnet_node.recipe_drivers import ( _artifact_sha256, @@ -437,6 +444,16 @@ def test_config_rejects_an_artifact_digest_mismatch(tmp_path: Path): _validate_config(config) +_TEST_SIGNING_KEY = Ed25519PrivateKey.generate() + + +def _resign_test_report(report: dict) -> None: + report["provenance"].pop("signature", None) + report["provenance"]["signature"] = base64.b64encode( + _TEST_SIGNING_KEY.sign(report_signing_payload(report)) + ).decode("ascii") + + def _lock_real_report(report: dict) -> PerformanceContract: report["evidence_class"] = "local-real" report["host"] = { @@ -449,7 +466,23 @@ def _lock_real_report(report: dict) -> PerformanceContract: entry["recipe"]["source_model_id"] = report["plan"]["model_id"] entry["recipe"]["source_model_revision"] = report["plan"]["model_revision"] entry["recipe"]["artifact_sha256"] = "a" * 64 - return PerformanceContract( + entry["load"]["backend_detail"] = f"fake-backend-{entry['recipe']['id']}" + + public_key = _TEST_SIGNING_KEY.public_key().public_bytes( + serialization.Encoding.Raw, serialization.PublicFormat.Raw + ) + config_sha256 = "c" * 64 + report["provenance"] = { + "schema_version": PROVENANCE_SCHEMA_VERSION, + "producer": REAL_REPORT_PRODUCER, + "run_id": "test-run-id", + "started_at": "2026-07-13T00:00:00Z", + "completed_at": "2026-07-13T00:01:00Z", + "config_sha256": config_sha256, + "signature_algorithm": "ed25519", + "signer_public_key_sha256": hashlib.sha256(public_key).hexdigest(), + } + contract = PerformanceContract( contract_version=1, locked_at="2026-07-13T00:00:00Z", locked_by="test", @@ -459,9 +492,29 @@ def _lock_real_report(report: dict) -> PerformanceContract: "required_evidence_class": "local-real", "required_recipes": [entry["recipe"]["id"] for entry in report["recipes"]], "required_concurrency_levels": [1, 4], + "required_config_sha256": config_sha256, + "required_signer_public_key": base64.b64encode(public_key).decode("ascii"), + "required_artifact_sha256": { + entry["recipe"]["id"]: entry["recipe"]["artifact_sha256"] + for entry in report["recipes"] + }, + "required_recipe_runtime": { + entry["recipe"]["id"]: { + field: entry["recipe"].get(field) + for field in ("runtime", "weight_format", "weight_quantization", "device") + } + for entry in report["recipes"] + }, + "required_backend_detail": { + entry["recipe"]["id"]: entry["load"]["backend_detail"] + for entry in report["recipes"] + }, + "required_host_identity": {"python": "3.12"}, }, stop_condition="test", ) + _resign_test_report(report) + return contract def test_locked_contract_rejects_synthetic_evidence(): @@ -480,6 +533,67 @@ def test_locked_contract_rejects_synthetic_evidence(): evaluate_contract(contract, report) +def test_non_synthetic_report_requires_canonical_provenance(): + reference = measure_recipe( + FakeDriver(), recipe("ref", Lane.QUALITY, reference=True), plan() + ) + with pytest.raises(BenchmarkError, match="signed provenance"): + build_report( + plan(), [reference], host={}, evidence_class="local-real" + ) + + +def test_signed_real_report_rejects_tampering_and_rebinding(): + texts = {prompt.text: "same greedy answer" for prompt in PROMPTS} + measurements = [ + measure_recipe(FakeDriver(texts=texts), recipe("ref", Lane.QUALITY, reference=True), plan()), + measure_recipe(FakeDriver(texts=texts), recipe("quality", Lane.QUALITY), plan()), + measure_recipe(FakeDriver(texts=texts), recipe("q4", Lane.PERFORMANCE_FIT), plan()), + ] + report = build_report(plan(), measurements, host={}, evidence_class="synthetic") + contract = _lock_real_report(report) + assert evaluate_contract(contract, report).verdict in {"promote", "optimize", "stop"} + + unsigned = copy.deepcopy(report) + unsigned["provenance"].pop("signature") + with pytest.raises(PerformanceContractError, match="signature"): + evaluate_contract(contract, unsigned) + + tampered = copy.deepcopy(report) + tampered["recipes"][0]["recipe"]["artifact_sha256"] = "b" * 64 + with pytest.raises(PerformanceContractError, match="signature verification failed"): + evaluate_contract(contract, tampered) + + rebound_artifact = copy.deepcopy(tampered) + _resign_test_report(rebound_artifact) + with pytest.raises(PerformanceContractError, match="artifact digest"): + evaluate_contract(contract, rebound_artifact) + + rebound_config = copy.deepcopy(report) + rebound_config["provenance"]["config_sha256"] = "d" * 64 + _resign_test_report(rebound_config) + with pytest.raises(PerformanceContractError, match="config digest"): + evaluate_contract(contract, rebound_config) + + rebound_runtime = copy.deepcopy(report) + rebound_runtime["recipes"][1]["recipe"]["runtime"] = "other-runtime" + _resign_test_report(rebound_runtime) + with pytest.raises(PerformanceContractError, match="runtime identity"): + evaluate_contract(contract, rebound_runtime) + + rebound_backend = copy.deepcopy(report) + rebound_backend["recipes"][1]["load"]["backend_detail"] = "other-backend" + _resign_test_report(rebound_backend) + with pytest.raises(PerformanceContractError, match="backend identity"): + evaluate_contract(contract, rebound_backend) + + rebound_host = copy.deepcopy(report) + rebound_host["host"]["python"] = "9.9" + _resign_test_report(rebound_host) + with pytest.raises(PerformanceContractError, match="host/runtime field"): + evaluate_contract(contract, rebound_host) + + def test_quality_lane_requires_every_prompt_to_be_compared(): texts = {prompt.text: "same greedy answer" for prompt in PROMPTS} reference = measure_recipe( @@ -503,6 +617,7 @@ def test_quality_lane_requires_every_prompt_to_be_compared(): ) quality_drift["compared_prompts"] = 1 quality_drift["per_prompt"] = quality_drift["per_prompt"][:1] + _resign_test_report(incomplete) evaluation = evaluate_contract(contract, incomplete) @@ -529,6 +644,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts(): changed_plan = copy.deepcopy(report) changed_plan["plan"]["prompts"][0]["text"] = "changed after locking" + _resign_test_report(changed_plan) with pytest.raises(PerformanceContractError, match="plan digest"): evaluate_contract(contract, changed_plan) @@ -537,6 +653,7 @@ def test_locked_contract_rejects_changed_plan_and_token_counts(): entry for entry in changed_tokens["recipes"] if entry["recipe"]["id"] == "quality" ) quality_entry["outcomes"][0]["decode_tokens"] += 1 + _resign_test_report(changed_tokens) with pytest.raises(PerformanceContractError, match="different prompt/decode token counts"): evaluate_contract(contract, changed_tokens)