popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ec36290863b94a32d9ae78ff7a05d8c0d23a93b5
neuron-tai/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json
Dobromir Popov b1c9deeb01 fix: cryptographically bind DGR-001 evidence
2026-07-13 19:38:14 +03:00

2492 lines
93 KiB
JSON
Raw Blame History

{
"drift": [
{
"advisory": false,
"compared_prompts": 3,
"exact_match_rate": 0.3333,
"lane": "quality",
"mean_similarity": 0.9471,
"per_prompt": [
{
"exact_match": false,
"first_divergence_char": 48,
"prompt_id": "long-summary",
"recipe_text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"reference_text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"similarity": 0.8703
},
{
"exact_match": true,
"first_divergence_char": -1,
"prompt_id": "medium-code",
"recipe_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"reference_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"similarity": 1.0
},
{
"exact_match": false,
"first_divergence_char": 63,
"prompt_id": "short-fact",
"recipe_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"reference_text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"similarity": 0.971
}
],
"recipe_id": "llama-cpp-near-lossless-quality",
"reference_id": "transformers-safetensors-reference"
},
{
"advisory": true,
"compared_prompts": 3,
"exact_match_rate": 0.0,
"lane": "performance-fit",
"mean_similarity": 0.4562,
"per_prompt": [
{
"exact_match": false,
"first_divergence_char": 4,
"prompt_id": "long-summary",
"recipe_text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"reference_text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"similarity": 0.2436
},
{
"exact_match": false,
"first_divergence_char": 4,
"prompt_id": "medium-code",
"recipe_text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"reference_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"similarity": 0.218
},
{
"exact_match": false,
"first_divergence_char": 63,
"prompt_id": "short-fact",
"recipe_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of",
"reference_text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"similarity": 0.9071
}
],
"recipe_id": "llama-cpp-quantized-performance-fit",
"reference_id": "transformers-safetensors-reference"
}
],
"evidence_class": "local-real",
"host": {
"accelerator_name": "Radeon 8060S Graphics",
"accelerator_runtime": "7.13.26183",
"benchmark_lane": "cpu-controlled-baseline",
"converter_sha256": "c819f18fb22927b49fabc3b35d1c9e21ee638b3817eccd1bd4efbcc7116eeb4d",
"cpu_count": 32,
"cuda_available": true,
"hostname": "fedora",
"llama_cpp_commit": "e920c523e3b8a0163fe498af5bf90df35ff51d25",
"llama_cpp_version": "9991",
"llama_server_identities": {
"/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server": {
"sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
"version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64"
}
},
"llama_server_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
"platform": "Linux-7.0.14-101.fc43.x86_64-x86_64-with-glibc2.42",
"python": "3.12.13",
"quantizer_sha256": "bd0cc8c7be6d48aad4755b31062e0e59a887cbadd43dbb8771853d5858bb198f",
"torch_version": "2.10.0+rocm7.13.0a20260513",
"transformers_version": "5.13.0"
},
"plan": {
"concurrency_levels": [
1,
4
],
"model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"plan_id": "dgr-001-controlled-whole-model-baseline-v1",
"prompts": [
{
"context_class": "short",
"id": "short-fact",
"text": "The capital of France is"
},
{
"context_class": "medium",
"id": "medium-code",
"text": "Complete this Python function without commentary:\n\ndef fibonacci(n):\n \"\"\"Return the nth Fibonacci number for n >= 0.\"\"\"\n"
},
{
"context_class": "long",
"id": "long-summary",
"text": "A distributed inference service divides a transformer across consumer machines. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution. Every request carries an immutable model identity and revision. Workers must reject incompatible protocol versions and resource demands before allocating large buffers. Activation tensors are chunked, checksummed, bounded by negotiated limits, and propagated with explicit flow-control credits. A caller may disconnect at any time, so cancellation must release queued work, in-flight transfers, and cache reservations without double billing. Retries can occur after network failures, requiring idempotent request identifiers and deterministic completion accounting. The system keeps the existing safetensors path as a correctness reference while a native GGUF path is measured. Benchmarks compare the same prompts, output lengths, sampling policy, device, and concurrency, and they separate near-lossless quality checks from quantized speed and fit claims. Summarize the design priorities in three concise bullet points."
}
],
"repeats": 3,
"sampling": {
"max_output_tokens": 32,
"seed": 1234,
"temperature": 0.0,
"top_k": 1,
"top_p": 1.0
},
"warmup_requests": 2
},
"provenance": {
"completed_at": "2026-07-13T16:27:19.647692Z",
"config_sha256": "00b2cce3e2f281bdf92fc5304ba5cac915a178ffccd3b9a25995ce39c00b90d3",
"producer": "meshnet_node.recipe_drivers.run_configured_benchmark/v1",
"run_id": "e4eedadf-22f6-4907-8990-985456961099",
"schema_version": 1,
"signature": "owev+/ToswP20C923G6E+srOCUBV5vrjmndVatr9CbTXakiFGqlHrTiEo+aymA4BcSwmG6KJTxlxO6WpLnpcAg==",
"signature_algorithm": "ed25519",
"signer_public_key_sha256": "8baca8742d9b3ed0c3fc54929c23f75ec8c1c739900aaf5334780d598ffa84de",
"started_at": "2026-07-13T16:26:22.361501Z"
},
"recipes": [
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 35.4722,
"concurrency": 1,
"decode_tokens_per_sec": 40.7545,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 818.3864,
"latency_p95_ms": 1258.0673,
"peak_rss_bytes": 1941458944,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 625.6467,
"requests": 9,
"ttft_p50_ms": 40.0018,
"ttft_p95_ms": 195.2551,
"wall_ms": 7865.3174
},
"4": {
"aggregate_decode_tokens_per_sec": 46.5375,
"concurrency": 4,
"decode_tokens_per_sec": 12.9506,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 2481.8662,
"latency_p95_ms": 3365.8395,
"peak_rss_bytes": 2104832000,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 264.0101,
"requests": 36,
"ttft_p50_ms": 97.0403,
"ttft_p95_ms": 429.0665,
"wall_ms": 23980.6838
}
},
"load": {
"artifact_bytes": 999586347,
"backend_detail": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cpu; intra-op threads 16",
"load_ms": 979.2172,
"rss_bytes": 877441024,
"vram_bytes": 0
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 789.1253,
"decode_tokens": 31,
"error": "",
"latency_ms": 818.3864,
"ok": true,
"prefill_ms": 29.2306,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 29.2306
},
{
"concurrency": 1,
"decode_ms": 783.4,
"decode_tokens": 31,
"error": "",
"latency_ms": 831.0575,
"ok": true,
"prefill_ms": 47.6305,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 47.6305
},
{
"concurrency": 1,
"decode_ms": 729.0472,
"decode_tokens": 31,
"error": "",
"latency_ms": 924.3293,
"ok": true,
"prefill_ms": 195.2551,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 195.2551
},
{
"concurrency": 1,
"decode_ms": 724.2915,
"decode_tokens": 31,
"error": "",
"latency_ms": 753.4753,
"ok": true,
"prefill_ms": 29.1565,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 29.1565
},
{
"concurrency": 1,
"decode_ms": 711.203,
"decode_tokens": 31,
"error": "",
"latency_ms": 745.9763,
"ok": true,
"prefill_ms": 34.747,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 34.747
},
{
"concurrency": 1,
"decode_ms": 776.5576,
"decode_tokens": 31,
"error": "",
"latency_ms": 960.6125,
"ok": true,
"prefill_ms": 184.0279,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 184.0279
},
{
"concurrency": 1,
"decode_ms": 642.2951,
"decode_tokens": 31,
"error": "",
"latency_ms": 668.9401,
"ok": true,
"prefill_ms": 26.6176,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 26.6176
},
{
"concurrency": 1,
"decode_ms": 722.4096,
"decode_tokens": 31,
"error": "",
"latency_ms": 762.4393,
"ok": true,
"prefill_ms": 40.0018,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 40.0018
},
{
"concurrency": 1,
"decode_ms": 1109.9513,
"decode_tokens": 31,
"error": "",
"latency_ms": 1258.0673,
"ok": true,
"prefill_ms": 148.0872,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 148.0872
},
{
"concurrency": 4,
"decode_ms": 2337.8807,
"decode_tokens": 31,
"error": "",
"latency_ms": 2500.639,
"ok": true,
"prefill_ms": 162.7268,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 162.7268
},
{
"concurrency": 4,
"decode_ms": 2303.5448,
"decode_tokens": 31,
"error": "",
"latency_ms": 2460.0828,
"ok": true,
"prefill_ms": 156.4924,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 156.4924
},
{
"concurrency": 4,
"decode_ms": 2316.9477,
"decode_tokens": 31,
"error": "",
"latency_ms": 2481.8662,
"ok": true,
"prefill_ms": 164.89,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 164.89
},
{
"concurrency": 4,
"decode_ms": 2306.1901,
"decode_tokens": 31,
"error": "",
"latency_ms": 2482.8428,
"ok": true,
"prefill_ms": 176.6204,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 176.6204
},
{
"concurrency": 4,
"decode_ms": 2294.0499,
"decode_tokens": 31,
"error": "",
"latency_ms": 2367.6722,
"ok": true,
"prefill_ms": 73.5665,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 73.5665
},
{
"concurrency": 4,
"decode_ms": 2310.4442,
"decode_tokens": 31,
"error": "",
"latency_ms": 2407.1392,
"ok": true,
"prefill_ms": 96.6606,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 96.6606
},
{
"concurrency": 4,
"decode_ms": 2311.9951,
"decode_tokens": 31,
"error": "",
"latency_ms": 2401.913,
"ok": true,
"prefill_ms": 89.876,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 89.876
},
{
"concurrency": 4,
"decode_ms": 2321.37,
"decode_tokens": 31,
"error": "",
"latency_ms": 2424.403,
"ok": true,
"prefill_ms": 103.0054,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 103.0054
},
{
"concurrency": 4,
"decode_ms": 2322.0048,
"decode_tokens": 31,
"error": "",
"latency_ms": 2691.1307,
"ok": true,
"prefill_ms": 369.0982,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 369.0982
},
{
"concurrency": 4,
"decode_ms": 2205.5395,
"decode_tokens": 31,
"error": "",
"latency_ms": 2634.641,
"ok": true,
"prefill_ms": 429.0665,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 429.0665
},
{
"concurrency": 4,
"decode_ms": 2196.7617,
"decode_tokens": 31,
"error": "",
"latency_ms": 2615.7086,
"ok": true,
"prefill_ms": 418.917,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 418.917
},
{
"concurrency": 4,
"decode_ms": 2247.9121,
"decode_tokens": 31,
"error": "",
"latency_ms": 2680.2786,
"ok": true,
"prefill_ms": 432.3199,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 432.3199
},
{
"concurrency": 4,
"decode_ms": 2203.407,
"decode_tokens": 31,
"error": "",
"latency_ms": 2267.7641,
"ok": true,
"prefill_ms": 64.3272,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 64.3272
},
{
"concurrency": 4,
"decode_ms": 2184.3202,
"decode_tokens": 31,
"error": "",
"latency_ms": 2248.9142,
"ok": true,
"prefill_ms": 64.5731,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 64.5731
},
{
"concurrency": 4,
"decode_ms": 2187.0349,
"decode_tokens": 31,
"error": "",
"latency_ms": 2257.8805,
"ok": true,
"prefill_ms": 70.8191,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 70.8191
},
{
"concurrency": 4,
"decode_ms": 2197.6307,
"decode_tokens": 31,
"error": "",
"latency_ms": 2268.1916,
"ok": true,
"prefill_ms": 70.5298,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 70.5298
},
{
"concurrency": 4,
"decode_ms": 2219.3714,
"decode_tokens": 31,
"error": "",
"latency_ms": 2299.2901,
"ok": true,
"prefill_ms": 79.8752,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 79.8752
},
{
"concurrency": 4,
"decode_ms": 2194.9725,
"decode_tokens": 31,
"error": "",
"latency_ms": 2283.2617,
"ok": true,
"prefill_ms": 88.2587,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 88.2587
},
{
"concurrency": 4,
"decode_ms": 2205.6012,
"decode_tokens": 31,
"error": "",
"latency_ms": 2294.631,
"ok": true,
"prefill_ms": 89.0024,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 89.0024
},
{
"concurrency": 4,
"decode_ms": 2299.5909,
"decode_tokens": 31,
"error": "",
"latency_ms": 2396.7511,
"ok": true,
"prefill_ms": 97.0403,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 97.0403
},
{
"concurrency": 4,
"decode_ms": 2269.6039,
"decode_tokens": 31,
"error": "",
"latency_ms": 2665.8279,
"ok": true,
"prefill_ms": 396.1888,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 396.1888
},
{
"concurrency": 4,
"decode_ms": 2223.5483,
"decode_tokens": 31,
"error": "",
"latency_ms": 2639.5593,
"ok": true,
"prefill_ms": 415.9796,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 415.9796
},
{
"concurrency": 4,
"decode_ms": 2230.8398,
"decode_tokens": 31,
"error": "",
"latency_ms": 2657.5963,
"ok": true,
"prefill_ms": 426.6935,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 426.6935
},
{
"concurrency": 4,
"decode_ms": 2221.9724,
"decode_tokens": 31,
"error": "",
"latency_ms": 2608.1412,
"ok": true,
"prefill_ms": 386.1249,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 386.1249
},
{
"concurrency": 4,
"decode_ms": 3091.2788,
"decode_tokens": 31,
"error": "",
"latency_ms": 3159.6612,
"ok": true,
"prefill_ms": 68.3579,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 68.3579
},
{
"concurrency": 4,
"decode_ms": 3053.4716,
"decode_tokens": 31,
"error": "",
"latency_ms": 3128.5477,
"ok": true,
"prefill_ms": 75.0442,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 75.0442
},
{
"concurrency": 4,
"decode_ms": 3080.1085,
"decode_tokens": 31,
"error": "",
"latency_ms": 3154.079,
"ok": true,
"prefill_ms": 73.9445,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 73.9445
},
{
"concurrency": 4,
"decode_ms": 3067.5886,
"decode_tokens": 31,
"error": "",
"latency_ms": 3148.8735,
"ok": true,
"prefill_ms": 81.2505,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the banks of the Se",
"ttft_ms": 81.2505
},
{
"concurrency": 4,
"decode_ms": 2323.7094,
"decode_tokens": 31,
"error": "",
"latency_ms": 2406.7763,
"ok": true,
"prefill_ms": 82.9929,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 82.9929
},
{
"concurrency": 4,
"decode_ms": 2314.5197,
"decode_tokens": 31,
"error": "",
"latency_ms": 2408.2229,
"ok": true,
"prefill_ms": 93.668,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 93.668
},
{
"concurrency": 4,
"decode_ms": 2304.5741,
"decode_tokens": 31,
"error": "",
"latency_ms": 2394.9373,
"ok": true,
"prefill_ms": 90.3384,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 90.3384
},
{
"concurrency": 4,
"decode_ms": 2328.6365,
"decode_tokens": 31,
"error": "",
"latency_ms": 2432.765,
"ok": true,
"prefill_ms": 104.0982,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 104.0982
},
{
"concurrency": 4,
"decode_ms": 2959.1949,
"decode_tokens": 31,
"error": "",
"latency_ms": 3365.8395,
"ok": true,
"prefill_ms": 406.5841,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 406.5841
},
{
"concurrency": 4,
"decode_ms": 2970.6675,
"decode_tokens": 31,
"error": "",
"latency_ms": 3397.0527,
"ok": true,
"prefill_ms": 426.3508,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 426.3508
},
{
"concurrency": 4,
"decode_ms": 2913.6743,
"decode_tokens": 31,
"error": "",
"latency_ms": 3326.1048,
"ok": true,
"prefill_ms": 412.3929,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 412.3929
},
{
"concurrency": 4,
"decode_ms": 2907.1259,
"decode_tokens": 31,
"error": "",
"latency_ms": 3299.2804,
"ok": true,
"prefill_ms": 392.114,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-safetensors-reference",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 392.114
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/safetensor/models/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775",
"artifact_sha256": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6",
"device": "cpu",
"id": "transformers-safetensors-reference",
"is_reference": true,
"lane": "quality",
"notes": "artifact_sha256 is the deterministic digest of every snapshot path and file byte",
"runtime": "transformers-5.13.0",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "safetensors",
"weight_quantization": "bfloat16"
},
"unavailable_reason": ""
},
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 86.7339,
"concurrency": 1,
"decode_tokens_per_sec": 98.5178,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 333.023,
"latency_p95_ms": 383.0597,
"peak_rss_bytes": 1110728704,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 1717.9451,
"requests": 9,
"ttft_p50_ms": 15.069,
"ttft_p95_ms": 63.766,
"wall_ms": 3216.7354
},
"4": {
"aggregate_decode_tokens_per_sec": 222.788,
"concurrency": 4,
"decode_tokens_per_sec": 76.6297,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 490.8738,
"latency_p95_ms": 646.26,
"peak_rss_bytes": 1139466240,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 859.8985,
"requests": 36,
"ttft_p50_ms": 32.445,
"ttft_p95_ms": 218.387,
"wall_ms": 5009.2465
}
},
"load": {
"artifact_bytes": 994156448,
"backend_detail": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0",
"load_ms": 513.098,
"rss_bytes": 1086771200,
"vram_bytes": 0
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 326.476,
"decode_tokens": 31,
"error": "",
"latency_ms": 339.6427,
"ok": true,
"prefill_ms": 11.875,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.2917,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 11.875
},
{
"concurrency": 1,
"decode_ms": 311.923,
"decode_tokens": 31,
"error": "",
"latency_ms": 329.4616,
"ok": true,
"prefill_ms": 16.03,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.5086,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 16.03
},
{
"concurrency": 1,
"decode_ms": 313.864,
"decode_tokens": 31,
"error": "",
"latency_ms": 380.2495,
"ok": true,
"prefill_ms": 63.766,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.6195,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 63.766
},
{
"concurrency": 1,
"decode_ms": 310.351,
"decode_tokens": 31,
"error": "",
"latency_ms": 324.069,
"ok": true,
"prefill_ms": 11.225,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.493,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 11.225
},
{
"concurrency": 1,
"decode_ms": 315.083,
"decode_tokens": 31,
"error": "",
"latency_ms": 331.4476,
"ok": true,
"prefill_ms": 15.069,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.2956,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 15.069
},
{
"concurrency": 1,
"decode_ms": 320.067,
"decode_tokens": 31,
"error": "",
"latency_ms": 383.0597,
"ok": true,
"prefill_ms": 60.269,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.7237,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 60.269
},
{
"concurrency": 1,
"decode_ms": 320.389,
"decode_tokens": 31,
"error": "",
"latency_ms": 333.023,
"ok": true,
"prefill_ms": 11.507,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.127,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 11.507
},
{
"concurrency": 1,
"decode_ms": 305.305,
"decode_tokens": 31,
"error": "",
"latency_ms": 321.8616,
"ok": true,
"prefill_ms": 15.038,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.5186,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 15.038
},
{
"concurrency": 1,
"decode_ms": 309.595,
"decode_tokens": 31,
"error": "",
"latency_ms": 369.7193,
"ok": true,
"prefill_ms": 57.7,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.4243,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 57.7
},
{
"concurrency": 4,
"decode_ms": 346.864,
"decode_tokens": 31,
"error": "",
"latency_ms": 363.9806,
"ok": true,
"prefill_ms": 14.475,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.6416,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 14.475
},
{
"concurrency": 4,
"decode_ms": 346.856,
"decode_tokens": 31,
"error": "",
"latency_ms": 364.0486,
"ok": true,
"prefill_ms": 14.752,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.4406,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 14.752
},
{
"concurrency": 4,
"decode_ms": 346.497,
"decode_tokens": 31,
"error": "",
"latency_ms": 363.7474,
"ok": true,
"prefill_ms": 15.223,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.0274,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 15.223
},
{
"concurrency": 4,
"decode_ms": 346.877,
"decode_tokens": 31,
"error": "",
"latency_ms": 363.1133,
"ok": true,
"prefill_ms": 14.63,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.6063,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 14.63
},
{
"concurrency": 4,
"decode_ms": 352.14,
"decode_tokens": 31,
"error": "",
"latency_ms": 387.0363,
"ok": true,
"prefill_ms": 32.445,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.4513,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 32.445
},
{
"concurrency": 4,
"decode_ms": 352.137,
"decode_tokens": 31,
"error": "",
"latency_ms": 386.453,
"ok": true,
"prefill_ms": 32.612,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.704,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 32.612
},
{
"concurrency": 4,
"decode_ms": 352.121,
"decode_tokens": 31,
"error": "",
"latency_ms": 386.3724,
"ok": true,
"prefill_ms": 32.771,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.4804,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 32.771
},
{
"concurrency": 4,
"decode_ms": 352.101,
"decode_tokens": 31,
"error": "",
"latency_ms": 386.3013,
"ok": true,
"prefill_ms": 32.919,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.2813,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 32.919
},
{
"concurrency": 4,
"decode_ms": 424.326,
"decode_tokens": 31,
"error": "",
"latency_ms": 647.1063,
"ok": true,
"prefill_ms": 218.058,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 4.7223,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 218.058
},
{
"concurrency": 4,
"decode_ms": 424.156,
"decode_tokens": 31,
"error": "",
"latency_ms": 646.26,
"ok": true,
"prefill_ms": 218.631,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.473,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 218.631
},
{
"concurrency": 4,
"decode_ms": 424.365,
"decode_tokens": 31,
"error": "",
"latency_ms": 645.6859,
"ok": true,
"prefill_ms": 218.224,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.0969,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 218.224
},
{
"concurrency": 4,
"decode_ms": 424.305,
"decode_tokens": 31,
"error": "",
"latency_ms": 645.6746,
"ok": true,
"prefill_ms": 218.387,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.9826,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 218.387
},
{
"concurrency": 4,
"decode_ms": 460.812,
"decode_tokens": 31,
"error": "",
"latency_ms": 489.7045,
"ok": true,
"prefill_ms": 26.141,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.7515,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 26.141
},
{
"concurrency": 4,
"decode_ms": 447.676,
"decode_tokens": 31,
"error": "",
"latency_ms": 500.4384,
"ok": true,
"prefill_ms": 28.144,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 24.6184,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 28.144
},
{
"concurrency": 4,
"decode_ms": 447.81,
"decode_tokens": 31,
"error": "",
"latency_ms": 496.1159,
"ok": true,
"prefill_ms": 28.233,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 20.0729,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 28.233
},
{
"concurrency": 4,
"decode_ms": 447.846,
"decode_tokens": 31,
"error": "",
"latency_ms": 490.8738,
"ok": true,
"prefill_ms": 28.369,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 14.6588,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 28.369
},
{
"concurrency": 4,
"decode_ms": 550.9,
"decode_tokens": 31,
"error": "",
"latency_ms": 584.8535,
"ok": true,
"prefill_ms": 29.515,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 4.4385,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 29.515
},
{
"concurrency": 4,
"decode_ms": 515.045,
"decode_tokens": 31,
"error": "",
"latency_ms": 573.8089,
"ok": true,
"prefill_ms": 47.181,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 11.5829,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 47.181
},
{
"concurrency": 4,
"decode_ms": 480.138,
"decode_tokens": 31,
"error": "",
"latency_ms": 569.6531,
"ok": true,
"prefill_ms": 45.676,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 43.8391,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 45.676
},
{
"concurrency": 4,
"decode_ms": 480.022,
"decode_tokens": 31,
"error": "",
"latency_ms": 566.2952,
"ok": true,
"prefill_ms": 45.891,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 40.3822,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 45.891
},
{
"concurrency": 4,
"decode_ms": 532.421,
"decode_tokens": 31,
"error": "",
"latency_ms": 594.8715,
"ok": true,
"prefill_ms": 59.373,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.0775,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 59.373
},
{
"concurrency": 4,
"decode_ms": 365.659,
"decode_tokens": 31,
"error": "",
"latency_ms": 575.7742,
"ok": true,
"prefill_ms": 178.428,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 31.6872,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 178.428
},
{
"concurrency": 4,
"decode_ms": 365.658,
"decode_tokens": 31,
"error": "",
"latency_ms": 571.4574,
"ok": true,
"prefill_ms": 178.58,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 27.2194,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 178.58
},
{
"concurrency": 4,
"decode_ms": 365.623,
"decode_tokens": 31,
"error": "",
"latency_ms": 570.8456,
"ok": true,
"prefill_ms": 178.746,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 26.4766,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 178.746
},
{
"concurrency": 4,
"decode_ms": 370.443,
"decode_tokens": 31,
"error": "",
"latency_ms": 383.5684,
"ok": true,
"prefill_ms": 11.612,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.5134,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 11.612
},
{
"concurrency": 4,
"decode_ms": 362.394,
"decode_tokens": 31,
"error": "",
"latency_ms": 386.4942,
"ok": true,
"prefill_ms": 21.305,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.7952,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 21.305
},
{
"concurrency": 4,
"decode_ms": 350.516,
"decode_tokens": 31,
"error": "",
"latency_ms": 395.4336,
"ok": true,
"prefill_ms": 23.0,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 21.9176,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 23.0
},
{
"concurrency": 4,
"decode_ms": 350.53,
"decode_tokens": 31,
"error": "",
"latency_ms": 393.0663,
"ok": true,
"prefill_ms": 23.11,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 19.4263,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 23.11
},
{
"concurrency": 4,
"decode_ms": 374.767,
"decode_tokens": 31,
"error": "",
"latency_ms": 391.7625,
"ok": true,
"prefill_ms": 14.813,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.1825,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 14.813
},
{
"concurrency": 4,
"decode_ms": 431.621,
"decode_tokens": 31,
"error": "",
"latency_ms": 465.0976,
"ok": true,
"prefill_ms": 25.343,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 8.1336,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 25.343
},
{
"concurrency": 4,
"decode_ms": 501.279,
"decode_tokens": 31,
"error": "",
"latency_ms": 543.2184,
"ok": true,
"prefill_ms": 31.046,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 10.8934,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 31.046
},
{
"concurrency": 4,
"decode_ms": 501.273,
"decode_tokens": 31,
"error": "",
"latency_ms": 542.1594,
"ok": true,
"prefill_ms": 31.193,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 9.6934,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 31.193
},
{
"concurrency": 4,
"decode_ms": 549.266,
"decode_tokens": 31,
"error": "",
"latency_ms": 628.2905,
"ok": true,
"prefill_ms": 69.487,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 9.5375,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 69.487
},
{
"concurrency": 4,
"decode_ms": 459.836,
"decode_tokens": 31,
"error": "",
"latency_ms": 550.1822,
"ok": true,
"prefill_ms": 79.286,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 11.0602,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 79.286
},
{
"concurrency": 4,
"decode_ms": 353.562,
"decode_tokens": 31,
"error": "",
"latency_ms": 478.789,
"ok": true,
"prefill_ms": 116.484,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 8.743,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 116.484
},
{
"concurrency": 4,
"decode_ms": 353.539,
"decode_tokens": 31,
"error": "",
"latency_ms": 478.1728,
"ok": true,
"prefill_ms": 116.378,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 8.2558,
"recipe_id": "llama-cpp-near-lossless-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 116.378
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-BF16.gguf",
"artifact_sha256": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862",
"device": "cpu",
"id": "llama-cpp-near-lossless-quality",
"is_reference": false,
"lane": "quality",
"notes": "Converted directly from the exact mounted safetensors revision while preserving BF16 weights with pinned llama.cpp",
"runtime": "llama.cpp-9991-e920c523",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "gguf",
"weight_quantization": "bfloat16"
},
"unavailable_reason": ""
},
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 139.2693,
"concurrency": 1,
"decode_tokens_per_sec": 207.712,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 168.3307,
"latency_p95_ms": 305.1338,
"peak_rss_bytes": 542081024,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 967.0195,
"requests": 9,
"ttft_p50_ms": 21.582,
"ttft_p95_ms": 147.859,
"wall_ms": 2003.3134
},
"4": {
"aggregate_decode_tokens_per_sec": 195.6789,
"concurrency": 4,
"decode_tokens_per_sec": 76.9497,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 437.9196,
"latency_p95_ms": 885.5355,
"peak_rss_bytes": 573259776,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 572.4424,
"requests": 36,
"ttft_p50_ms": 48.127,
"ttft_p95_ms": 416.531,
"wall_ms": 5703.2207
}
},
"load": {
"artifact_bytes": 397807520,
"backend_detail": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd; threads 16; parallel slots 4; ctx/slot 512; gpu layers 0",
"load_ms": 504.9034,
"rss_bytes": 519925760,
"vram_bytes": 0
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 154.112,
"decode_tokens": 31,
"error": "",
"latency_ms": 166.3913,
"ok": true,
"prefill_ms": 11.143,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.1363,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of",
"ttft_ms": 11.143
},
{
"concurrency": 1,
"decode_ms": 160.212,
"decode_tokens": 31,
"error": "",
"latency_ms": 185.7731,
"ok": true,
"prefill_ms": 24.1,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.4611,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 24.1
},
{
"concurrency": 1,
"decode_ms": 154.87,
"decode_tokens": 31,
"error": "",
"latency_ms": 305.1338,
"ok": true,
"prefill_ms": 147.859,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.4048,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 147.859
},
{
"concurrency": 1,
"decode_ms": 141.334,
"decode_tokens": 31,
"error": "",
"latency_ms": 152.5336,
"ok": true,
"prefill_ms": 9.418,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.7816,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of",
"ttft_ms": 9.418
},
{
"concurrency": 1,
"decode_ms": 141.596,
"decode_tokens": 31,
"error": "",
"latency_ms": 164.0665,
"ok": true,
"prefill_ms": 21.016,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.4545,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 21.016
},
{
"concurrency": 1,
"decode_ms": 152.059,
"decode_tokens": 31,
"error": "",
"latency_ms": 296.5754,
"ok": true,
"prefill_ms": 142.622,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 1.8944,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 142.622
},
{
"concurrency": 1,
"decode_ms": 143.413,
"decode_tokens": 31,
"error": "",
"latency_ms": 155.4718,
"ok": true,
"prefill_ms": 10.774,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.2848,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of",
"ttft_ms": 10.774
},
{
"concurrency": 1,
"decode_ms": 145.251,
"decode_tokens": 31,
"error": "",
"latency_ms": 168.3307,
"ok": true,
"prefill_ms": 21.582,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.4977,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 21.582
},
{
"concurrency": 1,
"decode_ms": 152.806,
"decode_tokens": 31,
"error": "",
"latency_ms": 293.6305,
"ok": true,
"prefill_ms": 138.543,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.2815,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 138.543
},
{
"concurrency": 4,
"decode_ms": 376.127,
"decode_tokens": 31,
"error": "",
"latency_ms": 401.7532,
"ok": true,
"prefill_ms": 23.322,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.3042,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 23.322
},
{
"concurrency": 4,
"decode_ms": 375.793,
"decode_tokens": 31,
"error": "",
"latency_ms": 401.5063,
"ok": true,
"prefill_ms": 23.77,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.9433,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 23.77
},
{
"concurrency": 4,
"decode_ms": 354.788,
"decode_tokens": 31,
"error": "",
"latency_ms": 406.3609,
"ok": true,
"prefill_ms": 26.2,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 25.3729,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 26.2
},
{
"concurrency": 4,
"decode_ms": 376.001,
"decode_tokens": 31,
"error": "",
"latency_ms": 400.5544,
"ok": true,
"prefill_ms": 23.17,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.3834,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 23.17
},
{
"concurrency": 4,
"decode_ms": 405.905,
"decode_tokens": 31,
"error": "",
"latency_ms": 428.1187,
"ok": true,
"prefill_ms": 20.392,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.8217,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 20.392
},
{
"concurrency": 4,
"decode_ms": 354.584,
"decode_tokens": 31,
"error": "",
"latency_ms": 437.9196,
"ok": true,
"prefill_ms": 61.802,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 21.5336,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 61.802
},
{
"concurrency": 4,
"decode_ms": 354.558,
"decode_tokens": 31,
"error": "",
"latency_ms": 437.755,
"ok": true,
"prefill_ms": 61.931,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 21.266,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 61.931
},
{
"concurrency": 4,
"decode_ms": 354.566,
"decode_tokens": 31,
"error": "",
"latency_ms": 435.8597,
"ok": true,
"prefill_ms": 61.676,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 19.6177,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 61.676
},
{
"concurrency": 4,
"decode_ms": 741.39,
"decode_tokens": 31,
"error": "",
"latency_ms": 885.5355,
"ok": true,
"prefill_ms": 141.568,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.5775,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 141.568
},
{
"concurrency": 4,
"decode_ms": 334.866,
"decode_tokens": 31,
"error": "",
"latency_ms": 868.3416,
"ok": true,
"prefill_ms": 416.421,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 117.0546,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 416.421
},
{
"concurrency": 4,
"decode_ms": 334.558,
"decode_tokens": 31,
"error": "",
"latency_ms": 867.4189,
"ok": true,
"prefill_ms": 416.972,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 115.8889,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 416.972
},
{
"concurrency": 4,
"decode_ms": 334.875,
"decode_tokens": 31,
"error": "",
"latency_ms": 866.8735,
"ok": true,
"prefill_ms": 416.531,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 115.4675,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 416.531
},
{
"concurrency": 4,
"decode_ms": 360.293,
"decode_tokens": 31,
"error": "",
"latency_ms": 373.9246,
"ok": true,
"prefill_ms": 11.202,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.4296,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 11.202
},
{
"concurrency": 4,
"decode_ms": 387.093,
"decode_tokens": 31,
"error": "",
"latency_ms": 412.5291,
"ok": true,
"prefill_ms": 21.333,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 4.1031,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 21.333
},
{
"concurrency": 4,
"decode_ms": 387.107,
"decode_tokens": 31,
"error": "",
"latency_ms": 411.9945,
"ok": true,
"prefill_ms": 21.211,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 3.6765,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 21.211
},
{
"concurrency": 4,
"decode_ms": 376.091,
"decode_tokens": 31,
"error": "",
"latency_ms": 417.7572,
"ok": true,
"prefill_ms": 21.212,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 20.4542,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 21.212
},
{
"concurrency": 4,
"decode_ms": 389.488,
"decode_tokens": 31,
"error": "",
"latency_ms": 431.0462,
"ok": true,
"prefill_ms": 30.89,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 10.6682,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 30.89
},
{
"concurrency": 4,
"decode_ms": 522.548,
"decode_tokens": 31,
"error": "",
"latency_ms": 573.2773,
"ok": true,
"prefill_ms": 48.127,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.6023,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 48.127
},
{
"concurrency": 4,
"decode_ms": 522.512,
"decode_tokens": 31,
"error": "",
"latency_ms": 572.773,
"ok": true,
"prefill_ms": 48.289,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.972,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 48.289
},
{
"concurrency": 4,
"decode_ms": 500.617,
"decode_tokens": 31,
"error": "",
"latency_ms": 576.4582,
"ok": true,
"prefill_ms": 32.565,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 43.2762,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 32.565
},
{
"concurrency": 4,
"decode_ms": 774.96,
"decode_tokens": 31,
"error": "",
"latency_ms": 938.0479,
"ok": true,
"prefill_ms": 155.553,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 7.5349,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 155.553
},
{
"concurrency": 4,
"decode_ms": 524.937,
"decode_tokens": 31,
"error": "",
"latency_ms": 826.886,
"ok": true,
"prefill_ms": 294.908,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 7.041,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 294.908
},
{
"concurrency": 4,
"decode_ms": 524.939,
"decode_tokens": 31,
"error": "",
"latency_ms": 826.2408,
"ok": true,
"prefill_ms": 294.79,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 6.5118,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 294.79
},
{
"concurrency": 4,
"decode_ms": 379.586,
"decode_tokens": 31,
"error": "",
"latency_ms": 827.0821,
"ok": true,
"prefill_ms": 155.842,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 291.6541,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 155.842
},
{
"concurrency": 4,
"decode_ms": 306.002,
"decode_tokens": 31,
"error": "",
"latency_ms": 330.1262,
"ok": true,
"prefill_ms": 20.307,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 3.8172,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the western coast of",
"ttft_ms": 20.307
},
{
"concurrency": 4,
"decode_ms": 388.127,
"decode_tokens": 31,
"error": "",
"latency_ms": 414.4426,
"ok": true,
"prefill_ms": 20.412,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 5.9036,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 20.412
},
{
"concurrency": 4,
"decode_ms": 388.131,
"decode_tokens": 31,
"error": "",
"latency_ms": 413.676,
"ok": true,
"prefill_ms": 20.524,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 5.021,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the left bank of the Seine River, in the",
"ttft_ms": 20.524
},
{
"concurrency": 4,
"decode_ms": 377.834,
"decode_tokens": 31,
"error": "",
"latency_ms": 417.606,
"ok": true,
"prefill_ms": 20.665,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 19.107,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located on the Seine River, which flows through the center of",
"ttft_ms": 20.665
},
{
"concurrency": 4,
"decode_ms": 337.163,
"decode_tokens": 31,
"error": "",
"latency_ms": 379.6094,
"ok": true,
"prefill_ms": 31.614,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 10.8324,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 31.614
},
{
"concurrency": 4,
"decode_ms": 530.866,
"decode_tokens": 31,
"error": "",
"latency_ms": 583.4067,
"ok": true,
"prefill_ms": 48.478,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 4.0627,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 48.478
},
{
"concurrency": 4,
"decode_ms": 530.849,
"decode_tokens": 31,
"error": "",
"latency_ms": 582.1522,
"ok": true,
"prefill_ms": 48.634,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.6692,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 48.634
},
{
"concurrency": 4,
"decode_ms": 509.052,
"decode_tokens": 31,
"error": "",
"latency_ms": 589.1578,
"ok": true,
"prefill_ms": 32.537,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 47.5688,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 32.537
},
{
"concurrency": 4,
"decode_ms": 340.616,
"decode_tokens": 31,
"error": "",
"latency_ms": 502.6207,
"ok": true,
"prefill_ms": 155.5,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 6.5047,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 155.5
},
{
"concurrency": 4,
"decode_ms": 602.847,
"decode_tokens": 31,
"error": "",
"latency_ms": 752.0484,
"ok": true,
"prefill_ms": 146.657,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.5444,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 146.657
},
{
"concurrency": 4,
"decode_ms": 320.521,
"decode_tokens": 31,
"error": "",
"latency_ms": 761.1155,
"ok": true,
"prefill_ms": 292.311,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 148.2835,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 292.311
},
{
"concurrency": 4,
"decode_ms": 320.539,
"decode_tokens": 31,
"error": "",
"latency_ms": 759.5872,
"ok": true,
"prefill_ms": 292.422,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 146.6262,
"recipe_id": "llama-cpp-quantized-performance-fit",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 292.422
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-Q4_K_M.gguf",
"artifact_sha256": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5",
"device": "cpu",
"id": "llama-cpp-quantized-performance-fit",
"is_reference": false,
"lane": "performance-fit",
"notes": "Quantized from the exact-revision F16 GGUF with pinned llama-quantize",
"runtime": "llama.cpp-9991-e920c523",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "gguf",
"weight_quantization": "Q4_K_M"
},
"unavailable_reason": ""
}
],
"reference_recipe_id": "transformers-safetensors-reference",
"schema_version": 1
}
Reference in New Issue View Git Blame Copy Permalink