popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ec36290863b94a32d9ae78ff7a05d8c0d23a93b5
neuron-tai/.scratch/distributed-gguf-runtime/evidence/DGR-001/gpu-diagnostic-results.json
Dobromir Popov ef2a9e67e8 feat: add signed ROCm diagnostic lane
2026-07-13 21:24:43 +03:00

3309 lines
124 KiB
JSON
Raw Blame History

{
"drift": [
{
"advisory": false,
"compared_prompts": 3,
"exact_match_rate": 1.0,
"lane": "quality",
"mean_similarity": 1.0,
"per_prompt": [
{
"exact_match": true,
"first_divergence_char": -1,
"prompt_id": "long-summary",
"recipe_text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"reference_text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"similarity": 1.0
},
{
"exact_match": true,
"first_divergence_char": -1,
"prompt_id": "medium-code",
"recipe_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"reference_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"similarity": 1.0
},
{
"exact_match": true,
"first_divergence_char": -1,
"prompt_id": "short-fact",
"recipe_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"reference_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"similarity": 1.0
}
],
"recipe_id": "llama-cpp-bf16-rocm-quality",
"reference_id": "transformers-fp32-rocm-quality-oracle"
},
{
"advisory": true,
"compared_prompts": 3,
"exact_match_rate": 0.3333,
"lane": "performance-fit",
"mean_similarity": 0.946,
"per_prompt": [
{
"exact_match": false,
"first_divergence_char": 48,
"prompt_id": "long-summary",
"recipe_text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"reference_text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"similarity": 0.8703
},
{
"exact_match": true,
"first_divergence_char": -1,
"prompt_id": "medium-code",
"recipe_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"reference_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"similarity": 1.0
},
{
"exact_match": false,
"first_divergence_char": 49,
"prompt_id": "short-fact",
"recipe_text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"reference_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"similarity": 0.9677
}
],
"recipe_id": "transformers-bf16-rocm-throughput",
"reference_id": "transformers-fp32-rocm-quality-oracle"
},
{
"advisory": true,
"compared_prompts": 3,
"exact_match_rate": 0.0,
"lane": "performance-fit",
"mean_similarity": 0.6284,
"per_prompt": [
{
"exact_match": false,
"first_divergence_char": 4,
"prompt_id": "long-summary",
"recipe_text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own model execution. 2. Workers must reject incompatible protocol versions",
"reference_text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"similarity": 0.3009
},
{
"exact_match": false,
"first_divergence_char": 76,
"prompt_id": "medium-code",
"recipe_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop through the Fibonacci sequence up to n\n ",
"reference_text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"similarity": 0.7522
},
{
"exact_match": false,
"first_divergence_char": 96,
"prompt_id": "short-fact",
"recipe_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located on the left bank of the Seine River, in",
"reference_text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"similarity": 0.8321
}
],
"recipe_id": "llama-cpp-q4-rocm-throughput",
"reference_id": "transformers-fp32-rocm-quality-oracle"
}
],
"evidence_class": "local-real",
"host": {
"accelerator_name": "Radeon 8060S Graphics",
"accelerator_runtime": "7.13.26183",
"benchmark_lane": "rocm-gpu-diagnostic",
"converter_sha256": "c819f18fb22927b49fabc3b35d1c9e21ee638b3817eccd1bd4efbcc7116eeb4d",
"cpu_count": 32,
"cuda_available": true,
"hostname": "fedora",
"llama_cpp_commit": "e920c523e3b8a0163fe498af5bf90df35ff51d25",
"llama_cpp_version": "9991",
"llama_server_identities": {
"/run/media/popov/d/DEV/llamacpp/llama.cpp/build-hip/bin/llama-server": {
"sha256": "b6bb4da687dbde86e243ba006cef05919b7b97255cd7e2371e1d451220aca139",
"version": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64"
}
},
"llama_server_sha256": "b6bb4da687dbde86e243ba006cef05919b7b97255cd7e2371e1d451220aca139",
"platform": "Linux-7.0.14-101.fc43.x86_64-x86_64-with-glibc2.42",
"python": "3.12.13",
"quantizer_sha256": "bd0cc8c7be6d48aad4755b31062e0e59a887cbadd43dbb8771853d5858bb198f",
"rocm_target": "gfx1151",
"torch_version": "2.10.0+rocm7.13.0a20260513",
"transformers_version": "5.13.0"
},
"plan": {
"concurrency_levels": [
1,
4
],
"model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"plan_id": "dgr-001-rocm-gpu-diagnostic-v1",
"prompts": [
{
"context_class": "short",
"id": "short-fact",
"text": "The capital of France is"
},
{
"context_class": "medium",
"id": "medium-code",
"text": "Complete this Python function without commentary:\n\ndef fibonacci(n):\n \"\"\"Return the nth Fibonacci number for n >= 0.\"\"\"\n"
},
{
"context_class": "long",
"id": "long-summary",
"text": "A distributed inference service divides a transformer across consumer machines. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution. Every request carries an immutable model identity and revision. Workers must reject incompatible protocol versions and resource demands before allocating large buffers. Activation tensors are chunked, checksummed, bounded by negotiated limits, and propagated with explicit flow-control credits. A caller may disconnect at any time, so cancellation must release queued work, in-flight transfers, and cache reservations without double billing. Retries can occur after network failures, requiring idempotent request identifiers and deterministic completion accounting. The system keeps the existing safetensors path as a correctness reference while a native GGUF path is measured. Benchmarks compare the same prompts, output lengths, sampling policy, device, and concurrency, and they separate near-lossless quality checks from quantized speed and fit claims. Summarize the design priorities in three concise bullet points."
}
],
"repeats": 3,
"sampling": {
"max_output_tokens": 32,
"seed": 1234,
"temperature": 0.0,
"top_k": 1,
"top_p": 1.0
},
"warmup_requests": 2
},
"provenance": {
"completed_at": "2026-07-13T17:40:44.155783Z",
"config_sha256": "b0f0c846c818f1307d034cee1f81daa311efc20985c32a4cdbbbd8ffe4153892",
"producer": "meshnet_node.recipe_drivers.run_configured_gpu_diagnostic/v1",
"run_id": "31bf44e7-ccd4-4277-84ac-c775dee65411",
"schema_version": 1,
"signature": "tkVeQzRV3Rh4jqfL5SRMZQhZ/WRSU5dE341KFqyB3NVTUTY6JSNHn/SwiBjbns2hDncirKFQMrvedplWCUP3Ag==",
"signature_algorithm": "ed25519",
"signer_public_key_sha256": "8baca8742d9b3ed0c3fc54929c23f75ec8c1c739900aaf5334780d598ffa84de",
"started_at": "2026-07-13T17:39:20.723216Z"
},
"recipes": [
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 29.6436,
"concurrency": 1,
"decode_tokens_per_sec": 35.712,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 937.52,
"latency_p95_ms": 1897.187,
"peak_rss_bytes": 1387806720,
"peak_vram_bytes": 2259130880,
"prefill_tokens_per_sec": 5746.6776,
"requests": 9,
"ttft_p50_ms": 11.0406,
"ttft_p95_ms": 35.5268,
"wall_ms": 9411.8232
},
"4": {
"aggregate_decode_tokens_per_sec": 35.4047,
"concurrency": 4,
"decode_tokens_per_sec": 9.3981,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 3367.1392,
"latency_p95_ms": 3694.6278,
"peak_rss_bytes": 1392402432,
"peak_vram_bytes": 2739758080,
"prefill_tokens_per_sec": 1985.4288,
"requests": 36,
"ttft_p50_ms": 27.4656,
"ttft_p95_ms": 80.4033,
"wall_ms": 31521.2591
}
},
"load": {
"artifact_bytes": 999586347,
"backend_detail": "torch 2.10.0+rocm7.13.0a20260513; dtype float32; device cuda; intra-op threads 16",
"load_ms": 1770.5108,
"rss_bytes": 1113800704,
"vram_bytes": 1976852992
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 1028.0642,
"decode_tokens": 31,
"error": "",
"latency_ms": 1080.5596,
"ok": true,
"prefill_ms": 34.4626,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 34.4626
},
{
"concurrency": 1,
"decode_ms": 862.3916,
"decode_tokens": 31,
"error": "",
"latency_ms": 1897.187,
"ok": true,
"prefill_ms": 35.5268,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 35.5268
},
{
"concurrency": 1,
"decode_ms": 909.6798,
"decode_tokens": 31,
"error": "",
"latency_ms": 996.1667,
"ok": true,
"prefill_ms": 19.6401,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 19.6401
},
{
"concurrency": 1,
"decode_ms": 818.6889,
"decode_tokens": 31,
"error": "",
"latency_ms": 849.625,
"ok": true,
"prefill_ms": 13.0604,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 13.0604
},
{
"concurrency": 1,
"decode_ms": 837.5539,
"decode_tokens": 31,
"error": "",
"latency_ms": 871.8977,
"ok": true,
"prefill_ms": 10.9147,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 10.9147
},
{
"concurrency": 1,
"decode_ms": 862.5615,
"decode_tokens": 31,
"error": "",
"latency_ms": 948.1076,
"ok": true,
"prefill_ms": 10.743,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 10.743
},
{
"concurrency": 1,
"decode_ms": 832.4664,
"decode_tokens": 31,
"error": "",
"latency_ms": 858.2611,
"ok": true,
"prefill_ms": 11.0406,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 11.0406
},
{
"concurrency": 1,
"decode_ms": 841.929,
"decode_tokens": 31,
"error": "",
"latency_ms": 875.9586,
"ok": true,
"prefill_ms": 10.0776,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 10.0776
},
{
"concurrency": 1,
"decode_ms": 852.7305,
"decode_tokens": 31,
"error": "",
"latency_ms": 937.52,
"ok": true,
"prefill_ms": 10.6233,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 10.6233
},
{
"concurrency": 4,
"decode_ms": 3306.7006,
"decode_tokens": 31,
"error": "",
"latency_ms": 3368.8742,
"ok": true,
"prefill_ms": 32.7682,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 32.7682
},
{
"concurrency": 4,
"decode_ms": 3278.8711,
"decode_tokens": 31,
"error": "",
"latency_ms": 3314.7711,
"ok": true,
"prefill_ms": 18.0986,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 18.0986
},
{
"concurrency": 4,
"decode_ms": 3282.5036,
"decode_tokens": 31,
"error": "",
"latency_ms": 3421.0453,
"ok": true,
"prefill_ms": 58.6108,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 58.6108
},
{
"concurrency": 4,
"decode_ms": 3300.8507,
"decode_tokens": 31,
"error": "",
"latency_ms": 3392.412,
"ok": true,
"prefill_ms": 48.6384,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 48.6384
},
{
"concurrency": 4,
"decode_ms": 3239.8849,
"decode_tokens": 31,
"error": "",
"latency_ms": 3299.947,
"ok": true,
"prefill_ms": 19.2849,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 19.2849
},
{
"concurrency": 4,
"decode_ms": 3234.3934,
"decode_tokens": 31,
"error": "",
"latency_ms": 3297.3359,
"ok": true,
"prefill_ms": 15.9917,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 15.9917
},
{
"concurrency": 4,
"decode_ms": 3255.6689,
"decode_tokens": 31,
"error": "",
"latency_ms": 3336.7213,
"ok": true,
"prefill_ms": 13.3303,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 13.3303
},
{
"concurrency": 4,
"decode_ms": 3196.076,
"decode_tokens": 31,
"error": "",
"latency_ms": 3307.8618,
"ok": true,
"prefill_ms": 14.4342,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 14.4342
},
{
"concurrency": 4,
"decode_ms": 3504.488,
"decode_tokens": 31,
"error": "",
"latency_ms": 3694.6278,
"ok": true,
"prefill_ms": 57.1251,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 57.1251
},
{
"concurrency": 4,
"decode_ms": 3474.5935,
"decode_tokens": 31,
"error": "",
"latency_ms": 3610.3597,
"ok": true,
"prefill_ms": 48.645,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 48.645
},
{
"concurrency": 4,
"decode_ms": 3420.9734,
"decode_tokens": 31,
"error": "",
"latency_ms": 3587.5118,
"ok": true,
"prefill_ms": 51.048,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 51.048
},
{
"concurrency": 4,
"decode_ms": 3335.9916,
"decode_tokens": 31,
"error": "",
"latency_ms": 3561.3291,
"ok": true,
"prefill_ms": 50.4607,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 50.4607
},
{
"concurrency": 4,
"decode_ms": 3185.6964,
"decode_tokens": 31,
"error": "",
"latency_ms": 3268.4157,
"ok": true,
"prefill_ms": 17.3899,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 17.3899
},
{
"concurrency": 4,
"decode_ms": 3216.7168,
"decode_tokens": 31,
"error": "",
"latency_ms": 3293.729,
"ok": true,
"prefill_ms": 21.8903,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 21.8903
},
{
"concurrency": 4,
"decode_ms": 3260.4239,
"decode_tokens": 31,
"error": "",
"latency_ms": 3367.1392,
"ok": true,
"prefill_ms": 26.3922,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 26.3922
},
{
"concurrency": 4,
"decode_ms": 3230.998,
"decode_tokens": 31,
"error": "",
"latency_ms": 3323.0277,
"ok": true,
"prefill_ms": 23.8221,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 23.8221
},
{
"concurrency": 4,
"decode_ms": 3268.9867,
"decode_tokens": 31,
"error": "",
"latency_ms": 3385.007,
"ok": true,
"prefill_ms": 14.3798,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 14.3798
},
{
"concurrency": 4,
"decode_ms": 3223.0252,
"decode_tokens": 31,
"error": "",
"latency_ms": 3298.8571,
"ok": true,
"prefill_ms": 27.0071,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 27.0071
},
{
"concurrency": 4,
"decode_ms": 3227.8985,
"decode_tokens": 31,
"error": "",
"latency_ms": 3323.0323,
"ok": true,
"prefill_ms": 28.4658,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 28.4658
},
{
"concurrency": 4,
"decode_ms": 3240.2344,
"decode_tokens": 31,
"error": "",
"latency_ms": 3351.3501,
"ok": true,
"prefill_ms": 28.7414,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 28.7414
},
{
"concurrency": 4,
"decode_ms": 3513.1364,
"decode_tokens": 31,
"error": "",
"latency_ms": 3654.0715,
"ok": true,
"prefill_ms": 29.3426,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 29.3426
},
{
"concurrency": 4,
"decode_ms": 3482.8109,
"decode_tokens": 31,
"error": "",
"latency_ms": 3680.7907,
"ok": true,
"prefill_ms": 31.205,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 31.205
},
{
"concurrency": 4,
"decode_ms": 3457.6179,
"decode_tokens": 31,
"error": "",
"latency_ms": 3710.027,
"ok": true,
"prefill_ms": 38.6106,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 38.6106
},
{
"concurrency": 4,
"decode_ms": 3352.3972,
"decode_tokens": 31,
"error": "",
"latency_ms": 3548.1681,
"ok": true,
"prefill_ms": 87.5472,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 87.5472
},
{
"concurrency": 4,
"decode_ms": 3252.8471,
"decode_tokens": 31,
"error": "",
"latency_ms": 3363.3887,
"ok": true,
"prefill_ms": 28.5597,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 28.5597
},
{
"concurrency": 4,
"decode_ms": 3229.1287,
"decode_tokens": 31,
"error": "",
"latency_ms": 3311.1931,
"ok": true,
"prefill_ms": 27.3453,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 27.3453
},
{
"concurrency": 4,
"decode_ms": 3236.7523,
"decode_tokens": 31,
"error": "",
"latency_ms": 3334.5212,
"ok": true,
"prefill_ms": 27.5122,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 27.5122
},
{
"concurrency": 4,
"decode_ms": 3292.8793,
"decode_tokens": 31,
"error": "",
"latency_ms": 3401.9064,
"ok": true,
"prefill_ms": 13.3991,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 13.3991
},
{
"concurrency": 4,
"decode_ms": 3235.4429,
"decode_tokens": 31,
"error": "",
"latency_ms": 3328.977,
"ok": true,
"prefill_ms": 27.4656,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 27.4656
},
{
"concurrency": 4,
"decode_ms": 3265.0306,
"decode_tokens": 31,
"error": "",
"latency_ms": 3358.4079,
"ok": true,
"prefill_ms": 27.2853,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 27.2853
},
{
"concurrency": 4,
"decode_ms": 3304.7263,
"decode_tokens": 31,
"error": "",
"latency_ms": 3413.4277,
"ok": true,
"prefill_ms": 27.3694,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 27.3694
},
{
"concurrency": 4,
"decode_ms": 3311.1315,
"decode_tokens": 31,
"error": "",
"latency_ms": 3414.0694,
"ok": true,
"prefill_ms": 13.3615,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 13.3615
},
{
"concurrency": 4,
"decode_ms": 3337.8156,
"decode_tokens": 31,
"error": "",
"latency_ms": 3521.5719,
"ok": true,
"prefill_ms": 29.0131,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 29.0131
},
{
"concurrency": 4,
"decode_ms": 3422.8115,
"decode_tokens": 31,
"error": "",
"latency_ms": 3546.2347,
"ok": true,
"prefill_ms": 24.7999,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 24.7999
},
{
"concurrency": 4,
"decode_ms": 3288.1038,
"decode_tokens": 31,
"error": "",
"latency_ms": 3445.1989,
"ok": true,
"prefill_ms": 80.4033,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 80.4033
},
{
"concurrency": 4,
"decode_ms": 3171.2666,
"decode_tokens": 31,
"error": "",
"latency_ms": 3334.7501,
"ok": true,
"prefill_ms": 64.8781,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-fp32-rocm-quality-oracle",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 64.8781
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/safetensor/models/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775",
"artifact_sha256": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6",
"device": "cuda",
"id": "transformers-fp32-rocm-quality-oracle",
"is_reference": true,
"lane": "quality",
"notes": "artifact_sha256 is the deterministic digest of every snapshot path and file byte",
"runtime": "transformers-5.13.0-rocm-float32",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "safetensors",
"weight_quantization": "bfloat16-weights-float32-accumulation"
},
"unavailable_reason": ""
},
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 127.4267,
"concurrency": 1,
"decode_tokens_per_sec": 147.9836,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 227.9197,
"latency_p95_ms": 294.9053,
"peak_rss_bytes": 838455296,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 4154.4473,
"requests": 9,
"ttft_p50_ms": 13.246,
"ttft_p95_ms": 83.424,
"wall_ms": 2189.4938
},
"4": {
"aggregate_decode_tokens_per_sec": 337.1289,
"concurrency": 4,
"decode_tokens_per_sec": 115.0852,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 305.8661,
"latency_p95_ms": 361.6061,
"peak_rss_bytes": 862527488,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 2205.3889,
"requests": 36,
"ttft_p50_ms": 25.131,
"ttft_p95_ms": 52.056,
"wall_ms": 3310.3068
}
},
"load": {
"artifact_bytes": 994156448,
"backend_detail": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 b6bb4da687dbde86e243ba006cef05919b7b97255cd7e2371e1d451220aca139; threads 16; parallel slots 4; ctx/slot 512; requested gpu layers 99; measured accelerator ROCm0: Radeon 8060S Graphics; measured offload 25/25 layers",
"load_ms": 770.3395,
"rss_bytes": 792096768,
"vram_bytes": 0
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 207.566,
"decode_tokens": 31,
"error": "",
"latency_ms": 217.0811,
"ok": true,
"prefill_ms": 8.172,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.3431,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 8.172
},
{
"concurrency": 1,
"decode_ms": 209.608,
"decode_tokens": 31,
"error": "",
"latency_ms": 294.9053,
"ok": true,
"prefill_ms": 83.424,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.8733,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 83.424
},
{
"concurrency": 1,
"decode_ms": 207.043,
"decode_tokens": 31,
"error": "",
"latency_ms": 228.3594,
"ok": true,
"prefill_ms": 17.983,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.3334,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 17.983
},
{
"concurrency": 1,
"decode_ms": 206.916,
"decode_tokens": 31,
"error": "",
"latency_ms": 217.6378,
"ok": true,
"prefill_ms": 8.096,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.6258,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 8.096
},
{
"concurrency": 1,
"decode_ms": 207.84,
"decode_tokens": 31,
"error": "",
"latency_ms": 222.1197,
"ok": true,
"prefill_ms": 13.246,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.0337,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 13.246
},
{
"concurrency": 1,
"decode_ms": 209.212,
"decode_tokens": 31,
"error": "",
"latency_ms": 230.1617,
"ok": true,
"prefill_ms": 17.722,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.2277,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 17.722
},
{
"concurrency": 1,
"decode_ms": 210.084,
"decode_tokens": 31,
"error": "",
"latency_ms": 220.2312,
"ok": true,
"prefill_ms": 8.085,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.0622,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 8.085
},
{
"concurrency": 1,
"decode_ms": 212.57,
"decode_tokens": 31,
"error": "",
"latency_ms": 227.9197,
"ok": true,
"prefill_ms": 13.158,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.1917,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 13.158
},
{
"concurrency": 1,
"decode_ms": 214.774,
"decode_tokens": 31,
"error": "",
"latency_ms": 236.7465,
"ok": true,
"prefill_ms": 18.689,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.2835,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 18.689
},
{
"concurrency": 4,
"decode_ms": 271.939,
"decode_tokens": 31,
"error": "",
"latency_ms": 290.2094,
"ok": true,
"prefill_ms": 16.213,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.0574,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 16.213
},
{
"concurrency": 4,
"decode_ms": 271.919,
"decode_tokens": 31,
"error": "",
"latency_ms": 289.5604,
"ok": true,
"prefill_ms": 15.902,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.7394,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 15.902
},
{
"concurrency": 4,
"decode_ms": 248.87,
"decode_tokens": 31,
"error": "",
"latency_ms": 297.9372,
"ok": true,
"prefill_ms": 31.623,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 17.4442,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 31.623
},
{
"concurrency": 4,
"decode_ms": 248.545,
"decode_tokens": 31,
"error": "",
"latency_ms": 297.6956,
"ok": true,
"prefill_ms": 32.061,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 17.0896,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 32.061
},
{
"concurrency": 4,
"decode_ms": 252.756,
"decode_tokens": 31,
"error": "",
"latency_ms": 281.6824,
"ok": true,
"prefill_ms": 26.275,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.6514,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 26.275
},
{
"concurrency": 4,
"decode_ms": 252.626,
"decode_tokens": 31,
"error": "",
"latency_ms": 281.4859,
"ok": true,
"prefill_ms": 26.109,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.7509,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 26.109
},
{
"concurrency": 4,
"decode_ms": 231.94,
"decode_tokens": 31,
"error": "",
"latency_ms": 283.806,
"ok": true,
"prefill_ms": 29.43,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 22.436,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 29.43
},
{
"concurrency": 4,
"decode_ms": 231.964,
"decode_tokens": 31,
"error": "",
"latency_ms": 283.8389,
"ok": true,
"prefill_ms": 29.56,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 22.3149,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 29.56
},
{
"concurrency": 4,
"decode_ms": 270.469,
"decode_tokens": 31,
"error": "",
"latency_ms": 310.9157,
"ok": true,
"prefill_ms": 35.154,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.2927,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 35.154
},
{
"concurrency": 4,
"decode_ms": 270.144,
"decode_tokens": 31,
"error": "",
"latency_ms": 310.1297,
"ok": true,
"prefill_ms": 35.026,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 4.9597,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 35.026
},
{
"concurrency": 4,
"decode_ms": 239.099,
"decode_tokens": 31,
"error": "",
"latency_ms": 306.6453,
"ok": true,
"prefill_ms": 40.241,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 27.3053,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 40.241
},
{
"concurrency": 4,
"decode_ms": 239.11,
"decode_tokens": 31,
"error": "",
"latency_ms": 305.8661,
"ok": true,
"prefill_ms": 40.075,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 26.6811,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 40.075
},
{
"concurrency": 4,
"decode_ms": 254.792,
"decode_tokens": 31,
"error": "",
"latency_ms": 266.7894,
"ok": true,
"prefill_ms": 8.477,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 3.5204,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 8.477
},
{
"concurrency": 4,
"decode_ms": 239.24,
"decode_tokens": 31,
"error": "",
"latency_ms": 274.243,
"ok": true,
"prefill_ms": 25.131,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 9.872,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 25.131
},
{
"concurrency": 4,
"decode_ms": 239.949,
"decode_tokens": 31,
"error": "",
"latency_ms": 270.1358,
"ok": true,
"prefill_ms": 24.211,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 5.9758,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 24.211
},
{
"concurrency": 4,
"decode_ms": 239.577,
"decode_tokens": 31,
"error": "",
"latency_ms": 270.2881,
"ok": true,
"prefill_ms": 24.683,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 6.0281,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 24.683
},
{
"concurrency": 4,
"decode_ms": 253.544,
"decode_tokens": 31,
"error": "",
"latency_ms": 269.3618,
"ok": true,
"prefill_ms": 13.279,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.5388,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 13.279
},
{
"concurrency": 4,
"decode_ms": 283.487,
"decode_tokens": 31,
"error": "",
"latency_ms": 312.6092,
"ok": true,
"prefill_ms": 22.384,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 6.7382,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 22.384
},
{
"concurrency": 4,
"decode_ms": 283.678,
"decode_tokens": 31,
"error": "",
"latency_ms": 312.5317,
"ok": true,
"prefill_ms": 22.587,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 6.2667,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 22.587
},
{
"concurrency": 4,
"decode_ms": 283.811,
"decode_tokens": 31,
"error": "",
"latency_ms": 310.3005,
"ok": true,
"prefill_ms": 22.751,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 3.7385,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 22.751
},
{
"concurrency": 4,
"decode_ms": 303.167,
"decode_tokens": 31,
"error": "",
"latency_ms": 344.8115,
"ok": true,
"prefill_ms": 26.831,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 14.8135,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 26.831
},
{
"concurrency": 4,
"decode_ms": 275.918,
"decode_tokens": 31,
"error": "",
"latency_ms": 334.631,
"ok": true,
"prefill_ms": 52.056,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 6.657,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 52.056
},
{
"concurrency": 4,
"decode_ms": 275.654,
"decode_tokens": 31,
"error": "",
"latency_ms": 333.8801,
"ok": true,
"prefill_ms": 52.366,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.8601,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 52.366
},
{
"concurrency": 4,
"decode_ms": 276.037,
"decode_tokens": 31,
"error": "",
"latency_ms": 332.9966,
"ok": true,
"prefill_ms": 51.79,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.1696,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 51.79
},
{
"concurrency": 4,
"decode_ms": 257.286,
"decode_tokens": 31,
"error": "",
"latency_ms": 284.1563,
"ok": true,
"prefill_ms": 16.579,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 10.2913,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 16.579
},
{
"concurrency": 4,
"decode_ms": 273.475,
"decode_tokens": 31,
"error": "",
"latency_ms": 294.6153,
"ok": true,
"prefill_ms": 16.492,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 4.6483,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 16.492
},
{
"concurrency": 4,
"decode_ms": 271.91,
"decode_tokens": 31,
"error": "",
"latency_ms": 304.1696,
"ok": true,
"prefill_ms": 16.971,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 15.2886,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 16.971
},
{
"concurrency": 4,
"decode_ms": 272.081,
"decode_tokens": 31,
"error": "",
"latency_ms": 303.6641,
"ok": true,
"prefill_ms": 16.662,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 14.9211,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 16.662
},
{
"concurrency": 4,
"decode_ms": 280.501,
"decode_tokens": 31,
"error": "",
"latency_ms": 316.214,
"ok": true,
"prefill_ms": 21.453,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 14.26,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 21.453
},
{
"concurrency": 4,
"decode_ms": 288.961,
"decode_tokens": 31,
"error": "",
"latency_ms": 313.7001,
"ok": true,
"prefill_ms": 20.857,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 3.8821,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 20.857
},
{
"concurrency": 4,
"decode_ms": 311.174,
"decode_tokens": 31,
"error": "",
"latency_ms": 342.5576,
"ok": true,
"prefill_ms": 21.589,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 9.7946,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 21.589
},
{
"concurrency": 4,
"decode_ms": 311.24,
"decode_tokens": 31,
"error": "",
"latency_ms": 342.5651,
"ok": true,
"prefill_ms": 21.721,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 9.6041,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 21.721
},
{
"concurrency": 4,
"decode_ms": 323.529,
"decode_tokens": 31,
"error": "",
"latency_ms": 361.6061,
"ok": true,
"prefill_ms": 26.043,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 12.0341,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 26.043
},
{
"concurrency": 4,
"decode_ms": 333.743,
"decode_tokens": 31,
"error": "",
"latency_ms": 385.4158,
"ok": true,
"prefill_ms": 33.661,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 18.0118,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 33.661
},
{
"concurrency": 4,
"decode_ms": 313.774,
"decode_tokens": 31,
"error": "",
"latency_ms": 356.1707,
"ok": true,
"prefill_ms": 32.822,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 9.5747,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 32.822
},
{
"concurrency": 4,
"decode_ms": 313.728,
"decode_tokens": 31,
"error": "",
"latency_ms": 356.1013,
"ok": true,
"prefill_ms": 33.118,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 9.2553,
"recipe_id": "llama-cpp-bf16-rocm-quality",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the service can handle requests efficiently by managing admission and routing, which are critical for the service's ability to",
"ttft_ms": 33.118
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-BF16.gguf",
"artifact_sha256": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862",
"device": "cuda",
"id": "llama-cpp-bf16-rocm-quality",
"is_reference": false,
"lane": "quality",
"notes": "Converted directly from the exact mounted safetensors revision while preserving BF16 weights with pinned llama.cpp",
"runtime": "llama.cpp-9991-e920c523-rocm-gfx1151",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "gguf",
"weight_quantization": "bfloat16"
},
"unavailable_reason": ""
},
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 73.4589,
"concurrency": 1,
"decode_tokens_per_sec": 81.1223,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 368.8704,
"latency_p95_ms": 610.539,
"peak_rss_bytes": 72605696,
"peak_vram_bytes": 2739758080,
"prefill_tokens_per_sec": 4787.3441,
"requests": 9,
"ttft_p50_ms": 13.7702,
"ttft_p95_ms": 22.2051,
"wall_ms": 3798.0447
},
"4": {
"aggregate_decode_tokens_per_sec": 91.2441,
"concurrency": 4,
"decode_tokens_per_sec": 24.3603,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 1279.9181,
"latency_p95_ms": 1555.697,
"peak_rss_bytes": 74674176,
"peak_vram_bytes": 2739758080,
"prefill_tokens_per_sec": 2666.5181,
"requests": 36,
"ttft_p50_ms": 29.7017,
"ttft_p95_ms": 58.4801,
"wall_ms": 12230.9241
}
},
"load": {
"artifact_bytes": 999586347,
"backend_detail": "torch 2.10.0+rocm7.13.0a20260513; dtype bfloat16; device cuda; intra-op threads 16",
"load_ms": 527.1005,
"rss_bytes": 66371584,
"vram_bytes": 2739758080
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 344.2464,
"decode_tokens": 31,
"error": "",
"latency_ms": 361.2113,
"ok": true,
"prefill_ms": 15.7786,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 15.7786
},
{
"concurrency": 1,
"decode_ms": 345.7384,
"decode_tokens": 31,
"error": "",
"latency_ms": 358.7145,
"ok": true,
"prefill_ms": 11.4908,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 11.4908
},
{
"concurrency": 1,
"decode_ms": 343.6121,
"decode_tokens": 31,
"error": "",
"latency_ms": 362.1829,
"ok": true,
"prefill_ms": 12.2759,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 12.2759
},
{
"concurrency": 1,
"decode_ms": 334.6859,
"decode_tokens": 31,
"error": "",
"latency_ms": 349.8262,
"ok": true,
"prefill_ms": 13.7702,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 13.7702
},
{
"concurrency": 1,
"decode_ms": 356.0055,
"decode_tokens": 31,
"error": "",
"latency_ms": 368.8704,
"ok": true,
"prefill_ms": 11.2695,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 11.2695
},
{
"concurrency": 1,
"decode_ms": 382.5716,
"decode_tokens": 31,
"error": "",
"latency_ms": 400.8131,
"ok": true,
"prefill_ms": 16.1844,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 16.1844
},
{
"concurrency": 1,
"decode_ms": 378.3914,
"decode_tokens": 31,
"error": "",
"latency_ms": 394.5781,
"ok": true,
"prefill_ms": 14.975,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 14.975
},
{
"concurrency": 1,
"decode_ms": 479.7719,
"decode_tokens": 31,
"error": "",
"latency_ms": 493.7611,
"ok": true,
"prefill_ms": 12.6857,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 12.6857
},
{
"concurrency": 1,
"decode_ms": 586.0558,
"decode_tokens": 31,
"error": "",
"latency_ms": 610.539,
"ok": true,
"prefill_ms": 22.2051,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 22.2051
},
{
"concurrency": 4,
"decode_ms": 1240.6234,
"decode_tokens": 31,
"error": "",
"latency_ms": 1300.3927,
"ok": true,
"prefill_ms": 58.4801,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 58.4801
},
{
"concurrency": 4,
"decode_ms": 1226.9004,
"decode_tokens": 31,
"error": "",
"latency_ms": 1278.8883,
"ok": true,
"prefill_ms": 50.916,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 50.916
},
{
"concurrency": 4,
"decode_ms": 1220.2502,
"decode_tokens": 31,
"error": "",
"latency_ms": 1281.9516,
"ok": true,
"prefill_ms": 59.9496,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 59.9496
},
{
"concurrency": 4,
"decode_ms": 1247.2456,
"decode_tokens": 31,
"error": "",
"latency_ms": 1300.3646,
"ok": true,
"prefill_ms": 51.8513,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 51.8513
},
{
"concurrency": 4,
"decode_ms": 1230.8629,
"decode_tokens": 31,
"error": "",
"latency_ms": 1261.5658,
"ok": true,
"prefill_ms": 29.044,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 29.044
},
{
"concurrency": 4,
"decode_ms": 1222.2133,
"decode_tokens": 31,
"error": "",
"latency_ms": 1254.5904,
"ok": true,
"prefill_ms": 29.7017,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 29.7017
},
{
"concurrency": 4,
"decode_ms": 1212.0522,
"decode_tokens": 31,
"error": "",
"latency_ms": 1249.3501,
"ok": true,
"prefill_ms": 28.2375,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 28.2375
},
{
"concurrency": 4,
"decode_ms": 1265.5141,
"decode_tokens": 31,
"error": "",
"latency_ms": 1305.635,
"ok": true,
"prefill_ms": 23.8198,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 23.8198
},
{
"concurrency": 4,
"decode_ms": 1325.2124,
"decode_tokens": 31,
"error": "",
"latency_ms": 1387.1324,
"ok": true,
"prefill_ms": 27.3617,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 27.3617
},
{
"concurrency": 4,
"decode_ms": 1320.5426,
"decode_tokens": 31,
"error": "",
"latency_ms": 1375.6438,
"ok": true,
"prefill_ms": 24.3676,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 24.3676
},
{
"concurrency": 4,
"decode_ms": 1336.0923,
"decode_tokens": 31,
"error": "",
"latency_ms": 1396.6943,
"ok": true,
"prefill_ms": 24.0531,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 24.0531
},
{
"concurrency": 4,
"decode_ms": 1309.2145,
"decode_tokens": 31,
"error": "",
"latency_ms": 1360.8499,
"ok": true,
"prefill_ms": 40.9188,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 0,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 40.9188
},
{
"concurrency": 4,
"decode_ms": 1224.9775,
"decode_tokens": 31,
"error": "",
"latency_ms": 1259.6556,
"ok": true,
"prefill_ms": 31.6828,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 31.6828
},
{
"concurrency": 4,
"decode_ms": 1209.8826,
"decode_tokens": 31,
"error": "",
"latency_ms": 1240.3993,
"ok": true,
"prefill_ms": 28.6973,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 28.6973
},
{
"concurrency": 4,
"decode_ms": 1241.0519,
"decode_tokens": 31,
"error": "",
"latency_ms": 1279.2029,
"ok": true,
"prefill_ms": 21.6672,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 21.6672
},
{
"concurrency": 4,
"decode_ms": 1225.2425,
"decode_tokens": 31,
"error": "",
"latency_ms": 1260.2329,
"ok": true,
"prefill_ms": 21.8891,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 21.8891
},
{
"concurrency": 4,
"decode_ms": 1207.5443,
"decode_tokens": 31,
"error": "",
"latency_ms": 1255.5812,
"ok": true,
"prefill_ms": 46.7943,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 46.7943
},
{
"concurrency": 4,
"decode_ms": 1215.6561,
"decode_tokens": 31,
"error": "",
"latency_ms": 1260.7109,
"ok": true,
"prefill_ms": 43.7571,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 43.7571
},
{
"concurrency": 4,
"decode_ms": 1234.242,
"decode_tokens": 31,
"error": "",
"latency_ms": 1275.0603,
"ok": true,
"prefill_ms": 29.8462,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 29.8462
},
{
"concurrency": 4,
"decode_ms": 1252.7279,
"decode_tokens": 31,
"error": "",
"latency_ms": 1295.1831,
"ok": true,
"prefill_ms": 28.7507,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 28.7507
},
{
"concurrency": 4,
"decode_ms": 1534.6603,
"decode_tokens": 31,
"error": "",
"latency_ms": 1575.6607,
"ok": true,
"prefill_ms": 21.0429,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 21.0429
},
{
"concurrency": 4,
"decode_ms": 1476.596,
"decode_tokens": 31,
"error": "",
"latency_ms": 1523.8462,
"ok": true,
"prefill_ms": 17.8517,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 17.8517
},
{
"concurrency": 4,
"decode_ms": 1511.9364,
"decode_tokens": 31,
"error": "",
"latency_ms": 1555.697,
"ok": true,
"prefill_ms": 30.4496,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 30.4496
},
{
"concurrency": 4,
"decode_ms": 1481.2282,
"decode_tokens": 31,
"error": "",
"latency_ms": 1530.0434,
"ok": true,
"prefill_ms": 24.4731,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 1,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 24.4731
},
{
"concurrency": 4,
"decode_ms": 1215.3153,
"decode_tokens": 31,
"error": "",
"latency_ms": 1256.6594,
"ok": true,
"prefill_ms": 38.5243,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 38.5243
},
{
"concurrency": 4,
"decode_ms": 1234.2491,
"decode_tokens": 31,
"error": "",
"latency_ms": 1279.9181,
"ok": true,
"prefill_ms": 33.2527,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 33.2527
},
{
"concurrency": 4,
"decode_ms": 1205.4323,
"decode_tokens": 31,
"error": "",
"latency_ms": 1249.299,
"ok": true,
"prefill_ms": 38.8544,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 38.8544
},
{
"concurrency": 4,
"decode_ms": 1286.3806,
"decode_tokens": 31,
"error": "",
"latency_ms": 1329.3891,
"ok": true,
"prefill_ms": 35.8302,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the second largest city in the world. It is located in the south of France, on the banks of the",
"ttft_ms": 35.8302
},
{
"concurrency": 4,
"decode_ms": 1227.7205,
"decode_tokens": 31,
"error": "",
"latency_ms": 1272.6158,
"ok": true,
"prefill_ms": 21.8068,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 21.8068
},
{
"concurrency": 4,
"decode_ms": 1237.1979,
"decode_tokens": 31,
"error": "",
"latency_ms": 1279.2811,
"ok": true,
"prefill_ms": 37.84,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 37.84
},
{
"concurrency": 4,
"decode_ms": 1247.6839,
"decode_tokens": 31,
"error": "",
"latency_ms": 1276.814,
"ok": true,
"prefill_ms": 17.76,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 17.76
},
{
"concurrency": 4,
"decode_ms": 1270.8182,
"decode_tokens": 31,
"error": "",
"latency_ms": 1307.7581,
"ok": true,
"prefill_ms": 20.9301,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 20.9301
},
{
"concurrency": 4,
"decode_ms": 1302.0124,
"decode_tokens": 31,
"error": "",
"latency_ms": 1352.3295,
"ok": true,
"prefill_ms": 26.8633,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 26.8633
},
{
"concurrency": 4,
"decode_ms": 1307.41,
"decode_tokens": 31,
"error": "",
"latency_ms": 1357.7264,
"ok": true,
"prefill_ms": 40.3117,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 40.3117
},
{
"concurrency": 4,
"decode_ms": 1278.3439,
"decode_tokens": 31,
"error": "",
"latency_ms": 1330.1471,
"ok": true,
"prefill_ms": 38.8708,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 38.8708
},
{
"concurrency": 4,
"decode_ms": 1217.6556,
"decode_tokens": 31,
"error": "",
"latency_ms": 1269.1355,
"ok": true,
"prefill_ms": 32.7943,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 0.0,
"recipe_id": "transformers-bf16-rocm-throughput",
"repeat": 2,
"text": " 1. **Admission and Routing**: Ensure that the system can handle requests efficiently by managing admission and routing, which are critical for the performance of the distributed",
"ttft_ms": 32.7943
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/safetensor/models/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775",
"artifact_sha256": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6",
"device": "cuda",
"id": "transformers-bf16-rocm-throughput",
"is_reference": false,
"lane": "performance-fit",
"notes": "artifact_sha256 is the deterministic digest of every snapshot path and file byte",
"runtime": "transformers-5.13.0-rocm-bfloat16",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "safetensors",
"weight_quantization": "bfloat16"
},
"unavailable_reason": ""
},
{
"available": true,
"concurrency": {
"1": {
"aggregate_decode_tokens_per_sec": 200.1035,
"concurrency": 1,
"decode_tokens_per_sec": 251.2489,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 136.1403,
"latency_p95_ms": 169.8598,
"peak_rss_bytes": 692973568,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 4219.2554,
"requests": 9,
"ttft_p50_ms": 11.799,
"ttft_p95_ms": 37.104,
"wall_ms": 1394.2787
},
"4": {
"aggregate_decode_tokens_per_sec": 511.3289,
"concurrency": 4,
"decode_tokens_per_sec": 189.7261,
"failure_reasons": [],
"failures": 0,
"latency_p50_ms": 192.1355,
"latency_p95_ms": 274.5073,
"peak_rss_bytes": 721944576,
"peak_vram_bytes": 0,
"prefill_tokens_per_sec": 2126.9041,
"requests": 36,
"ttft_p50_ms": 21.383,
"ttft_p95_ms": 101.016,
"wall_ms": 2182.5484
}
},
"load": {
"artifact_bytes": 397807520,
"backend_detail": "version: 9991 (e920c523) | built with GNU 15.2.1 for Linux x86_64; binary sha256 b6bb4da687dbde86e243ba006cef05919b7b97255cd7e2371e1d451220aca139; threads 16; parallel slots 4; ctx/slot 512; requested gpu layers 99; measured accelerator ROCm0: Radeon 8060S Graphics; measured offload 25/25 layers",
"load_ms": 756.1035,
"rss_bytes": 663019520,
"vram_bytes": 0
},
"outcomes": [
{
"concurrency": 1,
"decode_ms": 128.099,
"decode_tokens": 31,
"error": "",
"latency_ms": 136.1403,
"ok": true,
"prefill_ms": 6.395,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.6463,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located on the left bank of the Seine River, in",
"ttft_ms": 6.395
},
{
"concurrency": 1,
"decode_ms": 131.155,
"decode_tokens": 31,
"error": "",
"latency_ms": 169.8598,
"ok": true,
"prefill_ms": 37.104,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.6008,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop through the Fibonacci sequence up to n\n ",
"ttft_ms": 37.104
},
{
"concurrency": 1,
"decode_ms": 129.936,
"decode_tokens": 31,
"error": "",
"latency_ms": 155.4482,
"ok": true,
"prefill_ms": 20.74,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 4.7722,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own model execution. 2. Workers must reject incompatible protocol versions",
"ttft_ms": 20.74
},
{
"concurrency": 1,
"decode_ms": 130.699,
"decode_tokens": 31,
"error": "",
"latency_ms": 140.6293,
"ok": true,
"prefill_ms": 6.311,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 3.6193,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located on the left bank of the Seine River, in",
"ttft_ms": 6.311
},
{
"concurrency": 1,
"decode_ms": 147.637,
"decode_tokens": 31,
"error": "",
"latency_ms": 161.2559,
"ok": true,
"prefill_ms": 11.799,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.8199,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop through the Fibonacci sequence up to n\n ",
"ttft_ms": 11.799
},
{
"concurrency": 1,
"decode_ms": 113.716,
"decode_tokens": 31,
"error": "",
"latency_ms": 133.7613,
"ok": true,
"prefill_ms": 17.621,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.4243,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 17.621
},
{
"concurrency": 1,
"decode_ms": 112.909,
"decode_tokens": 31,
"error": "",
"latency_ms": 120.5551,
"ok": true,
"prefill_ms": 5.291,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.3551,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest city in the world. It is located on the left bank of the Seine River, in",
"ttft_ms": 5.291
},
{
"concurrency": 1,
"decode_ms": 112.725,
"decode_tokens": 31,
"error": "",
"latency_ms": 126.0947,
"ok": true,
"prefill_ms": 11.172,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.1977,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop through the Fibonacci sequence up to n\n ",
"ttft_ms": 11.172
},
{
"concurrency": 1,
"decode_ms": 112.741,
"decode_tokens": 31,
"error": "",
"latency_ms": 133.4263,
"ok": true,
"prefill_ms": 18.059,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.6263,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 18.059
},
{
"concurrency": 4,
"decode_ms": 159.539,
"decode_tokens": 31,
"error": "",
"latency_ms": 167.0196,
"ok": true,
"prefill_ms": 5.439,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.0416,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 5.439
},
{
"concurrency": 4,
"decode_ms": 152.896,
"decode_tokens": 31,
"error": "",
"latency_ms": 170.9966,
"ok": true,
"prefill_ms": 10.999,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 7.1016,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 10.999
},
{
"concurrency": 4,
"decode_ms": 152.854,
"decode_tokens": 31,
"error": "",
"latency_ms": 171.1728,
"ok": true,
"prefill_ms": 11.124,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 7.1948,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 11.124
},
{
"concurrency": 4,
"decode_ms": 152.51,
"decode_tokens": 31,
"error": "",
"latency_ms": 170.6718,
"ok": true,
"prefill_ms": 11.514,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 6.6478,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 11.514
},
{
"concurrency": 4,
"decode_ms": 164.509,
"decode_tokens": 31,
"error": "",
"latency_ms": 178.058,
"ok": true,
"prefill_ms": 10.859,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 2.69,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n while n >",
"ttft_ms": 10.859
},
{
"concurrency": 4,
"decode_ms": 177.405,
"decode_tokens": 31,
"error": "",
"latency_ms": 196.9544,
"ok": true,
"prefill_ms": 14.499,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 5.0504,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n for _ in",
"ttft_ms": 14.499
},
{
"concurrency": 4,
"decode_ms": 177.419,
"decode_tokens": 31,
"error": "",
"latency_ms": 197.1498,
"ok": true,
"prefill_ms": 14.8,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 4.9308,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until the nth Fibonacci number is reached\n ",
"ttft_ms": 14.8
},
{
"concurrency": 4,
"decode_ms": 177.428,
"decode_tokens": 31,
"error": "",
"latency_ms": 196.5071,
"ok": true,
"prefill_ms": 14.646,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 4.4331,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until the nth Fibonacci number is reached\n ",
"ttft_ms": 14.646
},
{
"concurrency": 4,
"decode_ms": 246.124,
"decode_tokens": 31,
"error": "",
"latency_ms": 275.6221,
"ok": true,
"prefill_ms": 22.421,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 7.0771,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 22.421
},
{
"concurrency": 4,
"decode_ms": 167.733,
"decode_tokens": 31,
"error": "",
"latency_ms": 274.5073,
"ok": true,
"prefill_ms": 100.872,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.9023,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 100.872
},
{
"concurrency": 4,
"decode_ms": 167.61,
"decode_tokens": 31,
"error": "",
"latency_ms": 274.1725,
"ok": true,
"prefill_ms": 101.172,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.3905,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 101.172
},
{
"concurrency": 4,
"decode_ms": 167.686,
"decode_tokens": 31,
"error": "",
"latency_ms": 274.1108,
"ok": true,
"prefill_ms": 101.016,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.4088,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 0,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 101.016
},
{
"concurrency": 4,
"decode_ms": 155.512,
"decode_tokens": 31,
"error": "",
"latency_ms": 170.5466,
"ok": true,
"prefill_ms": 9.619,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 5.4156,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the Atlantic coast. It",
"ttft_ms": 9.619
},
{
"concurrency": 4,
"decode_ms": 177.321,
"decode_tokens": 31,
"error": "",
"latency_ms": 192.3165,
"ok": true,
"prefill_ms": 10.558,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 4.4375,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 10.558
},
{
"concurrency": 4,
"decode_ms": 177.393,
"decode_tokens": 31,
"error": "",
"latency_ms": 192.1355,
"ok": true,
"prefill_ms": 10.314,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 4.4285,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 10.314
},
{
"concurrency": 4,
"decode_ms": 177.365,
"decode_tokens": 31,
"error": "",
"latency_ms": 191.6336,
"ok": true,
"prefill_ms": 10.43,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 3.8386,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 10.43
},
{
"concurrency": 4,
"decode_ms": 158.946,
"decode_tokens": 31,
"error": "",
"latency_ms": 179.8625,
"ok": true,
"prefill_ms": 15.661,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 5.2555,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 15.661
},
{
"concurrency": 4,
"decode_ms": 181.816,
"decode_tokens": 31,
"error": "",
"latency_ms": 203.3843,
"ok": true,
"prefill_ms": 14.647,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 6.9213,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until the nth Fibonacci number is reached\n ",
"ttft_ms": 14.647
},
{
"concurrency": 4,
"decode_ms": 181.864,
"decode_tokens": 31,
"error": "",
"latency_ms": 203.079,
"ok": true,
"prefill_ms": 14.5,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 6.715,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n for _ in",
"ttft_ms": 14.5
},
{
"concurrency": 4,
"decode_ms": 181.741,
"decode_tokens": 31,
"error": "",
"latency_ms": 203.0444,
"ok": true,
"prefill_ms": 14.807,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 6.4964,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until the nth Fibonacci number is reached\n ",
"ttft_ms": 14.807
},
{
"concurrency": 4,
"decode_ms": 145.56,
"decode_tokens": 31,
"error": "",
"latency_ms": 174.8495,
"ok": true,
"prefill_ms": 22.031,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 7.2585,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 22.031
},
{
"concurrency": 4,
"decode_ms": 163.05,
"decode_tokens": 31,
"error": "",
"latency_ms": 213.1526,
"ok": true,
"prefill_ms": 46.438,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.6646,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 46.438
},
{
"concurrency": 4,
"decode_ms": 163.048,
"decode_tokens": 31,
"error": "",
"latency_ms": 212.5585,
"ok": true,
"prefill_ms": 46.146,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.3645,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 46.146
},
{
"concurrency": 4,
"decode_ms": 163.0,
"decode_tokens": 31,
"error": "",
"latency_ms": 211.3843,
"ok": true,
"prefill_ms": 46.297,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 2.0873,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 1,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 46.297
},
{
"concurrency": 4,
"decode_ms": 154.023,
"decode_tokens": 31,
"error": "",
"latency_ms": 212.5402,
"ok": true,
"prefill_ms": 13.01,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 45.5072,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " Paris. It is the largest city in Europe and the third largest in the world. It is located in the south of France, on the western coast of the",
"ttft_ms": 13.01
},
{
"concurrency": 4,
"decode_ms": 156.115,
"decode_tokens": 31,
"error": "",
"latency_ms": 180.6458,
"ok": true,
"prefill_ms": 21.654,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.8768,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " Paris. The capital of the United States is Washington, D. C. The capital of the United Kingdom is London. The capital of the United States is Washington",
"ttft_ms": 21.654
},
{
"concurrency": 4,
"decode_ms": 156.153,
"decode_tokens": 31,
"error": "",
"latency_ms": 179.9028,
"ok": true,
"prefill_ms": 21.383,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 2.3668,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " Paris. The capital of the United States is Washington, D. C. The capital of the United Kingdom is London. The capital of the United States is Washington",
"ttft_ms": 21.383
},
{
"concurrency": 4,
"decode_ms": 156.136,
"decode_tokens": 31,
"error": "",
"latency_ms": 179.5942,
"ok": true,
"prefill_ms": 21.523,
"prompt_id": "short-fact",
"prompt_tokens": 5,
"queue_wait_ms": 1.9352,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " Paris. The capital of the United States is Washington, D. C. The capital of the United Kingdom is London. The capital of the United States is Washington",
"ttft_ms": 21.523
},
{
"concurrency": 4,
"decode_ms": 156.104,
"decode_tokens": 31,
"error": "",
"latency_ms": 179.6367,
"ok": true,
"prefill_ms": 21.805,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 1.7277,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " if n == 0:\n return 0\n elif n == 1:\n return 1\n else:\n return fibonacci(n-",
"ttft_ms": 21.805
},
{
"concurrency": 4,
"decode_ms": 155.73,
"decode_tokens": 31,
"error": "",
"latency_ms": 190.917,
"ok": true,
"prefill_ms": 29.996,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 5.191,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until the nth Fibonacci number is reached\n ",
"ttft_ms": 29.996
},
{
"concurrency": 4,
"decode_ms": 155.653,
"decode_tokens": 31,
"error": "",
"latency_ms": 191.3392,
"ok": true,
"prefill_ms": 30.308,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 5.3782,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until the nth Fibonacci number is reached\n ",
"ttft_ms": 30.308
},
{
"concurrency": 4,
"decode_ms": 155.739,
"decode_tokens": 31,
"error": "",
"latency_ms": 191.0055,
"ok": true,
"prefill_ms": 29.843,
"prompt_id": "medium-code",
"prompt_tokens": 24,
"queue_wait_ms": 5.4235,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " # Initialize the first two Fibonacci numbers\n a, b = 0, 1\n # Loop until n is reached\n for _ in",
"ttft_ms": 29.843
},
{
"concurrency": 4,
"decode_ms": 155.711,
"decode_tokens": 31,
"error": "",
"latency_ms": 190.956,
"ok": true,
"prefill_ms": 30.141,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 5.104,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 30.141
},
{
"concurrency": 4,
"decode_ms": 147.957,
"decode_tokens": 31,
"error": "",
"latency_ms": 197.317,
"ok": true,
"prefill_ms": 45.543,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.817,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 45.543
},
{
"concurrency": 4,
"decode_ms": 148.089,
"decode_tokens": 31,
"error": "",
"latency_ms": 196.8288,
"ok": true,
"prefill_ms": 45.216,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.5238,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 45.216
},
{
"concurrency": 4,
"decode_ms": 148.042,
"decode_tokens": 31,
"error": "",
"latency_ms": 196.7137,
"ok": true,
"prefill_ms": 45.375,
"prompt_id": "long-summary",
"prompt_tokens": 191,
"queue_wait_ms": 3.2967,
"recipe_id": "llama-cpp-q4-rocm-throughput",
"repeat": 2,
"text": " 1. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution.\n2. Workers must reject incompatible protocol versions",
"ttft_ms": 45.375
}
],
"recipe": {
"artifact_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-Q4_K_M.gguf",
"artifact_sha256": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5",
"device": "cuda",
"id": "llama-cpp-q4-rocm-throughput",
"is_reference": false,
"lane": "performance-fit",
"notes": "Quantized from the exact-revision F16 GGUF with pinned llama-quantize",
"runtime": "llama.cpp-9991-e920c523-rocm-gfx1151",
"source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
"source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
"weight_format": "gguf",
"weight_quantization": "Q4_K_M"
},
"unavailable_reason": ""
}
],
"reference_recipe_id": "transformers-fp32-rocm-quality-oracle",
"schema_version": 1
}
Reference in New Issue View Git Blame Copy Permalink