180 lines
3.4 KiB
JSON
180 lines
3.4 KiB
JSON
{
|
|
"concurrency_sweep": {
|
|
"corruption_free": true,
|
|
"reference_outputs": {
|
|
"s0": [
|
|
27,
|
|
8,
|
|
27,
|
|
8,
|
|
27,
|
|
8,
|
|
1,
|
|
1
|
|
],
|
|
"s1": [
|
|
26,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
3,
|
|
39,
|
|
39
|
|
],
|
|
"s2": [
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
12,
|
|
30,
|
|
12
|
|
],
|
|
"s3": [
|
|
29,
|
|
41,
|
|
42,
|
|
47,
|
|
47,
|
|
42,
|
|
47,
|
|
42
|
|
],
|
|
"s4": [
|
|
23,
|
|
11,
|
|
44,
|
|
29,
|
|
29,
|
|
29,
|
|
41,
|
|
29
|
|
],
|
|
"s5": [
|
|
35,
|
|
11,
|
|
0,
|
|
1,
|
|
11,
|
|
0,
|
|
11,
|
|
15
|
|
],
|
|
"s6": [
|
|
39,
|
|
39,
|
|
28,
|
|
39,
|
|
39,
|
|
39,
|
|
28,
|
|
28
|
|
],
|
|
"s7": [
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
39,
|
|
8,
|
|
47
|
|
]
|
|
},
|
|
"results": [
|
|
{
|
|
"avg_batch_occupancy": 1.0,
|
|
"cache_misses": 0,
|
|
"concurrency": 1,
|
|
"decode_batches": 56,
|
|
"decode_tokens": 56,
|
|
"max_batch_occupancy": 1,
|
|
"peak_kv_bytes": 15360,
|
|
"prefill_tokens": 32,
|
|
"rejected_admissions": 0,
|
|
"ticks": 64,
|
|
"tokens_per_tick": 1.375
|
|
},
|
|
{
|
|
"avg_batch_occupancy": 1.75,
|
|
"cache_misses": 0,
|
|
"concurrency": 2,
|
|
"decode_batches": 32,
|
|
"decode_tokens": 56,
|
|
"max_batch_occupancy": 2,
|
|
"peak_kv_bytes": 29184,
|
|
"prefill_tokens": 32,
|
|
"rejected_admissions": 0,
|
|
"ticks": 33,
|
|
"tokens_per_tick": 2.6667
|
|
},
|
|
{
|
|
"avg_batch_occupancy": 3.1111,
|
|
"cache_misses": 0,
|
|
"concurrency": 4,
|
|
"decode_batches": 18,
|
|
"decode_tokens": 56,
|
|
"max_batch_occupancy": 4,
|
|
"peak_kv_bytes": 52224,
|
|
"prefill_tokens": 32,
|
|
"rejected_admissions": 0,
|
|
"ticks": 19,
|
|
"tokens_per_tick": 4.6316
|
|
},
|
|
{
|
|
"avg_batch_occupancy": 4.0,
|
|
"cache_misses": 0,
|
|
"concurrency": 8,
|
|
"decode_batches": 14,
|
|
"decode_tokens": 56,
|
|
"max_batch_occupancy": 7,
|
|
"peak_kv_bytes": 75264,
|
|
"prefill_tokens": 32,
|
|
"rejected_admissions": 0,
|
|
"ticks": 15,
|
|
"tokens_per_tick": 5.8667
|
|
}
|
|
],
|
|
"saturation_concurrency": 8,
|
|
"schema_version": 1
|
|
},
|
|
"evidence_kind": "synthetic-unit",
|
|
"mid_run_telemetry_concurrency_4": {
|
|
"active_sessions": 4,
|
|
"batch_occupancy_avg": 4.0,
|
|
"batch_occupancy_last": 4,
|
|
"batch_occupancy_max": 4,
|
|
"completed_sessions": 0,
|
|
"decode_tokens_per_sec": 1637.355,
|
|
"decode_tokens_total": 20,
|
|
"kv_budget_bytes": 67108864,
|
|
"kv_pressure": 0.0008,
|
|
"kv_total_bytes": 55296,
|
|
"prefill_tokens_per_sec": 1309.884,
|
|
"prefill_tokens_total": 16,
|
|
"queue_depth": 4,
|
|
"rejected_admissions_total": 0,
|
|
"rejected_by_reason": {},
|
|
"scratch_budget_bytes": 4,
|
|
"scratch_pressure": 1.0,
|
|
"scratch_used_bytes": 4,
|
|
"ticks": 6,
|
|
"weight_bytes": 0
|
|
},
|
|
"model": {
|
|
"hidden": 32,
|
|
"n_heads": 4,
|
|
"n_layers": 6,
|
|
"reference": "pure-numpy KV-cached dense-Llama (tests/test_hot_kv_state)",
|
|
"vocab": 48
|
|
},
|
|
"schema_version": 1,
|
|
"workload": {
|
|
"max_new_tokens": 8,
|
|
"prompt_len": 4,
|
|
"sessions": 8
|
|
}
|
|
}
|