popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: checkpoint batching and release-gate stories

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-16 17:24:36 +03:00
parent 737bade989
commit 02b3709311
18 changed files with 4580 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

179
.scratch/distributed-gguf-runtime/evidence/DGR-012/results.json Normal file
View File

@@ -0,0 +1,179 @@
{
"concurrency_sweep": {
"corruption_free": true,
"reference_outputs": {
"s0": [
27,
8,
27,
8,
27,
8,
1,
1
],
"s1": [
26,
39,
39,
39,
39,
3,
39,
39
],
"s2": [
12,
12,
12,
12,
12,
12,
30,
12
],
"s3": [
29,
41,
42,
47,
47,
42,
47,
42
],
"s4": [
23,
11,
44,
29,
29,
29,
41,
29
],
"s5": [
35,
11,
0,
1,
11,
0,
11,
15
],
"s6": [
39,
39,
28,
39,
39,
39,
28,
28
],
"s7": [
39,
39,
39,
39,
39,
39,
8,
47
]
},
"results": [
{
"avg_batch_occupancy": 1.0,
"cache_misses": 0,
"concurrency": 1,
"decode_batches": 56,
"decode_tokens": 56,
"max_batch_occupancy": 1,
"peak_kv_bytes": 15360,
"prefill_tokens": 32,
"rejected_admissions": 0,
"ticks": 64,
"tokens_per_tick": 1.375
},
{
"avg_batch_occupancy": 1.75,
"cache_misses": 0,
"concurrency": 2,
"decode_batches": 32,
"decode_tokens": 56,
"max_batch_occupancy": 2,
"peak_kv_bytes": 29184,
"prefill_tokens": 32,
"rejected_admissions": 0,
"ticks": 33,
"tokens_per_tick": 2.6667
},
{
"avg_batch_occupancy": 3.1111,
"cache_misses": 0,
"concurrency": 4,
"decode_batches": 18,
"decode_tokens": 56,
"max_batch_occupancy": 4,
"peak_kv_bytes": 52224,
"prefill_tokens": 32,
"rejected_admissions": 0,
"ticks": 19,
"tokens_per_tick": 4.6316
},
{
"avg_batch_occupancy": 4.0,
"cache_misses": 0,
"concurrency": 8,
"decode_batches": 14,
"decode_tokens": 56,
"max_batch_occupancy": 7,
"peak_kv_bytes": 75264,
"prefill_tokens": 32,
"rejected_admissions": 0,
"ticks": 15,
"tokens_per_tick": 5.8667
}
],
"saturation_concurrency": 8,
"schema_version": 1
},
"evidence_kind": "synthetic-unit",
"mid_run_telemetry_concurrency_4": {
"active_sessions": 4,
"batch_occupancy_avg": 4.0,
"batch_occupancy_last": 4,
"batch_occupancy_max": 4,
"completed_sessions": 0,
"decode_tokens_per_sec": 1637.355,
"decode_tokens_total": 20,
"kv_budget_bytes": 67108864,
"kv_pressure": 0.0008,
"kv_total_bytes": 55296,
"prefill_tokens_per_sec": 1309.884,
"prefill_tokens_total": 16,
"queue_depth": 4,
"rejected_admissions_total": 0,
"rejected_by_reason": {},
"scratch_budget_bytes": 4,
"scratch_pressure": 1.0,
"scratch_used_bytes": 4,
"ticks": 6,
"weight_bytes": 0
},
"model": {
"hidden": 32,
"n_heads": 4,
"n_layers": 6,
"reference": "pure-numpy KV-cached dense-Llama (tests/test_hot_kv_state)",
"vocab": 48
},
"schema_version": 1,
"workload": {
"max_new_tokens": 8,
"prompt_len": 4,
"sessions": 8
}
}