235 lines
8.0 KiB
Python
235 lines
8.0 KiB
Python
#!/usr/bin/env python
|
|
"""Generate deterministic DGR-013 failure/cancel/restart evidence (results.json).
|
|
|
|
Runs the real hardened per-session stream (``HardenedSessionRunner`` over the
|
|
DGR-007 ``KvBoundaryAdapter`` + ``HotKvStateManager``) through each failure mode
|
|
with the same pure-numpy dense-Llama reference the default gate uses. No model
|
|
download, no GPU, no torch, no network, no API credit.
|
|
|
|
Run from the repo root with the worktree venv:
|
|
|
|
.venv/bin/python .scratch/distributed-gguf-runtime/evidence/DGR-013/generate_evidence.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
import numpy as np
|
|
|
|
# Make the worktree packages and the DGR-007 numpy reference importable, exactly
|
|
# as pytest's prepend-import + conftest do.
|
|
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", ".."))
|
|
sys.path.insert(0, os.path.join(ROOT, "packages", "node"))
|
|
sys.path.insert(0, os.path.join(ROOT, "tests"))
|
|
|
|
from meshnet_node.hot_kv_state import ( # noqa: E402
|
|
HotKvStateConfig,
|
|
HotKvStateManager,
|
|
KvBoundaryAdapter,
|
|
StaleRouteEpochError,
|
|
kv_recipe_for,
|
|
)
|
|
from meshnet_node.batch_scheduler import GenerationRequest # noqa: E402
|
|
from meshnet_node.failure_semantics import ( # noqa: E402
|
|
CancellationToken,
|
|
FailureKind,
|
|
HardenedSessionRunner,
|
|
RestartController,
|
|
StreamTerminated,
|
|
WorkLedger,
|
|
WorkStatus,
|
|
)
|
|
|
|
from test_hot_kv_state import _KvDenseLlama, _KvReferenceShard # noqa: E402
|
|
|
|
|
|
class _FaultyShard(_KvReferenceShard):
|
|
def __init__(self, model, start, end, *, fail_at_call=None):
|
|
super().__init__(model, start, end)
|
|
self._fail_at_call = fail_at_call
|
|
self.calls = 0
|
|
|
|
def run_layers_cached(self, hidden, *, positions, past_kv):
|
|
self.calls += 1
|
|
if self._fail_at_call is not None and self.calls == self._fail_at_call:
|
|
raise RuntimeError("worker died mid-step")
|
|
return super().run_layers_cached(hidden, positions=positions, past_kv=past_kv)
|
|
|
|
|
|
class _Clock:
|
|
def __init__(self):
|
|
self.now = 0.0
|
|
|
|
def __call__(self):
|
|
return self.now
|
|
|
|
def advance(self, d):
|
|
self.now += d
|
|
|
|
|
|
def _adapter(model, *, config=None, shard=None):
|
|
shard = shard or _KvReferenceShard(model, 0, model.n_layers - 1)
|
|
manager = HotKvStateManager(kv_recipe_for(shard), config=config)
|
|
return KvBoundaryAdapter(shard, manager)
|
|
|
|
|
|
def _gen(sid, prompt, n, epoch=0):
|
|
return GenerationRequest(
|
|
session_id=sid, route_epoch=epoch,
|
|
prompt_token_ids=tuple(prompt), max_new_tokens=n,
|
|
)
|
|
|
|
|
|
def _kv_released(manager, sid, epoch):
|
|
from meshnet_node.hot_kv_state import CacheMiss
|
|
return isinstance(manager.resolve(sid, epoch), CacheMiss)
|
|
|
|
|
|
def main() -> None:
|
|
model = _KvDenseLlama()
|
|
prompt = [7, 3, 9, 1]
|
|
n_new = 8
|
|
ledger = WorkLedger()
|
|
scenarios = []
|
|
|
|
# 1. Clean baseline.
|
|
ad = _adapter(model)
|
|
r = HardenedSessionRunner(ad, work_ledger=ledger).run(_gen("clean", prompt, n_new))
|
|
scenarios.append({
|
|
"scenario": "clean",
|
|
"status": r.status.value,
|
|
"tokens": r.token_count,
|
|
"matches_reference": list(r.tokens) == model.stateless_greedy(prompt, n_new),
|
|
"kv_released": _kv_released(ad.manager, "clean", 0),
|
|
})
|
|
|
|
# 2. Deadline terminates a blocked stream.
|
|
clk = _Clock()
|
|
ad = _adapter(model)
|
|
r = HardenedSessionRunner(ad, clock=clk).run(
|
|
_gen("deadline", prompt, 50), deadline=3.0,
|
|
before_step=lambda _s: clk.advance(1.0),
|
|
)
|
|
scenarios.append({
|
|
"scenario": "deadline", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value, "tokens": r.token_count,
|
|
"kv_released": _kv_released(ad.manager, "deadline", 0),
|
|
})
|
|
|
|
# 3. Heartbeat/health loss terminates a blocked stream.
|
|
clk = _Clock()
|
|
ad = _adapter(model)
|
|
r = HardenedSessionRunner(ad, clock=clk).run(
|
|
_gen("heartbeat", prompt, 50), heartbeat_timeout=1.5,
|
|
heartbeat=lambda step: step < 2,
|
|
before_step=lambda _s: clk.advance(1.0),
|
|
)
|
|
scenarios.append({
|
|
"scenario": "heartbeat_loss", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value, "tokens": r.token_count,
|
|
"kv_released": _kv_released(ad.manager, "heartbeat", 0),
|
|
})
|
|
|
|
# 4. Explicit client cancellation releases KV.
|
|
ad = _adapter(model)
|
|
tok = CancellationToken()
|
|
r = HardenedSessionRunner(ad, work_ledger=ledger).run(
|
|
_gen("cancel", prompt, 50), cancel_token=tok,
|
|
before_step=lambda step: tok.cancel("client-hangup") if step == 3 else None,
|
|
)
|
|
scenarios.append({
|
|
"scenario": "cancel", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value, "tokens": r.token_count,
|
|
"kv_released": _kv_released(ad.manager, "cancel", 0),
|
|
})
|
|
|
|
# 5. Worker death mid-step -> unverified.
|
|
ad = _adapter(model, shard=_FaultyShard(model, 0, model.n_layers - 1, fail_at_call=4))
|
|
r = HardenedSessionRunner(ad, work_ledger=ledger).run(_gen("worker", prompt, n_new))
|
|
scenarios.append({
|
|
"scenario": "worker_death", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value, "tokens": r.token_count,
|
|
"restartable": r.restartable, "kv_released": _kv_released(ad.manager, "worker", 0),
|
|
})
|
|
|
|
# 6. Stream reset -> failed, restartable.
|
|
ad = _adapter(model)
|
|
def reset(step):
|
|
if step == 2:
|
|
raise StreamTerminated(FailureKind.STREAM_RESET, "peer reset")
|
|
r = HardenedSessionRunner(ad).run(_gen("reset", prompt, n_new), before_step=reset)
|
|
scenarios.append({
|
|
"scenario": "stream_reset", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value, "restartable": r.restartable,
|
|
})
|
|
|
|
# 7. Stale epoch -> failed.
|
|
ad = _adapter(model)
|
|
ad.manager.open("stale", 5)
|
|
r = HardenedSessionRunner(ad).run(_gen("stale", prompt, n_new, epoch=3))
|
|
scenarios.append({
|
|
"scenario": "stale_epoch", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value,
|
|
})
|
|
|
|
# 8. Cache miss mid-stream -> restartable.
|
|
ad = _adapter(model)
|
|
mgr = ad.manager
|
|
r = HardenedSessionRunner(ad).run(
|
|
_gen("miss", prompt, 12),
|
|
before_step=lambda step: mgr.release("miss", 0) if step == 4 else None,
|
|
)
|
|
scenarios.append({
|
|
"scenario": "cache_miss", "status": r.status.value,
|
|
"failure_kind": r.failure_kind.value, "tokens": r.token_count,
|
|
"restartable": r.restartable,
|
|
})
|
|
|
|
# 9. Alpha failover: restart from token zero, no unverified KV import.
|
|
faulty = _FaultyShard(model, 0, model.n_layers - 1, fail_at_call=3)
|
|
ad = _adapter(model, shard=faulty)
|
|
runner = HardenedSessionRunner(ad, work_ledger=ledger)
|
|
controller = RestartController([ad.manager])
|
|
fo = runner.run_with_failover(_gen("failover", prompt, n_new, epoch=0), controller,
|
|
max_restarts=2)
|
|
old_epoch_stale = False
|
|
try:
|
|
ad.manager.resolve("failover", 0)
|
|
except StaleRouteEpochError:
|
|
old_epoch_stale = True
|
|
scenarios.append({
|
|
"scenario": "alpha_failover",
|
|
"final_status": fo.outcome.status.value,
|
|
"final_epoch": fo.outcome.route_epoch,
|
|
"restarts": fo.restarts,
|
|
"restarted_from_token_zero": list(fo.outcome.tokens) == model.stateless_greedy(prompt, n_new),
|
|
"old_epoch_stale": old_epoch_stale,
|
|
"attempt_statuses": [a.status.value for a in fo.attempts],
|
|
})
|
|
|
|
result = {
|
|
"schema_version": 1,
|
|
"evidence_kind": "synthetic-unit",
|
|
"model": {
|
|
"architecture": model.architecture_adapter,
|
|
"n_layers": model.n_layers, "vocab": model.vocab, "hidden": model.hidden,
|
|
},
|
|
"scenarios": scenarios,
|
|
"work_ledger": ledger.to_dict(),
|
|
}
|
|
|
|
out_path = os.path.join(os.path.dirname(__file__), "results.json")
|
|
with open(out_path, "w") as fh:
|
|
json.dump(result, fh, indent=2)
|
|
fh.write("\n")
|
|
counts = ledger.counts_by_status()
|
|
print(f"wrote {out_path}")
|
|
print(f"work statuses: {counts} billable_tokens={ledger.billable_tokens()}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|