382 lines
14 KiB
Python
382 lines
14 KiB
Python
"""AH-007: on-demand hop-boundary commitments + first-divergent-hop bisection.
|
|
|
|
`_final_text_node` (blame `max(shard_end)`) is wrong for multi-hop pipelines:
|
|
a corrupt early hop still produces a route whose *last* hop reported perfectly
|
|
honest activations. These tests build a two-hop route, corrupt one hop's
|
|
committed fingerprint, and assert the referee blames that hop specifically —
|
|
not always the last one.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import http.server
|
|
import json
|
|
import socketserver
|
|
import threading
|
|
from types import SimpleNamespace
|
|
|
|
from meshnet_validator import ToplocAuditConfig, ValidatorProcess
|
|
from meshnet_validator.audit import build_activation_proofs
|
|
|
|
|
|
class FakeToploc:
|
|
"""Fingerprint = the activations themselves; verify is exact equality,
|
|
matching the encode/verify contract exercised in test_toploc_audit.py."""
|
|
|
|
def build_proofs_base64(self, activations, *, decode_batching_size, topk, skip_prefill):
|
|
return {
|
|
"activation_fingerprint": tuple(tuple(row) for row in activations),
|
|
"decode_batching_size": decode_batching_size,
|
|
"topk": topk,
|
|
"skip_prefill": skip_prefill,
|
|
}
|
|
|
|
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
|
|
fingerprint = proofs.get("activation_fingerprint") if isinstance(proofs, dict) else None
|
|
return proofs == {
|
|
"activation_fingerprint": tuple(tuple(row) for row in activations),
|
|
"decode_batching_size": decode_batching_size,
|
|
"topk": topk,
|
|
"skip_prefill": skip_prefill,
|
|
} or (
|
|
fingerprint is not None
|
|
and tuple(tuple(row) for row in fingerprint) == tuple(tuple(row) for row in activations)
|
|
and proofs.get("decode_batching_size") == decode_batching_size
|
|
and proofs.get("topk") == topk
|
|
and proofs.get("skip_prefill") == skip_prefill
|
|
)
|
|
|
|
|
|
class FakeValidationLog:
|
|
def __init__(self, events) -> None:
|
|
self._events = events
|
|
|
|
def list_completed_inferences(self, *, after_index: int = -1):
|
|
return [event for event in self._events if event.index > after_index]
|
|
|
|
|
|
class FakeRegistry:
|
|
def __init__(self) -> None:
|
|
self.slashes: list[dict] = []
|
|
self.audit_outcomes: list[dict] = []
|
|
|
|
def get_wallet(self, wallet_address: str):
|
|
return SimpleNamespace(banned=False)
|
|
|
|
def submit_slash_proof(self, **kwargs):
|
|
self.slashes.append(kwargs)
|
|
return kwargs
|
|
|
|
def record_audit_outcome(self, wallet_address: str, *, passed: bool):
|
|
self.audit_outcomes.append({"wallet_address": wallet_address, "passed": passed})
|
|
|
|
|
|
class FakeContracts:
|
|
def __init__(self, events) -> None:
|
|
self.validation = FakeValidationLog(events)
|
|
self.registry = FakeRegistry()
|
|
|
|
|
|
class CommitmentStubServer:
|
|
def __init__(self, commitments: dict[str, dict]) -> None:
|
|
self.requests: list[dict] = []
|
|
outer = self
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
def log_message(self, fmt, *args):
|
|
pass
|
|
|
|
def do_POST(self):
|
|
if self.path != "/v1/audit/toploc/commitment":
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
return
|
|
length = int(self.headers.get("Content-Length", 0))
|
|
body = json.loads(self.rfile.read(length) or b"{}")
|
|
outer.requests.append(body)
|
|
payload = json.dumps(commitments[body["session_id"]]).encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(payload)))
|
|
self.end_headers()
|
|
self.wfile.write(payload)
|
|
|
|
self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
|
|
self._server.daemon_threads = True
|
|
self._thread: threading.Thread | None = None
|
|
|
|
def start(self) -> str:
|
|
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
|
self._thread.start()
|
|
return f"http://127.0.0.1:{self._server.server_address[1]}"
|
|
|
|
def stop(self) -> None:
|
|
self._server.shutdown()
|
|
self._server.server_close()
|
|
|
|
|
|
def _hop_route_event(*, index, hop0_claim_activations, hop1_claim_activations, config, backend, ts=None):
|
|
"""Two-hop route where each hop carries its own TOPLOC commitment built
|
|
from the given (possibly corrupted) activations."""
|
|
hop0_claim = build_activation_proofs(hop0_claim_activations, config=config, backend=backend)
|
|
hop1_claim = build_activation_proofs(hop1_claim_activations, config=config, backend=backend)
|
|
route_nodes = [
|
|
{
|
|
"wallet_address": "wallet-hop0",
|
|
"shard_start": 0,
|
|
"shard_end": 15,
|
|
"toploc_proof": hop0_claim.as_mapping(),
|
|
},
|
|
{
|
|
"wallet_address": "wallet-hop1",
|
|
"shard_start": 16,
|
|
"shard_end": 31,
|
|
"toploc_proof": hop1_claim.as_mapping(),
|
|
},
|
|
]
|
|
event = SimpleNamespace(
|
|
index=index,
|
|
session_id=f"session-hop-{index}",
|
|
model="Qwen2.5-0.5B-Instruct",
|
|
messages=[{"role": "user", "content": "hello"}],
|
|
observed_output="two-hop pipeline output",
|
|
route_nodes=route_nodes,
|
|
claimed_token_ids=[101, 202, 303],
|
|
)
|
|
if ts is not None:
|
|
event.ts = ts
|
|
return event
|
|
|
|
|
|
class HopReferenceValidator(ValidatorProcess):
|
|
"""Stands in for the reference node: returns canned ground-truth
|
|
activations at each hop cut-point instead of making an HTTP call."""
|
|
|
|
def __init__(self, *args, reference_activations_by_hop, **kwargs) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.reference_activations_by_hop = reference_activations_by_hop
|
|
self.hops_calls: list[dict] = []
|
|
self.text_reference_calls: list[list[dict]] = []
|
|
|
|
def _run_teacher_forced_prefill_hops(self, **kwargs):
|
|
self.hops_calls.append(kwargs)
|
|
return self.reference_activations_by_hop
|
|
|
|
def _run_reference(self, messages: list[dict]) -> str:
|
|
self.text_reference_calls.append(messages)
|
|
return "two-hop pipeline output"
|
|
|
|
|
|
def test_bisection_blames_first_divergent_hop_not_last_hop():
|
|
"Red: corrupt hop-0 only.\n\nTags: general"
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
backend = FakeToploc()
|
|
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
|
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
|
corrupted_hop0_claim_activations = [[9.0, 9.0], [9.0, 9.0]] # diverges from reference_hop0
|
|
|
|
event = _hop_route_event(
|
|
index=0,
|
|
hop0_claim_activations=corrupted_hop0_claim_activations,
|
|
hop1_claim_activations=reference_hop1,
|
|
config=config,
|
|
backend=backend,
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = HopReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=backend,
|
|
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
|
)
|
|
|
|
receipts = validator.validate_once()
|
|
|
|
assert len(receipts) == 1
|
|
assert contracts.registry.slashes[0]["wallet_address"] == "wallet-hop0"
|
|
assert "hop 0" in contracts.registry.slashes[0]["reason"]
|
|
|
|
|
|
def test_bisection_blames_the_actual_faulty_hop_when_it_is_the_second_hop():
|
|
"Integration test: multi-hop pipeline, fault injected at a known\n\nTags: general"
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
backend = FakeToploc()
|
|
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
|
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
|
corrupted_hop1_claim_activations = [[0.0, 0.0], [0.0, 0.0]]
|
|
|
|
event = _hop_route_event(
|
|
index=0,
|
|
hop0_claim_activations=reference_hop0,
|
|
hop1_claim_activations=corrupted_hop1_claim_activations,
|
|
config=config,
|
|
backend=backend,
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = HopReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=backend,
|
|
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
|
)
|
|
|
|
receipts = validator.validate_once()
|
|
|
|
assert len(receipts) == 1
|
|
assert contracts.registry.slashes[0]["wallet_address"] == "wallet-hop1"
|
|
|
|
|
|
def test_honest_two_hop_route_is_not_slashed():
|
|
"Honest two hop route is not slashed\n\nTags: routing"
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
backend = FakeToploc()
|
|
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
|
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
|
|
|
event = _hop_route_event(
|
|
index=0,
|
|
hop0_claim_activations=reference_hop0,
|
|
hop1_claim_activations=reference_hop1,
|
|
config=config,
|
|
backend=backend,
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = HopReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=backend,
|
|
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
|
)
|
|
|
|
assert validator.validate_once() == []
|
|
assert contracts.registry.slashes == []
|
|
|
|
|
|
def test_expired_commitment_window_falls_back_to_text_only_audit():
|
|
"ADR-0018 §3: nodes only retain boundary activations briefly.\n\nTags: general"
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16, commitment_ttl_seconds=1.0)
|
|
backend = FakeToploc()
|
|
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
|
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
|
|
|
event = _hop_route_event(
|
|
index=0,
|
|
hop0_claim_activations=reference_hop0,
|
|
hop1_claim_activations=reference_hop1,
|
|
config=config,
|
|
backend=backend,
|
|
ts=0.0, # far outside any TTL relative to real wall-clock time
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = HopReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=backend,
|
|
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
|
)
|
|
|
|
assert validator.validate_once() == []
|
|
assert validator.hops_calls == []
|
|
assert len(validator.text_reference_calls) == 1
|
|
|
|
|
|
def test_hop_commitments_are_not_requested_unless_the_event_is_audit_selected():
|
|
"On-demand: the (expensive) per-hop commitment retrieval only happens\n\nTags: general"
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
backend = FakeToploc()
|
|
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
|
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
|
|
|
event = _hop_route_event(
|
|
index=0,
|
|
hop0_claim_activations=reference_hop0,
|
|
hop1_claim_activations=reference_hop1,
|
|
config=config,
|
|
backend=backend,
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = HopReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=0.0, # never selects any event for audit
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=backend,
|
|
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
|
)
|
|
|
|
assert validator.validate_once() == []
|
|
assert validator.hops_calls == []
|
|
assert validator.text_reference_calls == []
|
|
assert validator.sampled_count == 0
|
|
|
|
def test_validator_fetches_missing_hop_commitments_on_demand_after_sampling():
|
|
"Production route events only carry hop metadata.\n\nTags: general"
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
backend = FakeToploc()
|
|
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
|
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
|
hop0_claim = build_activation_proofs(reference_hop0, config=config, backend=backend)
|
|
hop1_claim = build_activation_proofs(reference_hop1, config=config, backend=backend)
|
|
|
|
stub = CommitmentStubServer({
|
|
"session-on-demand": {
|
|
"toploc_proof": hop0_claim.as_mapping(),
|
|
"claimed_token_ids": [101, 202, 303],
|
|
}
|
|
})
|
|
endpoint0 = stub.start()
|
|
try:
|
|
event = SimpleNamespace(
|
|
index=0,
|
|
session_id="session-on-demand",
|
|
model="Qwen2.5-0.5B-Instruct",
|
|
messages=[{"role": "user", "content": "hello"}],
|
|
observed_output="two-hop pipeline output",
|
|
route_nodes=[
|
|
{"wallet_address": "wallet-hop0", "endpoint": endpoint0, "shard_start": 0, "shard_end": 15},
|
|
{
|
|
"wallet_address": "wallet-hop1",
|
|
"endpoint": "http://127.0.0.1:1",
|
|
"shard_start": 16,
|
|
"shard_end": 31,
|
|
"toploc_proof": hop1_claim.as_mapping(),
|
|
"claimed_token_ids": [101, 202, 303],
|
|
},
|
|
],
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = HopReferenceValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=backend,
|
|
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
|
)
|
|
|
|
assert validator.validate_once() == []
|
|
assert stub.requests == [{
|
|
"session_id": "session-on-demand",
|
|
"model": "Qwen2.5-0.5B-Instruct",
|
|
"messages": [{"role": "user", "content": "hello"}],
|
|
"shard_start": 0,
|
|
"shard_end": 15,
|
|
}]
|
|
assert validator.hops_calls
|
|
finally:
|
|
stub.stop()
|