fix(validator): connect audit commitments to request flow
Record completed tracker requests into the validation log with ordinary route metadata, then have the validator fetch missing hop TOPLOC commitments on demand only after sampling. This closes the remaining alpha review gap where bisection worked only for synthetic events. Verification: .venv/bin/python -m compileall -q packages tests; pytest tests/test_hop_bisection.py tests/test_billing_ledger.py tests/test_toploc_audit.py -q (31 passed); full pytest: 316 passed, 3 skipped, 1 env failure from meshnet-node pid 1263451 occupying port 7000.
This commit is contained in:
Dobromir Popov
@@ -583,6 +583,79 @@ def test_proxy_chat_rejects_request_above_spend_cap_before_routing():
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_proxy_chat_records_validation_event_with_plain_route_metadata():
|
||||
class FakeRegistry:
|
||||
def get_wallet(self, wallet_address):
|
||||
return type("Wallet", (), {"banned": False})()
|
||||
|
||||
class FakeValidation:
|
||||
def __init__(self):
|
||||
self.events = []
|
||||
|
||||
def record_completed_inference(self, **kwargs):
|
||||
self.events.append(kwargs)
|
||||
return kwargs
|
||||
|
||||
class FakeContracts:
|
||||
def __init__(self):
|
||||
self.registry = FakeRegistry()
|
||||
self.validation = FakeValidation()
|
||||
|
||||
contracts = FakeContracts()
|
||||
tracker = TrackerServer(
|
||||
model_presets={
|
||||
MODEL: {
|
||||
"layers_start": 0,
|
||||
"layers_end": 11,
|
||||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
||||
}
|
||||
},
|
||||
contracts=contracts,
|
||||
)
|
||||
port = tracker.start()
|
||||
tracker_url = f"http://127.0.0.1:{port}"
|
||||
stub = _UsageStubNode(total_tokens=1000)
|
||||
stub_port = stub.start()
|
||||
try:
|
||||
reg = json.dumps({
|
||||
"endpoint": f"http://127.0.0.1:{stub_port}",
|
||||
"shard_start": 0,
|
||||
"shard_end": 11,
|
||||
"model": MODEL,
|
||||
"hardware_profile": {},
|
||||
"score": 1.0,
|
||||
"tracker_mode": True,
|
||||
"wallet_address": "wallet-head",
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{tracker_url}/v1/nodes/register",
|
||||
data=reg,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
r.read()
|
||||
|
||||
_chat(tracker_url, api_key=None)
|
||||
|
||||
assert len(contracts.validation.events) == 1
|
||||
event = contracts.validation.events[0]
|
||||
assert event["model"] == MODEL
|
||||
assert event["messages"] == [{"role": "user", "content": "hi"}]
|
||||
assert event["observed_output"] == "ok"
|
||||
assert event["route_nodes"] == [{
|
||||
"node_id": next(iter(tracker._registry)),
|
||||
"endpoint": f"http://127.0.0.1:{stub_port}",
|
||||
"wallet_address": "wallet-head",
|
||||
"shard_start": 0,
|
||||
"shard_end": 11,
|
||||
}]
|
||||
assert "toploc_proof" not in event["route_nodes"][0]
|
||||
finally:
|
||||
stub.stop()
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_billing_gossip_endpoint_applies_events(billed_tracker):
|
||||
tracker_url, ledger, _ = billed_tracker
|
||||
peer = BillingLedger(starting_credit=0.03, default_price_per_1k=0.02)
|
||||
|
||||
@@ -9,6 +9,10 @@ not always the last one.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import socketserver
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
from meshnet_validator import ToplocAuditConfig, ValidatorProcess
|
||||
@@ -28,12 +32,19 @@ class FakeToploc:
|
||||
}
|
||||
|
||||
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
|
||||
fingerprint = proofs.get("activation_fingerprint") if isinstance(proofs, dict) else None
|
||||
return proofs == {
|
||||
"activation_fingerprint": tuple(tuple(row) for row in activations),
|
||||
"decode_batching_size": decode_batching_size,
|
||||
"topk": topk,
|
||||
"skip_prefill": skip_prefill,
|
||||
}
|
||||
} or (
|
||||
fingerprint is not None
|
||||
and tuple(tuple(row) for row in fingerprint) == tuple(tuple(row) for row in activations)
|
||||
and proofs.get("decode_batching_size") == decode_batching_size
|
||||
and proofs.get("topk") == topk
|
||||
and proofs.get("skip_prefill") == skip_prefill
|
||||
)
|
||||
|
||||
|
||||
class FakeValidationLog:
|
||||
@@ -66,6 +77,44 @@ class FakeContracts:
|
||||
self.registry = FakeRegistry()
|
||||
|
||||
|
||||
class CommitmentStubServer:
|
||||
def __init__(self, commitments: dict[str, dict]) -> None:
|
||||
self.requests: list[dict] = []
|
||||
outer = self
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
if self.path != "/v1/audit/toploc/commitment":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
outer.requests.append(body)
|
||||
payload = json.dumps(commitments[body["session_id"]]).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
|
||||
self._server.daemon_threads = True
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> str:
|
||||
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
||||
self._thread.start()
|
||||
return f"http://127.0.0.1:{self._server.server_address[1]}"
|
||||
|
||||
def stop(self) -> None:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
|
||||
|
||||
def _hop_route_event(*, index, hop0_claim_activations, hop1_claim_activations, config, backend, ts=None):
|
||||
"""Two-hop route where each hop carries its own TOPLOC commitment built
|
||||
from the given (possibly corrupted) activations."""
|
||||
@@ -277,3 +326,63 @@ def test_hop_commitments_are_not_requested_unless_the_event_is_audit_selected():
|
||||
assert validator.hops_calls == []
|
||||
assert validator.text_reference_calls == []
|
||||
assert validator.sampled_count == 0
|
||||
|
||||
def test_validator_fetches_missing_hop_commitments_on_demand_after_sampling():
|
||||
"""Production route events only carry hop metadata. Once sampled, the
|
||||
validator asks each route node for its retained boundary commitment window
|
||||
and then runs the same first-divergent-hop bisection path."""
|
||||
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
||||
backend = FakeToploc()
|
||||
reference_hop0 = [[1.0, 2.0], [3.0, 4.0]]
|
||||
reference_hop1 = [[5.0, 6.0], [7.0, 8.0]]
|
||||
hop0_claim = build_activation_proofs(reference_hop0, config=config, backend=backend)
|
||||
hop1_claim = build_activation_proofs(reference_hop1, config=config, backend=backend)
|
||||
|
||||
stub = CommitmentStubServer({
|
||||
"session-on-demand": {
|
||||
"toploc_proof": hop0_claim.as_mapping(),
|
||||
"claimed_token_ids": [101, 202, 303],
|
||||
}
|
||||
})
|
||||
endpoint0 = stub.start()
|
||||
try:
|
||||
event = SimpleNamespace(
|
||||
index=0,
|
||||
session_id="session-on-demand",
|
||||
model="Qwen2.5-0.5B-Instruct",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
observed_output="two-hop pipeline output",
|
||||
route_nodes=[
|
||||
{"wallet_address": "wallet-hop0", "endpoint": endpoint0, "shard_start": 0, "shard_end": 15},
|
||||
{
|
||||
"wallet_address": "wallet-hop1",
|
||||
"endpoint": "http://127.0.0.1:1",
|
||||
"shard_start": 16,
|
||||
"shard_end": 31,
|
||||
"toploc_proof": hop1_claim.as_mapping(),
|
||||
"claimed_token_ids": [101, 202, 303],
|
||||
},
|
||||
],
|
||||
)
|
||||
contracts = FakeContracts([event])
|
||||
validator = HopReferenceValidator(
|
||||
contracts=contracts,
|
||||
reference_node_url="http://reference.invalid",
|
||||
sample_rate=1.0,
|
||||
random_seed=7,
|
||||
toploc_config=config,
|
||||
toploc_backend=backend,
|
||||
reference_activations_by_hop=[reference_hop0, reference_hop1],
|
||||
)
|
||||
|
||||
assert validator.validate_once() == []
|
||||
assert stub.requests == [{
|
||||
"session_id": "session-on-demand",
|
||||
"model": "Qwen2.5-0.5B-Instruct",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"shard_start": 0,
|
||||
"shard_end": 15,
|
||||
}]
|
||||
assert validator.hops_calls
|
||||
finally:
|
||||
stub.stop()
|
||||
|
||||
Reference in New Issue
Block a user