fix(validator): connect audit commitments to request flow

Record completed tracker requests into the validation log with ordinary route metadata, then have the validator fetch missing hop TOPLOC commitments on demand only after sampling. This closes the remaining alpha review gap where bisection worked only for synthetic events.

Verification: .venv/bin/python -m compileall -q packages tests; pytest tests/test_hop_bisection.py tests/test_billing_ledger.py tests/test_toploc_audit.py -q (31 passed); full pytest: 316 passed, 3 skipped, 1 env failure from meshnet-node pid 1263451 occupying port 7000.
This commit is contained in:
Dobromir Popov
2026-07-05 22:16:31 +03:00
parent de6ce1d514
commit af56dec7bd
4 changed files with 302 additions and 6 deletions

View File

@@ -1330,6 +1330,20 @@ def _observed_non_stream_completion_tokens(payload: dict) -> int:
return observed
def _observed_output_from_non_stream_payload(payload: dict) -> str:
choices = payload.get("choices")
if not isinstance(choices, list) or not choices:
return ""
first = choices[0]
if not isinstance(first, dict):
return ""
message = first.get("message")
if isinstance(message, dict) and isinstance(message.get("content"), str):
return message["content"]
text = first.get("text")
return text if isinstance(text, str) else ""
def _observed_stream_tokens(payload: dict) -> int:
choices = payload.get("choices")
if not isinstance(choices, list):
@@ -2169,11 +2183,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
flush=True,
)
observed_output = ""
try:
tokens = _billable_non_stream_tokens(json.loads(resp_body), body)
response_payload = json.loads(resp_body)
tokens = _billable_non_stream_tokens(response_payload, body)
observed_output = _observed_output_from_non_stream_payload(response_payload)
except json.JSONDecodeError:
tokens = 0
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
self._bill_completed(api_key, model, tokens, node_work)
self.send_response(200)
self.send_header("Content-Type", content_type)
@@ -2217,6 +2235,42 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if observed is not None:
node.model_tokens_per_sec[model] = float(observed)
def _record_validation_event(
self,
session_id: str,
model: str,
request_body: dict,
observed_output: str,
route_nodes: list[_NodeEntry],
) -> None:
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
validation = getattr(server.contracts, "validation", None) if server.contracts is not None else None
if validation is None:
return
messages = request_body.get("messages")
if not isinstance(messages, list):
messages = []
event_nodes = [
{
"node_id": node.node_id,
"endpoint": node.endpoint,
"wallet_address": node.wallet_address,
"shard_start": node.shard_start,
"shard_end": node.shard_end,
}
for node in route_nodes
]
try:
validation.record_completed_inference(
session_id=session_id,
model=model,
messages=[dict(message) for message in messages if isinstance(message, dict)],
observed_output=observed_output,
route_nodes=event_nodes,
)
except Exception as exc:
print(f"[tracker] validation event recording failed for {session_id}: {exc}", flush=True)
def _bill_completed(
self,
api_key: str | None,

View File

@@ -142,6 +142,7 @@ class ValidatorProcess:
return text
def _validate_event(self, event: Any) -> "_AuditResult":
event = self._event_with_on_demand_commitments(event)
hop_commitments = _hop_commitments_from_event(event)
if hop_commitments is not None and self._commitment_expired(event):
# ADR-0018 §3: the on-demand retention window has passed — nodes
@@ -168,15 +169,15 @@ class ValidatorProcess:
# call so existing single-hop reference integrations keep working.
only = hop_commitments[0]
reference_activations_by_hop = [self._run_teacher_forced_prefill(
model=event.model,
messages=event.messages,
model=_event_value(event, "model"),
messages=_event_value(event, "messages"),
claimed_token_ids=only.token_ids,
claim=only.claim,
)]
else:
reference_activations_by_hop = self._run_teacher_forced_prefill_hops(
model=event.model,
messages=event.messages,
model=_event_value(event, "model"),
messages=_event_value(event, "messages"),
hop_commitments=hop_commitments,
)
culprit_index = _first_divergent_hop(
@@ -197,6 +198,65 @@ class ValidatorProcess:
culprit_node=None if ok else hop_commitments[culprit_index].node,
)
def _event_with_on_demand_commitments(self, event: Any) -> Any:
"""Fetch missing per-hop TOPLOC commitments only after audit sampling.
Tracker validation events deliberately carry ordinary route metadata,
not a pre-announced audit flag. When this validator samples an event, it
asks each hop for its short-lived boundary commitment and splices the
returned proof into a local event copy for the bisection verifier.
"""
route_nodes = _event_value(event, "route_nodes") or []
if not isinstance(route_nodes, list) or not route_nodes:
return event
updated_nodes: list[dict] = []
changed = False
for node in route_nodes:
if not isinstance(node, dict):
updated_nodes.append(node)
continue
updated = dict(node)
if _mapping_value(updated, "toploc_proof") is None:
commitment = self._fetch_hop_commitment(event, updated)
if commitment is not None:
updated.update(commitment)
changed = True
updated_nodes.append(updated)
if not changed:
return event
if isinstance(event, dict):
copied = dict(event)
else:
copied = dict(vars(event))
copied["route_nodes"] = updated_nodes
return copied
def _fetch_hop_commitment(self, event: Any, node: dict) -> dict[str, Any] | None:
endpoint = node.get("endpoint")
if not isinstance(endpoint, str) or not endpoint:
return None
try:
response = _post_json(
f"{endpoint.rstrip('/')}/v1/audit/toploc/commitment",
{
"session_id": _event_value(event, "session_id"),
"model": _event_value(event, "model"),
"messages": _event_value(event, "messages") or [],
"shard_start": node.get("shard_start"),
"shard_end": node.get("shard_end"),
},
timeout=2.0,
)
except (OSError, ValueError, json.JSONDecodeError):
return None
proof = response.get("toploc_proof") or response.get("activation_proof")
token_ids = response.get("claimed_token_ids") or response.get("output_token_ids")
if not isinstance(proof, dict):
return None
if not isinstance(token_ids, list) or not all(isinstance(token, int) for token in token_ids):
return None
return {"toploc_proof": proof, "claimed_token_ids": token_ids}
def _commitment_expired(self, event: Any) -> bool:
ts = _event_value(event, "ts")
if ts is None: