fix(validator): connect audit commitments to request flow
Record completed tracker requests into the validation log with ordinary route metadata, then have the validator fetch missing hop TOPLOC commitments on demand only after sampling. This closes the remaining alpha review gap where bisection worked only for synthetic events. Verification: .venv/bin/python -m compileall -q packages tests; pytest tests/test_hop_bisection.py tests/test_billing_ledger.py tests/test_toploc_audit.py -q (31 passed); full pytest: 316 passed, 3 skipped, 1 env failure from meshnet-node pid 1263451 occupying port 7000.
This commit is contained in:
@@ -1330,6 +1330,20 @@ def _observed_non_stream_completion_tokens(payload: dict) -> int:
|
||||
return observed
|
||||
|
||||
|
||||
def _observed_output_from_non_stream_payload(payload: dict) -> str:
|
||||
choices = payload.get("choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return ""
|
||||
first = choices[0]
|
||||
if not isinstance(first, dict):
|
||||
return ""
|
||||
message = first.get("message")
|
||||
if isinstance(message, dict) and isinstance(message.get("content"), str):
|
||||
return message["content"]
|
||||
text = first.get("text")
|
||||
return text if isinstance(text, str) else ""
|
||||
|
||||
|
||||
def _observed_stream_tokens(payload: dict) -> int:
|
||||
choices = payload.get("choices")
|
||||
if not isinstance(choices, list):
|
||||
@@ -2169,11 +2183,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
|
||||
flush=True,
|
||||
)
|
||||
observed_output = ""
|
||||
try:
|
||||
tokens = _billable_non_stream_tokens(json.loads(resp_body), body)
|
||||
response_payload = json.loads(resp_body)
|
||||
tokens = _billable_non_stream_tokens(response_payload, body)
|
||||
observed_output = _observed_output_from_non_stream_payload(response_payload)
|
||||
except json.JSONDecodeError:
|
||||
tokens = 0
|
||||
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
||||
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
|
||||
self._bill_completed(api_key, model, tokens, node_work)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
@@ -2217,6 +2235,42 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
if observed is not None:
|
||||
node.model_tokens_per_sec[model] = float(observed)
|
||||
|
||||
def _record_validation_event(
|
||||
self,
|
||||
session_id: str,
|
||||
model: str,
|
||||
request_body: dict,
|
||||
observed_output: str,
|
||||
route_nodes: list[_NodeEntry],
|
||||
) -> None:
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
validation = getattr(server.contracts, "validation", None) if server.contracts is not None else None
|
||||
if validation is None:
|
||||
return
|
||||
messages = request_body.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
messages = []
|
||||
event_nodes = [
|
||||
{
|
||||
"node_id": node.node_id,
|
||||
"endpoint": node.endpoint,
|
||||
"wallet_address": node.wallet_address,
|
||||
"shard_start": node.shard_start,
|
||||
"shard_end": node.shard_end,
|
||||
}
|
||||
for node in route_nodes
|
||||
]
|
||||
try:
|
||||
validation.record_completed_inference(
|
||||
session_id=session_id,
|
||||
model=model,
|
||||
messages=[dict(message) for message in messages if isinstance(message, dict)],
|
||||
observed_output=observed_output,
|
||||
route_nodes=event_nodes,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[tracker] validation event recording failed for {session_id}: {exc}", flush=True)
|
||||
|
||||
def _bill_completed(
|
||||
self,
|
||||
api_key: str | None,
|
||||
|
||||
@@ -142,6 +142,7 @@ class ValidatorProcess:
|
||||
return text
|
||||
|
||||
def _validate_event(self, event: Any) -> "_AuditResult":
|
||||
event = self._event_with_on_demand_commitments(event)
|
||||
hop_commitments = _hop_commitments_from_event(event)
|
||||
if hop_commitments is not None and self._commitment_expired(event):
|
||||
# ADR-0018 §3: the on-demand retention window has passed — nodes
|
||||
@@ -168,15 +169,15 @@ class ValidatorProcess:
|
||||
# call so existing single-hop reference integrations keep working.
|
||||
only = hop_commitments[0]
|
||||
reference_activations_by_hop = [self._run_teacher_forced_prefill(
|
||||
model=event.model,
|
||||
messages=event.messages,
|
||||
model=_event_value(event, "model"),
|
||||
messages=_event_value(event, "messages"),
|
||||
claimed_token_ids=only.token_ids,
|
||||
claim=only.claim,
|
||||
)]
|
||||
else:
|
||||
reference_activations_by_hop = self._run_teacher_forced_prefill_hops(
|
||||
model=event.model,
|
||||
messages=event.messages,
|
||||
model=_event_value(event, "model"),
|
||||
messages=_event_value(event, "messages"),
|
||||
hop_commitments=hop_commitments,
|
||||
)
|
||||
culprit_index = _first_divergent_hop(
|
||||
@@ -197,6 +198,65 @@ class ValidatorProcess:
|
||||
culprit_node=None if ok else hop_commitments[culprit_index].node,
|
||||
)
|
||||
|
||||
def _event_with_on_demand_commitments(self, event: Any) -> Any:
|
||||
"""Fetch missing per-hop TOPLOC commitments only after audit sampling.
|
||||
|
||||
Tracker validation events deliberately carry ordinary route metadata,
|
||||
not a pre-announced audit flag. When this validator samples an event, it
|
||||
asks each hop for its short-lived boundary commitment and splices the
|
||||
returned proof into a local event copy for the bisection verifier.
|
||||
"""
|
||||
route_nodes = _event_value(event, "route_nodes") or []
|
||||
if not isinstance(route_nodes, list) or not route_nodes:
|
||||
return event
|
||||
updated_nodes: list[dict] = []
|
||||
changed = False
|
||||
for node in route_nodes:
|
||||
if not isinstance(node, dict):
|
||||
updated_nodes.append(node)
|
||||
continue
|
||||
updated = dict(node)
|
||||
if _mapping_value(updated, "toploc_proof") is None:
|
||||
commitment = self._fetch_hop_commitment(event, updated)
|
||||
if commitment is not None:
|
||||
updated.update(commitment)
|
||||
changed = True
|
||||
updated_nodes.append(updated)
|
||||
if not changed:
|
||||
return event
|
||||
if isinstance(event, dict):
|
||||
copied = dict(event)
|
||||
else:
|
||||
copied = dict(vars(event))
|
||||
copied["route_nodes"] = updated_nodes
|
||||
return copied
|
||||
|
||||
def _fetch_hop_commitment(self, event: Any, node: dict) -> dict[str, Any] | None:
|
||||
endpoint = node.get("endpoint")
|
||||
if not isinstance(endpoint, str) or not endpoint:
|
||||
return None
|
||||
try:
|
||||
response = _post_json(
|
||||
f"{endpoint.rstrip('/')}/v1/audit/toploc/commitment",
|
||||
{
|
||||
"session_id": _event_value(event, "session_id"),
|
||||
"model": _event_value(event, "model"),
|
||||
"messages": _event_value(event, "messages") or [],
|
||||
"shard_start": node.get("shard_start"),
|
||||
"shard_end": node.get("shard_end"),
|
||||
},
|
||||
timeout=2.0,
|
||||
)
|
||||
except (OSError, ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
proof = response.get("toploc_proof") or response.get("activation_proof")
|
||||
token_ids = response.get("claimed_token_ids") or response.get("output_token_ids")
|
||||
if not isinstance(proof, dict):
|
||||
return None
|
||||
if not isinstance(token_ids, list) or not all(isinstance(token, int) for token in token_ids):
|
||||
return None
|
||||
return {"toploc_proof": proof, "claimed_token_ids": token_ids}
|
||||
|
||||
def _commitment_expired(self, event: Any) -> bool:
|
||||
ts = _event_value(event, "ts")
|
||||
if ts is None:
|
||||
|
||||
Reference in New Issue
Block a user