fix(validator): connect audit commitments to request flow
Record completed tracker requests into the validation log with ordinary route metadata, then have the validator fetch missing hop TOPLOC commitments on demand only after sampling. This closes the remaining alpha review gap where bisection worked only for synthetic events. Verification: .venv/bin/python -m compileall -q packages tests; pytest tests/test_hop_bisection.py tests/test_billing_ledger.py tests/test_toploc_audit.py -q (31 passed); full pytest: 316 passed, 3 skipped, 1 env failure from meshnet-node pid 1263451 occupying port 7000.
This commit is contained in:
@@ -1330,6 +1330,20 @@ def _observed_non_stream_completion_tokens(payload: dict) -> int:
|
||||
return observed
|
||||
|
||||
|
||||
def _observed_output_from_non_stream_payload(payload: dict) -> str:
|
||||
choices = payload.get("choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
return ""
|
||||
first = choices[0]
|
||||
if not isinstance(first, dict):
|
||||
return ""
|
||||
message = first.get("message")
|
||||
if isinstance(message, dict) and isinstance(message.get("content"), str):
|
||||
return message["content"]
|
||||
text = first.get("text")
|
||||
return text if isinstance(text, str) else ""
|
||||
|
||||
|
||||
def _observed_stream_tokens(payload: dict) -> int:
|
||||
choices = payload.get("choices")
|
||||
if not isinstance(choices, list):
|
||||
@@ -2169,11 +2183,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
|
||||
flush=True,
|
||||
)
|
||||
observed_output = ""
|
||||
try:
|
||||
tokens = _billable_non_stream_tokens(json.loads(resp_body), body)
|
||||
response_payload = json.loads(resp_body)
|
||||
tokens = _billable_non_stream_tokens(response_payload, body)
|
||||
observed_output = _observed_output_from_non_stream_payload(response_payload)
|
||||
except json.JSONDecodeError:
|
||||
tokens = 0
|
||||
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
||||
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
|
||||
self._bill_completed(api_key, model, tokens, node_work)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", content_type)
|
||||
@@ -2217,6 +2235,42 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
if observed is not None:
|
||||
node.model_tokens_per_sec[model] = float(observed)
|
||||
|
||||
def _record_validation_event(
|
||||
self,
|
||||
session_id: str,
|
||||
model: str,
|
||||
request_body: dict,
|
||||
observed_output: str,
|
||||
route_nodes: list[_NodeEntry],
|
||||
) -> None:
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
validation = getattr(server.contracts, "validation", None) if server.contracts is not None else None
|
||||
if validation is None:
|
||||
return
|
||||
messages = request_body.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
messages = []
|
||||
event_nodes = [
|
||||
{
|
||||
"node_id": node.node_id,
|
||||
"endpoint": node.endpoint,
|
||||
"wallet_address": node.wallet_address,
|
||||
"shard_start": node.shard_start,
|
||||
"shard_end": node.shard_end,
|
||||
}
|
||||
for node in route_nodes
|
||||
]
|
||||
try:
|
||||
validation.record_completed_inference(
|
||||
session_id=session_id,
|
||||
model=model,
|
||||
messages=[dict(message) for message in messages if isinstance(message, dict)],
|
||||
observed_output=observed_output,
|
||||
route_nodes=event_nodes,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[tracker] validation event recording failed for {session_id}: {exc}", flush=True)
|
||||
|
||||
def _bill_completed(
|
||||
self,
|
||||
api_key: str | None,
|
||||
|
||||
Reference in New Issue
Block a user