feat(alpha): complete hardening backlog
Complete the alpha-hardening Ralph task set, including tracker billing/accounting guards, validator fraud-audit primitives, wallet binding proof support, documentation runbooks, and updated tests. Verification: .venv/bin/python -m compileall -q packages tests; .venv/bin/python -m pytest -q --tb=short (313 passed, 3 skipped, 1 failed: tests/test_mining_cli.py::test_legacy_start_without_port_uses_next_available_port because meshnet-node pid 1263451 is already listening on port 7000).
This commit is contained in:
Dobromir Popov
@@ -8,6 +8,10 @@ import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .audit import ToplocAuditConfig, ToplocProofClaim, verify_activation_proofs
|
||||
from .sampling import AdaptiveAuditSampler, AuditRateConfig
|
||||
from .tripwire import detect_output_tripwire
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
|
||||
@@ -27,6 +31,9 @@ class ValidatorProcess:
|
||||
webhook_url: str | None = None,
|
||||
interval_seconds: float = 1.0,
|
||||
billing: Any | None = None,
|
||||
toploc_config: ToplocAuditConfig | None = None,
|
||||
toploc_backend: Any | None = None,
|
||||
audit_sampler: AdaptiveAuditSampler | None = None,
|
||||
) -> None:
|
||||
if not 0.0 <= sample_rate <= 1.0:
|
||||
raise ValueError("sample_rate must be between 0 and 1")
|
||||
@@ -48,6 +55,9 @@ class ValidatorProcess:
|
||||
self._strike_threshold = strike_threshold
|
||||
self._webhook_url = webhook_url
|
||||
self._interval_seconds = interval_seconds
|
||||
self._toploc_config = toploc_config or ToplocAuditConfig()
|
||||
self._toploc_backend = toploc_backend
|
||||
self._audit_sampler = audit_sampler
|
||||
self._random = random.Random(random_seed)
|
||||
self._last_event_index = -1
|
||||
self._running = False
|
||||
@@ -62,15 +72,47 @@ class ValidatorProcess:
|
||||
)
|
||||
for event in events:
|
||||
self._last_event_index = max(self._last_event_index, event.index)
|
||||
if self._random.random() >= self._sample_rate:
|
||||
if not self._should_sample(event):
|
||||
continue
|
||||
self.sampled_count += 1
|
||||
reference_output = self._run_reference(event.messages)
|
||||
if _outputs_match(event.observed_output, reference_output, self._tolerance):
|
||||
audit_result = self._validate_event(event)
|
||||
if audit_result.ok:
|
||||
self._record_clean_audit(event)
|
||||
continue
|
||||
receipts.extend(self._slash_route(event.route_nodes, event.observed_output, reference_output))
|
||||
receipts.extend(self._slash_node(
|
||||
audit_result.culprit_node,
|
||||
event.observed_output,
|
||||
audit_result.reference_output,
|
||||
reason=audit_result.reason,
|
||||
))
|
||||
return receipts
|
||||
|
||||
def _should_sample(self, event: Any) -> bool:
|
||||
"""ADR-0018 §1/§6-7: flat sample_rate stays the default; when an
|
||||
AdaptiveAuditSampler is configured, the decision is reputation- and
|
||||
tenure-weighted and budget-balanced against the fleet-wide target
|
||||
instead of a uniform coin flip."""
|
||||
if self._audit_sampler is None:
|
||||
return self._random.random() < self._sample_rate
|
||||
|
||||
tripwire = detect_output_tripwire(_event_value(event, "observed_output") or "")
|
||||
wallets = _route_wallets(event)
|
||||
if not wallets:
|
||||
return self._audit_sampler.should_audit(
|
||||
completed_job_count=0, reputation=1.0, tripwire=tripwire,
|
||||
)
|
||||
# A route is only as trustworthy as its least-trusted hop -- audit
|
||||
# against whichever wallet on the route looks riskiest.
|
||||
riskiest = min(
|
||||
(self._contracts.registry.get_wallet(wallet) for wallet in wallets),
|
||||
key=lambda wallet: wallet.reputation,
|
||||
)
|
||||
return self._audit_sampler.should_audit(
|
||||
completed_job_count=riskiest.completed_job_count,
|
||||
reputation=riskiest.reputation,
|
||||
tripwire=tripwire,
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
if self._running:
|
||||
raise RuntimeError("ValidatorProcess is already running")
|
||||
@@ -99,14 +141,141 @@ class ValidatorProcess:
|
||||
raise ValueError("reference node response did not contain text")
|
||||
return text
|
||||
|
||||
def _slash_route(
|
||||
def _validate_event(self, event: Any) -> "_AuditResult":
|
||||
hop_commitments = _hop_commitments_from_event(event)
|
||||
if hop_commitments is not None and self._commitment_expired(event):
|
||||
# ADR-0018 §3: the on-demand retention window has passed — nodes
|
||||
# are no longer expected to hold the boundary activations needed
|
||||
# to verify this commitment, so fall back to the text-only path.
|
||||
hop_commitments = None
|
||||
|
||||
if hop_commitments is None:
|
||||
reference_output = self._run_reference(event.messages)
|
||||
ok = _outputs_match(event.observed_output, reference_output, self._tolerance)
|
||||
return _AuditResult(
|
||||
ok=ok,
|
||||
reference_output=reference_output,
|
||||
reason="reference output diverged",
|
||||
# Text comparison has no per-hop signal; the last hop is the
|
||||
# best-effort guess (text-only fallback), never used when
|
||||
# hop-boundary commitments make real bisection possible.
|
||||
culprit_node=None if ok else _final_text_node(event.route_nodes),
|
||||
)
|
||||
|
||||
if len(hop_commitments) == 1:
|
||||
# Single-commitment route (AH-006 whole-route format, or a
|
||||
# genuinely one-hop pipeline): reuse the original teacher-forced
|
||||
# call so existing single-hop reference integrations keep working.
|
||||
only = hop_commitments[0]
|
||||
reference_activations_by_hop = [self._run_teacher_forced_prefill(
|
||||
model=event.model,
|
||||
messages=event.messages,
|
||||
claimed_token_ids=only.token_ids,
|
||||
claim=only.claim,
|
||||
)]
|
||||
else:
|
||||
reference_activations_by_hop = self._run_teacher_forced_prefill_hops(
|
||||
model=event.model,
|
||||
messages=event.messages,
|
||||
hop_commitments=hop_commitments,
|
||||
)
|
||||
culprit_index = _first_divergent_hop(
|
||||
hop_commitments,
|
||||
reference_activations_by_hop,
|
||||
config=self._toploc_config,
|
||||
backend=self._toploc_backend,
|
||||
)
|
||||
ok = culprit_index is None
|
||||
return _AuditResult(
|
||||
ok=ok,
|
||||
reference_output=(
|
||||
"TOPLOC activation proof accepted"
|
||||
if ok
|
||||
else f"TOPLOC activation proof mismatch at hop {culprit_index}"
|
||||
),
|
||||
reason="TOPLOC activation proof mismatch",
|
||||
culprit_node=None if ok else hop_commitments[culprit_index].node,
|
||||
)
|
||||
|
||||
def _commitment_expired(self, event: Any) -> bool:
|
||||
ts = _event_value(event, "ts")
|
||||
if ts is None:
|
||||
return False
|
||||
return (time.time() - float(ts)) > self._toploc_config.commitment_ttl_seconds
|
||||
|
||||
def _run_teacher_forced_prefill(
|
||||
self,
|
||||
route_nodes: list[dict],
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
claimed_token_ids: list[int],
|
||||
claim: ToplocProofClaim,
|
||||
) -> list[Any]:
|
||||
response = _post_json(
|
||||
f"{self._reference_node_url}/v1/audit/toploc",
|
||||
{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"claimed_token_ids": claimed_token_ids,
|
||||
"dtype": claim.dtype,
|
||||
"quantization": claim.quantization,
|
||||
"decode_batching_size": claim.decode_batching_size,
|
||||
"topk": claim.topk,
|
||||
"skip_prefill": claim.skip_prefill,
|
||||
},
|
||||
)
|
||||
activations = response.get("activations")
|
||||
if not isinstance(activations, list):
|
||||
raise ValueError("reference node audit response did not contain activations")
|
||||
return activations
|
||||
|
||||
def _run_teacher_forced_prefill_hops(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
hop_commitments: list["_HopCommitment"],
|
||||
) -> list[list[Any]]:
|
||||
"""Teacher-force the claimed tokens once and collect reference
|
||||
activations at every hop's boundary layer (ADR-0018 §4 / research §1.2:
|
||||
one referee forward pass, compared at each cut-point)."""
|
||||
reference_claim = hop_commitments[0].claim
|
||||
response = _post_json(
|
||||
f"{self._reference_node_url}/v1/audit/toploc",
|
||||
{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"claimed_token_ids": hop_commitments[-1].token_ids,
|
||||
"hop_boundaries": [hop.shard_end for hop in hop_commitments],
|
||||
"dtype": reference_claim.dtype,
|
||||
"quantization": reference_claim.quantization,
|
||||
"decode_batching_size": reference_claim.decode_batching_size,
|
||||
"topk": reference_claim.topk,
|
||||
"skip_prefill": reference_claim.skip_prefill,
|
||||
},
|
||||
)
|
||||
activations_by_hop = response.get("activations_by_hop")
|
||||
if not isinstance(activations_by_hop, list) or len(activations_by_hop) != len(hop_commitments):
|
||||
raise ValueError("reference node audit response did not contain per-hop activations")
|
||||
return activations_by_hop
|
||||
|
||||
def _record_clean_audit(self, event: Any) -> None:
|
||||
"""ADR-0018 §6: reputation derives only from tracker-verified audit
|
||||
outcomes — a clean audit credits every node on the verified route."""
|
||||
for wallet_address in _route_wallets(event):
|
||||
if self._contracts.registry.get_wallet(wallet_address).banned:
|
||||
continue
|
||||
self._contracts.registry.record_audit_outcome(wallet_address, passed=True)
|
||||
|
||||
def _slash_node(
|
||||
self,
|
||||
node: dict | None,
|
||||
observed_output: str,
|
||||
reference_output: str,
|
||||
*,
|
||||
reason: str = "reference output diverged",
|
||||
) -> list[Any]:
|
||||
receipts: list[Any] = []
|
||||
node = _final_text_node(route_nodes)
|
||||
wallet_address = node.get("wallet_address") if node else None
|
||||
if not wallet_address:
|
||||
return receipts
|
||||
@@ -117,11 +286,14 @@ class ValidatorProcess:
|
||||
slash_amount=self._slash_amount,
|
||||
strike_threshold=self._strike_threshold,
|
||||
reason=(
|
||||
"reference output diverged "
|
||||
f"{reason} "
|
||||
f"(observed={observed_output!r}, reference={reference_output!r})"
|
||||
),
|
||||
webhook_url=self._webhook_url,
|
||||
))
|
||||
# ADR-0018 §6: reputation loss is separate from the strike/ban that
|
||||
# submit_slash_proof already recorded above — never double-strike.
|
||||
self._contracts.registry.record_audit_outcome(wallet_address, passed=False)
|
||||
# ADR-0015: the pending balance is the collateral — forfeit it in the
|
||||
# same validation cycle as the strike.
|
||||
if self._billing is not None:
|
||||
@@ -134,12 +306,149 @@ class ValidatorProcess:
|
||||
return receipts
|
||||
|
||||
|
||||
def _route_wallets(event: Any) -> list[str]:
|
||||
"""Unique wallet addresses across a route, in hop order."""
|
||||
route_nodes = _event_value(event, "route_nodes") or []
|
||||
seen: set[str] = set()
|
||||
wallets: list[str] = []
|
||||
for node in route_nodes:
|
||||
wallet_address = node.get("wallet_address") if isinstance(node, dict) else None
|
||||
if wallet_address and wallet_address not in seen:
|
||||
seen.add(wallet_address)
|
||||
wallets.append(wallet_address)
|
||||
return wallets
|
||||
|
||||
|
||||
def _final_text_node(route_nodes: list[dict]) -> dict | None:
|
||||
"""Text-only fallback blame: when the audit has no per-hop fingerprints
|
||||
to bisect (free-running text comparison only), guess the last hop.
|
||||
Never used once hop-boundary commitments make real bisection possible."""
|
||||
if not route_nodes:
|
||||
return None
|
||||
return max(route_nodes, key=lambda node: int(node.get("shard_end", 0)))
|
||||
|
||||
|
||||
class _AuditResult:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ok: bool,
|
||||
reference_output: str,
|
||||
reason: str,
|
||||
culprit_node: dict | None = None,
|
||||
) -> None:
|
||||
self.ok = ok
|
||||
self.reference_output = reference_output
|
||||
self.reason = reason
|
||||
self.culprit_node = culprit_node
|
||||
|
||||
|
||||
class _HopCommitment:
|
||||
"""One hop's on-demand TOPLOC commitment plus the route node it blames."""
|
||||
|
||||
def __init__(self, node: dict | None, claim: ToplocProofClaim, token_ids: list[int]) -> None:
|
||||
self.node = node
|
||||
self.claim = claim
|
||||
self.token_ids = token_ids
|
||||
self.shard_end = int(node.get("shard_end", 0)) if node else None
|
||||
|
||||
|
||||
def _hop_commitments_from_event(event: Any) -> list[_HopCommitment] | None:
|
||||
"""Per-hop bisection commitments (ADR-0018 §3/§4): each route node reports
|
||||
its own output-boundary fingerprint. Falls back to the AH-006 whole-route
|
||||
commitment format (one fingerprint, no bisection) when hops don't carry
|
||||
individual commitments."""
|
||||
route_nodes = _event_value(event, "route_nodes") or []
|
||||
per_hop_nodes = [
|
||||
node for node in route_nodes
|
||||
if isinstance(node, dict) and _mapping_value(node, "toploc_proof") is not None
|
||||
]
|
||||
if per_hop_nodes and len(per_hop_nodes) == len(route_nodes):
|
||||
ordered = sorted(route_nodes, key=lambda node: int(node.get("shard_start", 0)))
|
||||
default_token_ids = _event_value(event, "claimed_token_ids")
|
||||
commitments = []
|
||||
for node in ordered:
|
||||
token_ids = node.get("claimed_token_ids", default_token_ids)
|
||||
if not isinstance(token_ids, list) or not all(isinstance(t, int) for t in token_ids):
|
||||
raise ValueError("TOPLOC hop commitments must include claimed_token_ids")
|
||||
commitments.append(_HopCommitment(
|
||||
node,
|
||||
ToplocProofClaim.from_mapping(node["toploc_proof"]),
|
||||
token_ids,
|
||||
))
|
||||
return commitments
|
||||
|
||||
whole_route = _toploc_audit_from_event(event)
|
||||
if whole_route is None:
|
||||
return None
|
||||
token_ids, claim = whole_route
|
||||
return [_HopCommitment(route_nodes[-1] if route_nodes else None, claim, token_ids)]
|
||||
|
||||
|
||||
def _first_divergent_hop(
|
||||
hop_commitments: list[_HopCommitment],
|
||||
reference_activations_by_hop: list[Any],
|
||||
*,
|
||||
config: ToplocAuditConfig,
|
||||
backend: Any | None,
|
||||
) -> int | None:
|
||||
"""First hop whose committed output fingerprint diverges from the
|
||||
referee's independently-computed reference activations at that same
|
||||
cut-point (research §1.2: no interactive game needed at hop granularity —
|
||||
the referee checks every cut-point in one replay)."""
|
||||
for index, commitment in enumerate(hop_commitments):
|
||||
ok = verify_activation_proofs(
|
||||
reference_activations_by_hop[index],
|
||||
commitment.claim,
|
||||
config=config,
|
||||
backend=backend,
|
||||
)
|
||||
if not ok:
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _toploc_audit_from_event(event: Any) -> tuple[list[int], ToplocProofClaim] | None:
|
||||
audit = _event_mapping(event, "audit")
|
||||
claim_data = (
|
||||
_event_mapping(event, "toploc_proof")
|
||||
or _event_mapping(event, "activation_proof")
|
||||
or _mapping_value(audit, "toploc_proof")
|
||||
or _mapping_value(audit, "activation_proof")
|
||||
or _mapping_value(audit, "toploc")
|
||||
)
|
||||
if claim_data is None:
|
||||
return None
|
||||
token_ids = (
|
||||
_event_value(event, "claimed_token_ids")
|
||||
or _event_value(event, "output_token_ids")
|
||||
or _mapping_value(audit, "claimed_token_ids")
|
||||
or _mapping_value(audit, "output_token_ids")
|
||||
)
|
||||
if not isinstance(token_ids, list) or not all(isinstance(token, int) for token in token_ids):
|
||||
raise ValueError("TOPLOC audit events must include claimed_token_ids")
|
||||
return token_ids, ToplocProofClaim.from_mapping(claim_data)
|
||||
|
||||
|
||||
def _event_value(event: Any, name: str) -> Any:
|
||||
if hasattr(event, name):
|
||||
return getattr(event, name)
|
||||
if isinstance(event, dict):
|
||||
return event.get(name)
|
||||
return None
|
||||
|
||||
|
||||
def _event_mapping(event: Any, name: str) -> dict[str, Any] | None:
|
||||
value = _event_value(event, name)
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def _mapping_value(mapping: dict[str, Any] | None, name: str) -> Any:
|
||||
if mapping is None:
|
||||
return None
|
||||
return mapping.get(name)
|
||||
|
||||
|
||||
def _outputs_match(observed: str, reference: str, tolerance: float) -> bool:
|
||||
observed_float = _parse_float(observed)
|
||||
reference_float = _parse_float(reference)
|
||||
@@ -167,4 +476,11 @@ def _post_json(url: str, payload: dict, timeout: float = 5.0) -> dict:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
__all__ = ["ValidatorProcess"]
|
||||
__all__ = [
|
||||
"ToplocAuditConfig",
|
||||
"ToplocProofClaim",
|
||||
"ValidatorProcess",
|
||||
"AdaptiveAuditSampler",
|
||||
"AuditRateConfig",
|
||||
"detect_output_tripwire",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user