feat(alpha): complete hardening backlog
Complete the alpha-hardening Ralph task set, including tracker billing/accounting guards, validator fraud-audit primitives, wallet binding proof support, documentation runbooks, and updated tests. Verification: .venv/bin/python -m compileall -q packages tests; .venv/bin/python -m pytest -q --tb=short (313 passed, 3 skipped, 1 failed: tests/test_mining_cli.py::test_legacy_start_without_port_uses_next_available_port because meshnet-node pid 1263451 is already listening on port 7000).
This commit is contained in:
@@ -8,6 +8,10 @@ import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .audit import ToplocAuditConfig, ToplocProofClaim, verify_activation_proofs
|
||||
from .sampling import AdaptiveAuditSampler, AuditRateConfig
|
||||
from .tripwire import detect_output_tripwire
|
||||
|
||||
__version__ = "0.1.0"
|
||||
|
||||
|
||||
@@ -27,6 +31,9 @@ class ValidatorProcess:
|
||||
webhook_url: str | None = None,
|
||||
interval_seconds: float = 1.0,
|
||||
billing: Any | None = None,
|
||||
toploc_config: ToplocAuditConfig | None = None,
|
||||
toploc_backend: Any | None = None,
|
||||
audit_sampler: AdaptiveAuditSampler | None = None,
|
||||
) -> None:
|
||||
if not 0.0 <= sample_rate <= 1.0:
|
||||
raise ValueError("sample_rate must be between 0 and 1")
|
||||
@@ -48,6 +55,9 @@ class ValidatorProcess:
|
||||
self._strike_threshold = strike_threshold
|
||||
self._webhook_url = webhook_url
|
||||
self._interval_seconds = interval_seconds
|
||||
self._toploc_config = toploc_config or ToplocAuditConfig()
|
||||
self._toploc_backend = toploc_backend
|
||||
self._audit_sampler = audit_sampler
|
||||
self._random = random.Random(random_seed)
|
||||
self._last_event_index = -1
|
||||
self._running = False
|
||||
@@ -62,15 +72,47 @@ class ValidatorProcess:
|
||||
)
|
||||
for event in events:
|
||||
self._last_event_index = max(self._last_event_index, event.index)
|
||||
if self._random.random() >= self._sample_rate:
|
||||
if not self._should_sample(event):
|
||||
continue
|
||||
self.sampled_count += 1
|
||||
reference_output = self._run_reference(event.messages)
|
||||
if _outputs_match(event.observed_output, reference_output, self._tolerance):
|
||||
audit_result = self._validate_event(event)
|
||||
if audit_result.ok:
|
||||
self._record_clean_audit(event)
|
||||
continue
|
||||
receipts.extend(self._slash_route(event.route_nodes, event.observed_output, reference_output))
|
||||
receipts.extend(self._slash_node(
|
||||
audit_result.culprit_node,
|
||||
event.observed_output,
|
||||
audit_result.reference_output,
|
||||
reason=audit_result.reason,
|
||||
))
|
||||
return receipts
|
||||
|
||||
def _should_sample(self, event: Any) -> bool:
|
||||
"""ADR-0018 §1/§6-7: flat sample_rate stays the default; when an
|
||||
AdaptiveAuditSampler is configured, the decision is reputation- and
|
||||
tenure-weighted and budget-balanced against the fleet-wide target
|
||||
instead of a uniform coin flip."""
|
||||
if self._audit_sampler is None:
|
||||
return self._random.random() < self._sample_rate
|
||||
|
||||
tripwire = detect_output_tripwire(_event_value(event, "observed_output") or "")
|
||||
wallets = _route_wallets(event)
|
||||
if not wallets:
|
||||
return self._audit_sampler.should_audit(
|
||||
completed_job_count=0, reputation=1.0, tripwire=tripwire,
|
||||
)
|
||||
# A route is only as trustworthy as its least-trusted hop -- audit
|
||||
# against whichever wallet on the route looks riskiest.
|
||||
riskiest = min(
|
||||
(self._contracts.registry.get_wallet(wallet) for wallet in wallets),
|
||||
key=lambda wallet: wallet.reputation,
|
||||
)
|
||||
return self._audit_sampler.should_audit(
|
||||
completed_job_count=riskiest.completed_job_count,
|
||||
reputation=riskiest.reputation,
|
||||
tripwire=tripwire,
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
if self._running:
|
||||
raise RuntimeError("ValidatorProcess is already running")
|
||||
@@ -99,14 +141,141 @@ class ValidatorProcess:
|
||||
raise ValueError("reference node response did not contain text")
|
||||
return text
|
||||
|
||||
def _slash_route(
|
||||
def _validate_event(self, event: Any) -> "_AuditResult":
|
||||
hop_commitments = _hop_commitments_from_event(event)
|
||||
if hop_commitments is not None and self._commitment_expired(event):
|
||||
# ADR-0018 §3: the on-demand retention window has passed — nodes
|
||||
# are no longer expected to hold the boundary activations needed
|
||||
# to verify this commitment, so fall back to the text-only path.
|
||||
hop_commitments = None
|
||||
|
||||
if hop_commitments is None:
|
||||
reference_output = self._run_reference(event.messages)
|
||||
ok = _outputs_match(event.observed_output, reference_output, self._tolerance)
|
||||
return _AuditResult(
|
||||
ok=ok,
|
||||
reference_output=reference_output,
|
||||
reason="reference output diverged",
|
||||
# Text comparison has no per-hop signal; the last hop is the
|
||||
# best-effort guess (text-only fallback), never used when
|
||||
# hop-boundary commitments make real bisection possible.
|
||||
culprit_node=None if ok else _final_text_node(event.route_nodes),
|
||||
)
|
||||
|
||||
if len(hop_commitments) == 1:
|
||||
# Single-commitment route (AH-006 whole-route format, or a
|
||||
# genuinely one-hop pipeline): reuse the original teacher-forced
|
||||
# call so existing single-hop reference integrations keep working.
|
||||
only = hop_commitments[0]
|
||||
reference_activations_by_hop = [self._run_teacher_forced_prefill(
|
||||
model=event.model,
|
||||
messages=event.messages,
|
||||
claimed_token_ids=only.token_ids,
|
||||
claim=only.claim,
|
||||
)]
|
||||
else:
|
||||
reference_activations_by_hop = self._run_teacher_forced_prefill_hops(
|
||||
model=event.model,
|
||||
messages=event.messages,
|
||||
hop_commitments=hop_commitments,
|
||||
)
|
||||
culprit_index = _first_divergent_hop(
|
||||
hop_commitments,
|
||||
reference_activations_by_hop,
|
||||
config=self._toploc_config,
|
||||
backend=self._toploc_backend,
|
||||
)
|
||||
ok = culprit_index is None
|
||||
return _AuditResult(
|
||||
ok=ok,
|
||||
reference_output=(
|
||||
"TOPLOC activation proof accepted"
|
||||
if ok
|
||||
else f"TOPLOC activation proof mismatch at hop {culprit_index}"
|
||||
),
|
||||
reason="TOPLOC activation proof mismatch",
|
||||
culprit_node=None if ok else hop_commitments[culprit_index].node,
|
||||
)
|
||||
|
||||
def _commitment_expired(self, event: Any) -> bool:
|
||||
ts = _event_value(event, "ts")
|
||||
if ts is None:
|
||||
return False
|
||||
return (time.time() - float(ts)) > self._toploc_config.commitment_ttl_seconds
|
||||
|
||||
def _run_teacher_forced_prefill(
|
||||
self,
|
||||
route_nodes: list[dict],
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
claimed_token_ids: list[int],
|
||||
claim: ToplocProofClaim,
|
||||
) -> list[Any]:
|
||||
response = _post_json(
|
||||
f"{self._reference_node_url}/v1/audit/toploc",
|
||||
{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"claimed_token_ids": claimed_token_ids,
|
||||
"dtype": claim.dtype,
|
||||
"quantization": claim.quantization,
|
||||
"decode_batching_size": claim.decode_batching_size,
|
||||
"topk": claim.topk,
|
||||
"skip_prefill": claim.skip_prefill,
|
||||
},
|
||||
)
|
||||
activations = response.get("activations")
|
||||
if not isinstance(activations, list):
|
||||
raise ValueError("reference node audit response did not contain activations")
|
||||
return activations
|
||||
|
||||
def _run_teacher_forced_prefill_hops(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
hop_commitments: list["_HopCommitment"],
|
||||
) -> list[list[Any]]:
|
||||
"""Teacher-force the claimed tokens once and collect reference
|
||||
activations at every hop's boundary layer (ADR-0018 §4 / research §1.2:
|
||||
one referee forward pass, compared at each cut-point)."""
|
||||
reference_claim = hop_commitments[0].claim
|
||||
response = _post_json(
|
||||
f"{self._reference_node_url}/v1/audit/toploc",
|
||||
{
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"claimed_token_ids": hop_commitments[-1].token_ids,
|
||||
"hop_boundaries": [hop.shard_end for hop in hop_commitments],
|
||||
"dtype": reference_claim.dtype,
|
||||
"quantization": reference_claim.quantization,
|
||||
"decode_batching_size": reference_claim.decode_batching_size,
|
||||
"topk": reference_claim.topk,
|
||||
"skip_prefill": reference_claim.skip_prefill,
|
||||
},
|
||||
)
|
||||
activations_by_hop = response.get("activations_by_hop")
|
||||
if not isinstance(activations_by_hop, list) or len(activations_by_hop) != len(hop_commitments):
|
||||
raise ValueError("reference node audit response did not contain per-hop activations")
|
||||
return activations_by_hop
|
||||
|
||||
def _record_clean_audit(self, event: Any) -> None:
|
||||
"""ADR-0018 §6: reputation derives only from tracker-verified audit
|
||||
outcomes — a clean audit credits every node on the verified route."""
|
||||
for wallet_address in _route_wallets(event):
|
||||
if self._contracts.registry.get_wallet(wallet_address).banned:
|
||||
continue
|
||||
self._contracts.registry.record_audit_outcome(wallet_address, passed=True)
|
||||
|
||||
def _slash_node(
|
||||
self,
|
||||
node: dict | None,
|
||||
observed_output: str,
|
||||
reference_output: str,
|
||||
*,
|
||||
reason: str = "reference output diverged",
|
||||
) -> list[Any]:
|
||||
receipts: list[Any] = []
|
||||
node = _final_text_node(route_nodes)
|
||||
wallet_address = node.get("wallet_address") if node else None
|
||||
if not wallet_address:
|
||||
return receipts
|
||||
@@ -117,11 +286,14 @@ class ValidatorProcess:
|
||||
slash_amount=self._slash_amount,
|
||||
strike_threshold=self._strike_threshold,
|
||||
reason=(
|
||||
"reference output diverged "
|
||||
f"{reason} "
|
||||
f"(observed={observed_output!r}, reference={reference_output!r})"
|
||||
),
|
||||
webhook_url=self._webhook_url,
|
||||
))
|
||||
# ADR-0018 §6: reputation loss is separate from the strike/ban that
|
||||
# submit_slash_proof already recorded above — never double-strike.
|
||||
self._contracts.registry.record_audit_outcome(wallet_address, passed=False)
|
||||
# ADR-0015: the pending balance is the collateral — forfeit it in the
|
||||
# same validation cycle as the strike.
|
||||
if self._billing is not None:
|
||||
@@ -134,12 +306,149 @@ class ValidatorProcess:
|
||||
return receipts
|
||||
|
||||
|
||||
def _route_wallets(event: Any) -> list[str]:
|
||||
"""Unique wallet addresses across a route, in hop order."""
|
||||
route_nodes = _event_value(event, "route_nodes") or []
|
||||
seen: set[str] = set()
|
||||
wallets: list[str] = []
|
||||
for node in route_nodes:
|
||||
wallet_address = node.get("wallet_address") if isinstance(node, dict) else None
|
||||
if wallet_address and wallet_address not in seen:
|
||||
seen.add(wallet_address)
|
||||
wallets.append(wallet_address)
|
||||
return wallets
|
||||
|
||||
|
||||
def _final_text_node(route_nodes: list[dict]) -> dict | None:
|
||||
"""Text-only fallback blame: when the audit has no per-hop fingerprints
|
||||
to bisect (free-running text comparison only), guess the last hop.
|
||||
Never used once hop-boundary commitments make real bisection possible."""
|
||||
if not route_nodes:
|
||||
return None
|
||||
return max(route_nodes, key=lambda node: int(node.get("shard_end", 0)))
|
||||
|
||||
|
||||
class _AuditResult:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ok: bool,
|
||||
reference_output: str,
|
||||
reason: str,
|
||||
culprit_node: dict | None = None,
|
||||
) -> None:
|
||||
self.ok = ok
|
||||
self.reference_output = reference_output
|
||||
self.reason = reason
|
||||
self.culprit_node = culprit_node
|
||||
|
||||
|
||||
class _HopCommitment:
|
||||
"""One hop's on-demand TOPLOC commitment plus the route node it blames."""
|
||||
|
||||
def __init__(self, node: dict | None, claim: ToplocProofClaim, token_ids: list[int]) -> None:
|
||||
self.node = node
|
||||
self.claim = claim
|
||||
self.token_ids = token_ids
|
||||
self.shard_end = int(node.get("shard_end", 0)) if node else None
|
||||
|
||||
|
||||
def _hop_commitments_from_event(event: Any) -> list[_HopCommitment] | None:
|
||||
"""Per-hop bisection commitments (ADR-0018 §3/§4): each route node reports
|
||||
its own output-boundary fingerprint. Falls back to the AH-006 whole-route
|
||||
commitment format (one fingerprint, no bisection) when hops don't carry
|
||||
individual commitments."""
|
||||
route_nodes = _event_value(event, "route_nodes") or []
|
||||
per_hop_nodes = [
|
||||
node for node in route_nodes
|
||||
if isinstance(node, dict) and _mapping_value(node, "toploc_proof") is not None
|
||||
]
|
||||
if per_hop_nodes and len(per_hop_nodes) == len(route_nodes):
|
||||
ordered = sorted(route_nodes, key=lambda node: int(node.get("shard_start", 0)))
|
||||
default_token_ids = _event_value(event, "claimed_token_ids")
|
||||
commitments = []
|
||||
for node in ordered:
|
||||
token_ids = node.get("claimed_token_ids", default_token_ids)
|
||||
if not isinstance(token_ids, list) or not all(isinstance(t, int) for t in token_ids):
|
||||
raise ValueError("TOPLOC hop commitments must include claimed_token_ids")
|
||||
commitments.append(_HopCommitment(
|
||||
node,
|
||||
ToplocProofClaim.from_mapping(node["toploc_proof"]),
|
||||
token_ids,
|
||||
))
|
||||
return commitments
|
||||
|
||||
whole_route = _toploc_audit_from_event(event)
|
||||
if whole_route is None:
|
||||
return None
|
||||
token_ids, claim = whole_route
|
||||
return [_HopCommitment(route_nodes[-1] if route_nodes else None, claim, token_ids)]
|
||||
|
||||
|
||||
def _first_divergent_hop(
|
||||
hop_commitments: list[_HopCommitment],
|
||||
reference_activations_by_hop: list[Any],
|
||||
*,
|
||||
config: ToplocAuditConfig,
|
||||
backend: Any | None,
|
||||
) -> int | None:
|
||||
"""First hop whose committed output fingerprint diverges from the
|
||||
referee's independently-computed reference activations at that same
|
||||
cut-point (research §1.2: no interactive game needed at hop granularity —
|
||||
the referee checks every cut-point in one replay)."""
|
||||
for index, commitment in enumerate(hop_commitments):
|
||||
ok = verify_activation_proofs(
|
||||
reference_activations_by_hop[index],
|
||||
commitment.claim,
|
||||
config=config,
|
||||
backend=backend,
|
||||
)
|
||||
if not ok:
|
||||
return index
|
||||
return None
|
||||
|
||||
|
||||
def _toploc_audit_from_event(event: Any) -> tuple[list[int], ToplocProofClaim] | None:
|
||||
audit = _event_mapping(event, "audit")
|
||||
claim_data = (
|
||||
_event_mapping(event, "toploc_proof")
|
||||
or _event_mapping(event, "activation_proof")
|
||||
or _mapping_value(audit, "toploc_proof")
|
||||
or _mapping_value(audit, "activation_proof")
|
||||
or _mapping_value(audit, "toploc")
|
||||
)
|
||||
if claim_data is None:
|
||||
return None
|
||||
token_ids = (
|
||||
_event_value(event, "claimed_token_ids")
|
||||
or _event_value(event, "output_token_ids")
|
||||
or _mapping_value(audit, "claimed_token_ids")
|
||||
or _mapping_value(audit, "output_token_ids")
|
||||
)
|
||||
if not isinstance(token_ids, list) or not all(isinstance(token, int) for token in token_ids):
|
||||
raise ValueError("TOPLOC audit events must include claimed_token_ids")
|
||||
return token_ids, ToplocProofClaim.from_mapping(claim_data)
|
||||
|
||||
|
||||
def _event_value(event: Any, name: str) -> Any:
|
||||
if hasattr(event, name):
|
||||
return getattr(event, name)
|
||||
if isinstance(event, dict):
|
||||
return event.get(name)
|
||||
return None
|
||||
|
||||
|
||||
def _event_mapping(event: Any, name: str) -> dict[str, Any] | None:
|
||||
value = _event_value(event, name)
|
||||
return value if isinstance(value, dict) else None
|
||||
|
||||
|
||||
def _mapping_value(mapping: dict[str, Any] | None, name: str) -> Any:
|
||||
if mapping is None:
|
||||
return None
|
||||
return mapping.get(name)
|
||||
|
||||
|
||||
def _outputs_match(observed: str, reference: str, tolerance: float) -> bool:
|
||||
observed_float = _parse_float(observed)
|
||||
reference_float = _parse_float(reference)
|
||||
@@ -167,4 +476,11 @@ def _post_json(url: str, payload: dict, timeout: float = 5.0) -> dict:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
__all__ = ["ValidatorProcess"]
|
||||
__all__ = [
|
||||
"ToplocAuditConfig",
|
||||
"ToplocProofClaim",
|
||||
"ValidatorProcess",
|
||||
"AdaptiveAuditSampler",
|
||||
"AuditRateConfig",
|
||||
"detect_output_tripwire",
|
||||
]
|
||||
|
||||
164
packages/validator/meshnet_validator/audit.py
Normal file
164
packages/validator/meshnet_validator/audit.py
Normal file
@@ -0,0 +1,164 @@
|
||||
"""TOPLOC activation proof helpers for validator-side audits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from importlib import import_module
|
||||
from typing import Any, Literal
|
||||
|
||||
|
||||
ProofEncoding = Literal["base64", "bytes"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToplocAuditConfig:
|
||||
"""Canonical audit parameters for one model preset."""
|
||||
|
||||
dtype: str = "bfloat16"
|
||||
quantization: str = "bfloat16"
|
||||
decode_batching_size: int = 32
|
||||
topk: int = 8
|
||||
skip_prefill: bool = True
|
||||
encoding: ProofEncoding = "base64"
|
||||
# ADR-0018 §3: nodes retain boundary activations only briefly; a commitment
|
||||
# older than this can no longer be verified against a live node and must
|
||||
# fall back to the text-only audit path.
|
||||
commitment_ttl_seconds: float = 30.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToplocProofClaim:
|
||||
"""Prover-provided TOPLOC proof and the parameters it was built with."""
|
||||
|
||||
proofs: Any
|
||||
dtype: str
|
||||
quantization: str
|
||||
decode_batching_size: int
|
||||
topk: int
|
||||
skip_prefill: bool = True
|
||||
encoding: ProofEncoding = "base64"
|
||||
|
||||
@classmethod
|
||||
def from_mapping(cls, value: dict[str, Any]) -> "ToplocProofClaim":
|
||||
return cls(
|
||||
proofs=value["proofs"],
|
||||
dtype=str(value.get("dtype", "bfloat16")),
|
||||
quantization=str(value.get("quantization", "bfloat16")),
|
||||
decode_batching_size=int(value.get("decode_batching_size", 32)),
|
||||
topk=int(value.get("topk", 8)),
|
||||
skip_prefill=bool(value.get("skip_prefill", True)),
|
||||
encoding=_proof_encoding(value.get("encoding", "base64")),
|
||||
)
|
||||
|
||||
def as_mapping(self) -> dict[str, Any]:
|
||||
return {
|
||||
"proofs": self.proofs,
|
||||
"dtype": self.dtype,
|
||||
"quantization": self.quantization,
|
||||
"decode_batching_size": self.decode_batching_size,
|
||||
"topk": self.topk,
|
||||
"skip_prefill": self.skip_prefill,
|
||||
"encoding": self.encoding,
|
||||
}
|
||||
|
||||
|
||||
def build_activation_proofs(
|
||||
activations: list[Any],
|
||||
*,
|
||||
config: ToplocAuditConfig | None = None,
|
||||
backend: Any | None = None,
|
||||
) -> ToplocProofClaim:
|
||||
"""Build a TOPLOC proof claim from captured activation tensors."""
|
||||
cfg = config or ToplocAuditConfig()
|
||||
module = backend or _load_toploc()
|
||||
function_name = f"build_proofs_{cfg.encoding}"
|
||||
build = getattr(module, function_name)
|
||||
proofs = _call_toploc(
|
||||
build,
|
||||
activations,
|
||||
decode_batching_size=cfg.decode_batching_size,
|
||||
topk=cfg.topk,
|
||||
skip_prefill=cfg.skip_prefill,
|
||||
)
|
||||
return ToplocProofClaim(
|
||||
proofs=proofs,
|
||||
dtype=cfg.dtype,
|
||||
quantization=cfg.quantization,
|
||||
decode_batching_size=cfg.decode_batching_size,
|
||||
topk=cfg.topk,
|
||||
skip_prefill=cfg.skip_prefill,
|
||||
encoding=cfg.encoding,
|
||||
)
|
||||
|
||||
|
||||
def verify_activation_proofs(
|
||||
reference_activations: list[Any],
|
||||
claim: ToplocProofClaim,
|
||||
*,
|
||||
config: ToplocAuditConfig | None = None,
|
||||
backend: Any | None = None,
|
||||
) -> bool:
|
||||
"""Verify prover TOPLOC proofs against reference teacher-forced activations."""
|
||||
cfg = config or ToplocAuditConfig(
|
||||
dtype=claim.dtype,
|
||||
quantization=claim.quantization,
|
||||
decode_batching_size=claim.decode_batching_size,
|
||||
topk=claim.topk,
|
||||
skip_prefill=claim.skip_prefill,
|
||||
encoding=claim.encoding,
|
||||
)
|
||||
if claim.dtype != cfg.dtype or claim.quantization != cfg.quantization:
|
||||
return False
|
||||
if claim.decode_batching_size != cfg.decode_batching_size or claim.topk != cfg.topk:
|
||||
return False
|
||||
if claim.skip_prefill != cfg.skip_prefill or claim.encoding != cfg.encoding:
|
||||
return False
|
||||
|
||||
module = backend or _load_toploc()
|
||||
function_name = f"verify_proofs_{claim.encoding}"
|
||||
verify = getattr(module, function_name)
|
||||
return bool(_call_toploc(
|
||||
verify,
|
||||
reference_activations,
|
||||
claim.proofs,
|
||||
decode_batching_size=claim.decode_batching_size,
|
||||
topk=claim.topk,
|
||||
skip_prefill=claim.skip_prefill,
|
||||
))
|
||||
|
||||
|
||||
def _load_toploc() -> Any:
|
||||
try:
|
||||
return import_module("toploc")
|
||||
except ModuleNotFoundError as exc:
|
||||
raise RuntimeError(
|
||||
"toploc is required for activation proof audits; install meshnet-validator with dependencies"
|
||||
) from exc
|
||||
|
||||
|
||||
def _call_toploc(function: Any, activations: list[Any], *args: Any, **kwargs: Any) -> Any:
|
||||
try:
|
||||
return function(activations, *args, **kwargs)
|
||||
except TypeError:
|
||||
if kwargs:
|
||||
ordered = [
|
||||
kwargs["decode_batching_size"],
|
||||
kwargs["topk"],
|
||||
kwargs["skip_prefill"],
|
||||
]
|
||||
return function(activations, *args, *ordered)
|
||||
raise
|
||||
|
||||
|
||||
def _proof_encoding(value: object) -> ProofEncoding:
|
||||
if value == "bytes":
|
||||
return "bytes"
|
||||
return "base64"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ToplocAuditConfig",
|
||||
"ToplocProofClaim",
|
||||
"build_activation_proofs",
|
||||
"verify_activation_proofs",
|
||||
]
|
||||
105
packages/validator/meshnet_validator/sampling.py
Normal file
105
packages/validator/meshnet_validator/sampling.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Reputation-weighted, budget-balanced audit sampling (ADR-0018 §1, §6-7).
|
||||
|
||||
The flat ``sample_rate`` on ``ValidatorProcess`` treats every wallet the same.
|
||||
This module scores each wallet's audit probability from its tenure
|
||||
(``completed_job_count``) and reputation -- newcomers and low-reputation
|
||||
wallets get sampled far more than veterans in good standing, who float down
|
||||
to a floor -- while a running budget balance keeps the *realized* fleet-wide
|
||||
average anchored to a configured target even as the wallet population mix
|
||||
drifts (research §6, §8 layers 2-4).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AuditRateConfig:
|
||||
"""Tunable audit-rate policy. Defaults match ADR-0018 §1/§6."""
|
||||
|
||||
target_rate: float = 0.05
|
||||
newcomer_rate: float = 0.25
|
||||
veteran_floor: float = 0.02
|
||||
probation_jobs: int = 50
|
||||
veteran_jobs: int = 500
|
||||
tripwire_multiplier: float = 3.0
|
||||
# Bounds the correction the budget balancer can apply in one step so a
|
||||
# short burst of skewed traffic can't swing everyone's rate wildly.
|
||||
max_budget_scale: float = 20.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not 0.0 <= self.target_rate <= 1.0:
|
||||
raise ValueError("target_rate must be between 0 and 1")
|
||||
if not 0.0 <= self.veteran_floor <= self.newcomer_rate <= 1.0:
|
||||
raise ValueError("veteran_floor must be <= newcomer_rate, both within [0, 1]")
|
||||
if self.probation_jobs < 0 or self.veteran_jobs <= self.probation_jobs:
|
||||
raise ValueError("veteran_jobs must be greater than probation_jobs")
|
||||
if self.tripwire_multiplier < 1.0:
|
||||
raise ValueError("tripwire_multiplier must be >= 1.0")
|
||||
|
||||
|
||||
class AdaptiveAuditSampler:
|
||||
"""Per-wallet audit probability + fleet-wide budget balancer.
|
||||
|
||||
``wallet_base_rate`` scores tenure and reputation into a rate between
|
||||
``veteran_floor`` and ``newcomer_rate``. ``should_audit`` scales that rate
|
||||
by a running budget-balance factor (target_rate / historical mean base
|
||||
rate) so the fleet-wide realized audit rate tracks ``target_rate`` even
|
||||
when the wallet mix is skewed toward veterans or newcomers, then applies
|
||||
the tripwire multiplier to *that single decision only* -- a tripwire flag
|
||||
never touches the shared budget-balance history, so it can't punish
|
||||
other wallets' audit rate.
|
||||
"""
|
||||
|
||||
def __init__(self, config: AuditRateConfig | None = None, *, random_seed: int | None = None) -> None:
|
||||
self.config = config or AuditRateConfig()
|
||||
self._random = random.Random(random_seed)
|
||||
self._base_rate_total = 0.0
|
||||
self._decisions = 0
|
||||
|
||||
def wallet_base_rate(self, *, completed_job_count: int, reputation: float) -> float:
|
||||
cfg = self.config
|
||||
if completed_job_count <= cfg.probation_jobs:
|
||||
tenure_rate = cfg.newcomer_rate
|
||||
elif completed_job_count >= cfg.veteran_jobs:
|
||||
tenure_rate = cfg.veteran_floor
|
||||
else:
|
||||
span = cfg.veteran_jobs - cfg.probation_jobs
|
||||
frac = (completed_job_count - cfg.probation_jobs) / span
|
||||
tenure_rate = cfg.newcomer_rate + (cfg.veteran_floor - cfg.newcomer_rate) * frac
|
||||
|
||||
reputation_gap = max(0.0, min(1.0, 1.0 - reputation))
|
||||
reputation_rate = cfg.veteran_floor + reputation_gap * (cfg.newcomer_rate - cfg.veteran_floor)
|
||||
|
||||
return max(tenure_rate, reputation_rate, cfg.veteran_floor)
|
||||
|
||||
def sample_rate_for(self, *, completed_job_count: int, reputation: float, tripwire: bool = False) -> float:
|
||||
"""Preview the effective audit probability without recording a decision."""
|
||||
base_rate = self.wallet_base_rate(completed_job_count=completed_job_count, reputation=reputation)
|
||||
return self._effective_rate(base_rate, tripwire=tripwire)
|
||||
|
||||
def should_audit(self, *, completed_job_count: int, reputation: float, tripwire: bool = False) -> bool:
|
||||
base_rate = self.wallet_base_rate(completed_job_count=completed_job_count, reputation=reputation)
|
||||
effective_rate = self._effective_rate(base_rate, tripwire=tripwire)
|
||||
self._base_rate_total += base_rate
|
||||
self._decisions += 1
|
||||
return self._random.random() < effective_rate
|
||||
|
||||
def _effective_rate(self, base_rate: float, *, tripwire: bool) -> float:
|
||||
rate = base_rate * self._budget_scale()
|
||||
if tripwire:
|
||||
rate *= self.config.tripwire_multiplier
|
||||
return max(self.config.veteran_floor, min(1.0, rate))
|
||||
|
||||
def _budget_scale(self) -> float:
|
||||
if self._decisions == 0:
|
||||
return 1.0
|
||||
mean_base_rate = self._base_rate_total / self._decisions
|
||||
if mean_base_rate <= 0:
|
||||
return 1.0
|
||||
return min(self.config.max_budget_scale, self.config.target_rate / mean_base_rate)
|
||||
|
||||
|
||||
__all__ = ["AuditRateConfig", "AdaptiveAuditSampler"]
|
||||
36
packages/validator/meshnet_validator/tripwire.py
Normal file
36
packages/validator/meshnet_validator/tripwire.py
Normal file
@@ -0,0 +1,36 @@
|
||||
"""Passive output-quality tripwires (ADR-0018 §7).
|
||||
|
||||
Cheap heuristics over every completed request's text -- no model access, no
|
||||
extra inference -- that flag likely-degenerate output (repetition loops,
|
||||
truncation) so ``AdaptiveAuditSampler`` can bump that single request's audit
|
||||
probability. A flag never strikes or bans a wallet by itself; it only raises
|
||||
the odds that request gets a real audit (research §8 layer 5).
|
||||
|
||||
True perplexity scoring needs the serving model's logits, which the validator
|
||||
does not have outside an audit call, so it stays roadmap-only (ADR-0018 §8);
|
||||
this covers the repetition/truncation half of the passive-tripwire layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def detect_output_tripwire(
|
||||
text: str,
|
||||
*,
|
||||
min_words: int = 4,
|
||||
repetition_threshold: float = 0.4,
|
||||
) -> bool:
|
||||
"""Flag empty output or a single word/token dominating the response."""
|
||||
if not text or not text.strip():
|
||||
return True
|
||||
words = text.split()
|
||||
if len(words) < min_words:
|
||||
return False
|
||||
counts: dict[str, int] = {}
|
||||
for word in words:
|
||||
counts[word] = counts.get(word, 0) + 1
|
||||
most_common = max(counts.values())
|
||||
return (most_common / len(words)) >= repetition_threshold
|
||||
|
||||
|
||||
__all__ = ["detect_output_tripwire"]
|
||||
Reference in New Issue
Block a user