280 lines
10 KiB
Python
280 lines
10 KiB
Python
"""AH-006: validator TOPLOC audit primitive."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import namedtuple
|
|
from types import SimpleNamespace
|
|
|
|
from meshnet_validator import ToplocAuditConfig, ValidatorProcess
|
|
from meshnet_validator.audit import (
|
|
build_activation_proofs,
|
|
verify_activation_proofs,
|
|
verify_activation_proofs_detailed,
|
|
)
|
|
|
|
|
|
class FakeToploc:
|
|
def __init__(self) -> None:
|
|
self.build_calls: list[dict] = []
|
|
self.verify_calls: list[dict] = []
|
|
|
|
def build_proofs_base64(
|
|
self,
|
|
activations,
|
|
*,
|
|
decode_batching_size: int,
|
|
topk: int,
|
|
skip_prefill: bool,
|
|
):
|
|
self.build_calls.append({
|
|
"decode_batching_size": decode_batching_size,
|
|
"topk": topk,
|
|
"skip_prefill": skip_prefill,
|
|
})
|
|
return {
|
|
"activation_fingerprint": tuple(tuple(row) for row in activations),
|
|
"decode_batching_size": decode_batching_size,
|
|
"topk": topk,
|
|
"skip_prefill": skip_prefill,
|
|
}
|
|
|
|
def verify_proofs_base64(
|
|
self,
|
|
activations,
|
|
proofs,
|
|
*,
|
|
decode_batching_size: int,
|
|
topk: int,
|
|
skip_prefill: bool,
|
|
):
|
|
self.verify_calls.append({
|
|
"decode_batching_size": decode_batching_size,
|
|
"topk": topk,
|
|
"skip_prefill": skip_prefill,
|
|
})
|
|
return proofs == {
|
|
"activation_fingerprint": tuple(tuple(row) for row in activations),
|
|
"decode_batching_size": decode_batching_size,
|
|
"topk": topk,
|
|
"skip_prefill": skip_prefill,
|
|
}
|
|
|
|
|
|
class FakeValidationLog:
|
|
def __init__(self, events) -> None:
|
|
self._events = events
|
|
|
|
def list_completed_inferences(self, *, after_index: int = -1):
|
|
return [event for event in self._events if event.index > after_index]
|
|
|
|
|
|
class FakeRegistry:
|
|
def __init__(self) -> None:
|
|
self.slashes: list[dict] = []
|
|
self.audit_outcomes: list[dict] = []
|
|
|
|
def get_wallet(self, wallet_address: str):
|
|
return SimpleNamespace(banned=False)
|
|
|
|
def submit_slash_proof(self, **kwargs):
|
|
self.slashes.append(kwargs)
|
|
return kwargs
|
|
|
|
def record_audit_outcome(self, wallet_address: str, *, passed: bool):
|
|
self.audit_outcomes.append({"wallet_address": wallet_address, "passed": passed})
|
|
|
|
|
|
class FakeContracts:
|
|
def __init__(self, events) -> None:
|
|
self.validation = FakeValidationLog(events)
|
|
self.registry = FakeRegistry()
|
|
|
|
|
|
class TeacherForcedValidator(ValidatorProcess):
|
|
def __init__(self, *args, reference_activations, **kwargs) -> None:
|
|
super().__init__(*args, **kwargs)
|
|
self.reference_activations = reference_activations
|
|
self.teacher_forced_calls: list[dict] = []
|
|
|
|
def _run_reference(self, messages: list[dict]) -> str:
|
|
raise AssertionError("TOPLOC audits must not free-generate reference text")
|
|
|
|
def _run_teacher_forced_prefill(self, **kwargs):
|
|
self.teacher_forced_calls.append(kwargs)
|
|
return self.reference_activations
|
|
|
|
|
|
def test_stub_activation_tensors_round_trip_through_toploc_proofs():
|
|
"Stub activation tensors round trip through toploc proofs\n\nTags: audit, calibration"
|
|
fake_toploc = FakeToploc()
|
|
activations = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
|
|
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
|
|
|
|
assert verify_activation_proofs(activations, claim, config=config, backend=fake_toploc) is True
|
|
assert fake_toploc.build_calls == [{
|
|
"decode_batching_size": 16,
|
|
"topk": 2,
|
|
"skip_prefill": True,
|
|
}]
|
|
assert fake_toploc.verify_calls == [{
|
|
"decode_batching_size": 16,
|
|
"topk": 2,
|
|
"skip_prefill": True,
|
|
}]
|
|
|
|
|
|
def test_validator_teacher_forces_claimed_tokens_for_toploc_audit():
|
|
"Validator teacher forces claimed tokens for toploc audit\n\nTags: audit, calibration"
|
|
fake_toploc = FakeToploc()
|
|
activations = [[0.25, 0.5], [0.75, 1.0]]
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
|
|
event = SimpleNamespace(
|
|
index=0,
|
|
session_id="session-toploc-ok",
|
|
model="Qwen2.5-0.5B-Instruct",
|
|
messages=[{"role": "user", "content": "hello"}],
|
|
observed_output="honest but hardware-divergent text",
|
|
route_nodes=[{"wallet_address": "wallet-good", "shard_end": 31}],
|
|
claimed_token_ids=[101, 202, 303],
|
|
toploc_proof=claim.as_mapping(),
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = TeacherForcedValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=config,
|
|
toploc_backend=fake_toploc,
|
|
reference_activations=activations,
|
|
)
|
|
|
|
assert validator.validate_once() == []
|
|
assert contracts.registry.slashes == []
|
|
assert validator.teacher_forced_calls == [{
|
|
"model": "Qwen2.5-0.5B-Instruct",
|
|
"messages": [{"role": "user", "content": "hello"}],
|
|
"claimed_token_ids": [101, 202, 303],
|
|
"claim": claim,
|
|
}]
|
|
|
|
|
|
def test_validator_rejects_swapped_precision_toploc_claim():
|
|
"Validator rejects swapped precision toploc claim\n\nTags: audit, calibration"
|
|
fake_toploc = FakeToploc()
|
|
activations = [[0.25, 0.5], [0.75, 1.0]]
|
|
canonical = ToplocAuditConfig(
|
|
dtype="bfloat16",
|
|
quantization="bfloat16",
|
|
topk=2,
|
|
decode_batching_size=16,
|
|
)
|
|
swapped = ToplocAuditConfig(
|
|
dtype="bfloat16",
|
|
quantization="int8",
|
|
topk=2,
|
|
decode_batching_size=16,
|
|
)
|
|
claim = build_activation_proofs(activations, config=swapped, backend=fake_toploc)
|
|
event = SimpleNamespace(
|
|
index=0,
|
|
session_id="session-toploc-bad",
|
|
model="Qwen2.5-0.5B-Instruct",
|
|
messages=[{"role": "user", "content": "hello"}],
|
|
observed_output="looks plausible",
|
|
route_nodes=[{"wallet_address": "wallet-bad", "shard_end": 31}],
|
|
audit={
|
|
"claimed_token_ids": [101, 202, 303],
|
|
"toploc_proof": claim.as_mapping(),
|
|
},
|
|
)
|
|
contracts = FakeContracts([event])
|
|
validator = TeacherForcedValidator(
|
|
contracts=contracts,
|
|
reference_node_url="http://reference.invalid",
|
|
sample_rate=1.0,
|
|
random_seed=7,
|
|
toploc_config=canonical,
|
|
toploc_backend=fake_toploc,
|
|
reference_activations=activations,
|
|
)
|
|
|
|
receipts = validator.validate_once()
|
|
|
|
assert len(receipts) == 1
|
|
assert contracts.registry.slashes[0]["wallet_address"] == "wallet-bad"
|
|
assert "TOPLOC activation proof mismatch" in contracts.registry.slashes[0]["reason"]
|
|
|
|
|
|
# AH-021: verify_activation_proofs_detailed surfaces the raw divergence
|
|
# metric a calibration corpus needs, instead of only a pass/fail bool.
|
|
|
|
ChunkResult = namedtuple("ChunkResult", ["exp_intersections", "mant_err_mean", "mant_err_median"])
|
|
|
|
|
|
class FakeToplocWithChunkResults:
|
|
"""Mimics the real `toploc` library: verify returns per-chunk results,
|
|
not a bool, so `bool(result)` alone (the AH-021 gap #1 bug) is always
|
|
true for any non-empty response regardless of divergence."""
|
|
|
|
def build_proofs_base64(self, activations, *, decode_batching_size, topk, skip_prefill):
|
|
return {"activations": activations}
|
|
|
|
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
|
|
return [
|
|
ChunkResult(exp_intersections=8, mant_err_mean=0.01, mant_err_median=0.008),
|
|
ChunkResult(exp_intersections=6, mant_err_mean=0.03, mant_err_median=0.02),
|
|
]
|
|
|
|
|
|
def test_verify_activation_proofs_detailed_aggregates_per_chunk_divergence():
|
|
"Verify activation proofs detailed aggregates per chunk divergence\n\nTags: audit, calibration"
|
|
fake_toploc = FakeToplocWithChunkResults()
|
|
activations = [[1.0, 2.0], [3.0, 4.0]]
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
|
|
|
|
result = verify_activation_proofs_detailed(activations, claim, config=config, backend=fake_toploc)
|
|
|
|
assert result.passed is True # non-empty list is truthy, same as legacy behavior
|
|
assert result.chunk_count == 2
|
|
assert result.exp_intersections == 6 # worst-case (min) across chunks
|
|
assert result.mant_err_mean == 0.02 # mean of per-chunk means
|
|
assert result.mant_err_median == 0.014 # mean of per-chunk medians
|
|
|
|
# verify_activation_proofs still returns just the bool for existing callers.
|
|
assert verify_activation_proofs(activations, claim, config=config, backend=fake_toploc) is True
|
|
|
|
|
|
def test_verify_activation_proofs_detailed_no_metric_from_plain_bool_backend():
|
|
"Verify activation proofs detailed no metric from plain bool backend\n\nTags: audit, calibration"
|
|
fake_toploc = FakeToploc()
|
|
activations = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
|
|
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
|
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
|
|
|
|
result = verify_activation_proofs_detailed(activations, claim, config=config, backend=fake_toploc)
|
|
|
|
assert result.passed is True
|
|
assert result.chunk_count == 0
|
|
assert result.exp_intersections is None
|
|
assert result.mant_err_mean is None
|
|
assert result.mant_err_median is None
|
|
|
|
|
|
def test_verify_activation_proofs_detailed_rejects_config_mismatch_without_calling_backend():
|
|
"Verify activation proofs detailed rejects config mismatch without calling backend\n\nTags: audit, calibration"
|
|
fake_toploc = FakeToplocWithChunkResults()
|
|
activations = [[1.0, 2.0]]
|
|
canonical = ToplocAuditConfig(dtype="bfloat16", quantization="bfloat16", topk=2, decode_batching_size=16)
|
|
swapped = ToplocAuditConfig(dtype="bfloat16", quantization="int8", topk=2, decode_batching_size=16)
|
|
claim = build_activation_proofs(activations, config=swapped, backend=fake_toploc)
|
|
|
|
result = verify_activation_proofs_detailed(activations, claim, config=canonical, backend=fake_toploc)
|
|
|
|
assert result.passed is False
|
|
assert result.chunk_count == 0
|