feat(alpha): complete hardening backlog
Complete the alpha-hardening Ralph task set, including tracker billing/accounting guards, validator fraud-audit primitives, wallet binding proof support, documentation runbooks, and updated tests. Verification: .venv/bin/python -m compileall -q packages tests; .venv/bin/python -m pytest -q --tb=short (313 passed, 3 skipped, 1 failed: tests/test_mining_cli.py::test_legacy_start_without_port_uses_next_available_port because meshnet-node pid 1263451 is already listening on port 7000).
This commit is contained in:
193
tests/test_adaptive_audit_sampling.py
Normal file
193
tests/test_adaptive_audit_sampling.py
Normal file
@@ -0,0 +1,193 @@
|
||||
"""AH-009 (FRAUD: reputation-weighted routing + adaptive audit rate).
|
||||
|
||||
Covers the audit-sampling half of the issue: per-wallet audit probability as
|
||||
a function of tenure/reputation, a fleet-wide budget balancer that keeps the
|
||||
realized audit rate anchored to a configured target, and passive tripwire
|
||||
escalation (ADR-0018 §1, §6-7).
|
||||
"""
|
||||
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_contracts import LocalSolanaContracts
|
||||
from meshnet_node.server import StubNodeServer
|
||||
from meshnet_validator import AdaptiveAuditSampler, AuditRateConfig, ValidatorProcess, detect_output_tripwire
|
||||
|
||||
MODEL = "stub-model"
|
||||
|
||||
|
||||
# ---- per-wallet base rate: newcomer high, veteran low, floor >= 2% ----
|
||||
|
||||
|
||||
def test_newcomer_gets_elevated_audit_rate():
|
||||
sampler = AdaptiveAuditSampler()
|
||||
rate = sampler.wallet_base_rate(completed_job_count=0, reputation=1.0)
|
||||
assert 0.20 <= rate <= 0.30
|
||||
|
||||
|
||||
def test_veteran_in_good_standing_floors_near_target():
|
||||
sampler = AdaptiveAuditSampler()
|
||||
rate = sampler.wallet_base_rate(completed_job_count=1000, reputation=1.0)
|
||||
assert rate == pytest.approx(0.02)
|
||||
|
||||
|
||||
def test_veteran_rate_never_drops_below_floor():
|
||||
sampler = AdaptiveAuditSampler(AuditRateConfig(veteran_floor=0.02))
|
||||
rate = sampler.wallet_base_rate(completed_job_count=10_000, reputation=1.0)
|
||||
assert rate >= 0.02
|
||||
|
||||
|
||||
def test_low_reputation_wallet_sampled_more_than_high_reputation_wallet():
|
||||
"""Red (test-first item 1): a uniform sampler ignores reputation. A
|
||||
low-reputation wallet must get a higher rate than a high-reputation one
|
||||
with the same tenure."""
|
||||
sampler = AdaptiveAuditSampler()
|
||||
low_rep_rate = sampler.sample_rate_for(completed_job_count=200, reputation=0.1)
|
||||
high_rep_rate = sampler.sample_rate_for(completed_job_count=200, reputation=1.0)
|
||||
assert low_rep_rate > high_rep_rate
|
||||
|
||||
|
||||
def test_low_reputation_escalates_even_for_a_tenured_wallet():
|
||||
sampler = AdaptiveAuditSampler()
|
||||
rate = sampler.wallet_base_rate(completed_job_count=1000, reputation=0.0)
|
||||
assert rate == pytest.approx(sampler.config.newcomer_rate)
|
||||
|
||||
|
||||
# ---- fleet-wide budget balance ----
|
||||
|
||||
|
||||
def test_fleet_wide_audit_rate_tracks_configured_target_within_one_point():
|
||||
"""Test-first item 2: over >=1000 requests with a fixed seed and a mixed
|
||||
wallet population, the measured fleet audit rate lands within +-1.0
|
||||
percentage point of the configured 5% target."""
|
||||
sampler = AdaptiveAuditSampler(random_seed=1234)
|
||||
rng = random.Random(99)
|
||||
|
||||
audited = 0
|
||||
total = 6000
|
||||
for _ in range(total):
|
||||
roll = rng.random()
|
||||
if roll < 0.7:
|
||||
wallet = dict(completed_job_count=800, reputation=1.0) # veteran
|
||||
elif roll < 0.9:
|
||||
wallet = dict(completed_job_count=150, reputation=0.9) # mid-tenure
|
||||
else:
|
||||
wallet = dict(completed_job_count=0, reputation=1.0) # newcomer
|
||||
if sampler.should_audit(**wallet):
|
||||
audited += 1
|
||||
|
||||
measured_rate = audited / total
|
||||
assert abs(measured_rate - sampler.config.target_rate) <= 0.01
|
||||
|
||||
|
||||
def test_fleet_wide_audit_rate_respects_custom_target():
|
||||
sampler = AdaptiveAuditSampler(AuditRateConfig(target_rate=0.10), random_seed=42)
|
||||
audited = sum(
|
||||
1
|
||||
for _ in range(2000)
|
||||
if sampler.should_audit(completed_job_count=800, reputation=1.0)
|
||||
)
|
||||
measured_rate = audited / 2000
|
||||
assert abs(measured_rate - 0.10) <= 0.01
|
||||
|
||||
|
||||
def test_sampling_is_deterministic_for_a_fixed_seed():
|
||||
sampler_a = AdaptiveAuditSampler(random_seed=7)
|
||||
sampler_b = AdaptiveAuditSampler(random_seed=7)
|
||||
decisions_a = [sampler_a.should_audit(completed_job_count=0, reputation=1.0) for _ in range(200)]
|
||||
decisions_b = [sampler_b.should_audit(completed_job_count=0, reputation=1.0) for _ in range(200)]
|
||||
assert decisions_a == decisions_b
|
||||
|
||||
|
||||
# ---- passive tripwires bump rate only ----
|
||||
|
||||
|
||||
def test_tripwire_flag_bumps_audit_rate_for_that_wallet():
|
||||
sampler = AdaptiveAuditSampler()
|
||||
normal_rate = sampler.sample_rate_for(completed_job_count=800, reputation=1.0, tripwire=False)
|
||||
flagged_rate = sampler.sample_rate_for(completed_job_count=800, reputation=1.0, tripwire=True)
|
||||
assert flagged_rate > normal_rate
|
||||
|
||||
|
||||
def test_tripwire_does_not_change_other_wallets_rate():
|
||||
"""A tripwire hit must never leak the multiplier into the shared
|
||||
budget-balance history -- only the wallet's un-boosted base rate is
|
||||
recorded, so a flagged decision affects the running budget scale exactly
|
||||
like a plain decision for the same wallet would, and never inflates or
|
||||
depresses everyone else's rate on top of that."""
|
||||
flagged = AdaptiveAuditSampler(random_seed=5)
|
||||
flagged.should_audit(completed_job_count=800, reputation=1.0, tripwire=True)
|
||||
|
||||
plain = AdaptiveAuditSampler(random_seed=5)
|
||||
plain.should_audit(completed_job_count=800, reputation=1.0, tripwire=False)
|
||||
|
||||
after_flagged = flagged.sample_rate_for(completed_job_count=800, reputation=1.0)
|
||||
after_plain = plain.sample_rate_for(completed_job_count=800, reputation=1.0)
|
||||
assert after_flagged == pytest.approx(after_plain)
|
||||
|
||||
|
||||
def test_detect_output_tripwire_flags_repetition_loop():
|
||||
degenerate = " ".join(["loop"] * 20)
|
||||
assert detect_output_tripwire(degenerate) is True
|
||||
|
||||
|
||||
def test_detect_output_tripwire_flags_empty_output():
|
||||
assert detect_output_tripwire("") is True
|
||||
|
||||
|
||||
def test_detect_output_tripwire_passes_normal_prose():
|
||||
normal = "The quick brown fox jumps over the lazy dog near the riverbank."
|
||||
assert detect_output_tripwire(normal) is False
|
||||
|
||||
|
||||
# ---- ValidatorProcess wiring: reputation-weighted sampling end to end ----
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def reference_node():
|
||||
node = StubNodeServer(shard_start=0, shard_end=31, is_last_shard=True)
|
||||
port = node.start()
|
||||
yield f"http://127.0.0.1:{port}"
|
||||
node.stop()
|
||||
|
||||
|
||||
def _record_event(contracts, session_id: str, wallet: str) -> None:
|
||||
contracts.validation.record_completed_inference(
|
||||
session_id=session_id,
|
||||
model=MODEL,
|
||||
messages=[{"role": "user", "content": "2+2"}],
|
||||
observed_output="4",
|
||||
route_nodes=[{"wallet_address": wallet, "shard_end": 31}],
|
||||
)
|
||||
|
||||
|
||||
def test_validator_uses_audit_sampler_when_configured(reference_node):
|
||||
"""A flagged low-reputation wallet gets audited far more often than a
|
||||
veteran in good standing when routed through the same validator."""
|
||||
contracts = LocalSolanaContracts()
|
||||
contracts.registry.record_completed_jobs("wallet-veteran", 800)
|
||||
for _ in range(9):
|
||||
contracts.registry.record_audit_outcome("wallet-shady", passed=False) # reputation -> 0.0
|
||||
|
||||
trials = 400
|
||||
shady_audits = 0
|
||||
veteran_audits = 0
|
||||
for i in range(trials):
|
||||
sampler = AdaptiveAuditSampler(random_seed=i)
|
||||
validator = ValidatorProcess(
|
||||
contracts=contracts,
|
||||
reference_node_url=reference_node,
|
||||
audit_sampler=sampler,
|
||||
random_seed=i,
|
||||
)
|
||||
shady_audits += int(validator._should_sample(_FakeEvent("wallet-shady")))
|
||||
veteran_audits += int(validator._should_sample(_FakeEvent("wallet-veteran")))
|
||||
|
||||
assert shady_audits > veteran_audits
|
||||
|
||||
|
||||
class _FakeEvent:
|
||||
def __init__(self, wallet: str) -> None:
|
||||
self.route_nodes = [{"wallet_address": wallet, "shard_end": 31}]
|
||||
self.observed_output = "4"
|
||||
Reference in New Issue
Block a user