neuron-tai/tests/test_adaptive_audit_sampling.py

"""AH-009 (FRAUD: reputation-weighted routing + adaptive audit rate).

Covers the audit-sampling half of the issue: per-wallet audit probability as
a function of tenure/reputation, a fleet-wide budget balancer that keeps the
realized audit rate anchored to a configured target, and passive tripwire
escalation (ADR-0018 §1, §6-7).
"""

import random

import pytest

from meshnet_contracts import LocalSolanaContracts
from meshnet_node.server import StubNodeServer
from meshnet_validator import AdaptiveAuditSampler, AuditRateConfig, ValidatorProcess, detect_output_tripwire

MODEL = "stub-model"


# ---- per-wallet base rate: newcomer high, veteran low, floor >= 2% ----


def test_newcomer_gets_elevated_audit_rate():
    sampler = AdaptiveAuditSampler()
    rate = sampler.wallet_base_rate(completed_job_count=0, reputation=1.0)
    assert 0.20 <= rate <= 0.30


def test_veteran_in_good_standing_floors_near_target():
    sampler = AdaptiveAuditSampler()
    rate = sampler.wallet_base_rate(completed_job_count=1000, reputation=1.0)
    assert rate == pytest.approx(0.02)


def test_veteran_rate_never_drops_below_floor():
    sampler = AdaptiveAuditSampler(AuditRateConfig(veteran_floor=0.02))
    rate = sampler.wallet_base_rate(completed_job_count=10_000, reputation=1.0)
    assert rate >= 0.02


def test_low_reputation_wallet_sampled_more_than_high_reputation_wallet():
    """Red (test-first item 1): a uniform sampler ignores reputation. A
    low-reputation wallet must get a higher rate than a high-reputation one
    with the same tenure."""
    sampler = AdaptiveAuditSampler()
    low_rep_rate = sampler.sample_rate_for(completed_job_count=200, reputation=0.1)
    high_rep_rate = sampler.sample_rate_for(completed_job_count=200, reputation=1.0)
    assert low_rep_rate > high_rep_rate


def test_low_reputation_escalates_even_for_a_tenured_wallet():
    sampler = AdaptiveAuditSampler()
    rate = sampler.wallet_base_rate(completed_job_count=1000, reputation=0.0)
    assert rate == pytest.approx(sampler.config.newcomer_rate)


# ---- fleet-wide budget balance ----


def test_fleet_wide_audit_rate_tracks_configured_target_within_one_point():
    """Test-first item 2: over >=1000 requests with a fixed seed and a mixed
    wallet population, the measured fleet audit rate lands within +-1.0
    percentage point of the configured 5% target."""
    sampler = AdaptiveAuditSampler(random_seed=1234)
    rng = random.Random(99)

    audited = 0
    total = 6000
    for _ in range(total):
        roll = rng.random()
        if roll < 0.7:
            wallet = dict(completed_job_count=800, reputation=1.0)  # veteran
        elif roll < 0.9:
            wallet = dict(completed_job_count=150, reputation=0.9)  # mid-tenure
        else:
            wallet = dict(completed_job_count=0, reputation=1.0)  # newcomer
        if sampler.should_audit(**wallet):
            audited += 1

    measured_rate = audited / total
    assert abs(measured_rate - sampler.config.target_rate) <= 0.01


def test_fleet_wide_audit_rate_respects_custom_target():
    sampler = AdaptiveAuditSampler(AuditRateConfig(target_rate=0.10), random_seed=42)
    audited = sum(
        1
        for _ in range(2000)
        if sampler.should_audit(completed_job_count=800, reputation=1.0)
    )
    measured_rate = audited / 2000
    assert abs(measured_rate - 0.10) <= 0.01


def test_sampling_is_deterministic_for_a_fixed_seed():
    sampler_a = AdaptiveAuditSampler(random_seed=7)
    sampler_b = AdaptiveAuditSampler(random_seed=7)
    decisions_a = [sampler_a.should_audit(completed_job_count=0, reputation=1.0) for _ in range(200)]
    decisions_b = [sampler_b.should_audit(completed_job_count=0, reputation=1.0) for _ in range(200)]
    assert decisions_a == decisions_b


# ---- passive tripwires bump rate only ----


def test_tripwire_flag_bumps_audit_rate_for_that_wallet():
    sampler = AdaptiveAuditSampler()
    normal_rate = sampler.sample_rate_for(completed_job_count=800, reputation=1.0, tripwire=False)
    flagged_rate = sampler.sample_rate_for(completed_job_count=800, reputation=1.0, tripwire=True)
    assert flagged_rate > normal_rate


def test_tripwire_does_not_change_other_wallets_rate():
    """A tripwire hit must never leak the multiplier into the shared
    budget-balance history -- only the wallet's un-boosted base rate is
    recorded, so a flagged decision affects the running budget scale exactly
    like a plain decision for the same wallet would, and never inflates or
    depresses everyone else's rate on top of that."""
    flagged = AdaptiveAuditSampler(random_seed=5)
    flagged.should_audit(completed_job_count=800, reputation=1.0, tripwire=True)

    plain = AdaptiveAuditSampler(random_seed=5)
    plain.should_audit(completed_job_count=800, reputation=1.0, tripwire=False)

    after_flagged = flagged.sample_rate_for(completed_job_count=800, reputation=1.0)
    after_plain = plain.sample_rate_for(completed_job_count=800, reputation=1.0)
    assert after_flagged == pytest.approx(after_plain)


def test_detect_output_tripwire_flags_repetition_loop():
    degenerate = " ".join(["loop"] * 20)
    assert detect_output_tripwire(degenerate) is True


def test_detect_output_tripwire_flags_empty_output():
    assert detect_output_tripwire("") is True


def test_detect_output_tripwire_passes_normal_prose():
    normal = "The quick brown fox jumps over the lazy dog near the riverbank."
    assert detect_output_tripwire(normal) is False


# ---- ValidatorProcess wiring: reputation-weighted sampling end to end ----


@pytest.fixture
def reference_node():
    node = StubNodeServer(shard_start=0, shard_end=31, is_last_shard=True)
    port = node.start()
    yield f"http://127.0.0.1:{port}"
    node.stop()


def _record_event(contracts, session_id: str, wallet: str) -> None:
    contracts.validation.record_completed_inference(
        session_id=session_id,
        model=MODEL,
        messages=[{"role": "user", "content": "2+2"}],
        observed_output="4",
        route_nodes=[{"wallet_address": wallet, "shard_end": 31}],
    )


def test_validator_uses_audit_sampler_when_configured(reference_node):
    """A flagged low-reputation wallet gets audited far more often than a
    veteran in good standing when routed through the same validator."""
    contracts = LocalSolanaContracts()
    contracts.registry.record_completed_jobs("wallet-veteran", 800)
    for _ in range(9):
        contracts.registry.record_audit_outcome("wallet-shady", passed=False)  # reputation -> 0.0

    trials = 400
    shady_audits = 0
    veteran_audits = 0
    for i in range(trials):
        sampler = AdaptiveAuditSampler(random_seed=i)
        validator = ValidatorProcess(
            contracts=contracts,
            reference_node_url=reference_node,
            audit_sampler=sampler,
            random_seed=i,
        )
        shady_audits += int(validator._should_sample(_FakeEvent("wallet-shady")))
        veteran_audits += int(validator._should_sample(_FakeEvent("wallet-veteran")))

    assert shady_audits > veteran_audits


class _FakeEvent:
    def __init__(self, wallet: str) -> None:
        self.route_nodes = [{"wallet_address": wallet, "shard_end": 31}]
        self.observed_output = "4"