f841dfaeed
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
81 lines
3.2 KiB
Python
81 lines
3.2 KiB
Python
"""AH-021: honest-noise TOPLOC calibration corpus storage + aggregation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from meshnet_tracker.calibration import ToplocCalibrationStore
|
|
|
|
|
|
def test_record_run_persists_and_reloads_from_sqlite(tmp_path):
|
|
db_path = str(tmp_path / "calibration.sqlite")
|
|
store = ToplocCalibrationStore(db_path=db_path)
|
|
store.record_run(
|
|
node_wallet="wallet-a",
|
|
gpu_model="RTX 4090",
|
|
dtype="bfloat16",
|
|
model="Qwen2.5-0.5B-Instruct",
|
|
passed=True,
|
|
exp_intersections=7.0,
|
|
mant_err_mean=0.01,
|
|
mant_err_median=0.008,
|
|
)
|
|
|
|
reloaded = ToplocCalibrationStore(db_path=db_path)
|
|
assert len(reloaded.runs()) == 1
|
|
assert reloaded.runs()[0]["node_wallet"] == "wallet-a"
|
|
assert reloaded.distinct_hardware_profiles() == {("RTX 4090", "bfloat16")}
|
|
|
|
|
|
def test_gate_status_requires_minimum_distinct_hardware_profiles():
|
|
store = ToplocCalibrationStore()
|
|
store.record_run(
|
|
node_wallet="wallet-a", gpu_model="RTX 4090", dtype="bfloat16", model="m",
|
|
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
|
|
)
|
|
assert store.gate_status(min_hardware_profiles=2)["ready"] is False
|
|
|
|
store.record_run(
|
|
node_wallet="wallet-b", gpu_model="A100", dtype="bfloat16", model="m",
|
|
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
|
|
)
|
|
status = store.gate_status(min_hardware_profiles=2)
|
|
assert status["ready"] is True
|
|
assert status["distinct_hardware_profiles"] == 2
|
|
|
|
|
|
def test_gate_status_empty_corpus_is_never_ready():
|
|
store = ToplocCalibrationStore()
|
|
assert store.gate_status(min_hardware_profiles=0)["ready"] is False
|
|
|
|
|
|
def test_envelope_derives_thresholds_from_worst_case_percentile_with_margin():
|
|
store = ToplocCalibrationStore()
|
|
# 100 honest runs; exp_intersections mostly 8, worst honest reading 5.
|
|
for i in range(100):
|
|
exp = 5.0 if i == 0 else 8.0
|
|
mant = 0.05 if i == 0 else 0.01
|
|
store.record_run(
|
|
node_wallet=f"wallet-{i}", gpu_model="RTX 4090", dtype="bfloat16", model="m",
|
|
passed=True, exp_intersections=exp, mant_err_mean=mant, mant_err_median=mant,
|
|
)
|
|
|
|
envelope = store.envelope(percentile=0.99, safety_margin=0.2)
|
|
assert envelope["sample_count"] == 100
|
|
# p1 (tail) of exp_intersections is pulled down by the one worst honest
|
|
# reading (5.0 vs the 8.0 bulk); a 20% safety margin shaves it further.
|
|
assert envelope["recommended_min_exp_intersections"] < 8.0
|
|
# p99 ceiling of mantissa error is likewise pulled up by the one worst
|
|
# honest reading (0.05 vs the 0.01 bulk), plus a 20% safety margin.
|
|
assert envelope["recommended_max_mant_err_mean"] > 0.01
|
|
# In-sample FPR: at most the one outlier row should be flagged by its own
|
|
# derived thresholds.
|
|
assert 0.0 <= envelope["estimated_false_positive_rate"] <= 0.02
|
|
|
|
|
|
def test_envelope_returns_none_when_no_samples():
|
|
store = ToplocCalibrationStore()
|
|
envelope = store.envelope()
|
|
assert envelope["recommended_min_exp_intersections"] is None
|
|
assert envelope["recommended_max_mant_err_mean"] is None
|
|
assert envelope["recommended_max_mant_err_median"] is None
|
|
assert envelope["estimated_false_positive_rate"] is None
|