feat(tracker): add alpha calibration and dynamic pricing
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
80
tests/test_toploc_calibration.py
Normal file
80
tests/test_toploc_calibration.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""AH-021: honest-noise TOPLOC calibration corpus storage + aggregation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from meshnet_tracker.calibration import ToplocCalibrationStore
|
||||
|
||||
|
||||
def test_record_run_persists_and_reloads_from_sqlite(tmp_path):
|
||||
db_path = str(tmp_path / "calibration.sqlite")
|
||||
store = ToplocCalibrationStore(db_path=db_path)
|
||||
store.record_run(
|
||||
node_wallet="wallet-a",
|
||||
gpu_model="RTX 4090",
|
||||
dtype="bfloat16",
|
||||
model="Qwen2.5-0.5B-Instruct",
|
||||
passed=True,
|
||||
exp_intersections=7.0,
|
||||
mant_err_mean=0.01,
|
||||
mant_err_median=0.008,
|
||||
)
|
||||
|
||||
reloaded = ToplocCalibrationStore(db_path=db_path)
|
||||
assert len(reloaded.runs()) == 1
|
||||
assert reloaded.runs()[0]["node_wallet"] == "wallet-a"
|
||||
assert reloaded.distinct_hardware_profiles() == {("RTX 4090", "bfloat16")}
|
||||
|
||||
|
||||
def test_gate_status_requires_minimum_distinct_hardware_profiles():
|
||||
store = ToplocCalibrationStore()
|
||||
store.record_run(
|
||||
node_wallet="wallet-a", gpu_model="RTX 4090", dtype="bfloat16", model="m",
|
||||
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
|
||||
)
|
||||
assert store.gate_status(min_hardware_profiles=2)["ready"] is False
|
||||
|
||||
store.record_run(
|
||||
node_wallet="wallet-b", gpu_model="A100", dtype="bfloat16", model="m",
|
||||
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
|
||||
)
|
||||
status = store.gate_status(min_hardware_profiles=2)
|
||||
assert status["ready"] is True
|
||||
assert status["distinct_hardware_profiles"] == 2
|
||||
|
||||
|
||||
def test_gate_status_empty_corpus_is_never_ready():
|
||||
store = ToplocCalibrationStore()
|
||||
assert store.gate_status(min_hardware_profiles=0)["ready"] is False
|
||||
|
||||
|
||||
def test_envelope_derives_thresholds_from_worst_case_percentile_with_margin():
|
||||
store = ToplocCalibrationStore()
|
||||
# 100 honest runs; exp_intersections mostly 8, worst honest reading 5.
|
||||
for i in range(100):
|
||||
exp = 5.0 if i == 0 else 8.0
|
||||
mant = 0.05 if i == 0 else 0.01
|
||||
store.record_run(
|
||||
node_wallet=f"wallet-{i}", gpu_model="RTX 4090", dtype="bfloat16", model="m",
|
||||
passed=True, exp_intersections=exp, mant_err_mean=mant, mant_err_median=mant,
|
||||
)
|
||||
|
||||
envelope = store.envelope(percentile=0.99, safety_margin=0.2)
|
||||
assert envelope["sample_count"] == 100
|
||||
# p1 (tail) of exp_intersections is pulled down by the one worst honest
|
||||
# reading (5.0 vs the 8.0 bulk); a 20% safety margin shaves it further.
|
||||
assert envelope["recommended_min_exp_intersections"] < 8.0
|
||||
# p99 ceiling of mantissa error is likewise pulled up by the one worst
|
||||
# honest reading (0.05 vs the 0.01 bulk), plus a 20% safety margin.
|
||||
assert envelope["recommended_max_mant_err_mean"] > 0.01
|
||||
# In-sample FPR: at most the one outlier row should be flagged by its own
|
||||
# derived thresholds.
|
||||
assert 0.0 <= envelope["estimated_false_positive_rate"] <= 0.02
|
||||
|
||||
|
||||
def test_envelope_returns_none_when_no_samples():
|
||||
store = ToplocCalibrationStore()
|
||||
envelope = store.envelope()
|
||||
assert envelope["recommended_min_exp_intersections"] is None
|
||||
assert envelope["recommended_max_mant_err_mean"] is None
|
||||
assert envelope["recommended_max_mant_err_median"] is None
|
||||
assert envelope["estimated_false_positive_rate"] is None
|
||||
Reference in New Issue
Block a user