feat(tracker): add alpha calibration and dynamic pricing
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
Dobromir Popov
223
packages/tracker/meshnet_tracker/calibration.py
Normal file
223
packages/tracker/meshnet_tracker/calibration.py
Normal file
@@ -0,0 +1,223 @@
|
||||
"""TOPLOC honest-noise calibration corpus (ADR-0018 consequences, issue 21).
|
||||
|
||||
Production TOPLOC audit thresholds must be derived from an empirical
|
||||
honest-noise baseline across the active fleet's hardware, not guessed
|
||||
(research-verifiable-inference.md §8 layer 3). This store persists one row
|
||||
per (node wallet, GPU model, dtype) calibration run so thresholds can be
|
||||
computed from a queryable corpus instead of a flat JSON dump, and re-derived
|
||||
whenever the fleet's hardware mix changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
DEFAULT_CALIBRATION_DB_PATH = "toploc_calibration.sqlite"
|
||||
# Headroom added on top of the observed p99 honest-noise envelope so normal
|
||||
# hardware variance doesn't trip the recommended threshold (ADR-0018 §3).
|
||||
DEFAULT_SAFETY_MARGIN = 0.20
|
||||
DEFAULT_PERCENTILE = 0.99
|
||||
|
||||
|
||||
class ToplocCalibrationStore:
|
||||
"""Thread-safe SQLite-backed corpus of per-node TOPLOC divergence runs."""
|
||||
|
||||
def __init__(self, db_path: str | None = None) -> None:
|
||||
self._db_path = db_path
|
||||
self._lock = threading.Lock()
|
||||
self._runs: list[dict] = []
|
||||
if self._db_path:
|
||||
self._init_db()
|
||||
self._load_from_db()
|
||||
|
||||
def record_run(
|
||||
self,
|
||||
*,
|
||||
node_wallet: str,
|
||||
gpu_model: str | None,
|
||||
dtype: str | None,
|
||||
model: str,
|
||||
passed: bool,
|
||||
exp_intersections: float | None,
|
||||
mant_err_mean: float | None,
|
||||
mant_err_median: float | None,
|
||||
ts: float | None = None,
|
||||
) -> dict:
|
||||
run = {
|
||||
"id": f"cal-{uuid.uuid4().hex}",
|
||||
"node_wallet": node_wallet,
|
||||
"gpu_model": gpu_model or "unknown",
|
||||
"dtype": dtype or "unknown",
|
||||
"model": model,
|
||||
"passed": bool(passed),
|
||||
"exp_intersections": exp_intersections,
|
||||
"mant_err_mean": mant_err_mean,
|
||||
"mant_err_median": mant_err_median,
|
||||
"ts": ts if ts is not None else time.time(),
|
||||
}
|
||||
with self._lock:
|
||||
self._runs.append(run)
|
||||
self._save_run(run)
|
||||
return run
|
||||
|
||||
def runs(self) -> list[dict]:
|
||||
with self._lock:
|
||||
return list(self._runs)
|
||||
|
||||
def distinct_hardware_profiles(self) -> set[tuple[str, str]]:
|
||||
with self._lock:
|
||||
return {(r["gpu_model"], r["dtype"]) for r in self._runs}
|
||||
|
||||
def gate_status(self, *, min_hardware_profiles: int) -> dict:
|
||||
"""Whether the corpus is broad enough to enable production thresholds.
|
||||
|
||||
Alpha exception (issue 21): with a small, fully hired/controlled VPS
|
||||
fleet, ``min_hardware_profiles`` may legitimately equal the fleet's
|
||||
actual distinct hardware count — this must be revisited before a
|
||||
public/volunteer launch broadens the hardware mix.
|
||||
"""
|
||||
distinct = len(self.distinct_hardware_profiles())
|
||||
return {
|
||||
"distinct_hardware_profiles": distinct,
|
||||
"min_hardware_profiles": min_hardware_profiles,
|
||||
"sample_count": len(self._runs),
|
||||
"ready": distinct > 0 and distinct >= min_hardware_profiles,
|
||||
}
|
||||
|
||||
def envelope(
|
||||
self,
|
||||
*,
|
||||
percentile: float = DEFAULT_PERCENTILE,
|
||||
safety_margin: float = DEFAULT_SAFETY_MARGIN,
|
||||
) -> dict:
|
||||
"""Recommended tolerance constants derived from the corpus.
|
||||
|
||||
`exp_intersections` (higher = better match) gets a floor at its
|
||||
worst-case (low) percentile minus margin; the mantissa errors
|
||||
(higher = worse) get a ceiling at their worst-case (high) percentile
|
||||
plus margin. Returns None for a metric with no samples yet.
|
||||
"""
|
||||
with self._lock:
|
||||
runs = list(self._runs)
|
||||
exp_vals = sorted(v for r in runs if (v := r["exp_intersections"]) is not None)
|
||||
mean_vals = sorted(v for r in runs if (v := r["mant_err_mean"]) is not None)
|
||||
median_vals = sorted(v for r in runs if (v := r["mant_err_median"]) is not None)
|
||||
min_exp = _floor(exp_vals, 1.0 - percentile, safety_margin)
|
||||
max_mean = _ceiling(mean_vals, percentile, safety_margin)
|
||||
max_median = _ceiling(median_vals, percentile, safety_margin)
|
||||
return {
|
||||
"sample_count": len(runs),
|
||||
"distinct_hardware_profiles": len(self.distinct_hardware_profiles()),
|
||||
"percentile": percentile,
|
||||
"safety_margin": safety_margin,
|
||||
"recommended_min_exp_intersections": min_exp,
|
||||
"recommended_max_mant_err_mean": max_mean,
|
||||
"recommended_max_mant_err_median": max_median,
|
||||
# In-sample estimate only: the fraction of this same honest
|
||||
# corpus that the recommended thresholds would themselves flag.
|
||||
# Not a substitute for independent validation data — but a
|
||||
# documented starting estimate per issue 21's acceptance
|
||||
# criteria, and a sanity check that the margin isn't too tight.
|
||||
"estimated_false_positive_rate": _false_positive_rate(
|
||||
runs, min_exp=min_exp, max_mean=max_mean, max_median=max_median,
|
||||
),
|
||||
}
|
||||
|
||||
# ---- persistence (billing.py pattern) ----
|
||||
|
||||
def _init_db(self) -> None:
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.execute(
|
||||
"CREATE TABLE IF NOT EXISTS toploc_calibration_runs "
|
||||
"(run_id TEXT PRIMARY KEY, node_wallet TEXT NOT NULL, "
|
||||
"gpu_model TEXT NOT NULL, dtype TEXT NOT NULL, payload TEXT NOT NULL, "
|
||||
"ts REAL NOT NULL)"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
def _load_from_db(self) -> None:
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
rows = con.execute(
|
||||
"SELECT payload FROM toploc_calibration_runs ORDER BY ts, run_id"
|
||||
).fetchall()
|
||||
con.close()
|
||||
for (payload,) in rows:
|
||||
try:
|
||||
self._runs.append(json.loads(payload))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
def _save_run(self, run: dict) -> None:
|
||||
if not self._db_path:
|
||||
return
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO toploc_calibration_runs "
|
||||
"(run_id, node_wallet, gpu_model, dtype, payload, ts) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(run["id"], run["node_wallet"], run["gpu_model"], run["dtype"], json.dumps(run), float(run["ts"])),
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def _percentile(sorted_vals: list[float], p: float) -> float:
|
||||
if len(sorted_vals) == 1:
|
||||
return sorted_vals[0]
|
||||
k = (len(sorted_vals) - 1) * p
|
||||
lo = int(k)
|
||||
hi = min(lo + 1, len(sorted_vals) - 1)
|
||||
if lo == hi:
|
||||
return sorted_vals[lo]
|
||||
return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * (k - lo)
|
||||
|
||||
|
||||
def _floor(sorted_vals: list[float], p: float, safety_margin: float) -> float | None:
|
||||
if not sorted_vals:
|
||||
return None
|
||||
return max(0.0, _percentile(sorted_vals, p) * (1.0 - safety_margin))
|
||||
|
||||
|
||||
def _ceiling(sorted_vals: list[float], p: float, safety_margin: float) -> float | None:
|
||||
if not sorted_vals:
|
||||
return None
|
||||
return _percentile(sorted_vals, p) * (1.0 + safety_margin)
|
||||
|
||||
|
||||
def _false_positive_rate(
|
||||
runs: list[dict],
|
||||
*,
|
||||
min_exp: float | None,
|
||||
max_mean: float | None,
|
||||
max_median: float | None,
|
||||
) -> float | None:
|
||||
"""Fraction of the (honest, by construction) corpus that would be
|
||||
flagged by the recommended thresholds — an in-sample false-positive
|
||||
rate estimate, not out-of-sample validation."""
|
||||
if not runs:
|
||||
return None
|
||||
flagged = 0
|
||||
for r in runs:
|
||||
exp = r["exp_intersections"]
|
||||
mean = r["mant_err_mean"]
|
||||
median = r["mant_err_median"]
|
||||
would_flag = (
|
||||
(min_exp is not None and exp is not None and exp < min_exp)
|
||||
or (max_mean is not None and mean is not None and mean > max_mean)
|
||||
or (max_median is not None and median is not None and median > max_median)
|
||||
)
|
||||
if would_flag:
|
||||
flagged += 1
|
||||
return flagged / len(runs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_CALIBRATION_DB_PATH",
|
||||
"DEFAULT_SAFETY_MARGIN",
|
||||
"DEFAULT_PERCENTILE",
|
||||
"ToplocCalibrationStore",
|
||||
]
|
||||
Reference in New Issue
Block a user