f841dfaeed
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
224 lines
8.1 KiB
Python
224 lines
8.1 KiB
Python
"""TOPLOC honest-noise calibration corpus (ADR-0018 consequences, issue 21).
|
|
|
|
Production TOPLOC audit thresholds must be derived from an empirical
|
|
honest-noise baseline across the active fleet's hardware, not guessed
|
|
(research-verifiable-inference.md §8 layer 3). This store persists one row
|
|
per (node wallet, GPU model, dtype) calibration run so thresholds can be
|
|
computed from a queryable corpus instead of a flat JSON dump, and re-derived
|
|
whenever the fleet's hardware mix changes.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sqlite3
|
|
import threading
|
|
import time
|
|
import uuid
|
|
|
|
DEFAULT_CALIBRATION_DB_PATH = "toploc_calibration.sqlite"
|
|
# Headroom added on top of the observed p99 honest-noise envelope so normal
|
|
# hardware variance doesn't trip the recommended threshold (ADR-0018 §3).
|
|
DEFAULT_SAFETY_MARGIN = 0.20
|
|
DEFAULT_PERCENTILE = 0.99
|
|
|
|
|
|
class ToplocCalibrationStore:
|
|
"""Thread-safe SQLite-backed corpus of per-node TOPLOC divergence runs."""
|
|
|
|
def __init__(self, db_path: str | None = None) -> None:
|
|
self._db_path = db_path
|
|
self._lock = threading.Lock()
|
|
self._runs: list[dict] = []
|
|
if self._db_path:
|
|
self._init_db()
|
|
self._load_from_db()
|
|
|
|
def record_run(
|
|
self,
|
|
*,
|
|
node_wallet: str,
|
|
gpu_model: str | None,
|
|
dtype: str | None,
|
|
model: str,
|
|
passed: bool,
|
|
exp_intersections: float | None,
|
|
mant_err_mean: float | None,
|
|
mant_err_median: float | None,
|
|
ts: float | None = None,
|
|
) -> dict:
|
|
run = {
|
|
"id": f"cal-{uuid.uuid4().hex}",
|
|
"node_wallet": node_wallet,
|
|
"gpu_model": gpu_model or "unknown",
|
|
"dtype": dtype or "unknown",
|
|
"model": model,
|
|
"passed": bool(passed),
|
|
"exp_intersections": exp_intersections,
|
|
"mant_err_mean": mant_err_mean,
|
|
"mant_err_median": mant_err_median,
|
|
"ts": ts if ts is not None else time.time(),
|
|
}
|
|
with self._lock:
|
|
self._runs.append(run)
|
|
self._save_run(run)
|
|
return run
|
|
|
|
def runs(self) -> list[dict]:
|
|
with self._lock:
|
|
return list(self._runs)
|
|
|
|
def distinct_hardware_profiles(self) -> set[tuple[str, str]]:
|
|
with self._lock:
|
|
return {(r["gpu_model"], r["dtype"]) for r in self._runs}
|
|
|
|
def gate_status(self, *, min_hardware_profiles: int) -> dict:
|
|
"""Whether the corpus is broad enough to enable production thresholds.
|
|
|
|
Alpha exception (issue 21): with a small, fully hired/controlled VPS
|
|
fleet, ``min_hardware_profiles`` may legitimately equal the fleet's
|
|
actual distinct hardware count — this must be revisited before a
|
|
public/volunteer launch broadens the hardware mix.
|
|
"""
|
|
distinct = len(self.distinct_hardware_profiles())
|
|
return {
|
|
"distinct_hardware_profiles": distinct,
|
|
"min_hardware_profiles": min_hardware_profiles,
|
|
"sample_count": len(self._runs),
|
|
"ready": distinct > 0 and distinct >= min_hardware_profiles,
|
|
}
|
|
|
|
def envelope(
|
|
self,
|
|
*,
|
|
percentile: float = DEFAULT_PERCENTILE,
|
|
safety_margin: float = DEFAULT_SAFETY_MARGIN,
|
|
) -> dict:
|
|
"""Recommended tolerance constants derived from the corpus.
|
|
|
|
`exp_intersections` (higher = better match) gets a floor at its
|
|
worst-case (low) percentile minus margin; the mantissa errors
|
|
(higher = worse) get a ceiling at their worst-case (high) percentile
|
|
plus margin. Returns None for a metric with no samples yet.
|
|
"""
|
|
with self._lock:
|
|
runs = list(self._runs)
|
|
exp_vals = sorted(v for r in runs if (v := r["exp_intersections"]) is not None)
|
|
mean_vals = sorted(v for r in runs if (v := r["mant_err_mean"]) is not None)
|
|
median_vals = sorted(v for r in runs if (v := r["mant_err_median"]) is not None)
|
|
min_exp = _floor(exp_vals, 1.0 - percentile, safety_margin)
|
|
max_mean = _ceiling(mean_vals, percentile, safety_margin)
|
|
max_median = _ceiling(median_vals, percentile, safety_margin)
|
|
return {
|
|
"sample_count": len(runs),
|
|
"distinct_hardware_profiles": len(self.distinct_hardware_profiles()),
|
|
"percentile": percentile,
|
|
"safety_margin": safety_margin,
|
|
"recommended_min_exp_intersections": min_exp,
|
|
"recommended_max_mant_err_mean": max_mean,
|
|
"recommended_max_mant_err_median": max_median,
|
|
# In-sample estimate only: the fraction of this same honest
|
|
# corpus that the recommended thresholds would themselves flag.
|
|
# Not a substitute for independent validation data — but a
|
|
# documented starting estimate per issue 21's acceptance
|
|
# criteria, and a sanity check that the margin isn't too tight.
|
|
"estimated_false_positive_rate": _false_positive_rate(
|
|
runs, min_exp=min_exp, max_mean=max_mean, max_median=max_median,
|
|
),
|
|
}
|
|
|
|
# ---- persistence (billing.py pattern) ----
|
|
|
|
def _init_db(self) -> None:
|
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
|
con.execute(
|
|
"CREATE TABLE IF NOT EXISTS toploc_calibration_runs "
|
|
"(run_id TEXT PRIMARY KEY, node_wallet TEXT NOT NULL, "
|
|
"gpu_model TEXT NOT NULL, dtype TEXT NOT NULL, payload TEXT NOT NULL, "
|
|
"ts REAL NOT NULL)"
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
|
|
def _load_from_db(self) -> None:
|
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
|
rows = con.execute(
|
|
"SELECT payload FROM toploc_calibration_runs ORDER BY ts, run_id"
|
|
).fetchall()
|
|
con.close()
|
|
for (payload,) in rows:
|
|
try:
|
|
self._runs.append(json.loads(payload))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
def _save_run(self, run: dict) -> None:
|
|
if not self._db_path:
|
|
return
|
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
|
con.execute(
|
|
"INSERT OR IGNORE INTO toploc_calibration_runs "
|
|
"(run_id, node_wallet, gpu_model, dtype, payload, ts) VALUES (?, ?, ?, ?, ?, ?)",
|
|
(run["id"], run["node_wallet"], run["gpu_model"], run["dtype"], json.dumps(run), float(run["ts"])),
|
|
)
|
|
con.commit()
|
|
con.close()
|
|
|
|
|
|
def _percentile(sorted_vals: list[float], p: float) -> float:
|
|
if len(sorted_vals) == 1:
|
|
return sorted_vals[0]
|
|
k = (len(sorted_vals) - 1) * p
|
|
lo = int(k)
|
|
hi = min(lo + 1, len(sorted_vals) - 1)
|
|
if lo == hi:
|
|
return sorted_vals[lo]
|
|
return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * (k - lo)
|
|
|
|
|
|
def _floor(sorted_vals: list[float], p: float, safety_margin: float) -> float | None:
|
|
if not sorted_vals:
|
|
return None
|
|
return max(0.0, _percentile(sorted_vals, p) * (1.0 - safety_margin))
|
|
|
|
|
|
def _ceiling(sorted_vals: list[float], p: float, safety_margin: float) -> float | None:
|
|
if not sorted_vals:
|
|
return None
|
|
return _percentile(sorted_vals, p) * (1.0 + safety_margin)
|
|
|
|
|
|
def _false_positive_rate(
|
|
runs: list[dict],
|
|
*,
|
|
min_exp: float | None,
|
|
max_mean: float | None,
|
|
max_median: float | None,
|
|
) -> float | None:
|
|
"""Fraction of the (honest, by construction) corpus that would be
|
|
flagged by the recommended thresholds — an in-sample false-positive
|
|
rate estimate, not out-of-sample validation."""
|
|
if not runs:
|
|
return None
|
|
flagged = 0
|
|
for r in runs:
|
|
exp = r["exp_intersections"]
|
|
mean = r["mant_err_mean"]
|
|
median = r["mant_err_median"]
|
|
would_flag = (
|
|
(min_exp is not None and exp is not None and exp < min_exp)
|
|
or (max_mean is not None and mean is not None and mean > max_mean)
|
|
or (max_median is not None and median is not None and median > max_median)
|
|
)
|
|
if would_flag:
|
|
flagged += 1
|
|
return flagged / len(runs)
|
|
|
|
|
|
__all__ = [
|
|
"DEFAULT_CALIBRATION_DB_PATH",
|
|
"DEFAULT_SAFETY_MARGIN",
|
|
"DEFAULT_PERCENTILE",
|
|
"ToplocCalibrationStore",
|
|
]
|