"""TOPLOC honest-noise calibration corpus (ADR-0018 consequences, issue 21). Production TOPLOC audit thresholds must be derived from an empirical honest-noise baseline across the active fleet's hardware, not guessed (research-verifiable-inference.md §8 layer 3). This store persists one row per (node wallet, GPU model, dtype) calibration run so thresholds can be computed from a queryable corpus instead of a flat JSON dump, and re-derived whenever the fleet's hardware mix changes. """ from __future__ import annotations import json import sqlite3 import threading import time import uuid DEFAULT_CALIBRATION_DB_PATH = "toploc_calibration.sqlite" # Headroom added on top of the observed p99 honest-noise envelope so normal # hardware variance doesn't trip the recommended threshold (ADR-0018 §3). DEFAULT_SAFETY_MARGIN = 0.20 DEFAULT_PERCENTILE = 0.99 class ToplocCalibrationStore: """Thread-safe SQLite-backed corpus of per-node TOPLOC divergence runs.""" def __init__(self, db_path: str | None = None) -> None: self._db_path = db_path self._lock = threading.Lock() self._runs: list[dict] = [] if self._db_path: self._init_db() self._load_from_db() def record_run( self, *, node_wallet: str, gpu_model: str | None, dtype: str | None, model: str, passed: bool, exp_intersections: float | None, mant_err_mean: float | None, mant_err_median: float | None, ts: float | None = None, ) -> dict: run = { "id": f"cal-{uuid.uuid4().hex}", "node_wallet": node_wallet, "gpu_model": gpu_model or "unknown", "dtype": dtype or "unknown", "model": model, "passed": bool(passed), "exp_intersections": exp_intersections, "mant_err_mean": mant_err_mean, "mant_err_median": mant_err_median, "ts": ts if ts is not None else time.time(), } with self._lock: self._runs.append(run) self._save_run(run) return run def runs(self) -> list[dict]: with self._lock: return list(self._runs) def distinct_hardware_profiles(self) -> set[tuple[str, str]]: with self._lock: return {(r["gpu_model"], r["dtype"]) for r in self._runs} def gate_status(self, *, min_hardware_profiles: int) -> dict: """Whether the corpus is broad enough to enable production thresholds. Alpha exception (issue 21): with a small, fully hired/controlled VPS fleet, ``min_hardware_profiles`` may legitimately equal the fleet's actual distinct hardware count — this must be revisited before a public/volunteer launch broadens the hardware mix. """ distinct = len(self.distinct_hardware_profiles()) return { "distinct_hardware_profiles": distinct, "min_hardware_profiles": min_hardware_profiles, "sample_count": len(self._runs), "ready": distinct > 0 and distinct >= min_hardware_profiles, } def envelope( self, *, percentile: float = DEFAULT_PERCENTILE, safety_margin: float = DEFAULT_SAFETY_MARGIN, ) -> dict: """Recommended tolerance constants derived from the corpus. `exp_intersections` (higher = better match) gets a floor at its worst-case (low) percentile minus margin; the mantissa errors (higher = worse) get a ceiling at their worst-case (high) percentile plus margin. Returns None for a metric with no samples yet. """ with self._lock: runs = list(self._runs) exp_vals = sorted(v for r in runs if (v := r["exp_intersections"]) is not None) mean_vals = sorted(v for r in runs if (v := r["mant_err_mean"]) is not None) median_vals = sorted(v for r in runs if (v := r["mant_err_median"]) is not None) min_exp = _floor(exp_vals, 1.0 - percentile, safety_margin) max_mean = _ceiling(mean_vals, percentile, safety_margin) max_median = _ceiling(median_vals, percentile, safety_margin) return { "sample_count": len(runs), "distinct_hardware_profiles": len(self.distinct_hardware_profiles()), "percentile": percentile, "safety_margin": safety_margin, "recommended_min_exp_intersections": min_exp, "recommended_max_mant_err_mean": max_mean, "recommended_max_mant_err_median": max_median, # In-sample estimate only: the fraction of this same honest # corpus that the recommended thresholds would themselves flag. # Not a substitute for independent validation data — but a # documented starting estimate per issue 21's acceptance # criteria, and a sanity check that the margin isn't too tight. "estimated_false_positive_rate": _false_positive_rate( runs, min_exp=min_exp, max_mean=max_mean, max_median=max_median, ), } # ---- persistence (billing.py pattern) ---- def _init_db(self) -> None: con = sqlite3.connect(self._db_path) # type: ignore[arg-type] con.execute( "CREATE TABLE IF NOT EXISTS toploc_calibration_runs " "(run_id TEXT PRIMARY KEY, node_wallet TEXT NOT NULL, " "gpu_model TEXT NOT NULL, dtype TEXT NOT NULL, payload TEXT NOT NULL, " "ts REAL NOT NULL)" ) con.commit() con.close() def _load_from_db(self) -> None: con = sqlite3.connect(self._db_path) # type: ignore[arg-type] rows = con.execute( "SELECT payload FROM toploc_calibration_runs ORDER BY ts, run_id" ).fetchall() con.close() for (payload,) in rows: try: self._runs.append(json.loads(payload)) except json.JSONDecodeError: continue def _save_run(self, run: dict) -> None: if not self._db_path: return con = sqlite3.connect(self._db_path) # type: ignore[arg-type] con.execute( "INSERT OR IGNORE INTO toploc_calibration_runs " "(run_id, node_wallet, gpu_model, dtype, payload, ts) VALUES (?, ?, ?, ?, ?, ?)", (run["id"], run["node_wallet"], run["gpu_model"], run["dtype"], json.dumps(run), float(run["ts"])), ) con.commit() con.close() def _percentile(sorted_vals: list[float], p: float) -> float: if len(sorted_vals) == 1: return sorted_vals[0] k = (len(sorted_vals) - 1) * p lo = int(k) hi = min(lo + 1, len(sorted_vals) - 1) if lo == hi: return sorted_vals[lo] return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * (k - lo) def _floor(sorted_vals: list[float], p: float, safety_margin: float) -> float | None: if not sorted_vals: return None return max(0.0, _percentile(sorted_vals, p) * (1.0 - safety_margin)) def _ceiling(sorted_vals: list[float], p: float, safety_margin: float) -> float | None: if not sorted_vals: return None return _percentile(sorted_vals, p) * (1.0 + safety_margin) def _false_positive_rate( runs: list[dict], *, min_exp: float | None, max_mean: float | None, max_median: float | None, ) -> float | None: """Fraction of the (honest, by construction) corpus that would be flagged by the recommended thresholds — an in-sample false-positive rate estimate, not out-of-sample validation.""" if not runs: return None flagged = 0 for r in runs: exp = r["exp_intersections"] mean = r["mant_err_mean"] median = r["mant_err_median"] would_flag = ( (min_exp is not None and exp is not None and exp < min_exp) or (max_mean is not None and mean is not None and mean > max_mean) or (max_median is not None and median is not None and median > max_median) ) if would_flag: flagged += 1 return flagged / len(runs) __all__ = [ "DEFAULT_CALIBRATION_DB_PATH", "DEFAULT_SAFETY_MARGIN", "DEFAULT_PERCENTILE", "ToplocCalibrationStore", ]