feat(tracker): add alpha calibration and dynamic pricing

Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021.

Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs.

Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
Dobromir Popov
2026-07-06 09:48:27 +03:00
parent 32514e84c9
commit f841dfaeed
18 changed files with 1996 additions and 25 deletions

View File

@@ -91,6 +91,28 @@ def build_activation_proofs(
)
@dataclass(frozen=True)
class ToplocVerificationResult:
"""Verification outcome plus the raw TOPLOC divergence metric.
The `toploc` library's `verify_proofs_*` returns a bool for simple
prover/verifier config mismatches, but for a real activation comparison
it returns one `VerificationResult(exp_intersections, mant_err_mean,
mant_err_median)` per chunk (README §"What it actually is"). Historically
only `bool(result)` was kept, which is always true for a non-empty list
of results regardless of how divergent they are (AH-021 gap #1). This
dataclass surfaces the raw per-chunk metrics (aggregated: worst-case
`exp_intersections`, mean `mant_err_mean`/`mant_err_median`) so a
calibration corpus can be built before any threshold is trusted.
"""
passed: bool
exp_intersections: float | None = None
mant_err_mean: float | None = None
mant_err_median: float | None = None
chunk_count: int = 0
def verify_activation_proofs(
reference_activations: list[Any],
claim: ToplocProofClaim,
@@ -99,6 +121,25 @@ def verify_activation_proofs(
backend: Any | None = None,
) -> bool:
"""Verify prover TOPLOC proofs against reference teacher-forced activations."""
return verify_activation_proofs_detailed(
reference_activations, claim, config=config, backend=backend,
).passed
def verify_activation_proofs_detailed(
reference_activations: list[Any],
claim: ToplocProofClaim,
*,
config: ToplocAuditConfig | None = None,
backend: Any | None = None,
) -> ToplocVerificationResult:
"""Verify prover TOPLOC proofs and surface the raw divergence metric.
Same pass/fail contract as `verify_activation_proofs` (kept as a thin
wrapper for existing call sites); this is the entry point for anything
that needs the underlying distance value, e.g. the AH-021 honest-noise
calibration corpus.
"""
cfg = config or ToplocAuditConfig(
dtype=claim.dtype,
quantization=claim.quantization,
@@ -108,23 +149,52 @@ def verify_activation_proofs(
encoding=claim.encoding,
)
if claim.dtype != cfg.dtype or claim.quantization != cfg.quantization:
return False
return ToplocVerificationResult(passed=False)
if claim.decode_batching_size != cfg.decode_batching_size or claim.topk != cfg.topk:
return False
return ToplocVerificationResult(passed=False)
if claim.skip_prefill != cfg.skip_prefill or claim.encoding != cfg.encoding:
return False
return ToplocVerificationResult(passed=False)
module = backend or _load_toploc()
function_name = f"verify_proofs_{claim.encoding}"
verify = getattr(module, function_name)
return bool(_call_toploc(
raw = _call_toploc(
verify,
reference_activations,
claim.proofs,
decode_batching_size=claim.decode_batching_size,
topk=claim.topk,
skip_prefill=claim.skip_prefill,
))
)
divergence = _extract_divergence(raw)
return ToplocVerificationResult(passed=bool(raw), **divergence)
def _extract_divergence(raw: Any) -> dict[str, Any]:
"""Aggregate per-chunk TOPLOC `VerificationResult`s, if present.
`raw` is a plain bool for the simple fake backends used in existing unit
tests (no per-chunk metric available). The real `toploc` library returns
a list of per-chunk results; `exp_intersections` is aggregated by min
(worst honest-noise case across chunks) and the mantissa errors by mean.
"""
chunks = raw if isinstance(raw, (list, tuple)) else None
if not chunks:
return {"exp_intersections": None, "mant_err_mean": None, "mant_err_median": None, "chunk_count": 0}
exp_vals = [v for v in (_chunk_field(c, "exp_intersections") for c in chunks) if v is not None]
mean_vals = [v for v in (_chunk_field(c, "mant_err_mean") for c in chunks) if v is not None]
median_vals = [v for v in (_chunk_field(c, "mant_err_median") for c in chunks) if v is not None]
return {
"exp_intersections": min(exp_vals) if exp_vals else None,
"mant_err_mean": (sum(mean_vals) / len(mean_vals)) if mean_vals else None,
"mant_err_median": (sum(median_vals) / len(median_vals)) if median_vals else None,
"chunk_count": len(chunks),
}
def _chunk_field(chunk: Any, name: str) -> float | None:
value = chunk.get(name) if isinstance(chunk, dict) else getattr(chunk, name, None)
return float(value) if isinstance(value, (int, float)) else None
def _load_toploc() -> Any:
@@ -159,6 +229,8 @@ def _proof_encoding(value: object) -> ProofEncoding:
__all__ = [
"ToplocAuditConfig",
"ToplocProofClaim",
"ToplocVerificationResult",
"build_activation_proofs",
"verify_activation_proofs",
"verify_activation_proofs_detailed",
]