feat(tracker): add alpha calibration and dynamic pricing

Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021.

Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs.

Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
Dobromir Popov
2026-07-06 09:48:27 +03:00
parent 32514e84c9
commit f841dfaeed
18 changed files with 1996 additions and 25 deletions

View File

@@ -51,8 +51,31 @@ skip_prefill = true
encoding = "base64"
```
`verify_activation_proofs_detailed()` (`meshnet_validator.audit`) surfaces the
raw TOPLOC divergence — `exp_intersections` (worst-case across chunks),
`mant_err_mean`, `mant_err_median` — alongside the pass/fail bool. This is
what the calibration corpus below is built from; existing callers that only
need the bool keep using `verify_activation_proofs()`.
**Do not enable production audit thresholds before issue 21 closes.**
Production audit thresholds remain gated on the honest-noise calibration
corpus in issue 21.
corpus in issue 21: the tracker's `POST /v1/calibration/toploc/run`
(admin/validator-only, mirrors `POST /v1/benchmark/hop-penalty`) dispatches a
fixed prompt to every solo-capable registered node, verifies each node's
on-demand commitment against a teacher-forced reference replay, and records
the divergence into a SQLite corpus (`meshnet_tracker.calibration.
ToplocCalibrationStore`) keyed by node wallet + GPU model + dtype.
`GET /v1/calibration/toploc/results` reports the corpus plus:
- `envelope`: p99 honest-noise value per metric with a 20% safety margin —
the recommended (not yet wired) tolerance constants.
- `gate_status.ready`: whether the corpus covers enough distinct hardware
profiles (`--toploc-calibration-gate-min-hardware-profiles`, default 1).
**Alpha exception:** with the hired-VPS-only launch fleet, `ready` may
legitimately mean "covers every node we currently operate" — this must be
revisited (raise the minimum) before a public/volunteer launch broadens
the hardware mix, since a new corpus is required whenever the fleet's
hardware composition changes.
Two operational notes:

View File

@@ -8,7 +8,13 @@ import time
import urllib.request
from typing import Any
from .audit import ToplocAuditConfig, ToplocProofClaim, verify_activation_proofs
from .audit import (
ToplocAuditConfig,
ToplocProofClaim,
ToplocVerificationResult,
verify_activation_proofs,
verify_activation_proofs_detailed,
)
from .sampling import AdaptiveAuditSampler, AuditRateConfig
from .tripwire import detect_output_tripwire

View File

@@ -91,6 +91,28 @@ def build_activation_proofs(
)
@dataclass(frozen=True)
class ToplocVerificationResult:
"""Verification outcome plus the raw TOPLOC divergence metric.
The `toploc` library's `verify_proofs_*` returns a bool for simple
prover/verifier config mismatches, but for a real activation comparison
it returns one `VerificationResult(exp_intersections, mant_err_mean,
mant_err_median)` per chunk (README §"What it actually is"). Historically
only `bool(result)` was kept, which is always true for a non-empty list
of results regardless of how divergent they are (AH-021 gap #1). This
dataclass surfaces the raw per-chunk metrics (aggregated: worst-case
`exp_intersections`, mean `mant_err_mean`/`mant_err_median`) so a
calibration corpus can be built before any threshold is trusted.
"""
passed: bool
exp_intersections: float | None = None
mant_err_mean: float | None = None
mant_err_median: float | None = None
chunk_count: int = 0
def verify_activation_proofs(
reference_activations: list[Any],
claim: ToplocProofClaim,
@@ -99,6 +121,25 @@ def verify_activation_proofs(
backend: Any | None = None,
) -> bool:
"""Verify prover TOPLOC proofs against reference teacher-forced activations."""
return verify_activation_proofs_detailed(
reference_activations, claim, config=config, backend=backend,
).passed
def verify_activation_proofs_detailed(
reference_activations: list[Any],
claim: ToplocProofClaim,
*,
config: ToplocAuditConfig | None = None,
backend: Any | None = None,
) -> ToplocVerificationResult:
"""Verify prover TOPLOC proofs and surface the raw divergence metric.
Same pass/fail contract as `verify_activation_proofs` (kept as a thin
wrapper for existing call sites); this is the entry point for anything
that needs the underlying distance value, e.g. the AH-021 honest-noise
calibration corpus.
"""
cfg = config or ToplocAuditConfig(
dtype=claim.dtype,
quantization=claim.quantization,
@@ -108,23 +149,52 @@ def verify_activation_proofs(
encoding=claim.encoding,
)
if claim.dtype != cfg.dtype or claim.quantization != cfg.quantization:
return False
return ToplocVerificationResult(passed=False)
if claim.decode_batching_size != cfg.decode_batching_size or claim.topk != cfg.topk:
return False
return ToplocVerificationResult(passed=False)
if claim.skip_prefill != cfg.skip_prefill or claim.encoding != cfg.encoding:
return False
return ToplocVerificationResult(passed=False)
module = backend or _load_toploc()
function_name = f"verify_proofs_{claim.encoding}"
verify = getattr(module, function_name)
return bool(_call_toploc(
raw = _call_toploc(
verify,
reference_activations,
claim.proofs,
decode_batching_size=claim.decode_batching_size,
topk=claim.topk,
skip_prefill=claim.skip_prefill,
))
)
divergence = _extract_divergence(raw)
return ToplocVerificationResult(passed=bool(raw), **divergence)
def _extract_divergence(raw: Any) -> dict[str, Any]:
"""Aggregate per-chunk TOPLOC `VerificationResult`s, if present.
`raw` is a plain bool for the simple fake backends used in existing unit
tests (no per-chunk metric available). The real `toploc` library returns
a list of per-chunk results; `exp_intersections` is aggregated by min
(worst honest-noise case across chunks) and the mantissa errors by mean.
"""
chunks = raw if isinstance(raw, (list, tuple)) else None
if not chunks:
return {"exp_intersections": None, "mant_err_mean": None, "mant_err_median": None, "chunk_count": 0}
exp_vals = [v for v in (_chunk_field(c, "exp_intersections") for c in chunks) if v is not None]
mean_vals = [v for v in (_chunk_field(c, "mant_err_mean") for c in chunks) if v is not None]
median_vals = [v for v in (_chunk_field(c, "mant_err_median") for c in chunks) if v is not None]
return {
"exp_intersections": min(exp_vals) if exp_vals else None,
"mant_err_mean": (sum(mean_vals) / len(mean_vals)) if mean_vals else None,
"mant_err_median": (sum(median_vals) / len(median_vals)) if median_vals else None,
"chunk_count": len(chunks),
}
def _chunk_field(chunk: Any, name: str) -> float | None:
value = chunk.get(name) if isinstance(chunk, dict) else getattr(chunk, name, None)
return float(value) if isinstance(value, (int, float)) else None
def _load_toploc() -> Any:
@@ -159,6 +229,8 @@ def _proof_encoding(value: object) -> ProofEncoding:
__all__ = [
"ToplocAuditConfig",
"ToplocProofClaim",
"ToplocVerificationResult",
"build_activation_proofs",
"verify_activation_proofs",
"verify_activation_proofs_detailed",
]