feat(tracker): add alpha calibration and dynamic pricing
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
@@ -51,8 +51,31 @@ skip_prefill = true
|
||||
encoding = "base64"
|
||||
```
|
||||
|
||||
`verify_activation_proofs_detailed()` (`meshnet_validator.audit`) surfaces the
|
||||
raw TOPLOC divergence — `exp_intersections` (worst-case across chunks),
|
||||
`mant_err_mean`, `mant_err_median` — alongside the pass/fail bool. This is
|
||||
what the calibration corpus below is built from; existing callers that only
|
||||
need the bool keep using `verify_activation_proofs()`.
|
||||
|
||||
**Do not enable production audit thresholds before issue 21 closes.**
|
||||
Production audit thresholds remain gated on the honest-noise calibration
|
||||
corpus in issue 21.
|
||||
corpus in issue 21: the tracker's `POST /v1/calibration/toploc/run`
|
||||
(admin/validator-only, mirrors `POST /v1/benchmark/hop-penalty`) dispatches a
|
||||
fixed prompt to every solo-capable registered node, verifies each node's
|
||||
on-demand commitment against a teacher-forced reference replay, and records
|
||||
the divergence into a SQLite corpus (`meshnet_tracker.calibration.
|
||||
ToplocCalibrationStore`) keyed by node wallet + GPU model + dtype.
|
||||
`GET /v1/calibration/toploc/results` reports the corpus plus:
|
||||
|
||||
- `envelope`: p99 honest-noise value per metric with a 20% safety margin —
|
||||
the recommended (not yet wired) tolerance constants.
|
||||
- `gate_status.ready`: whether the corpus covers enough distinct hardware
|
||||
profiles (`--toploc-calibration-gate-min-hardware-profiles`, default 1).
|
||||
**Alpha exception:** with the hired-VPS-only launch fleet, `ready` may
|
||||
legitimately mean "covers every node we currently operate" — this must be
|
||||
revisited (raise the minimum) before a public/volunteer launch broadens
|
||||
the hardware mix, since a new corpus is required whenever the fleet's
|
||||
hardware composition changes.
|
||||
|
||||
Two operational notes:
|
||||
|
||||
|
||||
@@ -8,7 +8,13 @@ import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .audit import ToplocAuditConfig, ToplocProofClaim, verify_activation_proofs
|
||||
from .audit import (
|
||||
ToplocAuditConfig,
|
||||
ToplocProofClaim,
|
||||
ToplocVerificationResult,
|
||||
verify_activation_proofs,
|
||||
verify_activation_proofs_detailed,
|
||||
)
|
||||
from .sampling import AdaptiveAuditSampler, AuditRateConfig
|
||||
from .tripwire import detect_output_tripwire
|
||||
|
||||
|
||||
@@ -91,6 +91,28 @@ def build_activation_proofs(
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToplocVerificationResult:
|
||||
"""Verification outcome plus the raw TOPLOC divergence metric.
|
||||
|
||||
The `toploc` library's `verify_proofs_*` returns a bool for simple
|
||||
prover/verifier config mismatches, but for a real activation comparison
|
||||
it returns one `VerificationResult(exp_intersections, mant_err_mean,
|
||||
mant_err_median)` per chunk (README §"What it actually is"). Historically
|
||||
only `bool(result)` was kept, which is always true for a non-empty list
|
||||
of results regardless of how divergent they are (AH-021 gap #1). This
|
||||
dataclass surfaces the raw per-chunk metrics (aggregated: worst-case
|
||||
`exp_intersections`, mean `mant_err_mean`/`mant_err_median`) so a
|
||||
calibration corpus can be built before any threshold is trusted.
|
||||
"""
|
||||
|
||||
passed: bool
|
||||
exp_intersections: float | None = None
|
||||
mant_err_mean: float | None = None
|
||||
mant_err_median: float | None = None
|
||||
chunk_count: int = 0
|
||||
|
||||
|
||||
def verify_activation_proofs(
|
||||
reference_activations: list[Any],
|
||||
claim: ToplocProofClaim,
|
||||
@@ -99,6 +121,25 @@ def verify_activation_proofs(
|
||||
backend: Any | None = None,
|
||||
) -> bool:
|
||||
"""Verify prover TOPLOC proofs against reference teacher-forced activations."""
|
||||
return verify_activation_proofs_detailed(
|
||||
reference_activations, claim, config=config, backend=backend,
|
||||
).passed
|
||||
|
||||
|
||||
def verify_activation_proofs_detailed(
|
||||
reference_activations: list[Any],
|
||||
claim: ToplocProofClaim,
|
||||
*,
|
||||
config: ToplocAuditConfig | None = None,
|
||||
backend: Any | None = None,
|
||||
) -> ToplocVerificationResult:
|
||||
"""Verify prover TOPLOC proofs and surface the raw divergence metric.
|
||||
|
||||
Same pass/fail contract as `verify_activation_proofs` (kept as a thin
|
||||
wrapper for existing call sites); this is the entry point for anything
|
||||
that needs the underlying distance value, e.g. the AH-021 honest-noise
|
||||
calibration corpus.
|
||||
"""
|
||||
cfg = config or ToplocAuditConfig(
|
||||
dtype=claim.dtype,
|
||||
quantization=claim.quantization,
|
||||
@@ -108,23 +149,52 @@ def verify_activation_proofs(
|
||||
encoding=claim.encoding,
|
||||
)
|
||||
if claim.dtype != cfg.dtype or claim.quantization != cfg.quantization:
|
||||
return False
|
||||
return ToplocVerificationResult(passed=False)
|
||||
if claim.decode_batching_size != cfg.decode_batching_size or claim.topk != cfg.topk:
|
||||
return False
|
||||
return ToplocVerificationResult(passed=False)
|
||||
if claim.skip_prefill != cfg.skip_prefill or claim.encoding != cfg.encoding:
|
||||
return False
|
||||
return ToplocVerificationResult(passed=False)
|
||||
|
||||
module = backend or _load_toploc()
|
||||
function_name = f"verify_proofs_{claim.encoding}"
|
||||
verify = getattr(module, function_name)
|
||||
return bool(_call_toploc(
|
||||
raw = _call_toploc(
|
||||
verify,
|
||||
reference_activations,
|
||||
claim.proofs,
|
||||
decode_batching_size=claim.decode_batching_size,
|
||||
topk=claim.topk,
|
||||
skip_prefill=claim.skip_prefill,
|
||||
))
|
||||
)
|
||||
divergence = _extract_divergence(raw)
|
||||
return ToplocVerificationResult(passed=bool(raw), **divergence)
|
||||
|
||||
|
||||
def _extract_divergence(raw: Any) -> dict[str, Any]:
|
||||
"""Aggregate per-chunk TOPLOC `VerificationResult`s, if present.
|
||||
|
||||
`raw` is a plain bool for the simple fake backends used in existing unit
|
||||
tests (no per-chunk metric available). The real `toploc` library returns
|
||||
a list of per-chunk results; `exp_intersections` is aggregated by min
|
||||
(worst honest-noise case across chunks) and the mantissa errors by mean.
|
||||
"""
|
||||
chunks = raw if isinstance(raw, (list, tuple)) else None
|
||||
if not chunks:
|
||||
return {"exp_intersections": None, "mant_err_mean": None, "mant_err_median": None, "chunk_count": 0}
|
||||
exp_vals = [v for v in (_chunk_field(c, "exp_intersections") for c in chunks) if v is not None]
|
||||
mean_vals = [v for v in (_chunk_field(c, "mant_err_mean") for c in chunks) if v is not None]
|
||||
median_vals = [v for v in (_chunk_field(c, "mant_err_median") for c in chunks) if v is not None]
|
||||
return {
|
||||
"exp_intersections": min(exp_vals) if exp_vals else None,
|
||||
"mant_err_mean": (sum(mean_vals) / len(mean_vals)) if mean_vals else None,
|
||||
"mant_err_median": (sum(median_vals) / len(median_vals)) if median_vals else None,
|
||||
"chunk_count": len(chunks),
|
||||
}
|
||||
|
||||
|
||||
def _chunk_field(chunk: Any, name: str) -> float | None:
|
||||
value = chunk.get(name) if isinstance(chunk, dict) else getattr(chunk, name, None)
|
||||
return float(value) if isinstance(value, (int, float)) else None
|
||||
|
||||
|
||||
def _load_toploc() -> Any:
|
||||
@@ -159,6 +229,8 @@ def _proof_encoding(value: object) -> ProofEncoding:
|
||||
__all__ = [
|
||||
"ToplocAuditConfig",
|
||||
"ToplocProofClaim",
|
||||
"ToplocVerificationResult",
|
||||
"build_activation_proofs",
|
||||
"verify_activation_proofs",
|
||||
"verify_activation_proofs_detailed",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user