feat(tracker): add alpha calibration and dynamic pricing
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
Dobromir Popov
223
packages/tracker/meshnet_tracker/calibration.py
Normal file
223
packages/tracker/meshnet_tracker/calibration.py
Normal file
@@ -0,0 +1,223 @@
|
||||
"""TOPLOC honest-noise calibration corpus (ADR-0018 consequences, issue 21).
|
||||
|
||||
Production TOPLOC audit thresholds must be derived from an empirical
|
||||
honest-noise baseline across the active fleet's hardware, not guessed
|
||||
(research-verifiable-inference.md §8 layer 3). This store persists one row
|
||||
per (node wallet, GPU model, dtype) calibration run so thresholds can be
|
||||
computed from a queryable corpus instead of a flat JSON dump, and re-derived
|
||||
whenever the fleet's hardware mix changes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
DEFAULT_CALIBRATION_DB_PATH = "toploc_calibration.sqlite"
|
||||
# Headroom added on top of the observed p99 honest-noise envelope so normal
|
||||
# hardware variance doesn't trip the recommended threshold (ADR-0018 §3).
|
||||
DEFAULT_SAFETY_MARGIN = 0.20
|
||||
DEFAULT_PERCENTILE = 0.99
|
||||
|
||||
|
||||
class ToplocCalibrationStore:
|
||||
"""Thread-safe SQLite-backed corpus of per-node TOPLOC divergence runs."""
|
||||
|
||||
def __init__(self, db_path: str | None = None) -> None:
|
||||
self._db_path = db_path
|
||||
self._lock = threading.Lock()
|
||||
self._runs: list[dict] = []
|
||||
if self._db_path:
|
||||
self._init_db()
|
||||
self._load_from_db()
|
||||
|
||||
def record_run(
|
||||
self,
|
||||
*,
|
||||
node_wallet: str,
|
||||
gpu_model: str | None,
|
||||
dtype: str | None,
|
||||
model: str,
|
||||
passed: bool,
|
||||
exp_intersections: float | None,
|
||||
mant_err_mean: float | None,
|
||||
mant_err_median: float | None,
|
||||
ts: float | None = None,
|
||||
) -> dict:
|
||||
run = {
|
||||
"id": f"cal-{uuid.uuid4().hex}",
|
||||
"node_wallet": node_wallet,
|
||||
"gpu_model": gpu_model or "unknown",
|
||||
"dtype": dtype or "unknown",
|
||||
"model": model,
|
||||
"passed": bool(passed),
|
||||
"exp_intersections": exp_intersections,
|
||||
"mant_err_mean": mant_err_mean,
|
||||
"mant_err_median": mant_err_median,
|
||||
"ts": ts if ts is not None else time.time(),
|
||||
}
|
||||
with self._lock:
|
||||
self._runs.append(run)
|
||||
self._save_run(run)
|
||||
return run
|
||||
|
||||
def runs(self) -> list[dict]:
|
||||
with self._lock:
|
||||
return list(self._runs)
|
||||
|
||||
def distinct_hardware_profiles(self) -> set[tuple[str, str]]:
|
||||
with self._lock:
|
||||
return {(r["gpu_model"], r["dtype"]) for r in self._runs}
|
||||
|
||||
def gate_status(self, *, min_hardware_profiles: int) -> dict:
|
||||
"""Whether the corpus is broad enough to enable production thresholds.
|
||||
|
||||
Alpha exception (issue 21): with a small, fully hired/controlled VPS
|
||||
fleet, ``min_hardware_profiles`` may legitimately equal the fleet's
|
||||
actual distinct hardware count — this must be revisited before a
|
||||
public/volunteer launch broadens the hardware mix.
|
||||
"""
|
||||
distinct = len(self.distinct_hardware_profiles())
|
||||
return {
|
||||
"distinct_hardware_profiles": distinct,
|
||||
"min_hardware_profiles": min_hardware_profiles,
|
||||
"sample_count": len(self._runs),
|
||||
"ready": distinct > 0 and distinct >= min_hardware_profiles,
|
||||
}
|
||||
|
||||
def envelope(
|
||||
self,
|
||||
*,
|
||||
percentile: float = DEFAULT_PERCENTILE,
|
||||
safety_margin: float = DEFAULT_SAFETY_MARGIN,
|
||||
) -> dict:
|
||||
"""Recommended tolerance constants derived from the corpus.
|
||||
|
||||
`exp_intersections` (higher = better match) gets a floor at its
|
||||
worst-case (low) percentile minus margin; the mantissa errors
|
||||
(higher = worse) get a ceiling at their worst-case (high) percentile
|
||||
plus margin. Returns None for a metric with no samples yet.
|
||||
"""
|
||||
with self._lock:
|
||||
runs = list(self._runs)
|
||||
exp_vals = sorted(v for r in runs if (v := r["exp_intersections"]) is not None)
|
||||
mean_vals = sorted(v for r in runs if (v := r["mant_err_mean"]) is not None)
|
||||
median_vals = sorted(v for r in runs if (v := r["mant_err_median"]) is not None)
|
||||
min_exp = _floor(exp_vals, 1.0 - percentile, safety_margin)
|
||||
max_mean = _ceiling(mean_vals, percentile, safety_margin)
|
||||
max_median = _ceiling(median_vals, percentile, safety_margin)
|
||||
return {
|
||||
"sample_count": len(runs),
|
||||
"distinct_hardware_profiles": len(self.distinct_hardware_profiles()),
|
||||
"percentile": percentile,
|
||||
"safety_margin": safety_margin,
|
||||
"recommended_min_exp_intersections": min_exp,
|
||||
"recommended_max_mant_err_mean": max_mean,
|
||||
"recommended_max_mant_err_median": max_median,
|
||||
# In-sample estimate only: the fraction of this same honest
|
||||
# corpus that the recommended thresholds would themselves flag.
|
||||
# Not a substitute for independent validation data — but a
|
||||
# documented starting estimate per issue 21's acceptance
|
||||
# criteria, and a sanity check that the margin isn't too tight.
|
||||
"estimated_false_positive_rate": _false_positive_rate(
|
||||
runs, min_exp=min_exp, max_mean=max_mean, max_median=max_median,
|
||||
),
|
||||
}
|
||||
|
||||
# ---- persistence (billing.py pattern) ----
|
||||
|
||||
def _init_db(self) -> None:
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.execute(
|
||||
"CREATE TABLE IF NOT EXISTS toploc_calibration_runs "
|
||||
"(run_id TEXT PRIMARY KEY, node_wallet TEXT NOT NULL, "
|
||||
"gpu_model TEXT NOT NULL, dtype TEXT NOT NULL, payload TEXT NOT NULL, "
|
||||
"ts REAL NOT NULL)"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
def _load_from_db(self) -> None:
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
rows = con.execute(
|
||||
"SELECT payload FROM toploc_calibration_runs ORDER BY ts, run_id"
|
||||
).fetchall()
|
||||
con.close()
|
||||
for (payload,) in rows:
|
||||
try:
|
||||
self._runs.append(json.loads(payload))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
def _save_run(self, run: dict) -> None:
|
||||
if not self._db_path:
|
||||
return
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO toploc_calibration_runs "
|
||||
"(run_id, node_wallet, gpu_model, dtype, payload, ts) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(run["id"], run["node_wallet"], run["gpu_model"], run["dtype"], json.dumps(run), float(run["ts"])),
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def _percentile(sorted_vals: list[float], p: float) -> float:
|
||||
if len(sorted_vals) == 1:
|
||||
return sorted_vals[0]
|
||||
k = (len(sorted_vals) - 1) * p
|
||||
lo = int(k)
|
||||
hi = min(lo + 1, len(sorted_vals) - 1)
|
||||
if lo == hi:
|
||||
return sorted_vals[lo]
|
||||
return sorted_vals[lo] + (sorted_vals[hi] - sorted_vals[lo]) * (k - lo)
|
||||
|
||||
|
||||
def _floor(sorted_vals: list[float], p: float, safety_margin: float) -> float | None:
|
||||
if not sorted_vals:
|
||||
return None
|
||||
return max(0.0, _percentile(sorted_vals, p) * (1.0 - safety_margin))
|
||||
|
||||
|
||||
def _ceiling(sorted_vals: list[float], p: float, safety_margin: float) -> float | None:
|
||||
if not sorted_vals:
|
||||
return None
|
||||
return _percentile(sorted_vals, p) * (1.0 + safety_margin)
|
||||
|
||||
|
||||
def _false_positive_rate(
|
||||
runs: list[dict],
|
||||
*,
|
||||
min_exp: float | None,
|
||||
max_mean: float | None,
|
||||
max_median: float | None,
|
||||
) -> float | None:
|
||||
"""Fraction of the (honest, by construction) corpus that would be
|
||||
flagged by the recommended thresholds — an in-sample false-positive
|
||||
rate estimate, not out-of-sample validation."""
|
||||
if not runs:
|
||||
return None
|
||||
flagged = 0
|
||||
for r in runs:
|
||||
exp = r["exp_intersections"]
|
||||
mean = r["mant_err_mean"]
|
||||
median = r["mant_err_median"]
|
||||
would_flag = (
|
||||
(min_exp is not None and exp is not None and exp < min_exp)
|
||||
or (max_mean is not None and mean is not None and mean > max_mean)
|
||||
or (max_median is not None and median is not None and median > max_median)
|
||||
)
|
||||
if would_flag:
|
||||
flagged += 1
|
||||
return flagged / len(runs)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_CALIBRATION_DB_PATH",
|
||||
"DEFAULT_SAFETY_MARGIN",
|
||||
"DEFAULT_PERCENTILE",
|
||||
"ToplocCalibrationStore",
|
||||
]
|
||||
@@ -6,6 +6,7 @@ import time
|
||||
|
||||
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
|
||||
from .billing import DEFAULT_BILLING_DB_PATH
|
||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
|
||||
from .server import TrackerServer, derive_relay_url_from_public_tracker_url
|
||||
|
||||
DEFAULT_REGISTRY_DB_PATH = "meshnet_registry.sqlite3"
|
||||
@@ -143,6 +144,55 @@ def main() -> None:
|
||||
"(default: MESHNET_HIVE_SECRET env; required for multi-tracker replication)"
|
||||
),
|
||||
)
|
||||
common.add_argument(
|
||||
"--toploc-calibration-db",
|
||||
default=None,
|
||||
metavar="PATH",
|
||||
help=(
|
||||
"SQLite path for the AH-021 honest-noise TOPLOC calibration corpus "
|
||||
"(enables POST /v1/calibration/toploc/run + GET /v1/calibration/toploc/results)"
|
||||
),
|
||||
)
|
||||
common.add_argument(
|
||||
"--toploc-reference-node-url",
|
||||
default=None,
|
||||
help="Reference node the calibration job teacher-forces claimed tokens against (see validator README)",
|
||||
)
|
||||
common.add_argument(
|
||||
"--toploc-calibration-gate-min-hardware-profiles",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Distinct (GPU model, dtype) profiles the corpus must cover before "
|
||||
"gate_status.ready is true (alpha exception: fleet size is acceptable)"
|
||||
),
|
||||
)
|
||||
common.add_argument(
|
||||
"--enable-hf-pricing",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Enable the daily dynamic pricing refresh (issue 23): for presets with a "
|
||||
"curated hf_aliases list, sets the client price to 80%% of the cheapest "
|
||||
"matching HuggingFace inference-marketplace rate. Presets without "
|
||||
"hf_aliases are unaffected and keep their static price."
|
||||
),
|
||||
)
|
||||
common.add_argument(
|
||||
"--hf-pricing-log-db",
|
||||
default=None,
|
||||
metavar="PATH",
|
||||
help=(
|
||||
"SQLite database path for the dynamic pricing change log "
|
||||
f"(default when --enable-hf-pricing is set: {DEFAULT_HF_PRICING_LOG_DB_PATH}; "
|
||||
"enables GET /v1/pricing/hf/history)"
|
||||
),
|
||||
)
|
||||
common.add_argument(
|
||||
"--hf-pricing-refresh-interval",
|
||||
type=float,
|
||||
default=86400.0,
|
||||
help="Seconds between dynamic pricing refresh passes (default: daily)",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="meshnet-tracker",
|
||||
@@ -189,6 +239,15 @@ def main() -> None:
|
||||
payout_dust_floor=args.payout_dust_floor,
|
||||
validator_service_token=args.validator_service_token,
|
||||
hive_secret=args.hive_secret,
|
||||
toploc_calibration_db=args.toploc_calibration_db,
|
||||
toploc_reference_node_url=args.toploc_reference_node_url,
|
||||
toploc_calibration_gate_min_hardware_profiles=args.toploc_calibration_gate_min_hardware_profiles,
|
||||
enable_hf_pricing=args.enable_hf_pricing,
|
||||
hf_pricing_log_db=(
|
||||
args.hf_pricing_log_db
|
||||
or (DEFAULT_HF_PRICING_LOG_DB_PATH if args.enable_hf_pricing else None)
|
||||
),
|
||||
hf_pricing_refresh_interval=args.hf_pricing_refresh_interval,
|
||||
)
|
||||
port = server.start()
|
||||
print(f"meshnet-tracker listening on http://{args.host}:{port}", flush=True)
|
||||
|
||||
314
packages/tracker/meshnet_tracker/hf_pricing.py
Normal file
314
packages/tracker/meshnet_tracker/hf_pricing.py
Normal file
@@ -0,0 +1,314 @@
|
||||
"""Dynamic per-model pricing benchmarked against HuggingFace inference rates (issue 23).
|
||||
|
||||
Client-facing price per model tracks the market: 80% of the cheapest
|
||||
comparable provider rate on HuggingFace's inference marketplace
|
||||
(https://huggingface.co/inference/models), refreshed daily. Nodes are
|
||||
unaffected — this only ever calls ``BillingLedger.set_price`` (the ledger's
|
||||
existing write path), never touches node payouts (ADR-0015's 90/10 split
|
||||
still applies to whatever price is charged).
|
||||
|
||||
Confirmed 2026-07-06: the pricing table is server-rendered into the initial
|
||||
HTML response (SvelteKit SSR) — a plain stdlib ``urllib.request`` GET plus
|
||||
HTML parsing is sufficient. No headless-browser fetch is required. Each
|
||||
table row carries an anchor whose href is
|
||||
``/<org>/<repo>/?inference_api=true&inference_provider=<provider>``, which is
|
||||
a cheaper and more stable extraction anchor than the display text (which
|
||||
duplicates the repo id at two responsive breakpoints).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from html.parser import HTMLParser
|
||||
from typing import Callable
|
||||
|
||||
HF_INFERENCE_MODELS_URL = "https://huggingface.co/inference/models"
|
||||
DEFAULT_HF_PRICING_LOG_DB_PATH = "hf_pricing_log.sqlite"
|
||||
DEFAULT_CLIENT_PRICE_FRACTION = 0.80 # charge 80% of the cheapest comparable rate
|
||||
|
||||
_ROW_HREF_RE = re.compile(
|
||||
r"^/(?P<repo>[^/]+/[^/?]+)/\?inference_api=true&inference_provider=(?P<provider>[^&\"]+)"
|
||||
)
|
||||
_PRICE_RE = re.compile(r"^\$[\d,]*\.?\d+$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HfPriceQuote:
|
||||
"""One (model, provider) row from the HF inference pricing table."""
|
||||
|
||||
repo_id: str
|
||||
provider: str
|
||||
input_per_1m: float
|
||||
output_per_1m: float
|
||||
|
||||
def blended_price_per_1k_tokens(self) -> float:
|
||||
"""Average of input/output $-per-1M-token rates, converted to $/1k.
|
||||
|
||||
The tracker bills a single per-1k-token rate (``BillingLedger``
|
||||
doesn't distinguish prompt vs. completion tokens), so this is the
|
||||
simplest fair proxy for "this provider's rate" in that unit.
|
||||
"""
|
||||
return (self.input_per_1m + self.output_per_1m) / 2.0 / 1000.0
|
||||
|
||||
def alias_keys(self) -> tuple[str, str]:
|
||||
"""Both the bare-repo and repo::provider forms an ``hf_aliases`` entry may use."""
|
||||
return (self.repo_id.lower(), f"{self.repo_id.lower()}::{self.provider.lower()}")
|
||||
|
||||
|
||||
class _HfPricingTableParser(HTMLParser):
|
||||
"""Extracts (repo_id, provider, input$/1M, output$/1M) rows from the raw HTML."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._in_tr = False
|
||||
self._row_match: tuple[str, str] | None = None
|
||||
self._row_prices: list[float] = []
|
||||
self._in_td = False
|
||||
self._td_text: list[str] = []
|
||||
self.quotes: list[HfPriceQuote] = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag == "tr":
|
||||
self._in_tr = True
|
||||
self._row_match = None
|
||||
self._row_prices = []
|
||||
elif tag == "a" and self._in_tr and self._row_match is None:
|
||||
href = dict(attrs).get("href") or ""
|
||||
m = _ROW_HREF_RE.match(href)
|
||||
if m:
|
||||
self._row_match = (
|
||||
urllib.parse.unquote(m.group("repo")),
|
||||
urllib.parse.unquote(m.group("provider")),
|
||||
)
|
||||
elif tag == "td":
|
||||
self._in_td = True
|
||||
self._td_text = []
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._in_td:
|
||||
self._td_text.append(data)
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag == "td":
|
||||
self._in_td = False
|
||||
text = "".join(self._td_text).strip()
|
||||
if _PRICE_RE.match(text):
|
||||
self._row_prices.append(float(text.replace("$", "").replace(",", "")))
|
||||
elif tag == "tr":
|
||||
self._in_tr = False
|
||||
if self._row_match and len(self._row_prices) >= 2:
|
||||
repo_id, provider = self._row_match
|
||||
self.quotes.append(
|
||||
HfPriceQuote(
|
||||
repo_id=repo_id,
|
||||
provider=provider,
|
||||
input_per_1m=self._row_prices[0],
|
||||
output_per_1m=self._row_prices[1],
|
||||
)
|
||||
)
|
||||
self._row_match = None
|
||||
self._row_prices = []
|
||||
|
||||
|
||||
def parse_hf_pricing_table(html: str) -> list[HfPriceQuote]:
|
||||
"""Pure parsing function — no network I/O, so it's directly unit-testable."""
|
||||
parser = _HfPricingTableParser()
|
||||
parser.feed(html)
|
||||
return parser.quotes
|
||||
|
||||
|
||||
def _default_fetch_html(url: str, *, timeout: float) -> str:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def fetch_hf_price_quotes(
|
||||
search_term: str,
|
||||
*,
|
||||
fetch_html: Callable[[str], str] | None = None,
|
||||
timeout: float = 15.0,
|
||||
) -> list[HfPriceQuote]:
|
||||
"""Fetch and parse the HF inference pricing table filtered by ``search_term``.
|
||||
|
||||
``fetch_html`` is the test injection point (mirrors the ``backend=``
|
||||
convention used elsewhere in this package) — it takes the full URL and
|
||||
returns the raw HTML text, so tests never hit the network.
|
||||
"""
|
||||
url = f"{HF_INFERENCE_MODELS_URL}?{urllib.parse.urlencode({'search': search_term})}"
|
||||
if fetch_html is not None:
|
||||
html = fetch_html(url)
|
||||
else:
|
||||
html = _default_fetch_html(url, timeout=timeout)
|
||||
return parse_hf_pricing_table(html)
|
||||
|
||||
|
||||
def cheapest_matching_quote(
|
||||
quotes: list[HfPriceQuote], aliases: list[str]
|
||||
) -> HfPriceQuote | None:
|
||||
"""Cheapest quote whose repo (optionally ``repo::provider``) is in ``aliases``.
|
||||
|
||||
An alias of ``"org/repo"`` matches that repo under any provider; an
|
||||
alias of ``"org/repo::provider"`` matches only that specific provider —
|
||||
useful when only one provider's deployment has been human-verified as a
|
||||
fair comparable (matching quantization/params).
|
||||
"""
|
||||
alias_set = {a.strip().lower() for a in aliases if isinstance(a, str) and a.strip()}
|
||||
if not alias_set:
|
||||
return None
|
||||
matches = [q for q in quotes if alias_set & set(q.alias_keys())]
|
||||
if not matches:
|
||||
return None
|
||||
return min(matches, key=lambda q: q.blended_price_per_1k_tokens())
|
||||
|
||||
|
||||
class HfPricingLog:
|
||||
"""Thread-safe SQLite-backed audit log of dynamic price changes (issue 23).
|
||||
|
||||
Every price change (old, new, source alias/provider, timestamp) is
|
||||
recorded here so a client dispute over a charge can be reconciled
|
||||
against exactly what the market-tracking job did and when — mirrors
|
||||
``calibration.py``'s persistence shape.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | None = None) -> None:
|
||||
self._db_path = db_path
|
||||
self._lock = threading.Lock()
|
||||
self._changes: list[dict] = []
|
||||
if self._db_path:
|
||||
self._init_db()
|
||||
self._load_from_db()
|
||||
|
||||
def record_change(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
old_price_per_1k: float,
|
||||
new_price_per_1k: float,
|
||||
source_repo_id: str,
|
||||
source_provider: str,
|
||||
ts: float | None = None,
|
||||
) -> dict:
|
||||
change = {
|
||||
"model": model,
|
||||
"old_price_per_1k": old_price_per_1k,
|
||||
"new_price_per_1k": new_price_per_1k,
|
||||
"source_repo_id": source_repo_id,
|
||||
"source_provider": source_provider,
|
||||
"ts": ts if ts is not None else time.time(),
|
||||
}
|
||||
with self._lock:
|
||||
self._changes.append(change)
|
||||
self._save_change(change)
|
||||
return change
|
||||
|
||||
def history(self, model: str | None = None, *, limit: int = 200) -> list[dict]:
|
||||
with self._lock:
|
||||
changes = list(self._changes)
|
||||
if model is not None:
|
||||
changes = [c for c in changes if c["model"] == model]
|
||||
return changes[-limit:]
|
||||
|
||||
# ---- persistence (billing.py / calibration.py pattern) ----
|
||||
|
||||
def _init_db(self) -> None:
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.execute(
|
||||
"CREATE TABLE IF NOT EXISTS hf_price_changes "
|
||||
"(id INTEGER PRIMARY KEY AUTOINCREMENT, model TEXT NOT NULL, "
|
||||
"payload TEXT NOT NULL, ts REAL NOT NULL)"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
def _load_from_db(self) -> None:
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
rows = con.execute(
|
||||
"SELECT payload FROM hf_price_changes ORDER BY ts, id"
|
||||
).fetchall()
|
||||
con.close()
|
||||
for (payload,) in rows:
|
||||
try:
|
||||
self._changes.append(json.loads(payload))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
def _save_change(self, change: dict) -> None:
|
||||
if not self._db_path:
|
||||
return
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.execute(
|
||||
"INSERT INTO hf_price_changes (model, payload, ts) VALUES (?, ?, ?)",
|
||||
(change["model"], json.dumps(change), float(change["ts"])),
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
|
||||
def hf_search_term(preset: dict, model_name: str) -> str:
|
||||
"""Best-effort search term for the HF pricing page's ``?search=`` filter."""
|
||||
hf_repo = preset.get("hf_repo")
|
||||
if isinstance(hf_repo, str) and hf_repo:
|
||||
return hf_repo.rsplit("/", 1)[-1]
|
||||
return model_name
|
||||
|
||||
|
||||
def refresh_preset_price(
|
||||
*,
|
||||
model_name: str,
|
||||
preset: dict,
|
||||
current_price: float,
|
||||
fetch_html: Callable[[str], str] | None = None,
|
||||
price_fraction: float = DEFAULT_CLIENT_PRICE_FRACTION,
|
||||
) -> dict | None:
|
||||
"""Compute the new price for one preset, or None if nothing should change.
|
||||
|
||||
Never raises — any fetch/parse failure or absence of a verified match is
|
||||
treated identically: keep the static default (deliverable's fallback
|
||||
requirement). Callers are responsible for actually applying the result
|
||||
(``BillingLedger.set_price`` + logging), so this function stays a pure
|
||||
"what should the new price be" computation and is trivially unit-testable.
|
||||
"""
|
||||
aliases = preset.get("hf_aliases")
|
||||
if not aliases:
|
||||
return None
|
||||
try:
|
||||
quotes = fetch_hf_price_quotes(
|
||||
hf_search_term(preset, model_name), fetch_html=fetch_html
|
||||
)
|
||||
quote = cheapest_matching_quote(quotes, aliases)
|
||||
except Exception:
|
||||
return None
|
||||
if quote is None:
|
||||
return None
|
||||
new_price = round(quote.blended_price_per_1k_tokens() * price_fraction, 6)
|
||||
if new_price <= 0:
|
||||
return None
|
||||
return {
|
||||
"model": model_name,
|
||||
"old_price_per_1k": current_price,
|
||||
"new_price_per_1k": new_price,
|
||||
"source_repo_id": quote.repo_id,
|
||||
"source_provider": quote.provider,
|
||||
}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"HF_INFERENCE_MODELS_URL",
|
||||
"DEFAULT_HF_PRICING_LOG_DB_PATH",
|
||||
"DEFAULT_CLIENT_PRICE_FRACTION",
|
||||
"HfPriceQuote",
|
||||
"HfPricingLog",
|
||||
"parse_hf_pricing_table",
|
||||
"fetch_hf_price_quotes",
|
||||
"cheapest_matching_quote",
|
||||
"hf_search_term",
|
||||
"refresh_preset_price",
|
||||
]
|
||||
@@ -11,6 +11,8 @@
|
||||
],
|
||||
"recommended": true,
|
||||
"deployment_status": "recommended",
|
||||
"hf_aliases": [],
|
||||
"hf_verified_match_note": "Pending human curation (issue 23) — no HF inference-marketplace listing has been confirmed as a comparable params/quantization match for this preset yet. Leave empty until a human signs off; an empty hf_aliases list keeps this model on the static default price.",
|
||||
"required_model_bytes": 638876385280,
|
||||
"download_size_bytes": 638876385280,
|
||||
"native_quantization": "int4",
|
||||
|
||||
@@ -40,6 +40,8 @@ from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
|
||||
from .auth import is_validator_token, sign_hive_request, verify_hive_request
|
||||
from .wallet_proof import binding_message, verify_wallet_signature
|
||||
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
|
||||
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||||
from .gossip import NodeGossip
|
||||
from .raft import RaftNode
|
||||
|
||||
@@ -88,6 +90,14 @@ DEFAULT_MODEL_PRESETS: dict[str, dict] = {
|
||||
**_load_model_presets_from_data(),
|
||||
}
|
||||
|
||||
def _clone_model_presets(presets: dict[str, dict]) -> dict[str, dict]:
|
||||
"""Shallow-copy each preset dict so a runtime mutation (e.g. issue 23's
|
||||
dynamic pricing refresh writing hf_last_price_per_1k/hf_last_updated)
|
||||
never leaks into the shared module-level DEFAULT_MODEL_PRESETS and from
|
||||
there into other TrackerServer instances in the same process."""
|
||||
return {name: dict(preset) for name, preset in presets.items()}
|
||||
|
||||
|
||||
DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
|
||||
DEFAULT_RAM_BYTES = 16 * 1024 * 1024 * 1024
|
||||
DEFAULT_QUANTIZATIONS = ["bfloat16"]
|
||||
@@ -976,6 +986,81 @@ def _nodes_and_bounds_for_model(
|
||||
return nodes, 0, max(node.num_layers for node in nodes) - 1
|
||||
|
||||
|
||||
def _fetch_toploc_commitment(
|
||||
node: _NodeEntry,
|
||||
*,
|
||||
session_id: str,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
) -> dict | None:
|
||||
"""Fetch a node's own on-demand TOPLOC boundary commitment (ADR-0018 §3),
|
||||
same protocol as `ValidatorProcess._fetch_hop_commitment`."""
|
||||
endpoint = node.endpoint
|
||||
if not isinstance(endpoint, str) or not endpoint:
|
||||
return None
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{endpoint.rstrip('/')}/v1/audit/toploc/commitment",
|
||||
data=json.dumps({
|
||||
"session_id": session_id,
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"shard_start": node.shard_start,
|
||||
"shard_end": node.shard_end,
|
||||
}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=5.0) as resp:
|
||||
response = json.loads(resp.read())
|
||||
except (OSError, ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
proof = response.get("toploc_proof") or response.get("activation_proof")
|
||||
token_ids = response.get("claimed_token_ids") or response.get("output_token_ids")
|
||||
if not isinstance(proof, dict):
|
||||
return None
|
||||
if not isinstance(token_ids, list) or not all(isinstance(t, int) for t in token_ids):
|
||||
return None
|
||||
return {"toploc_proof": proof, "claimed_token_ids": token_ids}
|
||||
|
||||
|
||||
def _fetch_toploc_reference_activations(
|
||||
reference_node_url: str,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
claimed_token_ids: list[int],
|
||||
claim: Any,
|
||||
) -> list | None:
|
||||
"""Teacher-force the claimed tokens on the reference node (same contract
|
||||
as `ValidatorProcess._run_teacher_forced_prefill` / validator README's
|
||||
"TOPLOC audit contract")."""
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{reference_node_url.rstrip('/')}/v1/audit/toploc",
|
||||
data=json.dumps({
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"claimed_token_ids": claimed_token_ids,
|
||||
"dtype": claim.dtype,
|
||||
"quantization": claim.quantization,
|
||||
"decode_batching_size": claim.decode_batching_size,
|
||||
"topk": claim.topk,
|
||||
"skip_prefill": claim.skip_prefill,
|
||||
}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=300.0) as resp:
|
||||
response = json.loads(resp.read())
|
||||
except (OSError, ValueError, json.JSONDecodeError):
|
||||
return None
|
||||
activations = response.get("activations")
|
||||
if not isinstance(activations, list):
|
||||
return None
|
||||
return activations
|
||||
|
||||
|
||||
def _load_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
|
||||
return {
|
||||
"action": "LOAD_SHARD",
|
||||
@@ -1422,6 +1507,11 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
validator_service_token: str | None = None,
|
||||
hive_secret: str | None = None,
|
||||
max_charge_per_request: float | None = None,
|
||||
toploc_calibration: "ToplocCalibrationStore | None" = None,
|
||||
toploc_reference_node_url: str | None = None,
|
||||
toploc_calibration_gate_min_hardware_profiles: int = 1,
|
||||
toploc_backend: Any | None = None,
|
||||
hf_pricing_log: "HfPricingLog | None" = None,
|
||||
) -> None:
|
||||
super().__init__(addr, handler)
|
||||
self.registry = registry
|
||||
@@ -1443,6 +1533,13 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
self.validator_service_token = validator_service_token
|
||||
self.hive_secret = hive_secret
|
||||
self.max_charge_per_request = max_charge_per_request
|
||||
self.toploc_calibration: ToplocCalibrationStore | None = toploc_calibration
|
||||
self.toploc_reference_node_url = (
|
||||
toploc_reference_node_url.rstrip("/") if toploc_reference_node_url else None
|
||||
)
|
||||
self.toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
|
||||
self.toploc_backend = toploc_backend
|
||||
self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
|
||||
|
||||
|
||||
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
@@ -1584,6 +1681,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
if self.path == "/v1/benchmark/hop-penalty":
|
||||
self._handle_benchmark_hop_penalty()
|
||||
return
|
||||
if self.path == "/v1/calibration/toploc/run":
|
||||
self._handle_toploc_calibration_run()
|
||||
return
|
||||
if self.path == "/v1/wallet/register":
|
||||
self._handle_wallet_register()
|
||||
return
|
||||
@@ -1632,6 +1732,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._handle_admin_accounts()
|
||||
elif parsed.path == "/v1/benchmark/results":
|
||||
self._handle_benchmark_results()
|
||||
elif parsed.path == "/v1/calibration/toploc/results":
|
||||
self._handle_toploc_calibration_results()
|
||||
elif parsed.path == "/v1/pricing/hf/history":
|
||||
self._handle_hf_pricing_history(parsed)
|
||||
elif parsed.path == "/v1/registry/wallets":
|
||||
self._handle_registry_wallets()
|
||||
elif parsed.path in ("/dashboard", "/dashboard/"):
|
||||
@@ -3103,6 +3207,206 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
results = []
|
||||
self._send_json(200, {"results": results if isinstance(results, list) else []})
|
||||
|
||||
def _handle_toploc_calibration_run(self):
|
||||
"""Privileged: honest-noise TOPLOC calibration dispatch (issue 21).
|
||||
|
||||
Fans the same fixed prompt through every currently registered node
|
||||
that can solo-serve the full model (one pinned-route hop, mirroring
|
||||
`_handle_benchmark_hop_penalty`'s dispatch pattern), then verifies
|
||||
each node's own on-demand TOPLOC commitment against a teacher-forced
|
||||
replay on the reference node — same audit contract the validator
|
||||
uses (`packages/validator/README.md` "TOPLOC audit contract"). Each
|
||||
node's raw divergence (not just pass/fail) is recorded into the
|
||||
calibration corpus, keyed by wallet + GPU model + dtype, so
|
||||
thresholds can eventually be derived instead of guessed.
|
||||
|
||||
Nodes that only hold a partial shard (need a multi-hop route) are
|
||||
skipped for this pass — solo dispatch isolates one node's hardware
|
||||
noise without a route composition confound — and nodes that don't
|
||||
answer the on-demand commitment fetch (endpoint down, or node-side
|
||||
TOPLOC serving not yet wired) are skipped and reported, not treated
|
||||
as a pass or a fail.
|
||||
"""
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
if not self._require_role("admin", "validator"):
|
||||
return
|
||||
if server.toploc_calibration is None:
|
||||
self._send_json(503, {"error": "toploc calibration store is not enabled on this tracker"})
|
||||
return
|
||||
if not server.toploc_reference_node_url:
|
||||
self._send_json(503, {"error": "toploc_reference_node_url is not configured on this tracker"})
|
||||
return
|
||||
auth = self.headers.get("Authorization")
|
||||
body = self._read_json_body()
|
||||
if body is None:
|
||||
return
|
||||
model = body.get("model", "")
|
||||
if not model:
|
||||
self._send_json(400, {"error": "model is required"})
|
||||
return
|
||||
prompt = body.get("prompt") or "Calibration: say OK."
|
||||
max_new_tokens = int(body.get("max_new_tokens", 32))
|
||||
seed = body.get("seed", 0)
|
||||
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
resolved = _nodes_and_bounds_for_model(server, model)
|
||||
if resolved is None or not resolved[0]:
|
||||
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
|
||||
return
|
||||
all_nodes, rs, re = resolved
|
||||
if server.contracts is not None:
|
||||
all_nodes = [
|
||||
node for node in all_nodes
|
||||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||||
]
|
||||
solo_nodes = [
|
||||
node for node in all_nodes
|
||||
if node.shard_start is not None and node.shard_end is not None
|
||||
and node.shard_start <= rs and node.shard_end >= re
|
||||
]
|
||||
|
||||
self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
node_results: list[dict] = []
|
||||
for node in solo_nodes:
|
||||
request_id = f"cal-{uuid.uuid4().hex}"
|
||||
request_body = json.dumps({
|
||||
"id": request_id,
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_new_tokens,
|
||||
"temperature": 0,
|
||||
"seed": seed,
|
||||
"route": [node.node_id],
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{self_url}/v1/chat/completions",
|
||||
data=request_body,
|
||||
headers={"Content-Type": "application/json", "Authorization": auth},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=300.0) as resp:
|
||||
json.loads(resp.read())
|
||||
except Exception as exc:
|
||||
node_results.append({"node_id": node.node_id, "wallet_address": node.wallet_address, "error": str(exc)})
|
||||
continue
|
||||
|
||||
outcome = self._verify_node_toploc_calibration(
|
||||
server, node, request_id=request_id, model=model, messages=messages,
|
||||
)
|
||||
node_results.append(outcome)
|
||||
|
||||
skipped_partial_shard = [
|
||||
node.node_id for node in all_nodes if node not in solo_nodes
|
||||
]
|
||||
record = {
|
||||
"timestamp": time.time(),
|
||||
"model": model,
|
||||
"prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
|
||||
"nodes": node_results,
|
||||
"skipped_partial_shard_node_ids": skipped_partial_shard,
|
||||
"gate_status": server.toploc_calibration.gate_status(
|
||||
min_hardware_profiles=self._toploc_calibration_gate_min_hardware_profiles(),
|
||||
),
|
||||
}
|
||||
self._send_json(200, record)
|
||||
|
||||
def _toploc_calibration_gate_min_hardware_profiles(self) -> int:
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
return server.toploc_calibration_gate_min_hardware_profiles
|
||||
|
||||
def _verify_node_toploc_calibration(
|
||||
self,
|
||||
server: "_TrackerHTTPServer",
|
||||
node: "_NodeEntry",
|
||||
*,
|
||||
request_id: str,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
) -> dict:
|
||||
"""One node's calibration outcome: fetch its on-demand commitment,
|
||||
teacher-force the claimed tokens on the reference node, verify, and
|
||||
persist the raw divergence into the corpus."""
|
||||
from meshnet_validator.audit import ToplocProofClaim, verify_activation_proofs_detailed
|
||||
|
||||
gpu_model = (node.hardware_profile or {}).get("gpu_name") or (node.hardware_profile or {}).get("device") or "unknown"
|
||||
dtype = node.quantization or "unknown"
|
||||
base_result = {
|
||||
"node_id": node.node_id,
|
||||
"wallet_address": node.wallet_address,
|
||||
"gpu_model": gpu_model,
|
||||
"dtype": dtype,
|
||||
}
|
||||
commitment = _fetch_toploc_commitment(
|
||||
node, session_id=request_id, model=model, messages=messages,
|
||||
)
|
||||
if commitment is None:
|
||||
return {**base_result, "skipped": "no on-demand toploc commitment available"}
|
||||
|
||||
try:
|
||||
claim = ToplocProofClaim.from_mapping(commitment["toploc_proof"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return {**base_result, "skipped": "malformed toploc commitment"}
|
||||
|
||||
reference_activations = _fetch_toploc_reference_activations(
|
||||
server.toploc_reference_node_url,
|
||||
model=model,
|
||||
messages=messages,
|
||||
claimed_token_ids=commitment["claimed_token_ids"],
|
||||
claim=claim,
|
||||
)
|
||||
if reference_activations is None:
|
||||
return {**base_result, "skipped": "reference node teacher-forced replay failed"}
|
||||
|
||||
result = verify_activation_proofs_detailed(reference_activations, claim, backend=server.toploc_backend)
|
||||
if node.wallet_address:
|
||||
server.toploc_calibration.record_run(
|
||||
node_wallet=node.wallet_address,
|
||||
gpu_model=gpu_model,
|
||||
dtype=dtype,
|
||||
model=model,
|
||||
passed=result.passed,
|
||||
exp_intersections=result.exp_intersections,
|
||||
mant_err_mean=result.mant_err_mean,
|
||||
mant_err_median=result.mant_err_median,
|
||||
)
|
||||
return {
|
||||
**base_result,
|
||||
"passed": result.passed,
|
||||
"exp_intersections": result.exp_intersections,
|
||||
"mant_err_mean": result.mant_err_mean,
|
||||
"mant_err_median": result.mant_err_median,
|
||||
"chunk_count": result.chunk_count,
|
||||
}
|
||||
|
||||
def _handle_toploc_calibration_results(self):
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
if not self._require_role("admin", "validator"):
|
||||
return
|
||||
if server.toploc_calibration is None:
|
||||
self._send_json(503, {"error": "toploc calibration store is not enabled on this tracker"})
|
||||
return
|
||||
min_profiles = self._toploc_calibration_gate_min_hardware_profiles()
|
||||
self._send_json(200, {
|
||||
"runs": server.toploc_calibration.runs(),
|
||||
"envelope": server.toploc_calibration.envelope(),
|
||||
"gate_status": server.toploc_calibration.gate_status(min_hardware_profiles=min_profiles),
|
||||
})
|
||||
|
||||
def _handle_hf_pricing_history(self, parsed: urllib.parse.ParseResult):
|
||||
"""Dispute-auditability log for the dynamic HF-benchmarked pricing (issue 23)."""
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
if not self._require_role("admin", "validator"):
|
||||
return
|
||||
if server.hf_pricing_log is None:
|
||||
self._send_json(503, {"error": "hf pricing log is not enabled on this tracker"})
|
||||
return
|
||||
params = urllib.parse.parse_qs(parsed.query)
|
||||
model = params.get("model", [None])[0]
|
||||
self._send_json(200, {"changes": server.hf_pricing_log.history(model)})
|
||||
|
||||
def _handle_assign(self, parsed: urllib.parse.ParseResult):
|
||||
"""Return an optimal shard assignment for a node given its hardware profile.
|
||||
|
||||
@@ -3547,13 +3851,23 @@ class TrackerServer:
|
||||
validator_service_token: str | None = None,
|
||||
hive_secret: str | None = None,
|
||||
max_charge_per_request: float | None = None,
|
||||
toploc_calibration: ToplocCalibrationStore | None = None,
|
||||
toploc_calibration_db: str | None = None,
|
||||
toploc_reference_node_url: str | None = None,
|
||||
toploc_calibration_gate_min_hardware_profiles: int = 1,
|
||||
toploc_backend: Any | None = None,
|
||||
enable_hf_pricing: bool = False,
|
||||
hf_pricing_log: HfPricingLog | None = None,
|
||||
hf_pricing_log_db: str | None = None,
|
||||
hf_pricing_refresh_interval: float = 86400.0,
|
||||
hf_pricing_fetch_html: Any | None = None,
|
||||
) -> None:
|
||||
self._host = host
|
||||
self._requested_port = port
|
||||
self._heartbeat_timeout = heartbeat_timeout
|
||||
self._rebalance_interval = rebalance_interval
|
||||
self._model_presets: dict = (
|
||||
model_presets if model_presets is not None else dict(DEFAULT_MODEL_PRESETS)
|
||||
model_presets if model_presets is not None else _clone_model_presets(DEFAULT_MODEL_PRESETS)
|
||||
)
|
||||
self._contracts = contracts
|
||||
self._minimum_stake = minimum_stake
|
||||
@@ -3619,6 +3933,20 @@ class TrackerServer:
|
||||
if hive_secret is not None
|
||||
else os.environ.get("MESHNET_HIVE_SECRET") or None
|
||||
)
|
||||
if toploc_calibration is None and toploc_calibration_db:
|
||||
toploc_calibration = ToplocCalibrationStore(db_path=toploc_calibration_db)
|
||||
self._toploc_calibration: ToplocCalibrationStore | None = toploc_calibration
|
||||
self._toploc_reference_node_url = toploc_reference_node_url
|
||||
self._toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
|
||||
self._toploc_backend = toploc_backend
|
||||
if hf_pricing_log is None and (enable_hf_pricing or hf_pricing_log_db):
|
||||
hf_pricing_log = HfPricingLog(db_path=hf_pricing_log_db or DEFAULT_HF_PRICING_LOG_DB_PATH)
|
||||
self._hf_pricing_log: HfPricingLog | None = hf_pricing_log
|
||||
self._enable_hf_pricing = enable_hf_pricing
|
||||
self._hf_pricing_refresh_interval = hf_pricing_refresh_interval
|
||||
self._hf_pricing_fetch_html = hf_pricing_fetch_html
|
||||
self._hf_pricing_stop = threading.Event()
|
||||
self._hf_pricing_thread: threading.Thread | None = None
|
||||
self.port: int | None = None
|
||||
|
||||
def start(self) -> int:
|
||||
@@ -3648,6 +3976,11 @@ class TrackerServer:
|
||||
validator_service_token=self._validator_service_token,
|
||||
hive_secret=self._hive_secret,
|
||||
max_charge_per_request=self._max_charge_per_request,
|
||||
toploc_calibration=self._toploc_calibration,
|
||||
toploc_reference_node_url=self._toploc_reference_node_url,
|
||||
toploc_calibration_gate_min_hardware_profiles=self._toploc_calibration_gate_min_hardware_profiles,
|
||||
toploc_backend=self._toploc_backend,
|
||||
hf_pricing_log=self._hf_pricing_log,
|
||||
)
|
||||
self.port = self._server.server_address[1]
|
||||
|
||||
@@ -3680,6 +4013,10 @@ class TrackerServer:
|
||||
self._settlement_stop.clear()
|
||||
self._settlement_thread = threading.Thread(target=self._settlement_loop, daemon=True)
|
||||
self._settlement_thread.start()
|
||||
if self._enable_hf_pricing and self._billing is not None:
|
||||
self._hf_pricing_stop.clear()
|
||||
self._hf_pricing_thread = threading.Thread(target=self._hf_pricing_loop, daemon=True)
|
||||
self._hf_pricing_thread.start()
|
||||
return self.port
|
||||
|
||||
def _settlement_loop(self) -> None:
|
||||
@@ -3789,6 +4126,52 @@ class TrackerServer:
|
||||
flush=True,
|
||||
)
|
||||
|
||||
def _hf_pricing_loop(self) -> None:
|
||||
"""Daily dynamic pricing refresh benchmarked against HF inference rates (issue 23).
|
||||
|
||||
For every preset with a curated, human-verified ``hf_aliases`` list,
|
||||
fetch current HF marketplace pricing and set the client price to 80%
|
||||
of the cheapest matching alias. Presets with no (or an empty)
|
||||
``hf_aliases`` are left entirely alone — they keep the static
|
||||
default price. Any single preset's fetch/parse failure is logged and
|
||||
skipped; it never raises into this loop or the request path.
|
||||
"""
|
||||
billing = self._billing
|
||||
assert billing is not None
|
||||
while not self._hf_pricing_stop.wait(self._hf_pricing_refresh_interval):
|
||||
for name, preset in list(self._model_presets.items()):
|
||||
if not isinstance(preset, dict) or not preset.get("hf_aliases"):
|
||||
continue
|
||||
try:
|
||||
result = refresh_preset_price(
|
||||
model_name=name,
|
||||
preset=preset,
|
||||
current_price=billing.price_for(name),
|
||||
fetch_html=self._hf_pricing_fetch_html,
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"[tracker] hf pricing refresh failed for {name!r}: {exc}", flush=True)
|
||||
continue
|
||||
if result is None:
|
||||
continue
|
||||
billing.set_price(name, result["new_price_per_1k"])
|
||||
preset["hf_last_price_per_1k"] = result["new_price_per_1k"]
|
||||
preset["hf_last_updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
if self._hf_pricing_log is not None:
|
||||
self._hf_pricing_log.record_change(
|
||||
model=name,
|
||||
old_price_per_1k=result["old_price_per_1k"],
|
||||
new_price_per_1k=result["new_price_per_1k"],
|
||||
source_repo_id=result["source_repo_id"],
|
||||
source_provider=result["source_provider"],
|
||||
)
|
||||
print(
|
||||
f"[tracker] hf pricing: {name} {result['old_price_per_1k']:.6f} -> "
|
||||
f"{result['new_price_per_1k']:.6f} USDT/1k tokens "
|
||||
f"(80% of {result['source_repo_id']}::{result['source_provider']})",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
def _raft_apply(self, command: str, payload: dict) -> None:
|
||||
"""Called by RaftNode when a log entry is committed — replicate to local registry."""
|
||||
if command != "register":
|
||||
@@ -3905,6 +4288,7 @@ class TrackerServer:
|
||||
self._stats_stop.set()
|
||||
self._deposit_stop.set()
|
||||
self._settlement_stop.set()
|
||||
self._hf_pricing_stop.set()
|
||||
if self._stats is not None:
|
||||
self._stats.save_to_db()
|
||||
if self._billing is not None:
|
||||
@@ -3928,6 +4312,9 @@ class TrackerServer:
|
||||
if self._settlement_thread is not None:
|
||||
self._settlement_thread.join(timeout=1)
|
||||
self._settlement_thread = None
|
||||
if self._hf_pricing_thread is not None:
|
||||
self._hf_pricing_thread.join(timeout=1)
|
||||
self._hf_pricing_thread = None
|
||||
self._server = None
|
||||
self._thread = None
|
||||
self._rebalance_thread = None
|
||||
|
||||
@@ -51,8 +51,31 @@ skip_prefill = true
|
||||
encoding = "base64"
|
||||
```
|
||||
|
||||
`verify_activation_proofs_detailed()` (`meshnet_validator.audit`) surfaces the
|
||||
raw TOPLOC divergence — `exp_intersections` (worst-case across chunks),
|
||||
`mant_err_mean`, `mant_err_median` — alongside the pass/fail bool. This is
|
||||
what the calibration corpus below is built from; existing callers that only
|
||||
need the bool keep using `verify_activation_proofs()`.
|
||||
|
||||
**Do not enable production audit thresholds before issue 21 closes.**
|
||||
Production audit thresholds remain gated on the honest-noise calibration
|
||||
corpus in issue 21.
|
||||
corpus in issue 21: the tracker's `POST /v1/calibration/toploc/run`
|
||||
(admin/validator-only, mirrors `POST /v1/benchmark/hop-penalty`) dispatches a
|
||||
fixed prompt to every solo-capable registered node, verifies each node's
|
||||
on-demand commitment against a teacher-forced reference replay, and records
|
||||
the divergence into a SQLite corpus (`meshnet_tracker.calibration.
|
||||
ToplocCalibrationStore`) keyed by node wallet + GPU model + dtype.
|
||||
`GET /v1/calibration/toploc/results` reports the corpus plus:
|
||||
|
||||
- `envelope`: p99 honest-noise value per metric with a 20% safety margin —
|
||||
the recommended (not yet wired) tolerance constants.
|
||||
- `gate_status.ready`: whether the corpus covers enough distinct hardware
|
||||
profiles (`--toploc-calibration-gate-min-hardware-profiles`, default 1).
|
||||
**Alpha exception:** with the hired-VPS-only launch fleet, `ready` may
|
||||
legitimately mean "covers every node we currently operate" — this must be
|
||||
revisited (raise the minimum) before a public/volunteer launch broadens
|
||||
the hardware mix, since a new corpus is required whenever the fleet's
|
||||
hardware composition changes.
|
||||
|
||||
Two operational notes:
|
||||
|
||||
|
||||
@@ -8,7 +8,13 @@ import time
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .audit import ToplocAuditConfig, ToplocProofClaim, verify_activation_proofs
|
||||
from .audit import (
|
||||
ToplocAuditConfig,
|
||||
ToplocProofClaim,
|
||||
ToplocVerificationResult,
|
||||
verify_activation_proofs,
|
||||
verify_activation_proofs_detailed,
|
||||
)
|
||||
from .sampling import AdaptiveAuditSampler, AuditRateConfig
|
||||
from .tripwire import detect_output_tripwire
|
||||
|
||||
|
||||
@@ -91,6 +91,28 @@ def build_activation_proofs(
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToplocVerificationResult:
|
||||
"""Verification outcome plus the raw TOPLOC divergence metric.
|
||||
|
||||
The `toploc` library's `verify_proofs_*` returns a bool for simple
|
||||
prover/verifier config mismatches, but for a real activation comparison
|
||||
it returns one `VerificationResult(exp_intersections, mant_err_mean,
|
||||
mant_err_median)` per chunk (README §"What it actually is"). Historically
|
||||
only `bool(result)` was kept, which is always true for a non-empty list
|
||||
of results regardless of how divergent they are (AH-021 gap #1). This
|
||||
dataclass surfaces the raw per-chunk metrics (aggregated: worst-case
|
||||
`exp_intersections`, mean `mant_err_mean`/`mant_err_median`) so a
|
||||
calibration corpus can be built before any threshold is trusted.
|
||||
"""
|
||||
|
||||
passed: bool
|
||||
exp_intersections: float | None = None
|
||||
mant_err_mean: float | None = None
|
||||
mant_err_median: float | None = None
|
||||
chunk_count: int = 0
|
||||
|
||||
|
||||
def verify_activation_proofs(
|
||||
reference_activations: list[Any],
|
||||
claim: ToplocProofClaim,
|
||||
@@ -99,6 +121,25 @@ def verify_activation_proofs(
|
||||
backend: Any | None = None,
|
||||
) -> bool:
|
||||
"""Verify prover TOPLOC proofs against reference teacher-forced activations."""
|
||||
return verify_activation_proofs_detailed(
|
||||
reference_activations, claim, config=config, backend=backend,
|
||||
).passed
|
||||
|
||||
|
||||
def verify_activation_proofs_detailed(
|
||||
reference_activations: list[Any],
|
||||
claim: ToplocProofClaim,
|
||||
*,
|
||||
config: ToplocAuditConfig | None = None,
|
||||
backend: Any | None = None,
|
||||
) -> ToplocVerificationResult:
|
||||
"""Verify prover TOPLOC proofs and surface the raw divergence metric.
|
||||
|
||||
Same pass/fail contract as `verify_activation_proofs` (kept as a thin
|
||||
wrapper for existing call sites); this is the entry point for anything
|
||||
that needs the underlying distance value, e.g. the AH-021 honest-noise
|
||||
calibration corpus.
|
||||
"""
|
||||
cfg = config or ToplocAuditConfig(
|
||||
dtype=claim.dtype,
|
||||
quantization=claim.quantization,
|
||||
@@ -108,23 +149,52 @@ def verify_activation_proofs(
|
||||
encoding=claim.encoding,
|
||||
)
|
||||
if claim.dtype != cfg.dtype or claim.quantization != cfg.quantization:
|
||||
return False
|
||||
return ToplocVerificationResult(passed=False)
|
||||
if claim.decode_batching_size != cfg.decode_batching_size or claim.topk != cfg.topk:
|
||||
return False
|
||||
return ToplocVerificationResult(passed=False)
|
||||
if claim.skip_prefill != cfg.skip_prefill or claim.encoding != cfg.encoding:
|
||||
return False
|
||||
return ToplocVerificationResult(passed=False)
|
||||
|
||||
module = backend or _load_toploc()
|
||||
function_name = f"verify_proofs_{claim.encoding}"
|
||||
verify = getattr(module, function_name)
|
||||
return bool(_call_toploc(
|
||||
raw = _call_toploc(
|
||||
verify,
|
||||
reference_activations,
|
||||
claim.proofs,
|
||||
decode_batching_size=claim.decode_batching_size,
|
||||
topk=claim.topk,
|
||||
skip_prefill=claim.skip_prefill,
|
||||
))
|
||||
)
|
||||
divergence = _extract_divergence(raw)
|
||||
return ToplocVerificationResult(passed=bool(raw), **divergence)
|
||||
|
||||
|
||||
def _extract_divergence(raw: Any) -> dict[str, Any]:
|
||||
"""Aggregate per-chunk TOPLOC `VerificationResult`s, if present.
|
||||
|
||||
`raw` is a plain bool for the simple fake backends used in existing unit
|
||||
tests (no per-chunk metric available). The real `toploc` library returns
|
||||
a list of per-chunk results; `exp_intersections` is aggregated by min
|
||||
(worst honest-noise case across chunks) and the mantissa errors by mean.
|
||||
"""
|
||||
chunks = raw if isinstance(raw, (list, tuple)) else None
|
||||
if not chunks:
|
||||
return {"exp_intersections": None, "mant_err_mean": None, "mant_err_median": None, "chunk_count": 0}
|
||||
exp_vals = [v for v in (_chunk_field(c, "exp_intersections") for c in chunks) if v is not None]
|
||||
mean_vals = [v for v in (_chunk_field(c, "mant_err_mean") for c in chunks) if v is not None]
|
||||
median_vals = [v for v in (_chunk_field(c, "mant_err_median") for c in chunks) if v is not None]
|
||||
return {
|
||||
"exp_intersections": min(exp_vals) if exp_vals else None,
|
||||
"mant_err_mean": (sum(mean_vals) / len(mean_vals)) if mean_vals else None,
|
||||
"mant_err_median": (sum(median_vals) / len(median_vals)) if median_vals else None,
|
||||
"chunk_count": len(chunks),
|
||||
}
|
||||
|
||||
|
||||
def _chunk_field(chunk: Any, name: str) -> float | None:
|
||||
value = chunk.get(name) if isinstance(chunk, dict) else getattr(chunk, name, None)
|
||||
return float(value) if isinstance(value, (int, float)) else None
|
||||
|
||||
|
||||
def _load_toploc() -> Any:
|
||||
@@ -159,6 +229,8 @@ def _proof_encoding(value: object) -> ProofEncoding:
|
||||
__all__ = [
|
||||
"ToplocAuditConfig",
|
||||
"ToplocProofClaim",
|
||||
"ToplocVerificationResult",
|
||||
"build_activation_proofs",
|
||||
"verify_activation_proofs",
|
||||
"verify_activation_proofs_detailed",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user