feat(tracker): add alpha calibration and dynamic pricing

Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021.

Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs.

Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
Dobromir Popov
2026-07-06 09:48:27 +03:00
parent 32514e84c9
commit f841dfaeed
18 changed files with 1996 additions and 25 deletions

150
tests/test_hf_pricing.py Normal file
View File

@@ -0,0 +1,150 @@
"""AH-023: dynamic per-model pricing benchmarked against HF inference rates.
Unit tests for the pure parsing/matching/computation pieces in
`meshnet_tracker.hf_pricing` — no network, no TrackerServer. The HTML
fixture below mirrors the row shape confirmed live against
https://huggingface.co/inference/models on 2026-07-06 (SSR'd table, one
`<a href="/org/repo/?inference_api=true&inference_provider=...">` anchor per
row, followed by `$input` / `$output` price cells).
"""
from __future__ import annotations
from meshnet_tracker.hf_pricing import (
HfPriceQuote,
HfPricingLog,
cheapest_matching_quote,
parse_hf_pricing_table,
refresh_preset_price,
)
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
return f"""
<tr>
<td class="group flex w-full items-center gap-x-2 whitespace-nowrap px-2 py-1">
<span title="{repo}">{repo}</span>
<a href="/{repo}/?inference_api=true&amp;inference_provider={provider}" target="_blank">link</a>
</td>
<td class="border px-2 py-1 text-left">{provider}</td>
<td class="border px-2 py-1 text-left">${input_price}</td>
<td class="border px-2 py-1 text-left">${output_price}</td>
<td class="border px-2 py-1 text-left">128000</td>
</tr>
"""
FIXTURE_HTML = f"""
<table class="w-full table-auto border">
<tr><th>Model</th><th>Provider</th><th>Input $/1M</th><th>Output $/1M</th></tr>
{_row("zai-org/GLM-5.2", "novita", "1.40", "4.40")}
{_row("zai-org/GLM-5.2", "deepinfra", "0.93", "3.00")}
{_row("zai-org/GLM-4.7-Flash", "novita", "0.07", "0.40")}
</table>
"""
def test_parse_hf_pricing_table_extracts_repo_provider_and_prices():
quotes = parse_hf_pricing_table(FIXTURE_HTML)
assert len(quotes) == 3
assert quotes[0] == HfPriceQuote("zai-org/GLM-5.2", "novita", 1.40, 4.40)
assert quotes[1] == HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
def test_blended_price_per_1k_tokens_is_average_of_input_output_over_1000():
quote = HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
assert quote.blended_price_per_1k_tokens() == (0.93 + 3.00) / 2 / 1000
def test_cheapest_matching_quote_picks_lowest_blended_price_among_aliases():
quotes = parse_hf_pricing_table(FIXTURE_HTML)
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2"])
assert cheapest.provider == "deepinfra"
def test_cheapest_matching_quote_honors_repo_provider_scoped_alias():
quotes = parse_hf_pricing_table(FIXTURE_HTML)
# Only the novita deployment was human-verified as comparable for this
# alias — the cheaper deepinfra row for the same repo must not match.
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2::novita"])
assert cheapest.provider == "novita"
def test_cheapest_matching_quote_returns_none_when_no_alias_matches():
quotes = parse_hf_pricing_table(FIXTURE_HTML)
assert cheapest_matching_quote(quotes, ["someone/unrelated-model"]) is None
def test_cheapest_matching_quote_returns_none_for_empty_aliases():
quotes = parse_hf_pricing_table(FIXTURE_HTML)
assert cheapest_matching_quote(quotes, []) is None
def test_refresh_preset_price_end_to_end_with_injected_fetch():
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
result = refresh_preset_price(
model_name="glm-5.2",
preset=preset,
current_price=0.02,
fetch_html=lambda url: FIXTURE_HTML,
)
assert result is not None
assert result["old_price_per_1k"] == 0.02
# 80% of the cheapest matched (deepinfra) blended rate.
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
assert result["new_price_per_1k"] == expected
assert result["source_repo_id"] == "zai-org/GLM-5.2"
assert result["source_provider"] == "deepinfra"
def test_refresh_preset_price_skips_presets_without_hf_aliases():
preset = {"hf_repo": "unsloth/Kimi-K2.7-Code"}
result = refresh_preset_price(
model_name="kimi-k2.7",
preset=preset,
current_price=0.02,
fetch_html=lambda url: FIXTURE_HTML,
)
assert result is None
def test_refresh_preset_price_falls_back_silently_on_fetch_failure():
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
def _boom(url: str) -> str:
raise ConnectionError("network unreachable")
result = refresh_preset_price(
model_name="glm-5.2", preset=preset, current_price=0.02, fetch_html=_boom,
)
assert result is None
def test_refresh_preset_price_falls_back_silently_when_no_match_found():
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["someone/unrelated-model"]}
result = refresh_preset_price(
model_name="glm-5.2",
preset=preset,
current_price=0.02,
fetch_html=lambda url: FIXTURE_HTML,
)
assert result is None
def test_hf_pricing_log_persists_and_is_queryable(tmp_path):
db_path = str(tmp_path / "hf_pricing_log.sqlite")
log = HfPricingLog(db_path=db_path)
log.record_change(
model="glm-5.2",
old_price_per_1k=0.02,
new_price_per_1k=0.00157,
source_repo_id="zai-org/GLM-5.2",
source_provider="deepinfra",
)
assert len(log.history()) == 1
assert log.history("glm-5.2")[0]["new_price_per_1k"] == 0.00157
assert log.history("some-other-model") == []
# Reopening against the same db path recovers the log (billing.py pattern).
reopened = HfPricingLog(db_path=db_path)
assert len(reopened.history()) == 1

View File

@@ -0,0 +1,142 @@
"""AH-023: end-to-end dynamic pricing refresh against a running TrackerServer.
Verifies the full path the issue's acceptance criteria demand: a curated
alias -> injected "live" fetch -> 80%-of-cheapest computed price ->
BillingLedger.set_price() -> preset metadata updated -> change logged and
queryable via GET /v1/pricing/hf/history. Also verifies the required
fallback: a preset with no hf_aliases is left completely alone.
"""
from __future__ import annotations
import json
import time
import urllib.error
import urllib.request
import pytest
from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer
PRICED_MODEL = "zai-org/GLM-5.2"
STATIC_MODEL = "openai-community/gpt2"
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
return f"""
<tr>
<td><span title="{repo}">{repo}</span>
<a href="/{repo}/?inference_api=true&amp;inference_provider={provider}">link</a>
</td>
<td>{provider}</td>
<td>${input_price}</td>
<td>${output_price}</td>
</tr>
"""
FIXTURE_HTML = f"""
<table>
{_row(PRICED_MODEL, "novita", "1.40", "4.40")}
{_row(PRICED_MODEL, "deepinfra", "0.93", "3.00")}
</table>
"""
def _get_json(url: str, headers: dict | None = None) -> dict:
req = urllib.request.Request(url, headers=headers or {})
with urllib.request.urlopen(req) as r:
return json.loads(r.read())
@pytest.fixture
def pricing_tracker(tmp_path):
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
tracker = TrackerServer(
model_presets={
PRICED_MODEL: {
"layers_start": 0,
"layers_end": 11,
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
"hf_aliases": [PRICED_MODEL],
},
STATIC_MODEL: {
"layers_start": 0,
"layers_end": 11,
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
},
},
billing=ledger,
validator_service_token="pricing-token",
enable_hf_pricing=True,
hf_pricing_log_db=str(tmp_path / "hf_pricing_log.sqlite"),
hf_pricing_refresh_interval=0.05,
hf_pricing_fetch_html=lambda url: FIXTURE_HTML,
)
port = tracker.start()
tracker_url = f"http://127.0.0.1:{port}"
yield tracker_url, ledger, tracker
tracker.stop()
def _wait_for_price_change(ledger: BillingLedger, model: str, *, timeout: float = 2.0) -> float:
deadline = time.time() + timeout
while time.time() < deadline:
price = ledger.price_for(model)
if price != 0.02:
return price
time.sleep(0.02)
raise AssertionError(f"price for {model!r} never changed from static default within {timeout}s")
def test_refresh_loop_repriced_model_with_curated_alias(pricing_tracker):
tracker_url, ledger, tracker = pricing_tracker
new_price = _wait_for_price_change(ledger, PRICED_MODEL)
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
assert new_price == expected
preset = tracker._model_presets[PRICED_MODEL]
assert preset["hf_last_price_per_1k"] == expected
assert preset["hf_last_updated"] # ISO timestamp string, non-empty
def test_refresh_loop_leaves_model_without_hf_aliases_on_static_price(pricing_tracker):
tracker_url, ledger, tracker = pricing_tracker
_wait_for_price_change(ledger, PRICED_MODEL) # let the loop run at least once
assert ledger.price_for(STATIC_MODEL) == 0.02
assert "hf_last_price_per_1k" not in tracker._model_presets[STATIC_MODEL]
def test_price_history_requires_auth(pricing_tracker):
tracker_url, ledger, tracker = pricing_tracker
_wait_for_price_change(ledger, PRICED_MODEL)
with pytest.raises(urllib.error.HTTPError) as exc_info:
_get_json(f"{tracker_url}/v1/pricing/hf/history")
assert exc_info.value.code == 401
def test_price_history_reports_old_new_source_and_timestamp(pricing_tracker):
tracker_url, ledger, tracker = pricing_tracker
_wait_for_price_change(ledger, PRICED_MODEL)
result = _get_json(
f"{tracker_url}/v1/pricing/hf/history?model={PRICED_MODEL}",
headers={"Authorization": "Bearer pricing-token"},
)
assert len(result["changes"]) >= 1
change = result["changes"][0]
assert change["old_price_per_1k"] == 0.02
assert change["new_price_per_1k"] == round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
assert change["source_repo_id"] == PRICED_MODEL
assert change["source_provider"] == "deepinfra"
assert change["ts"] > 0
def test_price_history_filters_by_model(pricing_tracker):
tracker_url, ledger, tracker = pricing_tracker
_wait_for_price_change(ledger, PRICED_MODEL)
result = _get_json(
f"{tracker_url}/v1/pricing/hf/history?model={STATIC_MODEL}",
headers={"Authorization": "Bearer pricing-token"},
)
assert result["changes"] == []

View File

@@ -2,10 +2,15 @@
from __future__ import annotations
from collections import namedtuple
from types import SimpleNamespace
from meshnet_validator import ToplocAuditConfig, ValidatorProcess
from meshnet_validator.audit import build_activation_proofs, verify_activation_proofs
from meshnet_validator.audit import (
build_activation_proofs,
verify_activation_proofs,
verify_activation_proofs_detailed,
)
class FakeToploc:
@@ -199,3 +204,70 @@ def test_validator_rejects_swapped_precision_toploc_claim():
assert len(receipts) == 1
assert contracts.registry.slashes[0]["wallet_address"] == "wallet-bad"
assert "TOPLOC activation proof mismatch" in contracts.registry.slashes[0]["reason"]
# AH-021: verify_activation_proofs_detailed surfaces the raw divergence
# metric a calibration corpus needs, instead of only a pass/fail bool.
ChunkResult = namedtuple("ChunkResult", ["exp_intersections", "mant_err_mean", "mant_err_median"])
class FakeToplocWithChunkResults:
"""Mimics the real `toploc` library: verify returns per-chunk results,
not a bool, so `bool(result)` alone (the AH-021 gap #1 bug) is always
true for any non-empty response regardless of divergence."""
def build_proofs_base64(self, activations, *, decode_batching_size, topk, skip_prefill):
return {"activations": activations}
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
return [
ChunkResult(exp_intersections=8, mant_err_mean=0.01, mant_err_median=0.008),
ChunkResult(exp_intersections=6, mant_err_mean=0.03, mant_err_median=0.02),
]
def test_verify_activation_proofs_detailed_aggregates_per_chunk_divergence():
fake_toploc = FakeToplocWithChunkResults()
activations = [[1.0, 2.0], [3.0, 4.0]]
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
result = verify_activation_proofs_detailed(activations, claim, config=config, backend=fake_toploc)
assert result.passed is True # non-empty list is truthy, same as legacy behavior
assert result.chunk_count == 2
assert result.exp_intersections == 6 # worst-case (min) across chunks
assert result.mant_err_mean == 0.02 # mean of per-chunk means
assert result.mant_err_median == 0.014 # mean of per-chunk medians
# verify_activation_proofs still returns just the bool for existing callers.
assert verify_activation_proofs(activations, claim, config=config, backend=fake_toploc) is True
def test_verify_activation_proofs_detailed_no_metric_from_plain_bool_backend():
fake_toploc = FakeToploc()
activations = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
result = verify_activation_proofs_detailed(activations, claim, config=config, backend=fake_toploc)
assert result.passed is True
assert result.chunk_count == 0
assert result.exp_intersections is None
assert result.mant_err_mean is None
assert result.mant_err_median is None
def test_verify_activation_proofs_detailed_rejects_config_mismatch_without_calling_backend():
fake_toploc = FakeToplocWithChunkResults()
activations = [[1.0, 2.0]]
canonical = ToplocAuditConfig(dtype="bfloat16", quantization="bfloat16", topk=2, decode_batching_size=16)
swapped = ToplocAuditConfig(dtype="bfloat16", quantization="int8", topk=2, decode_batching_size=16)
claim = build_activation_proofs(activations, config=swapped, backend=fake_toploc)
result = verify_activation_proofs_detailed(activations, claim, config=canonical, backend=fake_toploc)
assert result.passed is False
assert result.chunk_count == 0

View File

@@ -0,0 +1,80 @@
"""AH-021: honest-noise TOPLOC calibration corpus storage + aggregation."""
from __future__ import annotations
from meshnet_tracker.calibration import ToplocCalibrationStore
def test_record_run_persists_and_reloads_from_sqlite(tmp_path):
db_path = str(tmp_path / "calibration.sqlite")
store = ToplocCalibrationStore(db_path=db_path)
store.record_run(
node_wallet="wallet-a",
gpu_model="RTX 4090",
dtype="bfloat16",
model="Qwen2.5-0.5B-Instruct",
passed=True,
exp_intersections=7.0,
mant_err_mean=0.01,
mant_err_median=0.008,
)
reloaded = ToplocCalibrationStore(db_path=db_path)
assert len(reloaded.runs()) == 1
assert reloaded.runs()[0]["node_wallet"] == "wallet-a"
assert reloaded.distinct_hardware_profiles() == {("RTX 4090", "bfloat16")}
def test_gate_status_requires_minimum_distinct_hardware_profiles():
store = ToplocCalibrationStore()
store.record_run(
node_wallet="wallet-a", gpu_model="RTX 4090", dtype="bfloat16", model="m",
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
)
assert store.gate_status(min_hardware_profiles=2)["ready"] is False
store.record_run(
node_wallet="wallet-b", gpu_model="A100", dtype="bfloat16", model="m",
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
)
status = store.gate_status(min_hardware_profiles=2)
assert status["ready"] is True
assert status["distinct_hardware_profiles"] == 2
def test_gate_status_empty_corpus_is_never_ready():
store = ToplocCalibrationStore()
assert store.gate_status(min_hardware_profiles=0)["ready"] is False
def test_envelope_derives_thresholds_from_worst_case_percentile_with_margin():
store = ToplocCalibrationStore()
# 100 honest runs; exp_intersections mostly 8, worst honest reading 5.
for i in range(100):
exp = 5.0 if i == 0 else 8.0
mant = 0.05 if i == 0 else 0.01
store.record_run(
node_wallet=f"wallet-{i}", gpu_model="RTX 4090", dtype="bfloat16", model="m",
passed=True, exp_intersections=exp, mant_err_mean=mant, mant_err_median=mant,
)
envelope = store.envelope(percentile=0.99, safety_margin=0.2)
assert envelope["sample_count"] == 100
# p1 (tail) of exp_intersections is pulled down by the one worst honest
# reading (5.0 vs the 8.0 bulk); a 20% safety margin shaves it further.
assert envelope["recommended_min_exp_intersections"] < 8.0
# p99 ceiling of mantissa error is likewise pulled up by the one worst
# honest reading (0.05 vs the 0.01 bulk), plus a 20% safety margin.
assert envelope["recommended_max_mant_err_mean"] > 0.01
# In-sample FPR: at most the one outlier row should be flagged by its own
# derived thresholds.
assert 0.0 <= envelope["estimated_false_positive_rate"] <= 0.02
def test_envelope_returns_none_when_no_samples():
store = ToplocCalibrationStore()
envelope = store.envelope()
assert envelope["recommended_min_exp_intersections"] is None
assert envelope["recommended_max_mant_err_mean"] is None
assert envelope["recommended_max_mant_err_median"] is None
assert envelope["estimated_false_positive_rate"] is None

View File

@@ -0,0 +1,341 @@
"""AH-021: tracker-scheduled TOPLOC honest-noise calibration dispatch.
Extends the US-030 fleet-dispatch pattern (`_handle_benchmark_hop_penalty`)
from pinned-route latency benchmarking to a job that hits every solo-capable
registered node with a fixed prompt, verifies each node's own on-demand
TOPLOC commitment against a teacher-forced reference replay, and records the
raw divergence into the calibration corpus.
"""
from __future__ import annotations
import http.server
import json
import socketserver
import threading
import time
import urllib.error
import urllib.request
import pytest
from meshnet_tracker.server import TrackerServer
from meshnet_validator.audit import ToplocAuditConfig, build_activation_proofs
MODEL = "openai-community/gpt2"
CONFIG = ToplocAuditConfig(topk=2, decode_batching_size=16, dtype="bfloat16", quantization="bfloat16")
class FakeToploc:
"""Exact-equality fake backend, matching other TOPLOC test suites."""
def build_proofs_base64(self, activations, *, decode_batching_size, topk, skip_prefill):
return {
"activation_fingerprint": tuple(tuple(row) for row in activations),
"decode_batching_size": decode_batching_size,
"topk": topk,
"skip_prefill": skip_prefill,
}
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
# `proofs` may have round-tripped through JSON (HTTP dispatch), which
# turns the fingerprint's tuples into lists — normalize before compare.
fingerprint = proofs.get("activation_fingerprint") if isinstance(proofs, dict) else None
return (
fingerprint is not None
and tuple(tuple(row) for row in fingerprint) == tuple(tuple(row) for row in activations)
and proofs.get("decode_batching_size") == decode_batching_size
and proofs.get("topk") == topk
and proofs.get("skip_prefill") == skip_prefill
)
BACKEND = FakeToploc()
class FakeCalibrationNode:
"""Stands in for a node: serves both /v1/chat/completions (tracker_mode
style) and its own on-demand TOPLOC commitment endpoint."""
def __init__(self, *, claim_activations, claimed_token_ids, response_text="ok", commitment_available=True):
self.requests: list[dict] = []
claim = build_activation_proofs(claim_activations, config=CONFIG, backend=BACKEND)
outer = self
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
pass
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length) or b"{}")
if self.path == "/v1/chat/completions":
self._send_json(200, {
"id": "chatcmpl-cal",
"object": "chat.completion",
"created": int(time.time()),
"model": body.get("model", MODEL),
"choices": [{
"index": 0,
"message": {"role": "assistant", "content": response_text},
"finish_reason": "stop",
}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
})
return
if self.path == "/v1/audit/toploc/commitment":
outer.requests.append(body)
if not commitment_available:
self.send_response(404)
self.end_headers()
return
self._send_json(200, {
"toploc_proof": claim.as_mapping(),
"claimed_token_ids": claimed_token_ids,
})
return
self.send_response(404)
self.end_headers()
def _send_json(self, status, data):
payload = json.dumps(data).encode()
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
self._server.daemon_threads = True
self._thread: threading.Thread | None = None
def start(self) -> str:
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
self._thread.start()
return f"http://127.0.0.1:{self._server.server_address[1]}"
def stop(self) -> None:
self._server.shutdown()
self._server.server_close()
class FakeReferenceNode:
"""Stands in for the reference node: teacher-forces the claimed tokens
and returns canned reference activations."""
def __init__(self, *, reference_activations):
self.requests: list[dict] = []
outer = self
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
pass
def do_POST(self):
if self.path != "/v1/audit/toploc":
self.send_response(404)
self.end_headers()
return
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length) or b"{}")
outer.requests.append(body)
payload = json.dumps({"activations": reference_activations}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
self._server.daemon_threads = True
self._thread: threading.Thread | None = None
def start(self) -> str:
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
self._thread.start()
return f"http://127.0.0.1:{self._server.server_address[1]}"
def stop(self) -> None:
self._server.shutdown()
self._server.server_close()
def _post_json(url: str, payload: dict, headers: dict | None = None) -> dict:
req = urllib.request.Request(
url,
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json", **(headers or {})},
method="POST",
)
with urllib.request.urlopen(req) as r:
return json.loads(r.read())
def _get_json(url: str, headers: dict | None = None) -> dict:
req = urllib.request.Request(url, headers=headers or {})
with urllib.request.urlopen(req) as r:
return json.loads(r.read())
@pytest.fixture
def calibration_setup(tmp_path):
reference_activations = [[1.0, 2.0], [3.0, 4.0]]
reference = FakeReferenceNode(reference_activations=reference_activations)
reference_url = reference.start()
calibration_db = str(tmp_path / "calibration.sqlite")
tracker = TrackerServer(
model_presets={
MODEL: {"layers_start": 0, "layers_end": 11, "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024}},
},
validator_service_token="cal-token",
toploc_calibration_db=calibration_db,
toploc_reference_node_url=reference_url,
toploc_calibration_gate_min_hardware_profiles=1,
toploc_backend=BACKEND,
)
port = tracker.start()
tracker_url = f"http://127.0.0.1:{port}"
honest_node = FakeCalibrationNode(
claim_activations=reference_activations, # matches reference -> passes
claimed_token_ids=[101, 202],
)
honest_node_url = honest_node.start()
reply = _post_json(f"{tracker_url}/v1/nodes/register", {
"endpoint": honest_node_url,
"shard_start": 0,
"shard_end": 11,
"model": MODEL,
"hardware_profile": {"gpu_name": "RTX 4090"},
"quantization": "bfloat16",
"wallet_address": "wallet-honest",
"score": 1.0,
"tracker_mode": True,
})
honest_node_id = reply["node_id"]
partial_node = FakeCalibrationNode(claim_activations=reference_activations, claimed_token_ids=[101])
partial_node_url = partial_node.start()
reply = _post_json(f"{tracker_url}/v1/nodes/register", {
"endpoint": partial_node_url,
"shard_start": 0,
"shard_end": 5,
"model": MODEL,
"hardware_profile": {"gpu_name": "A100"},
"quantization": "bfloat16",
"wallet_address": "wallet-partial",
"score": 1.0,
"tracker_mode": True,
})
partial_node_id = reply["node_id"]
yield tracker_url, calibration_db, honest_node_id, partial_node_id
honest_node.stop()
partial_node.stop()
reference.stop()
tracker.stop()
def test_calibration_run_requires_auth(calibration_setup):
tracker_url, _, _, _ = calibration_setup
with pytest.raises(urllib.error.HTTPError) as exc_info:
_post_json(f"{tracker_url}/v1/calibration/toploc/run", {"model": MODEL})
assert exc_info.value.code == 401
def test_calibration_run_dispatches_only_solo_capable_nodes(calibration_setup):
tracker_url, _, honest_node_id, partial_node_id = calibration_setup
record = _post_json(
f"{tracker_url}/v1/calibration/toploc/run",
{"model": MODEL, "prompt": "2+2", "max_new_tokens": 4},
headers={"Authorization": "Bearer cal-token"},
)
assert record["skipped_partial_shard_node_ids"] == [partial_node_id]
assert len(record["nodes"]) == 1
result = record["nodes"][0]
assert result["node_id"] == honest_node_id
assert result["wallet_address"] == "wallet-honest"
assert result["gpu_model"] == "RTX 4090"
assert result["dtype"] == "bfloat16"
assert result["passed"] is True
def test_calibration_run_persists_corpus_and_results_endpoint_reports_it(calibration_setup):
tracker_url, calibration_db, _, _ = calibration_setup
_post_json(
f"{tracker_url}/v1/calibration/toploc/run",
{"model": MODEL},
headers={"Authorization": "Bearer cal-token"},
)
results = _get_json(
f"{tracker_url}/v1/calibration/toploc/results",
headers={"Authorization": "Bearer cal-token"},
)
assert len(results["runs"]) == 1
assert results["runs"][0]["node_wallet"] == "wallet-honest"
assert results["gate_status"]["distinct_hardware_profiles"] == 1
assert results["gate_status"]["ready"] is True
assert results["envelope"]["sample_count"] == 1
def test_calibration_run_missing_reference_node_url_is_503(tmp_path):
tracker = TrackerServer(
model_presets={MODEL: {"layers_start": 0, "layers_end": 11}},
validator_service_token="cal-token",
toploc_calibration_db=str(tmp_path / "calibration.sqlite"),
)
port = tracker.start()
try:
with pytest.raises(urllib.error.HTTPError) as exc_info:
_post_json(
f"http://127.0.0.1:{port}/v1/calibration/toploc/run",
{"model": MODEL},
headers={"Authorization": "Bearer cal-token"},
)
assert exc_info.value.code == 503
finally:
tracker.stop()
def test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed(tmp_path):
reference = FakeReferenceNode(reference_activations=[[1.0, 2.0]])
reference_url = reference.start()
tracker = TrackerServer(
model_presets={MODEL: {"layers_start": 0, "layers_end": 11}},
validator_service_token="cal-token",
toploc_calibration_db=str(tmp_path / "calibration.sqlite"),
toploc_reference_node_url=reference_url,
)
port = tracker.start()
tracker_url = f"http://127.0.0.1:{port}"
node = FakeCalibrationNode(
claim_activations=[[1.0, 2.0]], claimed_token_ids=[101], commitment_available=False,
)
node_url = node.start()
_post_json(f"{tracker_url}/v1/nodes/register", {
"endpoint": node_url,
"shard_start": 0,
"shard_end": 11,
"model": MODEL,
"hardware_profile": {},
"wallet_address": "wallet-no-commitment",
"score": 1.0,
"tracker_mode": True,
"node_id": "node-no-commitment",
})
try:
record = _post_json(
f"{tracker_url}/v1/calibration/toploc/run",
{"model": MODEL},
headers={"Authorization": "Bearer cal-token"},
)
assert len(record["nodes"]) == 1
assert "skipped" in record["nodes"][0]
assert record["gate_status"]["sample_count"] == 0
finally:
node.stop()
reference.stop()
tracker.stop()