feat(tracker): add alpha calibration and dynamic pricing
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
150
tests/test_hf_pricing.py
Normal file
150
tests/test_hf_pricing.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""AH-023: dynamic per-model pricing benchmarked against HF inference rates.
|
||||
|
||||
Unit tests for the pure parsing/matching/computation pieces in
|
||||
`meshnet_tracker.hf_pricing` — no network, no TrackerServer. The HTML
|
||||
fixture below mirrors the row shape confirmed live against
|
||||
https://huggingface.co/inference/models on 2026-07-06 (SSR'd table, one
|
||||
`<a href="/org/repo/?inference_api=true&inference_provider=...">` anchor per
|
||||
row, followed by `$input` / `$output` price cells).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from meshnet_tracker.hf_pricing import (
|
||||
HfPriceQuote,
|
||||
HfPricingLog,
|
||||
cheapest_matching_quote,
|
||||
parse_hf_pricing_table,
|
||||
refresh_preset_price,
|
||||
)
|
||||
|
||||
|
||||
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
|
||||
return f"""
|
||||
<tr>
|
||||
<td class="group flex w-full items-center gap-x-2 whitespace-nowrap px-2 py-1">
|
||||
<span title="{repo}">{repo}</span>
|
||||
<a href="/{repo}/?inference_api=true&inference_provider={provider}" target="_blank">link</a>
|
||||
</td>
|
||||
<td class="border px-2 py-1 text-left">{provider}</td>
|
||||
<td class="border px-2 py-1 text-left">${input_price}</td>
|
||||
<td class="border px-2 py-1 text-left">${output_price}</td>
|
||||
<td class="border px-2 py-1 text-left">128000</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
|
||||
FIXTURE_HTML = f"""
|
||||
<table class="w-full table-auto border">
|
||||
<tr><th>Model</th><th>Provider</th><th>Input $/1M</th><th>Output $/1M</th></tr>
|
||||
{_row("zai-org/GLM-5.2", "novita", "1.40", "4.40")}
|
||||
{_row("zai-org/GLM-5.2", "deepinfra", "0.93", "3.00")}
|
||||
{_row("zai-org/GLM-4.7-Flash", "novita", "0.07", "0.40")}
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
def test_parse_hf_pricing_table_extracts_repo_provider_and_prices():
|
||||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||||
assert len(quotes) == 3
|
||||
assert quotes[0] == HfPriceQuote("zai-org/GLM-5.2", "novita", 1.40, 4.40)
|
||||
assert quotes[1] == HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
|
||||
|
||||
|
||||
def test_blended_price_per_1k_tokens_is_average_of_input_output_over_1000():
|
||||
quote = HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
|
||||
assert quote.blended_price_per_1k_tokens() == (0.93 + 3.00) / 2 / 1000
|
||||
|
||||
|
||||
def test_cheapest_matching_quote_picks_lowest_blended_price_among_aliases():
|
||||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||||
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2"])
|
||||
assert cheapest.provider == "deepinfra"
|
||||
|
||||
|
||||
def test_cheapest_matching_quote_honors_repo_provider_scoped_alias():
|
||||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||||
# Only the novita deployment was human-verified as comparable for this
|
||||
# alias — the cheaper deepinfra row for the same repo must not match.
|
||||
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2::novita"])
|
||||
assert cheapest.provider == "novita"
|
||||
|
||||
|
||||
def test_cheapest_matching_quote_returns_none_when_no_alias_matches():
|
||||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||||
assert cheapest_matching_quote(quotes, ["someone/unrelated-model"]) is None
|
||||
|
||||
|
||||
def test_cheapest_matching_quote_returns_none_for_empty_aliases():
|
||||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||||
assert cheapest_matching_quote(quotes, []) is None
|
||||
|
||||
|
||||
def test_refresh_preset_price_end_to_end_with_injected_fetch():
|
||||
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
|
||||
result = refresh_preset_price(
|
||||
model_name="glm-5.2",
|
||||
preset=preset,
|
||||
current_price=0.02,
|
||||
fetch_html=lambda url: FIXTURE_HTML,
|
||||
)
|
||||
assert result is not None
|
||||
assert result["old_price_per_1k"] == 0.02
|
||||
# 80% of the cheapest matched (deepinfra) blended rate.
|
||||
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
||||
assert result["new_price_per_1k"] == expected
|
||||
assert result["source_repo_id"] == "zai-org/GLM-5.2"
|
||||
assert result["source_provider"] == "deepinfra"
|
||||
|
||||
|
||||
def test_refresh_preset_price_skips_presets_without_hf_aliases():
|
||||
preset = {"hf_repo": "unsloth/Kimi-K2.7-Code"}
|
||||
result = refresh_preset_price(
|
||||
model_name="kimi-k2.7",
|
||||
preset=preset,
|
||||
current_price=0.02,
|
||||
fetch_html=lambda url: FIXTURE_HTML,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_refresh_preset_price_falls_back_silently_on_fetch_failure():
|
||||
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
|
||||
|
||||
def _boom(url: str) -> str:
|
||||
raise ConnectionError("network unreachable")
|
||||
|
||||
result = refresh_preset_price(
|
||||
model_name="glm-5.2", preset=preset, current_price=0.02, fetch_html=_boom,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_refresh_preset_price_falls_back_silently_when_no_match_found():
|
||||
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["someone/unrelated-model"]}
|
||||
result = refresh_preset_price(
|
||||
model_name="glm-5.2",
|
||||
preset=preset,
|
||||
current_price=0.02,
|
||||
fetch_html=lambda url: FIXTURE_HTML,
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_hf_pricing_log_persists_and_is_queryable(tmp_path):
|
||||
db_path = str(tmp_path / "hf_pricing_log.sqlite")
|
||||
log = HfPricingLog(db_path=db_path)
|
||||
log.record_change(
|
||||
model="glm-5.2",
|
||||
old_price_per_1k=0.02,
|
||||
new_price_per_1k=0.00157,
|
||||
source_repo_id="zai-org/GLM-5.2",
|
||||
source_provider="deepinfra",
|
||||
)
|
||||
assert len(log.history()) == 1
|
||||
assert log.history("glm-5.2")[0]["new_price_per_1k"] == 0.00157
|
||||
assert log.history("some-other-model") == []
|
||||
|
||||
# Reopening against the same db path recovers the log (billing.py pattern).
|
||||
reopened = HfPricingLog(db_path=db_path)
|
||||
assert len(reopened.history()) == 1
|
||||
142
tests/test_hf_pricing_dispatch.py
Normal file
142
tests/test_hf_pricing_dispatch.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""AH-023: end-to-end dynamic pricing refresh against a running TrackerServer.
|
||||
|
||||
Verifies the full path the issue's acceptance criteria demand: a curated
|
||||
alias -> injected "live" fetch -> 80%-of-cheapest computed price ->
|
||||
BillingLedger.set_price() -> preset metadata updated -> change logged and
|
||||
queryable via GET /v1/pricing/hf/history. Also verifies the required
|
||||
fallback: a preset with no hf_aliases is left completely alone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_tracker.billing import BillingLedger
|
||||
from meshnet_tracker.server import TrackerServer
|
||||
|
||||
PRICED_MODEL = "zai-org/GLM-5.2"
|
||||
STATIC_MODEL = "openai-community/gpt2"
|
||||
|
||||
|
||||
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
|
||||
return f"""
|
||||
<tr>
|
||||
<td><span title="{repo}">{repo}</span>
|
||||
<a href="/{repo}/?inference_api=true&inference_provider={provider}">link</a>
|
||||
</td>
|
||||
<td>{provider}</td>
|
||||
<td>${input_price}</td>
|
||||
<td>${output_price}</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
|
||||
FIXTURE_HTML = f"""
|
||||
<table>
|
||||
{_row(PRICED_MODEL, "novita", "1.40", "4.40")}
|
||||
{_row(PRICED_MODEL, "deepinfra", "0.93", "3.00")}
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
def _get_json(url: str, headers: dict | None = None) -> dict:
|
||||
req = urllib.request.Request(url, headers=headers or {})
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pricing_tracker(tmp_path):
|
||||
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
|
||||
tracker = TrackerServer(
|
||||
model_presets={
|
||||
PRICED_MODEL: {
|
||||
"layers_start": 0,
|
||||
"layers_end": 11,
|
||||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
||||
"hf_aliases": [PRICED_MODEL],
|
||||
},
|
||||
STATIC_MODEL: {
|
||||
"layers_start": 0,
|
||||
"layers_end": 11,
|
||||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
||||
},
|
||||
},
|
||||
billing=ledger,
|
||||
validator_service_token="pricing-token",
|
||||
enable_hf_pricing=True,
|
||||
hf_pricing_log_db=str(tmp_path / "hf_pricing_log.sqlite"),
|
||||
hf_pricing_refresh_interval=0.05,
|
||||
hf_pricing_fetch_html=lambda url: FIXTURE_HTML,
|
||||
)
|
||||
port = tracker.start()
|
||||
tracker_url = f"http://127.0.0.1:{port}"
|
||||
yield tracker_url, ledger, tracker
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def _wait_for_price_change(ledger: BillingLedger, model: str, *, timeout: float = 2.0) -> float:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
price = ledger.price_for(model)
|
||||
if price != 0.02:
|
||||
return price
|
||||
time.sleep(0.02)
|
||||
raise AssertionError(f"price for {model!r} never changed from static default within {timeout}s")
|
||||
|
||||
|
||||
def test_refresh_loop_repriced_model_with_curated_alias(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
new_price = _wait_for_price_change(ledger, PRICED_MODEL)
|
||||
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
||||
assert new_price == expected
|
||||
|
||||
preset = tracker._model_presets[PRICED_MODEL]
|
||||
assert preset["hf_last_price_per_1k"] == expected
|
||||
assert preset["hf_last_updated"] # ISO timestamp string, non-empty
|
||||
|
||||
|
||||
def test_refresh_loop_leaves_model_without_hf_aliases_on_static_price(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL) # let the loop run at least once
|
||||
assert ledger.price_for(STATIC_MODEL) == 0.02
|
||||
assert "hf_last_price_per_1k" not in tracker._model_presets[STATIC_MODEL]
|
||||
|
||||
|
||||
def test_price_history_requires_auth(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL)
|
||||
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
||||
_get_json(f"{tracker_url}/v1/pricing/hf/history")
|
||||
assert exc_info.value.code == 401
|
||||
|
||||
|
||||
def test_price_history_reports_old_new_source_and_timestamp(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL)
|
||||
result = _get_json(
|
||||
f"{tracker_url}/v1/pricing/hf/history?model={PRICED_MODEL}",
|
||||
headers={"Authorization": "Bearer pricing-token"},
|
||||
)
|
||||
assert len(result["changes"]) >= 1
|
||||
change = result["changes"][0]
|
||||
assert change["old_price_per_1k"] == 0.02
|
||||
assert change["new_price_per_1k"] == round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
||||
assert change["source_repo_id"] == PRICED_MODEL
|
||||
assert change["source_provider"] == "deepinfra"
|
||||
assert change["ts"] > 0
|
||||
|
||||
|
||||
def test_price_history_filters_by_model(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL)
|
||||
result = _get_json(
|
||||
f"{tracker_url}/v1/pricing/hf/history?model={STATIC_MODEL}",
|
||||
headers={"Authorization": "Bearer pricing-token"},
|
||||
)
|
||||
assert result["changes"] == []
|
||||
@@ -2,10 +2,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import namedtuple
|
||||
from types import SimpleNamespace
|
||||
|
||||
from meshnet_validator import ToplocAuditConfig, ValidatorProcess
|
||||
from meshnet_validator.audit import build_activation_proofs, verify_activation_proofs
|
||||
from meshnet_validator.audit import (
|
||||
build_activation_proofs,
|
||||
verify_activation_proofs,
|
||||
verify_activation_proofs_detailed,
|
||||
)
|
||||
|
||||
|
||||
class FakeToploc:
|
||||
@@ -199,3 +204,70 @@ def test_validator_rejects_swapped_precision_toploc_claim():
|
||||
assert len(receipts) == 1
|
||||
assert contracts.registry.slashes[0]["wallet_address"] == "wallet-bad"
|
||||
assert "TOPLOC activation proof mismatch" in contracts.registry.slashes[0]["reason"]
|
||||
|
||||
|
||||
# AH-021: verify_activation_proofs_detailed surfaces the raw divergence
|
||||
# metric a calibration corpus needs, instead of only a pass/fail bool.
|
||||
|
||||
ChunkResult = namedtuple("ChunkResult", ["exp_intersections", "mant_err_mean", "mant_err_median"])
|
||||
|
||||
|
||||
class FakeToplocWithChunkResults:
|
||||
"""Mimics the real `toploc` library: verify returns per-chunk results,
|
||||
not a bool, so `bool(result)` alone (the AH-021 gap #1 bug) is always
|
||||
true for any non-empty response regardless of divergence."""
|
||||
|
||||
def build_proofs_base64(self, activations, *, decode_batching_size, topk, skip_prefill):
|
||||
return {"activations": activations}
|
||||
|
||||
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
|
||||
return [
|
||||
ChunkResult(exp_intersections=8, mant_err_mean=0.01, mant_err_median=0.008),
|
||||
ChunkResult(exp_intersections=6, mant_err_mean=0.03, mant_err_median=0.02),
|
||||
]
|
||||
|
||||
|
||||
def test_verify_activation_proofs_detailed_aggregates_per_chunk_divergence():
|
||||
fake_toploc = FakeToplocWithChunkResults()
|
||||
activations = [[1.0, 2.0], [3.0, 4.0]]
|
||||
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
||||
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
|
||||
|
||||
result = verify_activation_proofs_detailed(activations, claim, config=config, backend=fake_toploc)
|
||||
|
||||
assert result.passed is True # non-empty list is truthy, same as legacy behavior
|
||||
assert result.chunk_count == 2
|
||||
assert result.exp_intersections == 6 # worst-case (min) across chunks
|
||||
assert result.mant_err_mean == 0.02 # mean of per-chunk means
|
||||
assert result.mant_err_median == 0.014 # mean of per-chunk medians
|
||||
|
||||
# verify_activation_proofs still returns just the bool for existing callers.
|
||||
assert verify_activation_proofs(activations, claim, config=config, backend=fake_toploc) is True
|
||||
|
||||
|
||||
def test_verify_activation_proofs_detailed_no_metric_from_plain_bool_backend():
|
||||
fake_toploc = FakeToploc()
|
||||
activations = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
|
||||
config = ToplocAuditConfig(topk=2, decode_batching_size=16)
|
||||
claim = build_activation_proofs(activations, config=config, backend=fake_toploc)
|
||||
|
||||
result = verify_activation_proofs_detailed(activations, claim, config=config, backend=fake_toploc)
|
||||
|
||||
assert result.passed is True
|
||||
assert result.chunk_count == 0
|
||||
assert result.exp_intersections is None
|
||||
assert result.mant_err_mean is None
|
||||
assert result.mant_err_median is None
|
||||
|
||||
|
||||
def test_verify_activation_proofs_detailed_rejects_config_mismatch_without_calling_backend():
|
||||
fake_toploc = FakeToplocWithChunkResults()
|
||||
activations = [[1.0, 2.0]]
|
||||
canonical = ToplocAuditConfig(dtype="bfloat16", quantization="bfloat16", topk=2, decode_batching_size=16)
|
||||
swapped = ToplocAuditConfig(dtype="bfloat16", quantization="int8", topk=2, decode_batching_size=16)
|
||||
claim = build_activation_proofs(activations, config=swapped, backend=fake_toploc)
|
||||
|
||||
result = verify_activation_proofs_detailed(activations, claim, config=canonical, backend=fake_toploc)
|
||||
|
||||
assert result.passed is False
|
||||
assert result.chunk_count == 0
|
||||
|
||||
80
tests/test_toploc_calibration.py
Normal file
80
tests/test_toploc_calibration.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""AH-021: honest-noise TOPLOC calibration corpus storage + aggregation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from meshnet_tracker.calibration import ToplocCalibrationStore
|
||||
|
||||
|
||||
def test_record_run_persists_and_reloads_from_sqlite(tmp_path):
|
||||
db_path = str(tmp_path / "calibration.sqlite")
|
||||
store = ToplocCalibrationStore(db_path=db_path)
|
||||
store.record_run(
|
||||
node_wallet="wallet-a",
|
||||
gpu_model="RTX 4090",
|
||||
dtype="bfloat16",
|
||||
model="Qwen2.5-0.5B-Instruct",
|
||||
passed=True,
|
||||
exp_intersections=7.0,
|
||||
mant_err_mean=0.01,
|
||||
mant_err_median=0.008,
|
||||
)
|
||||
|
||||
reloaded = ToplocCalibrationStore(db_path=db_path)
|
||||
assert len(reloaded.runs()) == 1
|
||||
assert reloaded.runs()[0]["node_wallet"] == "wallet-a"
|
||||
assert reloaded.distinct_hardware_profiles() == {("RTX 4090", "bfloat16")}
|
||||
|
||||
|
||||
def test_gate_status_requires_minimum_distinct_hardware_profiles():
|
||||
store = ToplocCalibrationStore()
|
||||
store.record_run(
|
||||
node_wallet="wallet-a", gpu_model="RTX 4090", dtype="bfloat16", model="m",
|
||||
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
|
||||
)
|
||||
assert store.gate_status(min_hardware_profiles=2)["ready"] is False
|
||||
|
||||
store.record_run(
|
||||
node_wallet="wallet-b", gpu_model="A100", dtype="bfloat16", model="m",
|
||||
passed=True, exp_intersections=8.0, mant_err_mean=0.01, mant_err_median=0.01,
|
||||
)
|
||||
status = store.gate_status(min_hardware_profiles=2)
|
||||
assert status["ready"] is True
|
||||
assert status["distinct_hardware_profiles"] == 2
|
||||
|
||||
|
||||
def test_gate_status_empty_corpus_is_never_ready():
|
||||
store = ToplocCalibrationStore()
|
||||
assert store.gate_status(min_hardware_profiles=0)["ready"] is False
|
||||
|
||||
|
||||
def test_envelope_derives_thresholds_from_worst_case_percentile_with_margin():
|
||||
store = ToplocCalibrationStore()
|
||||
# 100 honest runs; exp_intersections mostly 8, worst honest reading 5.
|
||||
for i in range(100):
|
||||
exp = 5.0 if i == 0 else 8.0
|
||||
mant = 0.05 if i == 0 else 0.01
|
||||
store.record_run(
|
||||
node_wallet=f"wallet-{i}", gpu_model="RTX 4090", dtype="bfloat16", model="m",
|
||||
passed=True, exp_intersections=exp, mant_err_mean=mant, mant_err_median=mant,
|
||||
)
|
||||
|
||||
envelope = store.envelope(percentile=0.99, safety_margin=0.2)
|
||||
assert envelope["sample_count"] == 100
|
||||
# p1 (tail) of exp_intersections is pulled down by the one worst honest
|
||||
# reading (5.0 vs the 8.0 bulk); a 20% safety margin shaves it further.
|
||||
assert envelope["recommended_min_exp_intersections"] < 8.0
|
||||
# p99 ceiling of mantissa error is likewise pulled up by the one worst
|
||||
# honest reading (0.05 vs the 0.01 bulk), plus a 20% safety margin.
|
||||
assert envelope["recommended_max_mant_err_mean"] > 0.01
|
||||
# In-sample FPR: at most the one outlier row should be flagged by its own
|
||||
# derived thresholds.
|
||||
assert 0.0 <= envelope["estimated_false_positive_rate"] <= 0.02
|
||||
|
||||
|
||||
def test_envelope_returns_none_when_no_samples():
|
||||
store = ToplocCalibrationStore()
|
||||
envelope = store.envelope()
|
||||
assert envelope["recommended_min_exp_intersections"] is None
|
||||
assert envelope["recommended_max_mant_err_mean"] is None
|
||||
assert envelope["recommended_max_mant_err_median"] is None
|
||||
assert envelope["estimated_false_positive_rate"] is None
|
||||
341
tests/test_toploc_calibration_dispatch.py
Normal file
341
tests/test_toploc_calibration_dispatch.py
Normal file
@@ -0,0 +1,341 @@
|
||||
"""AH-021: tracker-scheduled TOPLOC honest-noise calibration dispatch.
|
||||
|
||||
Extends the US-030 fleet-dispatch pattern (`_handle_benchmark_hop_penalty`)
|
||||
from pinned-route latency benchmarking to a job that hits every solo-capable
|
||||
registered node with a fixed prompt, verifies each node's own on-demand
|
||||
TOPLOC commitment against a teacher-forced reference replay, and records the
|
||||
raw divergence into the calibration corpus.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.server
|
||||
import json
|
||||
import socketserver
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_tracker.server import TrackerServer
|
||||
from meshnet_validator.audit import ToplocAuditConfig, build_activation_proofs
|
||||
|
||||
MODEL = "openai-community/gpt2"
|
||||
CONFIG = ToplocAuditConfig(topk=2, decode_batching_size=16, dtype="bfloat16", quantization="bfloat16")
|
||||
|
||||
|
||||
class FakeToploc:
|
||||
"""Exact-equality fake backend, matching other TOPLOC test suites."""
|
||||
|
||||
def build_proofs_base64(self, activations, *, decode_batching_size, topk, skip_prefill):
|
||||
return {
|
||||
"activation_fingerprint": tuple(tuple(row) for row in activations),
|
||||
"decode_batching_size": decode_batching_size,
|
||||
"topk": topk,
|
||||
"skip_prefill": skip_prefill,
|
||||
}
|
||||
|
||||
def verify_proofs_base64(self, activations, proofs, *, decode_batching_size, topk, skip_prefill):
|
||||
# `proofs` may have round-tripped through JSON (HTTP dispatch), which
|
||||
# turns the fingerprint's tuples into lists — normalize before compare.
|
||||
fingerprint = proofs.get("activation_fingerprint") if isinstance(proofs, dict) else None
|
||||
return (
|
||||
fingerprint is not None
|
||||
and tuple(tuple(row) for row in fingerprint) == tuple(tuple(row) for row in activations)
|
||||
and proofs.get("decode_batching_size") == decode_batching_size
|
||||
and proofs.get("topk") == topk
|
||||
and proofs.get("skip_prefill") == skip_prefill
|
||||
)
|
||||
|
||||
|
||||
BACKEND = FakeToploc()
|
||||
|
||||
|
||||
class FakeCalibrationNode:
|
||||
"""Stands in for a node: serves both /v1/chat/completions (tracker_mode
|
||||
style) and its own on-demand TOPLOC commitment endpoint."""
|
||||
|
||||
def __init__(self, *, claim_activations, claimed_token_ids, response_text="ok", commitment_available=True):
|
||||
self.requests: list[dict] = []
|
||||
claim = build_activation_proofs(claim_activations, config=CONFIG, backend=BACKEND)
|
||||
outer = self
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
if self.path == "/v1/chat/completions":
|
||||
self._send_json(200, {
|
||||
"id": "chatcmpl-cal",
|
||||
"object": "chat.completion",
|
||||
"created": int(time.time()),
|
||||
"model": body.get("model", MODEL),
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": response_text},
|
||||
"finish_reason": "stop",
|
||||
}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
})
|
||||
return
|
||||
if self.path == "/v1/audit/toploc/commitment":
|
||||
outer.requests.append(body)
|
||||
if not commitment_available:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
self._send_json(200, {
|
||||
"toploc_proof": claim.as_mapping(),
|
||||
"claimed_token_ids": claimed_token_ids,
|
||||
})
|
||||
return
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def _send_json(self, status, data):
|
||||
payload = json.dumps(data).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
|
||||
self._server.daemon_threads = True
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> str:
|
||||
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
||||
self._thread.start()
|
||||
return f"http://127.0.0.1:{self._server.server_address[1]}"
|
||||
|
||||
def stop(self) -> None:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
|
||||
|
||||
class FakeReferenceNode:
|
||||
"""Stands in for the reference node: teacher-forces the claimed tokens
|
||||
and returns canned reference activations."""
|
||||
|
||||
def __init__(self, *, reference_activations):
|
||||
self.requests: list[dict] = []
|
||||
outer = self
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
if self.path != "/v1/audit/toploc":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(length) or b"{}")
|
||||
outer.requests.append(body)
|
||||
payload = json.dumps({"activations": reference_activations}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
|
||||
self._server.daemon_threads = True
|
||||
self._thread: threading.Thread | None = None
|
||||
|
||||
def start(self) -> str:
|
||||
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
||||
self._thread.start()
|
||||
return f"http://127.0.0.1:{self._server.server_address[1]}"
|
||||
|
||||
def stop(self) -> None:
|
||||
self._server.shutdown()
|
||||
self._server.server_close()
|
||||
|
||||
|
||||
def _post_json(url: str, payload: dict, headers: dict | None = None) -> dict:
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
data=json.dumps(payload).encode(),
|
||||
headers={"Content-Type": "application/json", **(headers or {})},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
def _get_json(url: str, headers: dict | None = None) -> dict:
|
||||
req = urllib.request.Request(url, headers=headers or {})
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def calibration_setup(tmp_path):
|
||||
reference_activations = [[1.0, 2.0], [3.0, 4.0]]
|
||||
reference = FakeReferenceNode(reference_activations=reference_activations)
|
||||
reference_url = reference.start()
|
||||
|
||||
calibration_db = str(tmp_path / "calibration.sqlite")
|
||||
tracker = TrackerServer(
|
||||
model_presets={
|
||||
MODEL: {"layers_start": 0, "layers_end": 11, "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024}},
|
||||
},
|
||||
validator_service_token="cal-token",
|
||||
toploc_calibration_db=calibration_db,
|
||||
toploc_reference_node_url=reference_url,
|
||||
toploc_calibration_gate_min_hardware_profiles=1,
|
||||
toploc_backend=BACKEND,
|
||||
)
|
||||
port = tracker.start()
|
||||
tracker_url = f"http://127.0.0.1:{port}"
|
||||
|
||||
honest_node = FakeCalibrationNode(
|
||||
claim_activations=reference_activations, # matches reference -> passes
|
||||
claimed_token_ids=[101, 202],
|
||||
)
|
||||
honest_node_url = honest_node.start()
|
||||
reply = _post_json(f"{tracker_url}/v1/nodes/register", {
|
||||
"endpoint": honest_node_url,
|
||||
"shard_start": 0,
|
||||
"shard_end": 11,
|
||||
"model": MODEL,
|
||||
"hardware_profile": {"gpu_name": "RTX 4090"},
|
||||
"quantization": "bfloat16",
|
||||
"wallet_address": "wallet-honest",
|
||||
"score": 1.0,
|
||||
"tracker_mode": True,
|
||||
})
|
||||
honest_node_id = reply["node_id"]
|
||||
|
||||
partial_node = FakeCalibrationNode(claim_activations=reference_activations, claimed_token_ids=[101])
|
||||
partial_node_url = partial_node.start()
|
||||
reply = _post_json(f"{tracker_url}/v1/nodes/register", {
|
||||
"endpoint": partial_node_url,
|
||||
"shard_start": 0,
|
||||
"shard_end": 5,
|
||||
"model": MODEL,
|
||||
"hardware_profile": {"gpu_name": "A100"},
|
||||
"quantization": "bfloat16",
|
||||
"wallet_address": "wallet-partial",
|
||||
"score": 1.0,
|
||||
"tracker_mode": True,
|
||||
})
|
||||
partial_node_id = reply["node_id"]
|
||||
|
||||
yield tracker_url, calibration_db, honest_node_id, partial_node_id
|
||||
|
||||
honest_node.stop()
|
||||
partial_node.stop()
|
||||
reference.stop()
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_calibration_run_requires_auth(calibration_setup):
|
||||
tracker_url, _, _, _ = calibration_setup
|
||||
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
||||
_post_json(f"{tracker_url}/v1/calibration/toploc/run", {"model": MODEL})
|
||||
assert exc_info.value.code == 401
|
||||
|
||||
|
||||
def test_calibration_run_dispatches_only_solo_capable_nodes(calibration_setup):
|
||||
tracker_url, _, honest_node_id, partial_node_id = calibration_setup
|
||||
record = _post_json(
|
||||
f"{tracker_url}/v1/calibration/toploc/run",
|
||||
{"model": MODEL, "prompt": "2+2", "max_new_tokens": 4},
|
||||
headers={"Authorization": "Bearer cal-token"},
|
||||
)
|
||||
assert record["skipped_partial_shard_node_ids"] == [partial_node_id]
|
||||
assert len(record["nodes"]) == 1
|
||||
result = record["nodes"][0]
|
||||
assert result["node_id"] == honest_node_id
|
||||
assert result["wallet_address"] == "wallet-honest"
|
||||
assert result["gpu_model"] == "RTX 4090"
|
||||
assert result["dtype"] == "bfloat16"
|
||||
assert result["passed"] is True
|
||||
|
||||
|
||||
def test_calibration_run_persists_corpus_and_results_endpoint_reports_it(calibration_setup):
|
||||
tracker_url, calibration_db, _, _ = calibration_setup
|
||||
_post_json(
|
||||
f"{tracker_url}/v1/calibration/toploc/run",
|
||||
{"model": MODEL},
|
||||
headers={"Authorization": "Bearer cal-token"},
|
||||
)
|
||||
results = _get_json(
|
||||
f"{tracker_url}/v1/calibration/toploc/results",
|
||||
headers={"Authorization": "Bearer cal-token"},
|
||||
)
|
||||
assert len(results["runs"]) == 1
|
||||
assert results["runs"][0]["node_wallet"] == "wallet-honest"
|
||||
assert results["gate_status"]["distinct_hardware_profiles"] == 1
|
||||
assert results["gate_status"]["ready"] is True
|
||||
assert results["envelope"]["sample_count"] == 1
|
||||
|
||||
|
||||
def test_calibration_run_missing_reference_node_url_is_503(tmp_path):
|
||||
tracker = TrackerServer(
|
||||
model_presets={MODEL: {"layers_start": 0, "layers_end": 11}},
|
||||
validator_service_token="cal-token",
|
||||
toploc_calibration_db=str(tmp_path / "calibration.sqlite"),
|
||||
)
|
||||
port = tracker.start()
|
||||
try:
|
||||
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
||||
_post_json(
|
||||
f"http://127.0.0.1:{port}/v1/calibration/toploc/run",
|
||||
{"model": MODEL},
|
||||
headers={"Authorization": "Bearer cal-token"},
|
||||
)
|
||||
assert exc_info.value.code == 503
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_calibration_run_node_without_commitment_endpoint_is_skipped_not_failed(tmp_path):
|
||||
reference = FakeReferenceNode(reference_activations=[[1.0, 2.0]])
|
||||
reference_url = reference.start()
|
||||
tracker = TrackerServer(
|
||||
model_presets={MODEL: {"layers_start": 0, "layers_end": 11}},
|
||||
validator_service_token="cal-token",
|
||||
toploc_calibration_db=str(tmp_path / "calibration.sqlite"),
|
||||
toploc_reference_node_url=reference_url,
|
||||
)
|
||||
port = tracker.start()
|
||||
tracker_url = f"http://127.0.0.1:{port}"
|
||||
node = FakeCalibrationNode(
|
||||
claim_activations=[[1.0, 2.0]], claimed_token_ids=[101], commitment_available=False,
|
||||
)
|
||||
node_url = node.start()
|
||||
_post_json(f"{tracker_url}/v1/nodes/register", {
|
||||
"endpoint": node_url,
|
||||
"shard_start": 0,
|
||||
"shard_end": 11,
|
||||
"model": MODEL,
|
||||
"hardware_profile": {},
|
||||
"wallet_address": "wallet-no-commitment",
|
||||
"score": 1.0,
|
||||
"tracker_mode": True,
|
||||
"node_id": "node-no-commitment",
|
||||
})
|
||||
|
||||
try:
|
||||
record = _post_json(
|
||||
f"{tracker_url}/v1/calibration/toploc/run",
|
||||
{"model": MODEL},
|
||||
headers={"Authorization": "Bearer cal-token"},
|
||||
)
|
||||
assert len(record["nodes"]) == 1
|
||||
assert "skipped" in record["nodes"][0]
|
||||
assert record["gate_status"]["sample_count"] == 0
|
||||
finally:
|
||||
node.stop()
|
||||
reference.stop()
|
||||
tracker.stop()
|
||||
Reference in New Issue
Block a user