feat(tracker): add alpha calibration and dynamic pricing
Add TOPLOC honest-noise calibration storage/dispatch and validator divergence reporting for AH-021. Add opt-in HuggingFace marketplace pricing refresh, price-change history, CLI flags, and AH-023 tracking docs. Verification: .venv/bin/python -m pytest tests/ -q -k 'not integration' => 346 passed, 2 skipped, 1 deselected; compileall packages tests passed; focused AH-021/AH-023 tests 32 passed.
This commit is contained in:
142
tests/test_hf_pricing_dispatch.py
Normal file
142
tests/test_hf_pricing_dispatch.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""AH-023: end-to-end dynamic pricing refresh against a running TrackerServer.
|
||||
|
||||
Verifies the full path the issue's acceptance criteria demand: a curated
|
||||
alias -> injected "live" fetch -> 80%-of-cheapest computed price ->
|
||||
BillingLedger.set_price() -> preset metadata updated -> change logged and
|
||||
queryable via GET /v1/pricing/hf/history. Also verifies the required
|
||||
fallback: a preset with no hf_aliases is left completely alone.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
from meshnet_tracker.billing import BillingLedger
|
||||
from meshnet_tracker.server import TrackerServer
|
||||
|
||||
PRICED_MODEL = "zai-org/GLM-5.2"
|
||||
STATIC_MODEL = "openai-community/gpt2"
|
||||
|
||||
|
||||
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
|
||||
return f"""
|
||||
<tr>
|
||||
<td><span title="{repo}">{repo}</span>
|
||||
<a href="/{repo}/?inference_api=true&inference_provider={provider}">link</a>
|
||||
</td>
|
||||
<td>{provider}</td>
|
||||
<td>${input_price}</td>
|
||||
<td>${output_price}</td>
|
||||
</tr>
|
||||
"""
|
||||
|
||||
|
||||
FIXTURE_HTML = f"""
|
||||
<table>
|
||||
{_row(PRICED_MODEL, "novita", "1.40", "4.40")}
|
||||
{_row(PRICED_MODEL, "deepinfra", "0.93", "3.00")}
|
||||
</table>
|
||||
"""
|
||||
|
||||
|
||||
def _get_json(url: str, headers: dict | None = None) -> dict:
|
||||
req = urllib.request.Request(url, headers=headers or {})
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pricing_tracker(tmp_path):
|
||||
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
|
||||
tracker = TrackerServer(
|
||||
model_presets={
|
||||
PRICED_MODEL: {
|
||||
"layers_start": 0,
|
||||
"layers_end": 11,
|
||||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
||||
"hf_aliases": [PRICED_MODEL],
|
||||
},
|
||||
STATIC_MODEL: {
|
||||
"layers_start": 0,
|
||||
"layers_end": 11,
|
||||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
||||
},
|
||||
},
|
||||
billing=ledger,
|
||||
validator_service_token="pricing-token",
|
||||
enable_hf_pricing=True,
|
||||
hf_pricing_log_db=str(tmp_path / "hf_pricing_log.sqlite"),
|
||||
hf_pricing_refresh_interval=0.05,
|
||||
hf_pricing_fetch_html=lambda url: FIXTURE_HTML,
|
||||
)
|
||||
port = tracker.start()
|
||||
tracker_url = f"http://127.0.0.1:{port}"
|
||||
yield tracker_url, ledger, tracker
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def _wait_for_price_change(ledger: BillingLedger, model: str, *, timeout: float = 2.0) -> float:
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
price = ledger.price_for(model)
|
||||
if price != 0.02:
|
||||
return price
|
||||
time.sleep(0.02)
|
||||
raise AssertionError(f"price for {model!r} never changed from static default within {timeout}s")
|
||||
|
||||
|
||||
def test_refresh_loop_repriced_model_with_curated_alias(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
new_price = _wait_for_price_change(ledger, PRICED_MODEL)
|
||||
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
||||
assert new_price == expected
|
||||
|
||||
preset = tracker._model_presets[PRICED_MODEL]
|
||||
assert preset["hf_last_price_per_1k"] == expected
|
||||
assert preset["hf_last_updated"] # ISO timestamp string, non-empty
|
||||
|
||||
|
||||
def test_refresh_loop_leaves_model_without_hf_aliases_on_static_price(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL) # let the loop run at least once
|
||||
assert ledger.price_for(STATIC_MODEL) == 0.02
|
||||
assert "hf_last_price_per_1k" not in tracker._model_presets[STATIC_MODEL]
|
||||
|
||||
|
||||
def test_price_history_requires_auth(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL)
|
||||
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
||||
_get_json(f"{tracker_url}/v1/pricing/hf/history")
|
||||
assert exc_info.value.code == 401
|
||||
|
||||
|
||||
def test_price_history_reports_old_new_source_and_timestamp(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL)
|
||||
result = _get_json(
|
||||
f"{tracker_url}/v1/pricing/hf/history?model={PRICED_MODEL}",
|
||||
headers={"Authorization": "Bearer pricing-token"},
|
||||
)
|
||||
assert len(result["changes"]) >= 1
|
||||
change = result["changes"][0]
|
||||
assert change["old_price_per_1k"] == 0.02
|
||||
assert change["new_price_per_1k"] == round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
||||
assert change["source_repo_id"] == PRICED_MODEL
|
||||
assert change["source_provider"] == "deepinfra"
|
||||
assert change["ts"] > 0
|
||||
|
||||
|
||||
def test_price_history_filters_by_model(pricing_tracker):
|
||||
tracker_url, ledger, tracker = pricing_tracker
|
||||
_wait_for_price_change(ledger, PRICED_MODEL)
|
||||
result = _get_json(
|
||||
f"{tracker_url}/v1/pricing/hf/history?model={STATIC_MODEL}",
|
||||
headers={"Authorization": "Bearer pricing-token"},
|
||||
)
|
||||
assert result["changes"] == []
|
||||
Reference in New Issue
Block a user