148 lines
5.2 KiB
Python
148 lines
5.2 KiB
Python
"""AH-023: end-to-end dynamic pricing refresh against a running TrackerServer.
|
|
|
|
Verifies the full path the issue's acceptance criteria demand: a curated
|
|
alias -> injected "live" fetch -> 80%-of-cheapest computed price ->
|
|
BillingLedger.set_price() -> preset metadata updated -> change logged and
|
|
queryable via GET /v1/pricing/hf/history. Also verifies the required
|
|
fallback: a preset with no hf_aliases is left completely alone.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
import pytest
|
|
|
|
from meshnet_tracker.billing import BillingLedger
|
|
from meshnet_tracker.server import TrackerServer
|
|
|
|
PRICED_MODEL = "zai-org/GLM-5.2"
|
|
STATIC_MODEL = "openai-community/gpt2"
|
|
|
|
|
|
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
|
|
return f"""
|
|
<tr>
|
|
<td><span title="{repo}">{repo}</span>
|
|
<a href="/{repo}/?inference_api=true&inference_provider={provider}">link</a>
|
|
</td>
|
|
<td>{provider}</td>
|
|
<td>${input_price}</td>
|
|
<td>${output_price}</td>
|
|
</tr>
|
|
"""
|
|
|
|
|
|
FIXTURE_HTML = f"""
|
|
<table>
|
|
{_row(PRICED_MODEL, "novita", "1.40", "4.40")}
|
|
{_row(PRICED_MODEL, "deepinfra", "0.93", "3.00")}
|
|
</table>
|
|
"""
|
|
|
|
|
|
def _get_json(url: str, headers: dict | None = None) -> dict:
|
|
req = urllib.request.Request(url, headers=headers or {})
|
|
with urllib.request.urlopen(req) as r:
|
|
return json.loads(r.read())
|
|
|
|
|
|
@pytest.fixture
|
|
def pricing_tracker(tmp_path):
|
|
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
|
|
tracker = TrackerServer(
|
|
model_presets={
|
|
PRICED_MODEL: {
|
|
"layers_start": 0,
|
|
"layers_end": 11,
|
|
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
|
"hf_aliases": [PRICED_MODEL],
|
|
},
|
|
STATIC_MODEL: {
|
|
"layers_start": 0,
|
|
"layers_end": 11,
|
|
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
|
},
|
|
},
|
|
billing=ledger,
|
|
validator_service_token="pricing-token",
|
|
enable_hf_pricing=True,
|
|
hf_pricing_log_db=str(tmp_path / "hf_pricing_log.sqlite"),
|
|
hf_pricing_refresh_interval=0.05,
|
|
hf_pricing_fetch_html=lambda url: FIXTURE_HTML,
|
|
)
|
|
port = tracker.start()
|
|
tracker_url = f"http://127.0.0.1:{port}"
|
|
yield tracker_url, ledger, tracker
|
|
tracker.stop()
|
|
|
|
|
|
def _wait_for_price_change(ledger: BillingLedger, model: str, *, timeout: float = 2.0) -> float:
|
|
deadline = time.time() + timeout
|
|
while time.time() < deadline:
|
|
price = ledger.price_for(model)
|
|
if price != 0.02:
|
|
return price
|
|
time.sleep(0.02)
|
|
raise AssertionError(f"price for {model!r} never changed from static default within {timeout}s")
|
|
|
|
|
|
def test_refresh_loop_repriced_model_with_curated_alias(pricing_tracker):
|
|
"Refresh loop repriced model with curated alias\n\nTags: general"
|
|
tracker_url, ledger, tracker = pricing_tracker
|
|
new_price = _wait_for_price_change(ledger, PRICED_MODEL)
|
|
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
|
assert new_price == pytest.approx(expected)
|
|
|
|
preset = tracker._model_presets[PRICED_MODEL]
|
|
assert preset["hf_last_price_per_1k"] == expected
|
|
assert preset["hf_last_updated"] # ISO timestamp string, non-empty
|
|
|
|
|
|
def test_refresh_loop_leaves_model_without_hf_aliases_on_static_price(pricing_tracker):
|
|
"Refresh loop leaves model without hf aliases on static price\n\nTags: general"
|
|
tracker_url, ledger, tracker = pricing_tracker
|
|
_wait_for_price_change(ledger, PRICED_MODEL) # let the loop run at least once
|
|
assert ledger.price_for(STATIC_MODEL) == 0.02
|
|
assert "hf_last_price_per_1k" not in tracker._model_presets[STATIC_MODEL]
|
|
|
|
|
|
def test_price_history_requires_auth(pricing_tracker):
|
|
"Price history requires auth\n\nTags: auth, security"
|
|
tracker_url, ledger, tracker = pricing_tracker
|
|
_wait_for_price_change(ledger, PRICED_MODEL)
|
|
with pytest.raises(urllib.error.HTTPError) as exc_info:
|
|
_get_json(f"{tracker_url}/v1/pricing/hf/history")
|
|
assert exc_info.value.code == 401
|
|
|
|
|
|
def test_price_history_reports_old_new_source_and_timestamp(pricing_tracker):
|
|
"Price history reports old new source and timestamp\n\nTags: general"
|
|
tracker_url, ledger, tracker = pricing_tracker
|
|
_wait_for_price_change(ledger, PRICED_MODEL)
|
|
result = _get_json(
|
|
f"{tracker_url}/v1/pricing/hf/history?model={PRICED_MODEL}",
|
|
headers={"Authorization": "Bearer pricing-token"},
|
|
)
|
|
assert len(result["changes"]) >= 1
|
|
change = result["changes"][0]
|
|
assert change["old_price_per_1k"] == 0.02
|
|
assert change["new_price_per_1k"] == round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
|
assert change["source_repo_id"] == PRICED_MODEL
|
|
assert change["source_provider"] == "deepinfra"
|
|
assert change["ts"] > 0
|
|
|
|
|
|
def test_price_history_filters_by_model(pricing_tracker):
|
|
"Price history filters by model\n\nTags: general"
|
|
tracker_url, ledger, tracker = pricing_tracker
|
|
_wait_for_price_change(ledger, PRICED_MODEL)
|
|
result = _get_json(
|
|
f"{tracker_url}/v1/pricing/hf/history?model={STATIC_MODEL}",
|
|
headers={"Authorization": "Bearer pricing-token"},
|
|
)
|
|
assert result["changes"] == []
|