210 lines
7.5 KiB
Python
210 lines
7.5 KiB
Python
"""AH-023: dynamic per-model pricing benchmarked against HF inference rates.
|
||
|
||
Unit tests for the pure parsing/matching/computation pieces in
|
||
`meshnet_tracker.hf_pricing` — no network, no TrackerServer. The HTML
|
||
fixture below mirrors the row shape confirmed live against
|
||
https://huggingface.co/inference/models on 2026-07-06 (SSR'd table, one
|
||
`<a href="/org/repo/?inference_api=true&inference_provider=...">` anchor per
|
||
row, followed by `$input` / `$output` price cells).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from meshnet_tracker.hf_pricing import (
|
||
HfPriceQuote,
|
||
HfPricingLog,
|
||
cheapest_matching_quote,
|
||
parse_hf_pricing_table,
|
||
refresh_preset_price,
|
||
)
|
||
|
||
|
||
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
|
||
return f"""
|
||
<tr>
|
||
<td class="group flex w-full items-center gap-x-2 whitespace-nowrap px-2 py-1">
|
||
<span title="{repo}">{repo}</span>
|
||
<a href="/{repo}/?inference_api=true&inference_provider={provider}" target="_blank">link</a>
|
||
</td>
|
||
<td class="border px-2 py-1 text-left">{provider}</td>
|
||
<td class="border px-2 py-1 text-left">${input_price}</td>
|
||
<td class="border px-2 py-1 text-left">${output_price}</td>
|
||
<td class="border px-2 py-1 text-left">128000</td>
|
||
</tr>
|
||
"""
|
||
|
||
|
||
FIXTURE_HTML = f"""
|
||
<table class="w-full table-auto border">
|
||
<tr><th>Model</th><th>Provider</th><th>Input $/1M</th><th>Output $/1M</th></tr>
|
||
{_row("zai-org/GLM-5.2", "novita", "1.40", "4.40")}
|
||
{_row("zai-org/GLM-5.2", "deepinfra", "0.93", "3.00")}
|
||
{_row("zai-org/GLM-4.7-Flash", "novita", "0.07", "0.40")}
|
||
</table>
|
||
"""
|
||
|
||
|
||
def test_parse_hf_pricing_table_extracts_repo_provider_and_prices():
|
||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||
assert len(quotes) == 3
|
||
assert quotes[0] == HfPriceQuote("zai-org/GLM-5.2", "novita", 1.40, 4.40)
|
||
assert quotes[1] == HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
|
||
|
||
|
||
def test_blended_price_per_1k_tokens_is_average_of_input_output_over_1000():
|
||
quote = HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
|
||
assert quote.blended_price_per_1k_tokens() == (0.93 + 3.00) / 2 / 1000
|
||
|
||
|
||
def test_cheapest_matching_quote_picks_lowest_blended_price_among_aliases():
|
||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2"])
|
||
assert cheapest.provider == "deepinfra"
|
||
|
||
|
||
def test_cheapest_matching_quote_honors_repo_provider_scoped_alias():
|
||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||
# Only the novita deployment was human-verified as comparable for this
|
||
# alias — the cheaper deepinfra row for the same repo must not match.
|
||
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2::novita"])
|
||
assert cheapest.provider == "novita"
|
||
|
||
|
||
def test_cheapest_matching_quote_returns_none_when_no_alias_matches():
|
||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||
assert cheapest_matching_quote(quotes, ["someone/unrelated-model"]) is None
|
||
|
||
|
||
def test_cheapest_matching_quote_returns_none_for_empty_aliases():
|
||
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
||
assert cheapest_matching_quote(quotes, []) is None
|
||
|
||
|
||
def test_refresh_preset_price_end_to_end_with_injected_fetch():
|
||
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
|
||
result = refresh_preset_price(
|
||
model_name="glm-5.2",
|
||
preset=preset,
|
||
current_price=0.02,
|
||
fetch_html=lambda url: FIXTURE_HTML,
|
||
)
|
||
assert result is not None
|
||
assert result["old_price_per_1k"] == 0.02
|
||
# 80% of the cheapest matched (deepinfra) blended rate.
|
||
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
||
assert result["new_price_per_1k"] == expected
|
||
assert result["source_repo_id"] == "zai-org/GLM-5.2"
|
||
assert result["source_provider"] == "deepinfra"
|
||
|
||
|
||
def test_refresh_preset_price_skips_presets_without_hf_aliases():
|
||
preset = {"hf_repo": "unsloth/Kimi-K2.7-Code"}
|
||
result = refresh_preset_price(
|
||
model_name="kimi-k2.7",
|
||
preset=preset,
|
||
current_price=0.02,
|
||
fetch_html=lambda url: FIXTURE_HTML,
|
||
)
|
||
assert result is None
|
||
|
||
|
||
def test_refresh_preset_price_falls_back_silently_on_fetch_failure():
|
||
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
|
||
|
||
def _boom(url: str) -> str:
|
||
raise ConnectionError("network unreachable")
|
||
|
||
result = refresh_preset_price(
|
||
model_name="glm-5.2", preset=preset, current_price=0.02, fetch_html=_boom,
|
||
)
|
||
assert result is None
|
||
|
||
|
||
def test_refresh_preset_price_falls_back_silently_when_no_match_found():
|
||
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["someone/unrelated-model"]}
|
||
result = refresh_preset_price(
|
||
model_name="glm-5.2",
|
||
preset=preset,
|
||
current_price=0.02,
|
||
fetch_html=lambda url: FIXTURE_HTML,
|
||
)
|
||
assert result is None
|
||
|
||
|
||
def test_hf_pricing_log_persists_and_is_queryable(tmp_path):
|
||
db_path = str(tmp_path / "hf_pricing_log.sqlite")
|
||
log = HfPricingLog(db_path=db_path)
|
||
log.record_change(
|
||
model="glm-5.2",
|
||
old_price_per_1k=0.02,
|
||
new_price_per_1k=0.00157,
|
||
source_repo_id="zai-org/GLM-5.2",
|
||
source_provider="deepinfra",
|
||
)
|
||
assert len(log.history()) == 1
|
||
assert log.history("glm-5.2")[0]["new_price_per_1k"] == 0.00157
|
||
assert log.history("some-other-model") == []
|
||
|
||
# Reopening against the same db path recovers the log (billing.py pattern).
|
||
reopened = HfPricingLog(db_path=db_path)
|
||
assert len(reopened.history()) == 1
|
||
|
||
|
||
def test_preset_price_keys_cover_name_repo_and_aliases():
|
||
from meshnet_tracker.server import _preset_price_keys
|
||
|
||
preset = {
|
||
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||
"aliases": ["qwen3.6-35b-a3b", "Qwen/Qwen3.6-35B-A3B"],
|
||
}
|
||
keys = _preset_price_keys("qwen3.6-35b-a3b", preset)
|
||
assert keys == {
|
||
"qwen3.6-35b-a3b",
|
||
"unsloth/Qwen3.6-35B-A3B",
|
||
"Qwen/Qwen3.6-35B-A3B",
|
||
}
|
||
assert _preset_price_keys("bare", {}) == {"bare"}
|
||
|
||
|
||
def test_qwen_preset_prices_apply_to_all_aliases(tmp_path):
|
||
"""Requests naming the repo id (what nodes register) bill at the preset price."""
|
||
from meshnet_tracker.server import TrackerServer
|
||
|
||
import pytest
|
||
|
||
tracker = TrackerServer(billing_db=str(tmp_path / "billing.sqlite"))
|
||
try:
|
||
billing = tracker._billing
|
||
assert billing is not None
|
||
for key in (
|
||
"qwen3.6-35b-a3b",
|
||
"unsloth/Qwen3.6-35B-A3B",
|
||
"Qwen/Qwen3.6-35B-A3B",
|
||
):
|
||
assert billing.price_for(key) == pytest.approx(0.00044), key
|
||
# Unknown models keep the default rate
|
||
assert billing.price_for("some/other-model") == pytest.approx(0.02)
|
||
finally:
|
||
pass
|
||
|
||
|
||
def test_qwen25_preset_price_is_ten_x_commercial_reference(tmp_path):
|
||
"""Qwen2.5-0.5B bills at 10× ~$0.20/1M reference ($0.002/1k), not the 0.02 default."""
|
||
import pytest
|
||
from meshnet_tracker.server import TrackerServer, _resolve_model_preset, DEFAULT_MODEL_PRESETS
|
||
|
||
name, preset = _resolve_model_preset(DEFAULT_MODEL_PRESETS, "Qwen/Qwen2.5-0.5B-Instruct")
|
||
assert name == "qwen2.5-0.5b-instruct"
|
||
assert preset["price_per_1k_tokens"] == pytest.approx(0.002)
|
||
|
||
tracker = TrackerServer(billing_db=str(tmp_path / "billing.sqlite"))
|
||
billing = tracker._billing
|
||
assert billing is not None
|
||
for key in (
|
||
"qwen2.5-0.5b",
|
||
"Qwen2.5-0.5B-Instruct",
|
||
"Qwen/Qwen2.5-0.5B-Instruct",
|
||
):
|
||
assert billing.price_for(key) == pytest.approx(0.002), key
|
||
assert billing.price_for("unrelated-model") == pytest.approx(0.02)
|