223 lines
8.4 KiB
Python
223 lines
8.4 KiB
Python
"""AH-023: dynamic per-model pricing benchmarked against HF inference rates.
|
|
|
|
Unit tests for the pure parsing/matching/computation pieces in
|
|
`meshnet_tracker.hf_pricing` — no network, no TrackerServer. The HTML
|
|
fixture below mirrors the row shape confirmed live against
|
|
https://huggingface.co/inference/models on 2026-07-06 (SSR'd table, one
|
|
`<a href="/org/repo/?inference_api=true&inference_provider=...">` anchor per
|
|
row, followed by `$input` / `$output` price cells).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from meshnet_tracker.hf_pricing import (
|
|
HfPriceQuote,
|
|
HfPricingLog,
|
|
cheapest_matching_quote,
|
|
parse_hf_pricing_table,
|
|
refresh_preset_price,
|
|
)
|
|
|
|
|
|
def _row(repo: str, provider: str, input_price: str, output_price: str) -> str:
|
|
return f"""
|
|
<tr>
|
|
<td class="group flex w-full items-center gap-x-2 whitespace-nowrap px-2 py-1">
|
|
<span title="{repo}">{repo}</span>
|
|
<a href="/{repo}/?inference_api=true&inference_provider={provider}" target="_blank">link</a>
|
|
</td>
|
|
<td class="border px-2 py-1 text-left">{provider}</td>
|
|
<td class="border px-2 py-1 text-left">${input_price}</td>
|
|
<td class="border px-2 py-1 text-left">${output_price}</td>
|
|
<td class="border px-2 py-1 text-left">128000</td>
|
|
</tr>
|
|
"""
|
|
|
|
|
|
FIXTURE_HTML = f"""
|
|
<table class="w-full table-auto border">
|
|
<tr><th>Model</th><th>Provider</th><th>Input $/1M</th><th>Output $/1M</th></tr>
|
|
{_row("zai-org/GLM-5.2", "novita", "1.40", "4.40")}
|
|
{_row("zai-org/GLM-5.2", "deepinfra", "0.93", "3.00")}
|
|
{_row("zai-org/GLM-4.7-Flash", "novita", "0.07", "0.40")}
|
|
</table>
|
|
"""
|
|
|
|
|
|
def test_parse_hf_pricing_table_extracts_repo_provider_and_prices():
|
|
"Parse hf pricing table extracts repo provider and prices\n\nTags: general"
|
|
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
|
assert len(quotes) == 3
|
|
assert quotes[0] == HfPriceQuote("zai-org/GLM-5.2", "novita", 1.40, 4.40)
|
|
assert quotes[1] == HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
|
|
|
|
|
|
def test_blended_price_per_1k_tokens_is_average_of_input_output_over_1000():
|
|
"Blended price per 1k tokens is average of input output over 1000\n\nTags: general"
|
|
quote = HfPriceQuote("zai-org/GLM-5.2", "deepinfra", 0.93, 3.00)
|
|
assert quote.blended_price_per_1k_tokens() == (0.93 + 3.00) / 2 / 1000
|
|
|
|
|
|
def test_cheapest_matching_quote_picks_lowest_blended_price_among_aliases():
|
|
"Cheapest matching quote picks lowest blended price among aliases\n\nTags: general"
|
|
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
|
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2"])
|
|
assert cheapest.provider == "deepinfra"
|
|
|
|
|
|
def test_cheapest_matching_quote_honors_repo_provider_scoped_alias():
|
|
"Cheapest matching quote honors repo provider scoped alias\n\nTags: general"
|
|
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
|
# Only the novita deployment was human-verified as comparable for this
|
|
# alias — the cheaper deepinfra row for the same repo must not match.
|
|
cheapest = cheapest_matching_quote(quotes, ["zai-org/GLM-5.2::novita"])
|
|
assert cheapest.provider == "novita"
|
|
|
|
|
|
def test_cheapest_matching_quote_returns_none_when_no_alias_matches():
|
|
"Cheapest matching quote returns none when no alias matches\n\nTags: general"
|
|
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
|
assert cheapest_matching_quote(quotes, ["someone/unrelated-model"]) is None
|
|
|
|
|
|
def test_cheapest_matching_quote_returns_none_for_empty_aliases():
|
|
"Cheapest matching quote returns none for empty aliases\n\nTags: general"
|
|
quotes = parse_hf_pricing_table(FIXTURE_HTML)
|
|
assert cheapest_matching_quote(quotes, []) is None
|
|
|
|
|
|
def test_refresh_preset_price_end_to_end_with_injected_fetch():
|
|
"Refresh preset price end to end with injected fetch\n\nTags: general"
|
|
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
|
|
result = refresh_preset_price(
|
|
model_name="glm-5.2",
|
|
preset=preset,
|
|
current_price=0.02,
|
|
fetch_html=lambda url: FIXTURE_HTML,
|
|
)
|
|
assert result is not None
|
|
assert result["old_price_per_1k"] == 0.02
|
|
# 80% of the cheapest matched (deepinfra) blended rate.
|
|
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
|
|
assert result["new_price_per_1k"] == expected
|
|
assert result["source_repo_id"] == "zai-org/GLM-5.2"
|
|
assert result["source_provider"] == "deepinfra"
|
|
|
|
|
|
def test_refresh_preset_price_skips_presets_without_hf_aliases():
|
|
"Refresh preset price skips presets without hf aliases\n\nTags: general"
|
|
preset = {"hf_repo": "unsloth/Kimi-K2.7-Code"}
|
|
result = refresh_preset_price(
|
|
model_name="kimi-k2.7",
|
|
preset=preset,
|
|
current_price=0.02,
|
|
fetch_html=lambda url: FIXTURE_HTML,
|
|
)
|
|
assert result is None
|
|
|
|
|
|
def test_refresh_preset_price_falls_back_silently_on_fetch_failure():
|
|
"Refresh preset price falls back silently on fetch failure\n\nTags: general"
|
|
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["zai-org/GLM-5.2"]}
|
|
|
|
def _boom(url: str) -> str:
|
|
raise ConnectionError("network unreachable")
|
|
|
|
result = refresh_preset_price(
|
|
model_name="glm-5.2", preset=preset, current_price=0.02, fetch_html=_boom,
|
|
)
|
|
assert result is None
|
|
|
|
|
|
def test_refresh_preset_price_falls_back_silently_when_no_match_found():
|
|
"Refresh preset price falls back silently when no match found\n\nTags: general"
|
|
preset = {"hf_repo": "zai-org/GLM-5.2", "hf_aliases": ["someone/unrelated-model"]}
|
|
result = refresh_preset_price(
|
|
model_name="glm-5.2",
|
|
preset=preset,
|
|
current_price=0.02,
|
|
fetch_html=lambda url: FIXTURE_HTML,
|
|
)
|
|
assert result is None
|
|
|
|
|
|
def test_hf_pricing_log_persists_and_is_queryable(tmp_path):
|
|
"Hf pricing log persists and is queryable\n\nTags: persistence"
|
|
db_path = str(tmp_path / "hf_pricing_log.sqlite")
|
|
log = HfPricingLog(db_path=db_path)
|
|
log.record_change(
|
|
model="glm-5.2",
|
|
old_price_per_1k=0.02,
|
|
new_price_per_1k=0.00157,
|
|
source_repo_id="zai-org/GLM-5.2",
|
|
source_provider="deepinfra",
|
|
)
|
|
assert len(log.history()) == 1
|
|
assert log.history("glm-5.2")[0]["new_price_per_1k"] == 0.00157
|
|
assert log.history("some-other-model") == []
|
|
|
|
# Reopening against the same db path recovers the log (billing.py pattern).
|
|
reopened = HfPricingLog(db_path=db_path)
|
|
assert len(reopened.history()) == 1
|
|
|
|
|
|
def test_preset_price_keys_cover_name_repo_and_aliases():
|
|
"Preset price keys cover name repo and aliases\n\nTags: general"
|
|
from meshnet_tracker.server import _preset_price_keys
|
|
|
|
preset = {
|
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
|
"aliases": ["qwen3.6-35b-a3b", "Qwen/Qwen3.6-35B-A3B"],
|
|
}
|
|
keys = _preset_price_keys("qwen3.6-35b-a3b", preset)
|
|
assert keys == {
|
|
"qwen3.6-35b-a3b",
|
|
"unsloth/Qwen3.6-35B-A3B",
|
|
"Qwen/Qwen3.6-35B-A3B",
|
|
}
|
|
assert _preset_price_keys("bare", {}) == {"bare"}
|
|
|
|
|
|
def test_qwen_preset_prices_apply_to_all_aliases(tmp_path):
|
|
"Requests naming the repo id (what nodes register) bill at the preset price.\n\nTags: general"
|
|
|
|
from meshnet_tracker.server import TrackerServer
|
|
|
|
import pytest
|
|
|
|
tracker = TrackerServer(billing_db=str(tmp_path / "billing.sqlite"))
|
|
try:
|
|
billing = tracker._billing
|
|
assert billing is not None
|
|
for key in (
|
|
"qwen3.6-35b-a3b",
|
|
"unsloth/Qwen3.6-35B-A3B",
|
|
"Qwen/Qwen3.6-35B-A3B",
|
|
):
|
|
assert billing.price_for(key) == pytest.approx(0.00044), key
|
|
# Unknown models keep the default rate
|
|
assert billing.price_for("some/other-model") == pytest.approx(0.02)
|
|
finally:
|
|
pass
|
|
|
|
|
|
def test_qwen25_preset_price_is_ten_x_commercial_reference(tmp_path):
|
|
"Qwen2.\n\nTags: general"
|
|
import pytest
|
|
from meshnet_tracker.server import TrackerServer, _resolve_model_preset, DEFAULT_MODEL_PRESETS
|
|
|
|
name, preset = _resolve_model_preset(DEFAULT_MODEL_PRESETS, "Qwen/Qwen2.5-0.5B-Instruct")
|
|
assert name == "qwen2.5-0.5b-instruct"
|
|
assert preset["price_per_1k_tokens"] == pytest.approx(0.002)
|
|
|
|
tracker = TrackerServer(billing_db=str(tmp_path / "billing.sqlite"))
|
|
billing = tracker._billing
|
|
assert billing is not None
|
|
for key in (
|
|
"qwen2.5-0.5b",
|
|
"Qwen2.5-0.5B-Instruct",
|
|
"Qwen/Qwen2.5-0.5B-Instruct",
|
|
):
|
|
assert billing.price_for(key) == pytest.approx(0.002), key
|
|
assert billing.price_for("unrelated-model") == pytest.approx(0.02)
|