popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-08 20:00:56 +03:00
parent 91e4bcf2c9 e44abc910d
commit a224644247
28 changed files with 14182 additions and 12368 deletions
Download Patch File Download Diff File Expand all files Collapse all files

768
tests/test_accounts.py
View File

@@ -1,384 +1,384 @@
"""Dashboard user accounts: registration, login, roles, API keys, usage.
Unit tests for AccountStore plus HTTP integration on the tracker:
register/login/logout, per-account balance and usage, API-key lifecycle
(revoked keys rejected by the OpenAI proxy), and the admin listing.
"""
import http.cookies
import json
import urllib.error
import urllib.request
import pytest
from meshnet_tracker.accounts import AccountStore
from meshnet_tracker.auth import sign_hive_request
from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer
HIVE_SECRET = "test-hive-secret"
# ---------------------------------------------------------------- unit tests
def test_first_account_is_admin_then_users():
store = AccountStore()
first = store.register(email="admin@example.com", password="secret-123")
second = store.register(email="user@example.com", password="secret-123")
assert first["role"] == "admin"
assert second["role"] == "user"
def test_register_requires_email_or_wallet_and_password_length():
store = AccountStore()
with pytest.raises(ValueError, match="email or a wallet"):
store.register(password="secret-123")
with pytest.raises(ValueError, match="invalid email"):
store.register(email="not-an-email", password="secret-123")
with pytest.raises(ValueError, match="at least 8"):
store.register(email="a@b.co", password="short")
def test_register_rejects_duplicate_identifiers():
store = AccountStore()
store.register(email="dup@example.com", password="secret-123")
with pytest.raises(ValueError, match="already exists"):
store.register(email="DUP@example.com", password="other-secret")
def test_login_by_email_or_wallet():
store = AccountStore()
account = store.register(
email="both@example.com", wallet="WalletXYZ", password="secret-123"
)
assert store.verify_login("both@example.com", "secret-123")["account_id"] == account["account_id"]
assert store.verify_login("WalletXYZ", "secret-123")["account_id"] == account["account_id"]
assert store.verify_login("both@example.com", "wrong-password") is None
assert store.verify_login("nobody@example.com", "secret-123") is None
def test_sessions_resolve_and_destroy():
store = AccountStore()
account = store.register(email="s@example.com", password="secret-123")
token = store.create_session(account["account_id"])
assert store.session_account(token)["account_id"] == account["account_id"]
store.destroy_session(token)
assert store.session_account(token) is None
assert store.session_account("bogus") is None
def test_sessions_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="cookie@example.com", password="secret-123")
token = store.create_session(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.session_account(token)["account_id"] == account["account_id"]
def test_api_key_lifecycle():
store = AccountStore()
account = store.register(email="k@example.com", password="secret-123")
other = store.register(email="other@example.com", password="secret-123")
key = store.create_api_key(account["account_id"])
assert key.startswith("sk-mesh-")
assert store.keys_for(account["account_id"]) == [key]
# someone else's account cannot revoke it
assert store.revoke_api_key(other["account_id"], key) is False
assert store.revoke_api_key(account["account_id"], key) is True
assert store.keys_for(account["account_id"]) == []
assert store.is_key_revoked(key)
def test_accounts_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="p@example.com", password="secret-123")
key = store.create_api_key(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.verify_login("p@example.com", "secret-123") is not None
assert reloaded.keys_for(account["account_id"]) == [key]
def test_account_events_replicate_and_dedupe():
leader = AccountStore()
follower = AccountStore()
account = leader.register(email="r@example.com", password="secret-123")
key = leader.create_api_key(account["account_id"])
leader.revoke_api_key(account["account_id"], key)
events, cursor = leader.events_since(0)
assert follower.apply_events(events) == len(events)
assert follower.apply_events(events) == 0 # replay is a no-op
assert follower.verify_login("r@example.com", "secret-123") is not None
assert follower.is_key_revoked(key)
more, _ = leader.events_since(cursor)
assert more == []
# ---------------------------------------------------------- HTTP integration
def _call(url, method="GET", body=None, token=None):
headers = {"Content-Type": "application/json"}
if token:
headers["Authorization"] = f"Bearer {token}"
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(url, data=data, headers=headers, method=method)
with urllib.request.urlopen(req) as r:
return json.loads(r.read())
@pytest.fixture
def account_tracker():
"""Tracker with credit features pinned OFF (defaults are devnet-friendly 1.0)."""
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
tracker = TrackerServer(
billing=ledger,
accounts=AccountStore(),
hive_secret=HIVE_SECRET,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
yield f"http://127.0.0.1:{port}", ledger
tracker.stop()
def test_register_login_and_account_view(account_tracker):
url, _ = account_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "admin@example.com", "password": "secret-123"})
assert reg["account"]["role"] == "admin"
assert reg["api_key"].startswith("sk-mesh-")
assert reg["session_token"]
login = _call(f"{url}/v1/auth/login", "POST",
{"identifier": "admin@example.com", "password": "secret-123"})
me = _call(f"{url}/v1/account", token=login["session_token"])
assert me["account"]["email"] == "admin@example.com"
assert me["api_keys"] == [reg["api_key"]]
assert me["total_balance"] == pytest.approx(0.0)
assert me["usage"]["requests"] == 0
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
accounts_db = str(tmp_path / "accounts.db")
tracker = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_call(f"{url}/v1/auth/register", "POST",
{"email": "cookie-http@example.com", "password": "secret-123"})
req = urllib.request.Request(
f"{url}/v1/auth/login",
data=json.dumps({
"identifier": "cookie-http@example.com",
"password": "secret-123",
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req) as r:
assert json.loads(r.read())["session_token"]
cookie_header = r.headers["Set-Cookie"]
finally:
tracker.stop()
cookie = http.cookies.SimpleCookie(cookie_header)
session_cookie = cookie["meshnet_session"].OutputString()
restarted = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
restarted_port = restarted.start()
restarted_url = f"http://127.0.0.1:{restarted_port}"
try:
req = urllib.request.Request(
f"{restarted_url}/v1/account",
headers={"Cookie": session_cookie},
method="GET",
)
with urllib.request.urlopen(req) as r:
me = json.loads(r.read())
finally:
restarted.stop()
assert me["account"]["email"] == "cookie-http@example.com"
def test_bad_credentials_and_missing_session_are_401(account_tracker):
url, _ = account_tracker
_call(f"{url}/v1/auth/register", "POST",
{"email": "a@example.com", "password": "secret-123"})
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/auth/login", "POST",
{"identifier": "a@example.com", "password": "wrong-pass"})
assert exc_info.value.code == 401
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/account")
assert exc_info.value.code == 401
def test_key_create_revoke_and_revoked_key_rejected_by_proxy(account_tracker):
url, _ = account_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "k@example.com", "password": "secret-123"})
token = reg["session_token"]
new_key = _call(f"{url}/v1/account/keys", "POST", {}, token=token)["api_key"]
me = _call(f"{url}/v1/account", token=token)
assert sorted(me["api_keys"]) == sorted([reg["api_key"], new_key])
_call(f"{url}/v1/account/keys/revoke", "POST", {"api_key": new_key}, token=token)
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/chat/completions", "POST",
{"model": "any", "messages": []}, token=new_key)
assert exc_info.value.code == 401
assert "revoked" in exc_info.value.read().decode()
def test_admin_listing_requires_admin_role(account_tracker):
url, _ = account_tracker
admin = _call(f"{url}/v1/auth/register", "POST",
{"email": "admin@example.com", "password": "secret-123"})
user = _call(f"{url}/v1/auth/register", "POST",
{"wallet": "WalletUser1", "password": "secret-123"})
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/admin/accounts", token=user["session_token"])
assert exc_info.value.code == 403
listing = _call(f"{url}/v1/admin/accounts", token=admin["session_token"])
accounts = listing["accounts"]
assert len(accounts) == 2
assert accounts[0]["role"] == "admin"
assert accounts[1]["wallet"] == "WalletUser1"
assert "balances" in accounts[0]
def test_accounts_gossip_endpoint_applies_events(account_tracker):
url, _ = account_tracker
peer = AccountStore()
peer.register(email="remote@example.com", password="secret-123")
events, _ = peer.events_since(0)
body = json.dumps({"events": events}).encode()
req = urllib.request.Request(
f"{url}/v1/accounts/gossip", data=body,
headers={"Content-Type": "application/json", **sign_hive_request(HIVE_SECRET, body)},
method="POST",
)
with urllib.request.urlopen(req) as r:
result = json.loads(r.read())
assert result["applied"] == len(events)
login = _call(f"{url}/v1/auth/login", "POST",
{"identifier": "remote@example.com", "password": "secret-123"})
assert login["account"]["email"] == "remote@example.com"
def test_accounts_endpoints_404_when_disabled():
tracker = TrackerServer() # no accounts, no billing
port = tracker.start()
try:
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"http://127.0.0.1:{port}/v1/auth/register", "POST",
{"email": "x@example.com", "password": "secret-123"})
assert exc_info.value.code == 404
finally:
tracker.stop()
# ------------------------------------------- US-039/US-040: credit and top-up
@pytest.fixture
def funded_tracker():
"""Tracker with Caller Credit and the devnet top-up faucet enabled."""
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
tracker = TrackerServer(
billing=ledger,
accounts=AccountStore(),
hive_secret=HIVE_SECRET,
starting_credit=1.0,
devnet_topup_amount=10.0,
)
port = tracker.start()
yield f"http://127.0.0.1:{port}", ledger
tracker.stop()
def test_caller_credit_granted_once_per_account(funded_tracker):
url, ledger = funded_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "c@example.com", "password": "secret-123"})
token = reg["session_token"]
first_key = reg["api_key"]
assert ledger.get_client_balance(first_key) == pytest.approx(1.0)
# A second key never re-grants — not even after revoking the first.
second = _call(f"{url}/v1/account/keys", "POST", {}, token=token)
assert second["caller_credit_granted"] is False
assert ledger.get_client_balance(second["api_key"]) == pytest.approx(0.0)
_call(f"{url}/v1/account/keys/revoke", "POST", {"api_key": first_key}, token=token)
third = _call(f"{url}/v1/account/keys", "POST", {}, token=token)
assert third["caller_credit_granted"] is False
assert ledger.get_client_balance(third["api_key"]) == pytest.approx(0.0)
def test_unknown_bearer_key_rejected_by_proxy(funded_tracker):
url, ledger = funded_tracker
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/chat/completions", "POST",
{"model": "any", "messages": []}, token="sk-mesh-made-up-key")
assert exc_info.value.code == 401
assert "unknown API key" in exc_info.value.read().decode()
# The invented key must not have become a billable client.
assert ledger.get_client_balance("sk-mesh-made-up-key") == pytest.approx(0.0)
def test_devnet_topup_credits_own_key_only(funded_tracker):
url, ledger = funded_tracker
owner = _call(f"{url}/v1/auth/register", "POST",
{"email": "own@example.com", "password": "secret-123"})
other = _call(f"{url}/v1/auth/register", "POST",
{"email": "oth@example.com", "password": "secret-123"})
me = _call(f"{url}/v1/account", token=owner["session_token"])
assert me["topup_amount"] == pytest.approx(10.0)
result = _call(f"{url}/v1/account/topup", "POST",
{"api_key": owner["api_key"]}, token=owner["session_token"])
assert result["credited"] == pytest.approx(10.0)
assert result["balance"] == pytest.approx(11.0) # 1.0 caller credit + 10.0
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/account/topup", "POST",
{"api_key": owner["api_key"]}, token=other["session_token"])
assert exc_info.value.code == 403
assert ledger.get_client_balance(owner["api_key"]) == pytest.approx(11.0)
def test_topup_404_when_disabled(account_tracker):
url, _ = account_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "t@example.com", "password": "secret-123"})
me = _call(f"{url}/v1/account", token=reg["session_token"])
assert me["topup_amount"] == pytest.approx(0.0)
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/account/topup", "POST",
{"api_key": reg["api_key"]}, token=reg["session_token"])
assert exc_info.value.code == 404
"""Dashboard user accounts: registration, login, roles, API keys, usage.
Unit tests for AccountStore plus HTTP integration on the tracker:
register/login/logout, per-account balance and usage, API-key lifecycle
(revoked keys rejected by the OpenAI proxy), and the admin listing.
"""
import http.cookies
import json
import urllib.error
import urllib.request
import pytest
from meshnet_tracker.accounts import AccountStore
from meshnet_tracker.auth import sign_hive_request
from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer
HIVE_SECRET = "test-hive-secret"
# ---------------------------------------------------------------- unit tests
def test_first_account_is_admin_then_users():
store = AccountStore()
first = store.register(email="admin@example.com", password="secret-123")
second = store.register(email="user@example.com", password="secret-123")
assert first["role"] == "admin"
assert second["role"] == "user"
def test_register_requires_email_or_wallet_and_password_length():
store = AccountStore()
with pytest.raises(ValueError, match="email or a wallet"):
store.register(password="secret-123")
with pytest.raises(ValueError, match="invalid email"):
store.register(email="not-an-email", password="secret-123")
with pytest.raises(ValueError, match="at least 8"):
store.register(email="a@b.co", password="short")
def test_register_rejects_duplicate_identifiers():
store = AccountStore()
store.register(email="dup@example.com", password="secret-123")
with pytest.raises(ValueError, match="already exists"):
store.register(email="DUP@example.com", password="other-secret")
def test_login_by_email_or_wallet():
store = AccountStore()
account = store.register(
email="both@example.com", wallet="WalletXYZ", password="secret-123"
)
assert store.verify_login("both@example.com", "secret-123")["account_id"] == account["account_id"]
assert store.verify_login("WalletXYZ", "secret-123")["account_id"] == account["account_id"]
assert store.verify_login("both@example.com", "wrong-password") is None
assert store.verify_login("nobody@example.com", "secret-123") is None
def test_sessions_resolve_and_destroy():
store = AccountStore()
account = store.register(email="s@example.com", password="secret-123")
token = store.create_session(account["account_id"])
assert store.session_account(token)["account_id"] == account["account_id"]
store.destroy_session(token)
assert store.session_account(token) is None
assert store.session_account("bogus") is None
def test_sessions_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="cookie@example.com", password="secret-123")
token = store.create_session(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.session_account(token)["account_id"] == account["account_id"]
def test_api_key_lifecycle():
store = AccountStore()
account = store.register(email="k@example.com", password="secret-123")
other = store.register(email="other@example.com", password="secret-123")
key = store.create_api_key(account["account_id"])
assert key.startswith("sk-mesh-")
assert store.keys_for(account["account_id"]) == [key]
# someone else's account cannot revoke it
assert store.revoke_api_key(other["account_id"], key) is False
assert store.revoke_api_key(account["account_id"], key) is True
assert store.keys_for(account["account_id"]) == []
assert store.is_key_revoked(key)
def test_accounts_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="p@example.com", password="secret-123")
key = store.create_api_key(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.verify_login("p@example.com", "secret-123") is not None
assert reloaded.keys_for(account["account_id"]) == [key]
def test_account_events_replicate_and_dedupe():
leader = AccountStore()
follower = AccountStore()
account = leader.register(email="r@example.com", password="secret-123")
key = leader.create_api_key(account["account_id"])
leader.revoke_api_key(account["account_id"], key)
events, cursor = leader.events_since(0)
assert follower.apply_events(events) == len(events)
assert follower.apply_events(events) == 0 # replay is a no-op
assert follower.verify_login("r@example.com", "secret-123") is not None
assert follower.is_key_revoked(key)
more, _ = leader.events_since(cursor)
assert more == []
# ---------------------------------------------------------- HTTP integration
def _call(url, method="GET", body=None, token=None):
headers = {"Content-Type": "application/json"}
if token:
headers["Authorization"] = f"Bearer {token}"
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(url, data=data, headers=headers, method=method)
with urllib.request.urlopen(req) as r:
return json.loads(r.read())
@pytest.fixture
def account_tracker():
"""Tracker with credit features pinned OFF (defaults are devnet-friendly 1.0)."""
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
tracker = TrackerServer(
billing=ledger,
accounts=AccountStore(),
hive_secret=HIVE_SECRET,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
yield f"http://127.0.0.1:{port}", ledger
tracker.stop()
def test_register_login_and_account_view(account_tracker):
url, _ = account_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "admin@example.com", "password": "secret-123"})
assert reg["account"]["role"] == "admin"
assert reg["api_key"].startswith("sk-mesh-")
assert reg["session_token"]
login = _call(f"{url}/v1/auth/login", "POST",
{"identifier": "admin@example.com", "password": "secret-123"})
me = _call(f"{url}/v1/account", token=login["session_token"])
assert me["account"]["email"] == "admin@example.com"
assert me["api_keys"] == [reg["api_key"]]
assert me["total_balance"] == pytest.approx(0.0)
assert me["usage"]["requests"] == 0
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
accounts_db = str(tmp_path / "accounts.db")
tracker = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_call(f"{url}/v1/auth/register", "POST",
{"email": "cookie-http@example.com", "password": "secret-123"})
req = urllib.request.Request(
f"{url}/v1/auth/login",
data=json.dumps({
"identifier": "cookie-http@example.com",
"password": "secret-123",
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req) as r:
assert json.loads(r.read())["session_token"]
cookie_header = r.headers["Set-Cookie"]
finally:
tracker.stop()
cookie = http.cookies.SimpleCookie(cookie_header)
session_cookie = cookie["meshnet_session"].OutputString()
restarted = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
restarted_port = restarted.start()
restarted_url = f"http://127.0.0.1:{restarted_port}"
try:
req = urllib.request.Request(
f"{restarted_url}/v1/account",
headers={"Cookie": session_cookie},
method="GET",
)
with urllib.request.urlopen(req) as r:
me = json.loads(r.read())
finally:
restarted.stop()
assert me["account"]["email"] == "cookie-http@example.com"
def test_bad_credentials_and_missing_session_are_401(account_tracker):
url, _ = account_tracker
_call(f"{url}/v1/auth/register", "POST",
{"email": "a@example.com", "password": "secret-123"})
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/auth/login", "POST",
{"identifier": "a@example.com", "password": "wrong-pass"})
assert exc_info.value.code == 401
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/account")
assert exc_info.value.code == 401
def test_key_create_revoke_and_revoked_key_rejected_by_proxy(account_tracker):
url, _ = account_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "k@example.com", "password": "secret-123"})
token = reg["session_token"]
new_key = _call(f"{url}/v1/account/keys", "POST", {}, token=token)["api_key"]
me = _call(f"{url}/v1/account", token=token)
assert sorted(me["api_keys"]) == sorted([reg["api_key"], new_key])
_call(f"{url}/v1/account/keys/revoke", "POST", {"api_key": new_key}, token=token)
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/chat/completions", "POST",
{"model": "any", "messages": []}, token=new_key)
assert exc_info.value.code == 401
assert "revoked" in exc_info.value.read().decode()
def test_admin_listing_requires_admin_role(account_tracker):
url, _ = account_tracker
admin = _call(f"{url}/v1/auth/register", "POST",
{"email": "admin@example.com", "password": "secret-123"})
user = _call(f"{url}/v1/auth/register", "POST",
{"wallet": "WalletUser1", "password": "secret-123"})
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/admin/accounts", token=user["session_token"])
assert exc_info.value.code == 403
listing = _call(f"{url}/v1/admin/accounts", token=admin["session_token"])
accounts = listing["accounts"]
assert len(accounts) == 2
assert accounts[0]["role"] == "admin"
assert accounts[1]["wallet"] == "WalletUser1"
assert "balances" in accounts[0]
def test_accounts_gossip_endpoint_applies_events(account_tracker):
url, _ = account_tracker
peer = AccountStore()
peer.register(email="remote@example.com", password="secret-123")
events, _ = peer.events_since(0)
body = json.dumps({"events": events}).encode()
req = urllib.request.Request(
f"{url}/v1/accounts/gossip", data=body,
headers={"Content-Type": "application/json", **sign_hive_request(HIVE_SECRET, body)},
method="POST",
)
with urllib.request.urlopen(req) as r:
result = json.loads(r.read())
assert result["applied"] == len(events)
login = _call(f"{url}/v1/auth/login", "POST",
{"identifier": "remote@example.com", "password": "secret-123"})
assert login["account"]["email"] == "remote@example.com"
def test_accounts_endpoints_404_when_disabled():
tracker = TrackerServer() # no accounts, no billing
port = tracker.start()
try:
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"http://127.0.0.1:{port}/v1/auth/register", "POST",
{"email": "x@example.com", "password": "secret-123"})
assert exc_info.value.code == 404
finally:
tracker.stop()
# ------------------------------------------- US-039/US-040: credit and top-up
@pytest.fixture
def funded_tracker():
"""Tracker with Caller Credit and the devnet top-up faucet enabled."""
ledger = BillingLedger(starting_credit=0.0, default_price_per_1k=0.02)
tracker = TrackerServer(
billing=ledger,
accounts=AccountStore(),
hive_secret=HIVE_SECRET,
starting_credit=1.0,
devnet_topup_amount=10.0,
)
port = tracker.start()
yield f"http://127.0.0.1:{port}", ledger
tracker.stop()
def test_caller_credit_granted_once_per_account(funded_tracker):
url, ledger = funded_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "c@example.com", "password": "secret-123"})
token = reg["session_token"]
first_key = reg["api_key"]
assert ledger.get_client_balance(first_key) == pytest.approx(1.0)
# A second key never re-grants — not even after revoking the first.
second = _call(f"{url}/v1/account/keys", "POST", {}, token=token)
assert second["caller_credit_granted"] is False
assert ledger.get_client_balance(second["api_key"]) == pytest.approx(0.0)
_call(f"{url}/v1/account/keys/revoke", "POST", {"api_key": first_key}, token=token)
third = _call(f"{url}/v1/account/keys", "POST", {}, token=token)
assert third["caller_credit_granted"] is False
assert ledger.get_client_balance(third["api_key"]) == pytest.approx(0.0)
def test_unknown_bearer_key_rejected_by_proxy(funded_tracker):
url, ledger = funded_tracker
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/chat/completions", "POST",
{"model": "any", "messages": []}, token="sk-mesh-made-up-key")
assert exc_info.value.code == 401
assert "unknown API key" in exc_info.value.read().decode()
# The invented key must not have become a billable client.
assert ledger.get_client_balance("sk-mesh-made-up-key") == pytest.approx(0.0)
def test_devnet_topup_credits_own_key_only(funded_tracker):
url, ledger = funded_tracker
owner = _call(f"{url}/v1/auth/register", "POST",
{"email": "own@example.com", "password": "secret-123"})
other = _call(f"{url}/v1/auth/register", "POST",
{"email": "oth@example.com", "password": "secret-123"})
me = _call(f"{url}/v1/account", token=owner["session_token"])
assert me["topup_amount"] == pytest.approx(10.0)
result = _call(f"{url}/v1/account/topup", "POST",
{"api_key": owner["api_key"]}, token=owner["session_token"])
assert result["credited"] == pytest.approx(10.0)
assert result["balance"] == pytest.approx(11.0) # 1.0 caller credit + 10.0
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/account/topup", "POST",
{"api_key": owner["api_key"]}, token=other["session_token"])
assert exc_info.value.code == 403
assert ledger.get_client_balance(owner["api_key"]) == pytest.approx(11.0)
def test_topup_404_when_disabled(account_tracker):
url, _ = account_tracker
reg = _call(f"{url}/v1/auth/register", "POST",
{"email": "t@example.com", "password": "secret-123"})
me = _call(f"{url}/v1/account", token=reg["session_token"])
assert me["topup_amount"] == pytest.approx(0.0)
with pytest.raises(urllib.error.HTTPError) as exc_info:
_call(f"{url}/v1/account/topup", "POST",
{"api_key": reg["api_key"]}, token=reg["session_token"])
assert exc_info.value.code == 404

1348
tests/test_billing_ledger.py
View File

File diff suppressed because it is too large Load Diff

5
tests/test_dashboard.py
View File

@@ -29,6 +29,9 @@ def test_dashboard_served_with_all_panels():
for panel in PANELS:
assert panel in html
assert "<script>" in html # polling client embedded, no build step
assert "resolveModelGroup" in html
assert "buildModelAliasMap" in html
assert "modelAliasKey(raw)" in html
finally:
tracker.stop()
@@ -45,7 +48,7 @@ def test_dashboard_chat_uses_streaming_fetch():
assert "stream: true" in html
assert ".body.getReader()" in html
assert 'data === "[DONE]"' in html
assert '=== "[DONE]"' in html
assert 'console.error("chat stream failed", err)' in html

306
tests/test_dynamic_routing.py Normal file
View File

@@ -0,0 +1,306 @@
"""ADR-0021: dynamic bandit-style route selection with learned statistics."""
import http.server
import json
import random
import threading
import types
import urllib.request
from meshnet_tracker.routing_stats import (
RouteCandidate,
RouteStatsStore,
RoutingConfig,
choose_route,
route_signature,
route_table,
)
from meshnet_tracker.server import TrackerServer, _enumerate_routes
def _post_json(url: str, payload: dict) -> dict:
req = urllib.request.Request(
url,
data=json.dumps(payload).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=10.0) as resp:
return json.loads(resp.read())
def _get_json(url: str) -> dict:
with urllib.request.urlopen(url, timeout=10.0) as resp:
return json.loads(resp.read())
def _fake_node(node_id, shard_start, shard_end, benchmark=100.0, endpoint=None):
return types.SimpleNamespace(
node_id=node_id,
endpoint=endpoint or f"http://{node_id}:7000",
model="qwen3.6-35b-a3b",
hf_repo="unsloth/Qwen3.6-35B-A3B",
shard_start=shard_start,
shard_end=shard_end,
num_layers=40,
benchmark_tokens_per_sec=benchmark,
model_tokens_per_sec={},
queue_depth=0,
proxy_inflight=0,
wallet_address=None,
relay_addr=None,
)
# ---- RouteStatsStore ----------------------------------------------------
def test_route_stats_sample_becomes_proven_and_decays():
store = RouteStatsStore(RoutingConfig(stats_half_life_seconds=100.0))
sig = "m|a[0-39]"
assert store.snapshot(sig, "m", now=0.0)["status"] == "unsampled"
assert store.record_sample("m", sig, tokens=100, elapsed_seconds=10.0, now=0.0)
snap = store.snapshot(sig, "m", now=1.0)
assert snap["status"] == "proven"
assert snap["tps"] == 10.0
# After many half-lives the sample mass decays below the proven threshold.
assert store.snapshot(sig, "m", now=1000.0)["status"] == "decayed"
def test_route_stats_rejects_near_empty_samples():
store = RouteStatsStore(RoutingConfig(min_sample_tokens=8))
assert not store.record_sample("m", "sig", tokens=3, elapsed_seconds=1.0)
assert store.snapshot("sig", "m")["samples"] == 0
def test_route_stats_epoch_bump_marks_stale():
store = RouteStatsStore()
sig = "m|a[0-39]"
store.record_sample("m", sig, tokens=100, elapsed_seconds=10.0, now=0.0)
assert store.snapshot(sig, "m", now=1.0)["status"] == "proven"
store.bump_epoch(["m"])
snap = store.snapshot(sig, "m", now=1.0)
assert snap["status"] == "stale"
assert snap["tps"] == 10.0 # EWMA kept as a prior for display
# A fresh sample under the new epoch re-proves the route.
store.record_sample("m", sig, tokens=100, elapsed_seconds=10.0, now=2.0)
assert store.snapshot(sig, "m", now=3.0)["status"] == "proven"
def test_route_stats_ewma_averages_samples():
store = RouteStatsStore(RoutingConfig(stats_half_life_seconds=1e9))
sig = "m|a"
store.record_sample("m", sig, tokens=100, elapsed_seconds=10.0, now=0.0) # 10 tps
store.record_sample("m", sig, tokens=200, elapsed_seconds=10.0, now=1.0) # 20 tps
snap = store.snapshot(sig, "m", now=2.0)
assert 14.9 < snap["tps"] < 15.1
# ---- choose_route --------------------------------------------------------
def _candidates_two_routes():
fast = RouteCandidate(nodes=[], signature="m|fast", prior_tps=100.0)
slow = RouteCandidate(nodes=[], signature="m|slow", prior_tps=50.0)
return fast, slow
def test_choose_route_without_samples_is_deterministic_best_prior():
store = RouteStatsStore()
fast, slow = _candidates_two_routes()
for _ in range(20):
picked, decision = choose_route([slow, fast], store, "m", rng=random.Random(7))
assert picked is fast
assert decision["mode"] == "scout"
def test_choose_route_traffic_proportional_to_tps():
store = RouteStatsStore(RoutingConfig(stats_half_life_seconds=1e9))
fast, slow = _candidates_two_routes()
now = 0.0
for _ in range(5):
now += 1.0
store.record_sample("m", fast.signature, tokens=150, elapsed_seconds=10.0, now=now)
store.record_sample("m", slow.signature, tokens=100, elapsed_seconds=10.0, now=now)
rng = random.Random(42)
picks = {"m|fast": 0, "m|slow": 0}
for _ in range(4000):
picked, decision = choose_route([fast, slow], store, "m", rng=rng, now=now)
assert decision["mode"] == "exploit"
picks[picked.signature] += 1
share = picks["m|fast"] / 4000
# 15 tps vs 10 tps at alpha=1 → expected fast share 0.6
assert 0.55 < share < 0.65
def test_choose_route_scouts_unproven_routes_at_explore_share():
store = RouteStatsStore(RoutingConfig(explore_share=0.25, stats_half_life_seconds=1e9))
fast, slow = _candidates_two_routes()
now = 1.0
store.record_sample("m", fast.signature, tokens=150, elapsed_seconds=10.0, now=now)
rng = random.Random(11)
scouted = 0
for _ in range(4000):
picked, decision = choose_route([fast, slow], store, "m", rng=rng, now=now)
if decision["mode"] == "scout":
scouted += 1
assert picked is slow
assert 0.20 < scouted / 4000 < 0.30
# ---- _enumerate_routes ---------------------------------------------------
def test_enumerate_routes_mixed_topology_yields_both_routes():
gpu = _fake_node("gpu", 0, 21, benchmark=11000.0)
cpu = _fake_node("cpu", 0, 39, benchmark=425.0)
candidates = _enumerate_routes([gpu, cpu], 0, 39, model="qwen3.6-35b-a3b")
signatures = {c.signature for c in candidates}
assert signatures == {
route_signature("qwen3.6-35b-a3b", [gpu, cpu]),
route_signature("qwen3.6-35b-a3b", [cpu]),
}
hybrid = next(c for c in candidates if len(c.nodes) == 2)
assert [n.node_id for n in hybrid.nodes] == ["gpu", "cpu"]
# Hybrid route's prior is its bottleneck hop, not the fast head.
assert hybrid.prior_tps == 425.0
def test_enumerate_routes_requires_head_at_first_layer():
tail_only = _fake_node("tail", 22, 39)
assert _enumerate_routes([tail_only], 0, 39, model="m") == []
def test_route_table_reports_coefficient_and_share():
store = RouteStatsStore(RoutingConfig(explore_share=0.3, stats_half_life_seconds=1e9))
fast, slow = _candidates_two_routes()
now = 1.0
for _ in range(3):
store.record_sample("m", fast.signature, tokens=150, elapsed_seconds=10.0, now=now)
store.record_sample("m", slow.signature, tokens=100, elapsed_seconds=10.0, now=now)
now += 1.0
rows = route_table([fast, slow], store, "m", now=now)
by_sig = {r["signature"]: r for r in rows}
assert by_sig["m|fast"]["coefficient"] == 1.0
assert abs(by_sig["m|slow"]["coefficient"] - (10.0 / 15.0)) < 0.01
# No scouts → full exploit budget split 0.6 / 0.4.
assert abs(by_sig["m|fast"]["expected_share"] - 0.6) < 0.01
assert abs(by_sig["m|slow"]["expected_share"] - 0.4) < 0.01
# ---- integration: proxy uses route head + /v1/routing --------------------
def test_proxy_head_is_route_head_and_routing_endpoint_lists_routes():
"""Mixed topology (partial head 0-21 + full node 0-39): the proxy target
must be the selected route's own head, downstream hops must continue at
head.shard_end + 1 (the ADR-0020 flaw), and /v1/routing must list both
candidate routes."""
class ChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, *args): # noqa: ARG002
pass
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
self.rfile.read(length)
route_header = self.headers.get("X-Meshnet-Route") or "[]"
body = json.dumps({
"choices": [{"message": {"role": "assistant", "content": route_header}}],
"usage": {"prompt_tokens": 10, "completion_tokens": 40},
}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
stubs = []
threads = []
for _ in range(2):
stub = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
thread = threading.Thread(target=stub.serve_forever, daemon=True)
thread.start()
stubs.append(stub)
threads.append(thread)
gpu_stub, cpu_stub = stubs
tracker = TrackerServer(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"aliases": ["Qwen3.6-35B-A3B"],
}
})
tracker_port = tracker.start()
try:
tracker._server.route_rng = random.Random(3)
for stub, shard_end, bench in ((gpu_stub, 21, 11000.0), (cpu_stub, 39, 425.0)):
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{stub.server_address[1]}",
"model": "qwen3.6-35b-a3b",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 0,
"shard_end": shard_end,
"tracker_mode": True,
"benchmark_tokens_per_sec": bench,
"hardware_profile": {},
"score": 1.0},
)
for _ in range(8):
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
{"model": "Qwen3.6-35B-A3B",
"messages": [{"role": "user", "content": "hi"}]},
)
console = _get_json(f"http://127.0.0.1:{tracker_port}/v1/console")
routing = _get_json(f"http://127.0.0.1:{tracker_port}/v1/routing")
finally:
tracker.stop()
for stub, thread in zip(stubs, threads):
stub.shutdown()
stub.server_close()
thread.join(timeout=1.0)
gpu_endpoint = f"http://127.0.0.1:{gpu_stub.server_address[1]}"
cpu_endpoint = f"http://127.0.0.1:{cpu_stub.server_address[1]}"
selected = [e for e in console["events"] if e["message"] == "proxy route selected"]
assert selected
for event in selected:
fields = event["fields"]
nodes = fields["nodes"]
# The proxy head must be the route's first hop (ADR-0020 regression).
assert fields["head_endpoint"] == nodes[0]["endpoint"]
downstream = json.loads(fields["downstream"])
if fields["head_endpoint"] == gpu_endpoint:
# Partial head: downstream continues at layer 22, never 0.
assert downstream == [{"endpoint": cpu_endpoint, "start_layer": 22}]
else:
assert fields["head_endpoint"] == cpu_endpoint
assert downstream == []
table = routing["models"]["qwen3.6-35b-a3b"]
assert len(table["routes"]) == 2
sampled = [r for r in table["routes"] if r["samples"] > 0]
assert sampled, "completed requests must produce route samples"
def test_endpoint_key_distinguishes_same_port_different_hosts():
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
assert _endpoint_key("http://192.168.0.20:7000") == "192.168.0.20:7000"
assert _endpoint_key("http://192.168.0.179:7000") == "192.168.0.179:7000"
assert _endpoint_key("http://192.168.0.20:7000") != _endpoint_key("http://192.168.0.179:7000")
class Backend:
shard_end = 21
hops = [{"endpoint": "http://192.168.0.179:7000", "start_layer": 0}]
assert _clamp_downstream_hops(hops, Backend()) == [
{"endpoint": "http://192.168.0.179:7000", "start_layer": 22},
]

109
tests/test_node_startup.py
View File

@@ -1680,6 +1680,66 @@ def test_preset_startup_rejects_pinned_shard_above_memory_budget(tmp_path, monke
tracker.stop()
def test_network_auto_join_clips_oversized_cpu_assignment(tmp_path, monkeypatch, capsys):
"""Old trackers may assign too many CPU layers; node clips before model load."""
import meshnet_node.startup as startup_mod
torch_calls: list[dict] = []
registrations: list[dict] = []
class FakeBackend:
total_layers = 40
class FakeTorchNodeServer:
def __init__(self, **kwargs):
torch_calls.append(kwargs)
self.backend = FakeBackend()
self.tracker_node_id = None
def start(self):
return 7000
def stop(self):
pass
oversized_assignment = {
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"model": "qwen3.6-35b-a3b",
"shard_start": 0,
"shard_end": 36,
"num_layers": 40,
"gap_found": False,
"bytes_per_layer": {"bfloat16": 1_797_594_419},
"model_sources": [],
}
monkeypatch.setattr(
startup_mod,
"detect_hardware",
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0, "ram_mb": 79 * 1024},
)
monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer)
monkeypatch.setattr(startup_mod, "_get_json", lambda *_args, **_kwargs: oversized_assignment)
monkeypatch.setattr(startup_mod, "_post_json", lambda _url, payload: registrations.append(payload) or {"node_id": "n1"})
monkeypatch.setattr(startup_mod, "_start_heartbeat", lambda *_args, **_kwargs: None)
monkeypatch.setattr(startup_mod, "model_metadata_for", lambda *_args, **_kwargs: {"num_layers": 40})
node = run_startup(
tracker_url="http://127.0.0.1:8080",
wallet_path=tmp_path / "wallet.json",
tracker_source_disabled=True,
)
try:
assert torch_calls[0]["shard_start"] == 0
assert torch_calls[0]["shard_end"] == 24
assert registrations[0]["shard_end"] == 24
output = capsys.readouterr().out
assert "CPU-safe runtime budget fits 25/40 layers" in output
assert "layers 0-24" in output
finally:
node.stop()
def test_preset_model_with_hf_repo_loads_torch_backend(tmp_path, monkeypatch, capsys):
"""Named presets that advertise hf_repo must load TorchNodeServer, not the stub server."""
import meshnet_node.startup as startup_mod
@@ -1996,6 +2056,55 @@ def test_network_assign_gap_found_field():
tracker.stop()
def test_network_assign_uses_conservative_cpu_runtime_budget():
"""CPU assignments leave headroom for partial-load overhead, not just raw weights."""
import json as _json
import urllib.request as _ur
tracker = TrackerServer(model_presets={
"qwen3.6-35b-a3b": {
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"aliases": ["unsloth/Qwen3.6-35B-A3B"],
"layers_start": 0,
"layers_end": 39,
"recommended": True,
"bytes_per_layer": {"bfloat16": 1_797_594_419},
},
})
port = tracker.start()
try:
data = _json.dumps({
"endpoint": "http://127.0.0.1:9200",
"model": "qwen3.6-35b-a3b",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 0,
"shard_end": 39,
"hardware_profile": {},
"score": 1.0,
}).encode()
req = _ur.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
with _ur.urlopen(req) as r:
r.read()
resp = _get_json(
f"http://127.0.0.1:{port}/v1/network/assign"
"?device=cpu&vram_mb=0&ram_mb=80896"
"&hf_repo=unsloth/Qwen3.6-35B-A3B"
)
assert resp["gap_found"] is False
assert resp["shard_start"] == 0
assert resp["shard_end"] == 24
finally:
tracker.stop()
def test_route_finds_hf_model_across_two_nodes():
"""Tracker /v1/route returns ordered route for HF model even without a preset."""
import json as _json

2016
tests/test_real_model_backend.py
View File

File diff suppressed because it is too large Load Diff

115
tests/test_tracker_routing.py
View File

@@ -911,6 +911,57 @@ def test_tracker_route_endpoint_ignores_model_case_and_outer_whitespace():
assert response["route"] == ["http://127.0.0.1:9101"]
def test_tracker_route_prefers_distributed_over_single_full_shard():
"""When a full 0-39 node and a partial 0-21 head coexist, /v1/route
should return both hops — not the full shard alone."""
tracker = TrackerServer(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"aliases": ["Qwen3.6-35B-A3B"],
}
})
tracker_port = tracker.start()
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://192.168.0.179:7000",
"model": "qwen3.6-35b-a3b",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 0,
"shard_end": 39,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://192.168.0.20:7000",
"model": "Qwen3.6-35B-A3B",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 0,
"shard_end": 21,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
response = _get_json(
f"http://127.0.0.1:{tracker_port}/v1/route?model=qwen3.6-35b-a3b"
)
finally:
tracker.stop()
assert response["route"] == [
"http://192.168.0.20:7000",
"http://192.168.0.179:7000",
]
assert [node["start_layer"] for node in response["nodes"]] == [0, 22]
def test_tracker_proxy_ignores_model_case_and_outer_whitespace():
class ChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
@@ -1568,6 +1619,70 @@ def test_tracker_heartbeat_updates_node():
tracker.stop()
def test_tracker_heartbeat_stores_current_requests():
"""Node-reported in-flight request snapshots appear on the network map."""
tracker = TrackerServer()
tracker_port = tracker.start()
try:
reg = _post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:9001",
"model": "progress-model",
"shard_start": 0,
"shard_end": 31,
"hardware_profile": {},
"score": 1.0,
},
)
node_id = reg["node_id"]
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/{node_id}/heartbeat",
{
"queue_depth": 1,
"current_requests": [{
"request_id": "req-abc123",
"model": "progress-model",
"kind": "chat",
"tokens": 17,
"elapsed_seconds": 42.5,
"tokens_per_sec": 0.4,
"routing_complete": True,
}],
},
)
network = _get_json(f"http://127.0.0.1:{tracker_port}/v1/network/map")
node = next(item for item in network["nodes"] if item["node_id"] == node_id)
assert node["stats"]["queue_depth"] == 1
assert node["stats"]["current_requests"] == [{
"request_id": "req-abc123",
"model": "progress-model",
"kind": "chat",
"tokens": 17,
"elapsed_seconds": 42.5,
"tokens_per_sec": 0.4,
"routing_complete": True,
}]
finally:
tracker.stop()
def test_normalize_current_requests_sanitizes_payload():
from meshnet_tracker.server import _normalize_current_requests
assert _normalize_current_requests(None) == []
assert _normalize_current_requests([
{"request_id": "req-1", "model": "m", "tokens": "9", "tokens_per_sec": "1.5"},
{"model": "missing-id"},
"bad",
]) == [{
"request_id": "req-1",
"model": "m",
"tokens": 9,
"tokens_per_sec": 1.5,
}]
def test_tracker_heartbeat_expiry():
"""Nodes that miss their heartbeat window are excluded from routes."""
tracker = TrackerServer(heartbeat_timeout=0.05) # 50 ms