# Conflicts:
#	packages/tracker/meshnet_tracker/cli.py
#	packages/tracker/meshnet_tracker/dashboard.html
#	packages/tracker/meshnet_tracker/server.py
#	tests/test_dashboard.py
This commit is contained in:
Dobromir Popov
2026-07-08 16:14:24 +02:00
12 changed files with 6759 additions and 6249 deletions

3
.gitignore vendored
View File

@@ -23,3 +23,6 @@ dist/
# Local tracker/node sqlite databases (never commit runtime state) # Local tracker/node sqlite databases (never commit runtime state)
*.sqlite *.sqlite
*.sqlite3 *.sqlite3
logs/tracker/error.log
logs/tracker/info.log
logs/tracker/warning.log

View File

@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, completion) self._send_json(200, completion)
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None: def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
"""Forward a raw request body to a head worker and stream the response back.""" """Forward a raw request body to a head worker and relay SSE without buffering."""
target_url = f"{url}/v1/chat/completions" target_url = f"{url}/v1/chat/completions"
req = urllib.request.Request( req = urllib.request.Request(
target_url, target_url,
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
try: try:
with urllib.request.urlopen(req, timeout=30.0) as r: with urllib.request.urlopen(req, timeout=30.0) as r:
content_type = r.headers.get("Content-Type", "application/json") content_type = r.headers.get("Content-Type", "application/json")
if "text/event-stream" in content_type:
self.send_response(r.status)
self.send_header("Content-Type", content_type)
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.end_headers()
while True:
line = r.readline()
if not line:
break
self.wfile.write(line)
self.wfile.flush()
return
resp_body = r.read() resp_body = r.read()
status = r.status status = r.status
except urllib.error.HTTPError as exc: except urllib.error.HTTPError as exc:

View File

@@ -8,7 +8,8 @@ regular user.
Mutations are append-only events with unique ids — the same replication Mutations are append-only events with unique ids — the same replication
model as ``BillingLedger`` — so accounts and API keys converge across the model as ``BillingLedger`` — so accounts and API keys converge across the
tracker hive via gossip, and every dashboard can serve registration/login. tracker hive via gossip, and every dashboard can serve registration/login.
Sessions are deliberately local to each tracker (bearer tokens in memory). Sessions are local to each tracker and persisted so dashboard cookies survive
tracker restarts.
""" """
from __future__ import annotations from __future__ import annotations
@@ -115,6 +116,8 @@ class AccountStore:
"account_id": account_id, "account_id": account_id,
"expires": time.time() + SESSION_TTL, "expires": time.time() + SESSION_TTL,
} }
self._dirty = True
self.save_to_db()
return token return token
def session_account(self, token: str | None) -> dict | None: def session_account(self, token: str | None) -> dict | None:
@@ -134,7 +137,9 @@ class AccountStore:
if not token: if not token:
return return
with self._lock: with self._lock:
self._sessions.pop(token, None) if self._sessions.pop(token, None) is not None:
self._dirty = True
self.save_to_db()
# ---- API keys ---- # ---- API keys ----
@@ -271,6 +276,10 @@ class AccountStore:
"CREATE TABLE IF NOT EXISTS account_events " "CREATE TABLE IF NOT EXISTS account_events "
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)" "(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
) )
con.execute(
"CREATE TABLE IF NOT EXISTS account_sessions "
"(token TEXT PRIMARY KEY, account_id TEXT NOT NULL, expires REAL NOT NULL)"
)
con.commit() con.commit()
con.close() con.close()
@@ -279,6 +288,10 @@ class AccountStore:
rows = con.execute( rows = con.execute(
"SELECT payload FROM account_events ORDER BY ts, event_id" "SELECT payload FROM account_events ORDER BY ts, event_id"
).fetchall() ).fetchall()
session_rows = con.execute(
"SELECT token, account_id, expires FROM account_sessions WHERE expires >= ?",
(time.time(),),
).fetchall()
con.close() con.close()
with self._lock: with self._lock:
for (payload,) in rows: for (payload,) in rows:
@@ -288,6 +301,11 @@ class AccountStore:
continue continue
if event.get("id") not in self._seen_event_ids: if event.get("id") not in self._seen_event_ids:
self._apply_locked(event) self._apply_locked(event)
self._sessions = {
token: {"account_id": account_id, "expires": float(expires)}
for token, account_id, expires in session_rows
if account_id in self._accounts
}
self._dirty = False self._dirty = False
def save_to_db(self) -> None: def save_to_db(self) -> None:
@@ -297,11 +315,21 @@ class AccountStore:
if not self._dirty: if not self._dirty:
return return
events = list(self._event_log) events = list(self._event_log)
sessions = [
(token, session["account_id"], float(session["expires"]))
for token, session in self._sessions.items()
if session["expires"] >= time.time()
]
self._dirty = False self._dirty = False
con = sqlite3.connect(self._db_path) # type: ignore[arg-type] con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
con.executemany( con.executemany(
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)", "INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events], [(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
) )
con.execute("DELETE FROM account_sessions")
con.executemany(
"INSERT INTO account_sessions (token, account_id, expires) VALUES (?, ?, ?)",
sessions,
)
con.commit() con.commit()
con.close() con.close()

View File

@@ -9,6 +9,12 @@ from pathlib import Path
from .accounts import DEFAULT_ACCOUNTS_DB_PATH from .accounts import DEFAULT_ACCOUNTS_DB_PATH
from .billing import DEFAULT_BILLING_DB_PATH from .billing import DEFAULT_BILLING_DB_PATH
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
from .logging_setup import (
DEFAULT_LOG_BACKUP_COUNT,
DEFAULT_LOG_DIR,
DEFAULT_LOG_MAX_BYTES,
configure_tracker_file_logging,
)
from .routing_stats import RoutingConfig from .routing_stats import RoutingConfig
from .server import ( from .server import (
DEFAULT_CALLER_CREDIT_USDT, DEFAULT_CALLER_CREDIT_USDT,
@@ -302,6 +308,34 @@ def main() -> None:
metavar="SECONDS", metavar="SECONDS",
help="Half-life for decaying route throughput observations (default 600)", help="Half-life for decaying route throughput observations (default 600)",
) )
common.add_argument(
"--log-dir",
default=DEFAULT_LOG_DIR,
metavar="PATH",
help=(
"Directory for rotating tracker logs "
f"(default: {DEFAULT_LOG_DIR}; files: info.log, warning.log, error.log)"
),
)
common.add_argument(
"--log-max-bytes",
type=int,
default=DEFAULT_LOG_MAX_BYTES,
metavar="BYTES",
help=f"Rotate each tracker log file after this many bytes (default: {DEFAULT_LOG_MAX_BYTES})",
)
common.add_argument(
"--log-backup-count",
type=int,
default=DEFAULT_LOG_BACKUP_COUNT,
metavar="N",
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
)
common.add_argument(
"--no-file-logs",
action="store_true",
help="Disable rotating tracker log files and only write to the terminal",
)
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="meshnet-tracker", prog="meshnet-tracker",
@@ -315,6 +349,13 @@ def main() -> None:
args = parser.parse_args() args = parser.parse_args()
if args.command in {None, "start"}: if args.command in {None, "start"}:
if not args.no_file_logs:
log_dir = configure_tracker_file_logging(
args.log_dir,
max_bytes=args.log_max_bytes,
backup_count=args.log_backup_count,
)
print(f"meshnet-tracker logs: {log_dir}", flush=True)
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()] cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url) relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
treasury = None treasury = None

View File

@@ -234,6 +234,7 @@
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section> <section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section> <section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
<section data-tab="chat" class="wide chat-section"> <section data-tab="chat" class="wide chat-section">
<h2 style="display:none">Chat / inference</h2>
<div class="chat-app"> <div class="chat-app">
<aside class="chat-sidebar"> <aside class="chat-sidebar">
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button> <button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
@@ -283,7 +284,7 @@ async function fetchJson(path) {
const headers = {}; const headers = {};
const token = localStorage.getItem("meshnet_session"); const token = localStorage.getItem("meshnet_session");
if (token) headers["Authorization"] = "Bearer " + token; if (token) headers["Authorization"] = "Bearer " + token;
const r = await fetch(path, { headers }); const r = await fetch(path, { headers, credentials: "same-origin" });
if (!r.ok) return null; if (!r.ok) return null;
return await r.json(); return await r.json();
} catch { return null; } } catch { return null; }
@@ -1109,6 +1110,7 @@ async function apiCall(path, method, body, bearerToken) {
const r = await fetch(path, { const r = await fetch(path, {
method: method || "GET", method: method || "GET",
headers, headers,
credentials: "same-origin",
body: body ? JSON.stringify(body) : undefined, body: body ? JSON.stringify(body) : undefined,
}); });
const data = await r.json().catch(() => ({})); const data = await r.json().catch(() => ({}));
@@ -1368,10 +1370,11 @@ async function sendChat() {
promptEl.value = ""; promptEl.value = "";
promptEl.style.height = "auto"; promptEl.style.height = "auto";
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel }); chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
chatHistory.push(assistantMessage);
renderChatHistory(); renderChatHistory();
persistActiveChatSession(); persistActiveChatSession();
renderChatStatus("sending request…"); renderChatStatus("sending request…");
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
let tokens = 0; let tokens = 0;
let usage = null; let usage = null;
const started = Date.now(); const started = Date.now();
@@ -1382,6 +1385,7 @@ async function sendChat() {
const resp = await fetch("/v1/chat/completions", { const resp = await fetch("/v1/chat/completions", {
method: "POST", method: "POST",
headers, headers,
credentials: "same-origin",
body: JSON.stringify(body), body: JSON.stringify(body),
signal: chatAbortController.signal, signal: chatAbortController.signal,
}); });
@@ -1396,12 +1400,10 @@ async function sendChat() {
const contentType = resp.headers.get("Content-Type") || ""; const contentType = resp.headers.get("Content-Type") || "";
if (!resp.body || !contentType.includes("text/event-stream")) { if (!resp.body || !contentType.includes("text/event-stream")) {
const data = await resp.json(); const data = await resp.json();
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || ""; assistantMessage.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
usage = data.usage || null; usage = data.usage || null;
tokens = (usage && usage.completion_tokens) || 0; tokens = (usage && usage.completion_tokens) || 0;
} else { } else {
chatHistory.push(assistantMsg);
renderChatHistory();
const reader = resp.body.getReader(); const reader = resp.body.getReader();
const decoder = new TextDecoder(); const decoder = new TextDecoder();
let buffered = ""; let buffered = "";
@@ -1425,30 +1427,31 @@ async function sendChat() {
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta; const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
const piece = delta && delta.content; const piece = delta && delta.content;
if (piece) { if (piece) {
assistantMsg.content += piece; assistantMessage.content += piece;
tokens += 1; tokens += 1;
updateStreamingChatBubble(assistantMsg.content); updateStreamingChatBubble(assistantMessage.content);
const secs = Math.max((Date.now() - started) / 1000, 0.001); const secs = Math.max((Date.now() - started) / 1000, 0.001);
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`); renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
} }
} }
} }
} }
finalizeAssistantMessage(assistantMsg); finalizeAssistantMessage(assistantMessage);
renderChatStatus(usage renderChatStatus(usage
? `done: ${usage.total_tokens ?? "?"} tokens` ? `done: ${usage.total_tokens ?? "?"} tokens`
: `done: ${tokens} tokens`); : `done: ${tokens} tokens`);
} catch (err) { } catch (err) {
if (err && err.name === "AbortError") { if (err && err.name === "AbortError") {
if (assistantMsg.content) { if (assistantMessage.content) {
finalizeAssistantMessage(assistantMsg); finalizeAssistantMessage(assistantMessage);
renderChatStatus(`stopped after ${tokens} tokens`); renderChatStatus(`stopped after ${tokens} tokens`);
} else { } else {
removeAssistantMessage(assistantMsg); removeAssistantMessage(assistantMessage);
persistActiveChatSession();
renderChatStatus("stopped"); renderChatStatus("stopped");
} }
} else { } else {
removeAssistantMessage(assistantMsg); removeAssistantMessage(assistantMessage);
const error = (err && err.message) || "request failed"; const error = (err && err.message) || "request failed";
chatHistory.push({ role: "error", content: error, model: selectedChatModel }); chatHistory.push({ role: "error", content: error, model: selectedChatModel });
renderChatHistory(); renderChatHistory();
@@ -1557,7 +1560,7 @@ renderChatModels();
renderChatHistory(); renderChatHistory();
renderChatAuthHint(); renderChatAuthHint();
setInterval(refresh, 4000); setInterval(refresh, 4000);
setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000); setInterval(() => { if (sessionToken || isLoggedIn) renderAccountPanel(); }, 8000);
</script> </script>
</body> </body>
</html> </html>

View File

@@ -0,0 +1,99 @@
"""Rotating file logging for the tracker CLI."""
from __future__ import annotations
import logging
import sys
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import TextIO
DEFAULT_LOG_DIR = "logs/tracker"
DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
DEFAULT_LOG_BACKUP_COUNT = 5
TRACKER_LOGGER_NAME = "meshnet.tracker"
class _ExactLevelFilter(logging.Filter):
def __init__(self, level: int) -> None:
super().__init__()
self._level = level
def filter(self, record: logging.LogRecord) -> bool:
return record.levelno == self._level
class _TeeStream:
def __init__(self, stream: TextIO, logger: logging.Logger, level: int) -> None:
self._stream = stream
self._logger = logger
self._level = level
self._buffer = ""
def write(self, text: str) -> int:
self._stream.write(text)
self._stream.flush()
self._buffer += text
while "\n" in self._buffer:
line, self._buffer = self._buffer.split("\n", 1)
line = line.rstrip()
if line:
self._logger.log(self._level, line)
return len(text)
def flush(self) -> None:
self._stream.flush()
line = self._buffer.rstrip()
if line:
self._logger.log(self._level, line)
self._buffer = ""
def isatty(self) -> bool:
return self._stream.isatty()
def _make_handler(path: Path, level: int, *, max_bytes: int, backup_count: int) -> RotatingFileHandler:
handler = RotatingFileHandler(
path,
maxBytes=max_bytes,
backupCount=backup_count,
encoding="utf-8",
)
handler.setLevel(level)
handler.addFilter(_ExactLevelFilter(level))
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
return handler
def configure_tracker_file_logging(
log_dir: str | Path = DEFAULT_LOG_DIR,
*,
max_bytes: int = DEFAULT_LOG_MAX_BYTES,
backup_count: int = DEFAULT_LOG_BACKUP_COUNT,
tee_stdio: bool = True,
) -> Path:
"""Configure rotatable info/warning/error log files and return the directory."""
path = Path(log_dir).expanduser()
path.mkdir(parents=True, exist_ok=True)
logger = logging.getLogger(TRACKER_LOGGER_NAME)
logger.setLevel(logging.INFO)
logger.propagate = False
logger.handlers.clear()
logger.addHandler(_make_handler(path / "info.log", logging.INFO, max_bytes=max_bytes, backup_count=backup_count))
logger.addHandler(_make_handler(path / "warning.log", logging.WARNING, max_bytes=max_bytes, backup_count=backup_count))
logger.addHandler(_make_handler(path / "error.log", logging.ERROR, max_bytes=max_bytes, backup_count=backup_count))
if tee_stdio:
if not isinstance(sys.stdout, _TeeStream):
sys.stdout = _TeeStream(sys.stdout, logger, logging.INFO) # type: ignore[assignment]
if not isinstance(sys.stderr, _TeeStream):
sys.stderr = _TeeStream(sys.stderr, logger, logging.ERROR) # type: ignore[assignment]
return path
def tracker_logger() -> logging.Logger:
return logging.getLogger(TRACKER_LOGGER_NAME)

File diff suppressed because it is too large Load Diff

View File

@@ -5,6 +5,7 @@ register/login/logout, per-account balance and usage, API-key lifecycle
(revoked keys rejected by the OpenAI proxy), and the admin listing. (revoked keys rejected by the OpenAI proxy), and the admin listing.
""" """
import http.cookies
import json import json
import urllib.error import urllib.error
import urllib.request import urllib.request
@@ -68,6 +69,17 @@ def test_sessions_resolve_and_destroy():
assert store.session_account("bogus") is None assert store.session_account("bogus") is None
def test_sessions_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="cookie@example.com", password="secret-123")
token = store.create_session(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.session_account(token)["account_id"] == account["account_id"]
def test_api_key_lifecycle(): def test_api_key_lifecycle():
store = AccountStore() store = AccountStore()
account = store.register(email="k@example.com", password="secret-123") account = store.register(email="k@example.com", password="secret-123")
@@ -156,6 +168,59 @@ def test_register_login_and_account_view(account_tracker):
assert me["usage"]["requests"] == 0 assert me["usage"]["requests"] == 0
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
accounts_db = str(tmp_path / "accounts.db")
tracker = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_call(f"{url}/v1/auth/register", "POST",
{"email": "cookie-http@example.com", "password": "secret-123"})
req = urllib.request.Request(
f"{url}/v1/auth/login",
data=json.dumps({
"identifier": "cookie-http@example.com",
"password": "secret-123",
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req) as r:
assert json.loads(r.read())["session_token"]
cookie_header = r.headers["Set-Cookie"]
finally:
tracker.stop()
cookie = http.cookies.SimpleCookie(cookie_header)
session_cookie = cookie["meshnet_session"].OutputString()
restarted = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
restarted_port = restarted.start()
restarted_url = f"http://127.0.0.1:{restarted_port}"
try:
req = urllib.request.Request(
f"{restarted_url}/v1/account",
headers={"Cookie": session_cookie},
method="GET",
)
with urllib.request.urlopen(req) as r:
me = json.loads(r.read())
finally:
restarted.stop()
assert me["account"]["email"] == "cookie-http@example.com"
def test_bad_credentials_and_missing_session_are_401(account_tracker): def test_bad_credentials_and_missing_session_are_401(account_tracker):
url, _ = account_tracker url, _ = account_tracker
_call(f"{url}/v1/auth/register", "POST", _call(f"{url}/v1/auth/register", "POST",

View File

@@ -1,138 +1,153 @@
"""US-035: tracker web dashboard — served from any tracker, embedded asset.""" """US-035: tracker web dashboard — served from any tracker, embedded asset."""
import json import json
import time import time
import urllib.request import urllib.request
from meshnet_contracts import LocalSolanaContracts from meshnet_contracts import LocalSolanaContracts
from meshnet_tracker.accounts import AccountStore from meshnet_tracker.accounts import AccountStore
from meshnet_tracker.billing import BillingLedger from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer from meshnet_tracker.server import TrackerServer
PANELS = [ PANELS = [
"Tracker hive", "Nodes &amp; coverage", "Client balances", "Tracker hive", "Nodes &amp; coverage", "Client balances",
"Node pending payouts", "Settlement history", "Node pending payouts", "Settlement history",
"Strikes / bans / forfeitures", "Model usage", "Call wall", "Strikes / bans / forfeitures", "Model usage", "Call wall",
"Usage summary", "Node throughput", "Request history", "Usage summary", "Node throughput", "Request history",
'id="tab-chat"', "Chat / inference",
"Console output", "Console output",
] ]
def test_dashboard_served_with_all_panels(): def test_dashboard_served_with_all_panels():
tracker = TrackerServer(billing=BillingLedger()) tracker = TrackerServer(billing=BillingLedger())
port = tracker.start() port = tracker.start()
try: try:
html = urllib.request.urlopen( html = urllib.request.urlopen(
f"http://127.0.0.1:{port}/dashboard" f"http://127.0.0.1:{port}/dashboard"
).read().decode() ).read().decode()
for panel in PANELS: for panel in PANELS:
assert panel in html assert panel in html
assert "<script>" in html # polling client embedded, no build step assert "<script>" in html # polling client embedded, no build step
finally: finally:
tracker.stop() tracker.stop()
def test_dashboard_served_by_follower(): def test_dashboard_chat_uses_streaming_fetch():
"""A tracker that is not the leader (unreachable peers → never elected) tracker = TrackerServer(billing=BillingLedger())
still serves the dashboard from its own replicated state.""" port = tracker.start()
tracker = TrackerServer( try:
billing=BillingLedger(), html = urllib.request.urlopen(
cluster_peers=["http://127.0.0.1:1", "http://127.0.0.1:2"], f"http://127.0.0.1:{port}/dashboard"
) ).read().decode()
port = tracker.start() finally:
try: tracker.stop()
response = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard")
assert response.status == 200 assert "stream: true" in html
assert "meshnet tracker" in response.read().decode() assert ".body.getReader()" in html
finally: assert '=== "[DONE]"' in html
tracker.stop()
def test_dashboard_served_by_follower():
def test_registry_wallets_endpoint(): """A tracker that is not the leader (unreachable peers → never elected)
contracts = LocalSolanaContracts() still serves the dashboard from its own replicated state."""
contracts.registry.submit_stake("wallet-a", 100) tracker = TrackerServer(
contracts.registry.record_strike("wallet-a") billing=BillingLedger(),
accounts = AccountStore() cluster_peers=["http://127.0.0.1:1", "http://127.0.0.1:2"],
admin = accounts.register(email="admin@example.com", password="admin-pass-123") )
session = accounts.create_session(admin["account_id"]) port = tracker.start()
tracker = TrackerServer(contracts=contracts, accounts=accounts) try:
port = tracker.start() response = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard")
try: assert response.status == 200
req = urllib.request.Request( assert "meshnet tracker" in response.read().decode()
f"http://127.0.0.1:{port}/v1/registry/wallets", finally:
headers={"Authorization": f"Bearer {session}"}, tracker.stop()
)
data = json.loads(urllib.request.urlopen(req).read())
assert data["wallets"]["wallet-a"]["strike_count"] == 1 def test_registry_wallets_endpoint():
assert data["wallets"]["wallet-a"]["banned"] is False contracts = LocalSolanaContracts()
finally: contracts.registry.submit_stake("wallet-a", 100)
tracker.stop() contracts.registry.record_strike("wallet-a")
accounts = AccountStore()
admin = accounts.register(email="admin@example.com", password="admin-pass-123")
def test_console_endpoint_exposes_tracker_events(): session = accounts.create_session(admin["account_id"])
tracker = TrackerServer() tracker = TrackerServer(contracts=contracts, accounts=accounts)
port = tracker.start() port = tracker.start()
try: try:
body = json.dumps({ req = urllib.request.Request(
"endpoint": "http://127.0.0.1:9001", f"http://127.0.0.1:{port}/v1/registry/wallets",
"model": "stub-model", headers={"Authorization": f"Bearer {session}"},
"shard_start": 0, )
"shard_end": 3, data = json.loads(urllib.request.urlopen(req).read())
"hardware_profile": {}, assert data["wallets"]["wallet-a"]["strike_count"] == 1
}).encode() assert data["wallets"]["wallet-a"]["banned"] is False
req = urllib.request.Request( finally:
f"http://127.0.0.1:{port}/v1/nodes/register", tracker.stop()
data=body,
headers={"Content-Type": "application/json"},
method="POST", def test_console_endpoint_exposes_tracker_events():
) tracker = TrackerServer()
urllib.request.urlopen(req).read() port = tracker.start()
try:
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read()) body = json.dumps({
finally: "endpoint": "http://127.0.0.1:9001",
tracker.stop() "model": "stub-model",
"shard_start": 0,
assert any(event["message"] == "node registered" for event in data["events"]) "shard_end": 3,
"hardware_profile": {},
}).encode()
def test_console_node_lifecycle_events_include_model_health(): req = urllib.request.Request(
tracker = TrackerServer(heartbeat_timeout=0.05) f"http://127.0.0.1:{port}/v1/nodes/register",
port = tracker.start() data=body,
try: headers={"Content-Type": "application/json"},
body = json.dumps({ method="POST",
"endpoint": "http://127.0.0.1:9002", )
"model": "console-health-test", urllib.request.urlopen(req).read()
"hf_repo": "example/console-health-test",
"num_layers": 4, data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
"shard_start": 0, finally:
"shard_end": 1, tracker.stop()
"hardware_profile": {},
}).encode() assert any(event["message"] == "node registered" for event in data["events"])
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=body, def test_console_node_lifecycle_events_include_model_health():
headers={"Content-Type": "application/json"}, tracker = TrackerServer(heartbeat_timeout=0.05)
method="POST", port = tracker.start()
) try:
urllib.request.urlopen(req).read() body = json.dumps({
"endpoint": "http://127.0.0.1:9002",
registered = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read()) "model": "console-health-test",
registered_event = next( "hf_repo": "example/console-health-test",
event for event in registered["events"] "num_layers": 4,
if event["message"] == "node registered" "shard_start": 0,
) "shard_end": 1,
assert registered_event["fields"]["model_health"]["served_model_copies"] == 0.5 "hardware_profile": {},
assert registered_event["fields"]["model_health"]["coverage_percentage"] == 50.0 }).encode()
req = urllib.request.Request(
time.sleep(0.06) f"http://127.0.0.1:{port}/v1/nodes/register",
urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map").read() data=body,
expired = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read()) headers={"Content-Type": "application/json"},
expired_event = next( method="POST",
event for event in expired["events"] )
if event["message"] == "node expired" urllib.request.urlopen(req).read()
)
assert expired_event["fields"]["model_health"]["served_model_copies"] == 0.0 registered = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
assert expired_event["fields"]["model_health"]["coverage_percentage"] == 0.0 registered_event = next(
finally: event for event in registered["events"]
tracker.stop() if event["message"] == "node registered"
)
assert registered_event["fields"]["model_health"]["served_model_copies"] == 0.5
assert registered_event["fields"]["model_health"]["coverage_percentage"] == 50.0
time.sleep(0.06)
urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map").read()
expired = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
expired_event = next(
event for event in expired["events"]
if event["message"] == "node expired"
)
assert expired_event["fields"]["model_health"]["served_model_copies"] == 0.0
assert expired_event["fields"]["model_health"]["coverage_percentage"] == 0.0
finally:
tracker.stop()

View File

@@ -142,6 +142,21 @@ def _send_chat_request(gateway_url: str, prompt: str) -> dict:
return json.loads(r.read()) return json.loads(r.read())
def _send_streaming_chat_request(gateway_url: str, prompt: str):
data = json.dumps({
"model": GPT2_MODEL,
"messages": [{"role": "user", "content": prompt}],
"stream": True,
}).encode()
req = urllib.request.Request(
f"{gateway_url}/v1/chat/completions",
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
return urllib.request.urlopen(req)
def test_all_responses_valid_openai_format(tracker_node_setup): def test_all_responses_valid_openai_format(tracker_node_setup):
"""Ten requests via gateway all return valid OpenAI chat completion format.""" """Ten requests via gateway all return valid OpenAI chat completion format."""
gateway_url, _, _ = tracker_node_setup gateway_url, _, _ = tracker_node_setup
@@ -155,6 +170,30 @@ def test_all_responses_valid_openai_format(tracker_node_setup):
assert isinstance(message.get("content"), str), f"request {i}: content must be a string" assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
def test_streaming_head_worker_response_is_not_buffered_with_content_length(tracker_node_setup):
"""Gateway must relay head-worker SSE as a live stream, not a buffered JSON-sized body."""
gateway_url, _, _ = tracker_node_setup
with _send_streaming_chat_request(gateway_url, "stream through head worker") as resp:
assert resp.status == 200
assert "text/event-stream" in resp.headers["Content-Type"]
assert "Content-Length" not in resp.headers
data_lines = []
while len(data_lines) < 4:
line = resp.readline().decode().strip()
if line.startswith("data: "):
data_lines.append(line)
if line == "data: [DONE]":
break
assert data_lines[-1] == "data: [DONE]"
content = "".join(
json.loads(line[6:])["choices"][0].get("delta", {}).get("content", "")
for line in data_lines[:-1]
)
assert "head-worker" in content
def test_both_tracker_nodes_receive_load(tracker_node_setup): def test_both_tracker_nodes_receive_load(tracker_node_setup):
"""Both head workers handle at least one request each out of ten.""" """Both head workers handle at least one request each out of ten."""
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup

View File

@@ -0,0 +1,67 @@
import logging
import sys
from meshnet_tracker.logging_setup import configure_tracker_file_logging, tracker_logger
def test_tracker_file_logging_writes_separate_level_files(tmp_path):
original_stdout = sys.stdout
original_stderr = sys.stderr
try:
log_dir = configure_tracker_file_logging(tmp_path, tee_stdio=False)
logger = tracker_logger()
logger.info("info-event")
logger.warning("warning-event")
logger.error("error-event")
for handler in logger.handlers:
handler.flush()
assert (log_dir / "info.log").read_text().count("info-event") == 1
assert "warning-event" not in (log_dir / "info.log").read_text()
assert "error-event" not in (log_dir / "info.log").read_text()
assert "warning-event" in (log_dir / "warning.log").read_text()
assert "info-event" not in (log_dir / "warning.log").read_text()
assert "error-event" not in (log_dir / "warning.log").read_text()
assert "error-event" in (log_dir / "error.log").read_text()
assert "info-event" not in (log_dir / "error.log").read_text()
assert "warning-event" not in (log_dir / "error.log").read_text()
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
def test_tracker_file_logging_tees_stdio_and_rotates(tmp_path):
original_stdout = sys.stdout
original_stderr = sys.stderr
try:
log_dir = configure_tracker_file_logging(
tmp_path,
max_bytes=120,
backup_count=1,
)
print("stdout goes to info", flush=True)
print("stderr goes to error", file=sys.stderr, flush=True)
for handler in tracker_logger().handlers:
handler.flush()
assert "stdout goes to info" in (log_dir / "info.log").read_text()
assert "stderr goes to error" in (log_dir / "error.log").read_text()
for index in range(12):
tracker_logger().info("rotating-info-line-%02d", index)
for handler in tracker_logger().handlers:
handler.flush()
assert (log_dir / "info.log.1").exists()
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
logger = tracker_logger()
for handler in logger.handlers:
handler.close()
logger.handlers.clear()
logger.setLevel(logging.NOTSET)

View File

@@ -502,6 +502,75 @@ def test_tracker_logs_stream_progress_before_request_completes():
node_thread.join(timeout=1.0) node_thread.join(timeout=1.0)
def test_tracker_stream_survives_idle_gap_between_sse_chunks():
first_chunk_sent = threading.Event()
class IdleStreamingChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, format, *args):
pass
def do_POST(self):
if self.path != "/v1/chat/completions":
self.send_response(404)
self.end_headers()
return
self.rfile.read(int(self.headers.get("Content-Length", 0)))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
self.end_headers()
first = json.dumps({
"choices": [{"delta": {"content": "hello"}}],
}).encode()
second = json.dumps({
"choices": [{"delta": {"content": " world"}}],
}).encode()
self.wfile.write(b"data: " + first + b"\n\n")
self.wfile.flush()
first_chunk_sent.set()
time.sleep(1.0)
self.wfile.write(b"data: " + second + b"\n\n")
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker_port = tracker.start()
response = None
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
"model": "idle-stream-model", "num_layers": 1,
"shard_start": 0, "shard_end": 0,
"hardware_profile": {}, "score": 1.0},
)
req = urllib.request.Request(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
data=json.dumps({
"model": "idle-stream-model",
"stream": True,
"messages": [{"role": "user", "content": "hi"}],
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
response = urllib.request.urlopen(req, timeout=3.0)
assert response.readline().startswith(b"data:")
assert first_chunk_sent.wait(timeout=1.0)
remaining = response.read().splitlines()
assert b"data: [DONE]" in remaining
finally:
if response is not None:
response.close()
tracker.stop()
node.shutdown()
node.server_close()
node_thread.join(timeout=1.0)
def test_tracker_dashboard_can_cancel_inflight_proxy(): def test_tracker_dashboard_can_cancel_inflight_proxy():
chunk_sent = threading.Event() chunk_sent = threading.Event()
release = threading.Event() release = threading.Event()