# Conflicts:
#	packages/tracker/meshnet_tracker/cli.py
#	packages/tracker/meshnet_tracker/dashboard.html
#	packages/tracker/meshnet_tracker/server.py
#	tests/test_dashboard.py
This commit is contained in:
Dobromir Popov
2026-07-08 16:14:24 +02:00
12 changed files with 6759 additions and 6249 deletions

3
.gitignore vendored
View File

@@ -23,3 +23,6 @@ dist/
# Local tracker/node sqlite databases (never commit runtime state) # Local tracker/node sqlite databases (never commit runtime state)
*.sqlite *.sqlite
*.sqlite3 *.sqlite3
logs/tracker/error.log
logs/tracker/info.log
logs/tracker/warning.log

View File

@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, completion) self._send_json(200, completion)
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None: def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
"""Forward a raw request body to a head worker and stream the response back.""" """Forward a raw request body to a head worker and relay SSE without buffering."""
target_url = f"{url}/v1/chat/completions" target_url = f"{url}/v1/chat/completions"
req = urllib.request.Request( req = urllib.request.Request(
target_url, target_url,
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
try: try:
with urllib.request.urlopen(req, timeout=30.0) as r: with urllib.request.urlopen(req, timeout=30.0) as r:
content_type = r.headers.get("Content-Type", "application/json") content_type = r.headers.get("Content-Type", "application/json")
if "text/event-stream" in content_type:
self.send_response(r.status)
self.send_header("Content-Type", content_type)
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.end_headers()
while True:
line = r.readline()
if not line:
break
self.wfile.write(line)
self.wfile.flush()
return
resp_body = r.read() resp_body = r.read()
status = r.status status = r.status
except urllib.error.HTTPError as exc: except urllib.error.HTTPError as exc:

View File

@@ -8,7 +8,8 @@ regular user.
Mutations are append-only events with unique ids — the same replication Mutations are append-only events with unique ids — the same replication
model as ``BillingLedger`` — so accounts and API keys converge across the model as ``BillingLedger`` — so accounts and API keys converge across the
tracker hive via gossip, and every dashboard can serve registration/login. tracker hive via gossip, and every dashboard can serve registration/login.
Sessions are deliberately local to each tracker (bearer tokens in memory). Sessions are local to each tracker and persisted so dashboard cookies survive
tracker restarts.
""" """
from __future__ import annotations from __future__ import annotations
@@ -115,6 +116,8 @@ class AccountStore:
"account_id": account_id, "account_id": account_id,
"expires": time.time() + SESSION_TTL, "expires": time.time() + SESSION_TTL,
} }
self._dirty = True
self.save_to_db()
return token return token
def session_account(self, token: str | None) -> dict | None: def session_account(self, token: str | None) -> dict | None:
@@ -134,7 +137,9 @@ class AccountStore:
if not token: if not token:
return return
with self._lock: with self._lock:
self._sessions.pop(token, None) if self._sessions.pop(token, None) is not None:
self._dirty = True
self.save_to_db()
# ---- API keys ---- # ---- API keys ----
@@ -271,6 +276,10 @@ class AccountStore:
"CREATE TABLE IF NOT EXISTS account_events " "CREATE TABLE IF NOT EXISTS account_events "
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)" "(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
) )
con.execute(
"CREATE TABLE IF NOT EXISTS account_sessions "
"(token TEXT PRIMARY KEY, account_id TEXT NOT NULL, expires REAL NOT NULL)"
)
con.commit() con.commit()
con.close() con.close()
@@ -279,6 +288,10 @@ class AccountStore:
rows = con.execute( rows = con.execute(
"SELECT payload FROM account_events ORDER BY ts, event_id" "SELECT payload FROM account_events ORDER BY ts, event_id"
).fetchall() ).fetchall()
session_rows = con.execute(
"SELECT token, account_id, expires FROM account_sessions WHERE expires >= ?",
(time.time(),),
).fetchall()
con.close() con.close()
with self._lock: with self._lock:
for (payload,) in rows: for (payload,) in rows:
@@ -288,6 +301,11 @@ class AccountStore:
continue continue
if event.get("id") not in self._seen_event_ids: if event.get("id") not in self._seen_event_ids:
self._apply_locked(event) self._apply_locked(event)
self._sessions = {
token: {"account_id": account_id, "expires": float(expires)}
for token, account_id, expires in session_rows
if account_id in self._accounts
}
self._dirty = False self._dirty = False
def save_to_db(self) -> None: def save_to_db(self) -> None:
@@ -297,11 +315,21 @@ class AccountStore:
if not self._dirty: if not self._dirty:
return return
events = list(self._event_log) events = list(self._event_log)
sessions = [
(token, session["account_id"], float(session["expires"]))
for token, session in self._sessions.items()
if session["expires"] >= time.time()
]
self._dirty = False self._dirty = False
con = sqlite3.connect(self._db_path) # type: ignore[arg-type] con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
con.executemany( con.executemany(
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)", "INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events], [(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
) )
con.execute("DELETE FROM account_sessions")
con.executemany(
"INSERT INTO account_sessions (token, account_id, expires) VALUES (?, ?, ?)",
sessions,
)
con.commit() con.commit()
con.close() con.close()

View File

@@ -9,6 +9,12 @@ from pathlib import Path
from .accounts import DEFAULT_ACCOUNTS_DB_PATH from .accounts import DEFAULT_ACCOUNTS_DB_PATH
from .billing import DEFAULT_BILLING_DB_PATH from .billing import DEFAULT_BILLING_DB_PATH
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
from .logging_setup import (
DEFAULT_LOG_BACKUP_COUNT,
DEFAULT_LOG_DIR,
DEFAULT_LOG_MAX_BYTES,
configure_tracker_file_logging,
)
from .routing_stats import RoutingConfig from .routing_stats import RoutingConfig
from .server import ( from .server import (
DEFAULT_CALLER_CREDIT_USDT, DEFAULT_CALLER_CREDIT_USDT,
@@ -302,6 +308,34 @@ def main() -> None:
metavar="SECONDS", metavar="SECONDS",
help="Half-life for decaying route throughput observations (default 600)", help="Half-life for decaying route throughput observations (default 600)",
) )
common.add_argument(
"--log-dir",
default=DEFAULT_LOG_DIR,
metavar="PATH",
help=(
"Directory for rotating tracker logs "
f"(default: {DEFAULT_LOG_DIR}; files: info.log, warning.log, error.log)"
),
)
common.add_argument(
"--log-max-bytes",
type=int,
default=DEFAULT_LOG_MAX_BYTES,
metavar="BYTES",
help=f"Rotate each tracker log file after this many bytes (default: {DEFAULT_LOG_MAX_BYTES})",
)
common.add_argument(
"--log-backup-count",
type=int,
default=DEFAULT_LOG_BACKUP_COUNT,
metavar="N",
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
)
common.add_argument(
"--no-file-logs",
action="store_true",
help="Disable rotating tracker log files and only write to the terminal",
)
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
prog="meshnet-tracker", prog="meshnet-tracker",
@@ -315,6 +349,13 @@ def main() -> None:
args = parser.parse_args() args = parser.parse_args()
if args.command in {None, "start"}: if args.command in {None, "start"}:
if not args.no_file_logs:
log_dir = configure_tracker_file_logging(
args.log_dir,
max_bytes=args.log_max_bytes,
backup_count=args.log_backup_count,
)
print(f"meshnet-tracker logs: {log_dir}", flush=True)
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()] cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url) relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
treasury = None treasury = None

View File

@@ -234,6 +234,7 @@
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section> <section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section> <section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
<section data-tab="chat" class="wide chat-section"> <section data-tab="chat" class="wide chat-section">
<h2 style="display:none">Chat / inference</h2>
<div class="chat-app"> <div class="chat-app">
<aside class="chat-sidebar"> <aside class="chat-sidebar">
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button> <button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
@@ -283,7 +284,7 @@ async function fetchJson(path) {
const headers = {}; const headers = {};
const token = localStorage.getItem("meshnet_session"); const token = localStorage.getItem("meshnet_session");
if (token) headers["Authorization"] = "Bearer " + token; if (token) headers["Authorization"] = "Bearer " + token;
const r = await fetch(path, { headers }); const r = await fetch(path, { headers, credentials: "same-origin" });
if (!r.ok) return null; if (!r.ok) return null;
return await r.json(); return await r.json();
} catch { return null; } } catch { return null; }
@@ -1109,6 +1110,7 @@ async function apiCall(path, method, body, bearerToken) {
const r = await fetch(path, { const r = await fetch(path, {
method: method || "GET", method: method || "GET",
headers, headers,
credentials: "same-origin",
body: body ? JSON.stringify(body) : undefined, body: body ? JSON.stringify(body) : undefined,
}); });
const data = await r.json().catch(() => ({})); const data = await r.json().catch(() => ({}));
@@ -1368,10 +1370,11 @@ async function sendChat() {
promptEl.value = ""; promptEl.value = "";
promptEl.style.height = "auto"; promptEl.style.height = "auto";
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel }); chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
chatHistory.push(assistantMessage);
renderChatHistory(); renderChatHistory();
persistActiveChatSession(); persistActiveChatSession();
renderChatStatus("sending request…"); renderChatStatus("sending request…");
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
let tokens = 0; let tokens = 0;
let usage = null; let usage = null;
const started = Date.now(); const started = Date.now();
@@ -1382,6 +1385,7 @@ async function sendChat() {
const resp = await fetch("/v1/chat/completions", { const resp = await fetch("/v1/chat/completions", {
method: "POST", method: "POST",
headers, headers,
credentials: "same-origin",
body: JSON.stringify(body), body: JSON.stringify(body),
signal: chatAbortController.signal, signal: chatAbortController.signal,
}); });
@@ -1396,12 +1400,10 @@ async function sendChat() {
const contentType = resp.headers.get("Content-Type") || ""; const contentType = resp.headers.get("Content-Type") || "";
if (!resp.body || !contentType.includes("text/event-stream")) { if (!resp.body || !contentType.includes("text/event-stream")) {
const data = await resp.json(); const data = await resp.json();
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || ""; assistantMessage.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
usage = data.usage || null; usage = data.usage || null;
tokens = (usage && usage.completion_tokens) || 0; tokens = (usage && usage.completion_tokens) || 0;
} else { } else {
chatHistory.push(assistantMsg);
renderChatHistory();
const reader = resp.body.getReader(); const reader = resp.body.getReader();
const decoder = new TextDecoder(); const decoder = new TextDecoder();
let buffered = ""; let buffered = "";
@@ -1425,30 +1427,31 @@ async function sendChat() {
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta; const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
const piece = delta && delta.content; const piece = delta && delta.content;
if (piece) { if (piece) {
assistantMsg.content += piece; assistantMessage.content += piece;
tokens += 1; tokens += 1;
updateStreamingChatBubble(assistantMsg.content); updateStreamingChatBubble(assistantMessage.content);
const secs = Math.max((Date.now() - started) / 1000, 0.001); const secs = Math.max((Date.now() - started) / 1000, 0.001);
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`); renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
} }
} }
} }
} }
finalizeAssistantMessage(assistantMsg); finalizeAssistantMessage(assistantMessage);
renderChatStatus(usage renderChatStatus(usage
? `done: ${usage.total_tokens ?? "?"} tokens` ? `done: ${usage.total_tokens ?? "?"} tokens`
: `done: ${tokens} tokens`); : `done: ${tokens} tokens`);
} catch (err) { } catch (err) {
if (err && err.name === "AbortError") { if (err && err.name === "AbortError") {
if (assistantMsg.content) { if (assistantMessage.content) {
finalizeAssistantMessage(assistantMsg); finalizeAssistantMessage(assistantMessage);
renderChatStatus(`stopped after ${tokens} tokens`); renderChatStatus(`stopped after ${tokens} tokens`);
} else { } else {
removeAssistantMessage(assistantMsg); removeAssistantMessage(assistantMessage);
persistActiveChatSession();
renderChatStatus("stopped"); renderChatStatus("stopped");
} }
} else { } else {
removeAssistantMessage(assistantMsg); removeAssistantMessage(assistantMessage);
const error = (err && err.message) || "request failed"; const error = (err && err.message) || "request failed";
chatHistory.push({ role: "error", content: error, model: selectedChatModel }); chatHistory.push({ role: "error", content: error, model: selectedChatModel });
renderChatHistory(); renderChatHistory();
@@ -1557,7 +1560,7 @@ renderChatModels();
renderChatHistory(); renderChatHistory();
renderChatAuthHint(); renderChatAuthHint();
setInterval(refresh, 4000); setInterval(refresh, 4000);
setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000); setInterval(() => { if (sessionToken || isLoggedIn) renderAccountPanel(); }, 8000);
</script> </script>
</body> </body>
</html> </html>

View File

@@ -0,0 +1,99 @@
"""Rotating file logging for the tracker CLI."""
from __future__ import annotations
import logging
import sys
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import TextIO
DEFAULT_LOG_DIR = "logs/tracker"
DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
DEFAULT_LOG_BACKUP_COUNT = 5
TRACKER_LOGGER_NAME = "meshnet.tracker"
class _ExactLevelFilter(logging.Filter):
def __init__(self, level: int) -> None:
super().__init__()
self._level = level
def filter(self, record: logging.LogRecord) -> bool:
return record.levelno == self._level
class _TeeStream:
def __init__(self, stream: TextIO, logger: logging.Logger, level: int) -> None:
self._stream = stream
self._logger = logger
self._level = level
self._buffer = ""
def write(self, text: str) -> int:
self._stream.write(text)
self._stream.flush()
self._buffer += text
while "\n" in self._buffer:
line, self._buffer = self._buffer.split("\n", 1)
line = line.rstrip()
if line:
self._logger.log(self._level, line)
return len(text)
def flush(self) -> None:
self._stream.flush()
line = self._buffer.rstrip()
if line:
self._logger.log(self._level, line)
self._buffer = ""
def isatty(self) -> bool:
return self._stream.isatty()
def _make_handler(path: Path, level: int, *, max_bytes: int, backup_count: int) -> RotatingFileHandler:
handler = RotatingFileHandler(
path,
maxBytes=max_bytes,
backupCount=backup_count,
encoding="utf-8",
)
handler.setLevel(level)
handler.addFilter(_ExactLevelFilter(level))
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
return handler
def configure_tracker_file_logging(
log_dir: str | Path = DEFAULT_LOG_DIR,
*,
max_bytes: int = DEFAULT_LOG_MAX_BYTES,
backup_count: int = DEFAULT_LOG_BACKUP_COUNT,
tee_stdio: bool = True,
) -> Path:
"""Configure rotatable info/warning/error log files and return the directory."""
path = Path(log_dir).expanduser()
path.mkdir(parents=True, exist_ok=True)
logger = logging.getLogger(TRACKER_LOGGER_NAME)
logger.setLevel(logging.INFO)
logger.propagate = False
logger.handlers.clear()
logger.addHandler(_make_handler(path / "info.log", logging.INFO, max_bytes=max_bytes, backup_count=backup_count))
logger.addHandler(_make_handler(path / "warning.log", logging.WARNING, max_bytes=max_bytes, backup_count=backup_count))
logger.addHandler(_make_handler(path / "error.log", logging.ERROR, max_bytes=max_bytes, backup_count=backup_count))
if tee_stdio:
if not isinstance(sys.stdout, _TeeStream):
sys.stdout = _TeeStream(sys.stdout, logger, logging.INFO) # type: ignore[assignment]
if not isinstance(sys.stderr, _TeeStream):
sys.stderr = _TeeStream(sys.stderr, logger, logging.ERROR) # type: ignore[assignment]
return path
def tracker_logger() -> logging.Logger:
return logging.getLogger(TRACKER_LOGGER_NAME)

View File

@@ -23,12 +23,14 @@ HTTP API contract:
Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}} Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}}
""" """
import http.cookies
import http.server import http.server
import hashlib import hashlib
import itertools import itertools
import json import json
import os import os
import random import random
import select
import socketserver import socketserver
import sqlite3 import sqlite3
import tarfile import tarfile
@@ -50,6 +52,7 @@ from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
from .gossip import NodeGossip from .gossip import NodeGossip
from .logging_setup import tracker_logger
from .routing_stats import ( from .routing_stats import (
RouteCandidate, RouteCandidate,
RouteStatsStore, RouteStatsStore,
@@ -64,6 +67,7 @@ from .raft import RaftNode
_CONSOLE_LIMIT = 300 _CONSOLE_LIMIT = 300
_PROXY_PROGRESS_LOG_INTERVAL = 5.0 _PROXY_PROGRESS_LOG_INTERVAL = 5.0
_SESSION_COOKIE_NAME = "meshnet_session"
def _preset_price_keys(name: str, preset: dict) -> set[str]: def _preset_price_keys(name: str, preset: dict) -> set[str]:
@@ -1487,7 +1491,9 @@ def _relay_http_request_frames(
*, *,
cancel_event: threading.Event | None = None, cancel_event: threading.Event | None = None,
ws_holder: list[Any] | None = None, ws_holder: list[Any] | None = None,
ws_lock: threading.Lock | None = None, # Quoted: threading.Lock is a factory function (not a class) before
# Python 3.13, so an unquoted `| None` union crashes at import time.
ws_lock: "threading.Lock | None" = None,
): ):
"""Send an HTTP-shaped request through a relay RPC WebSocket, yielding """Send an HTTP-shaped request through a relay RPC WebSocket, yielding
response frames until a terminal one (US-036). response frames until a terminal one (US-036).
@@ -2015,6 +2021,38 @@ def _api_key_from_headers(headers) -> str | None:
return auth.strip() or None return auth.strip() or None
def _session_token_from_headers(headers) -> str | None:
token = _api_key_from_headers(headers)
if token:
return token
cookie_header = headers.get("Cookie")
if not cookie_header:
return None
cookie = http.cookies.SimpleCookie()
try:
cookie.load(cookie_header)
except http.cookies.CookieError:
return None
morsel = cookie.get(_SESSION_COOKIE_NAME)
if morsel is None:
return None
return morsel.value.strip() or None
def _session_cookie_header(token: str | None) -> str:
cookie = http.cookies.SimpleCookie()
cookie[_SESSION_COOKIE_NAME] = token or ""
morsel = cookie[_SESSION_COOKIE_NAME]
morsel["path"] = "/"
morsel["httponly"] = True
morsel["samesite"] = "Lax"
if token:
morsel["max-age"] = str(int(7 * 86400))
else:
morsel["max-age"] = "0"
return morsel.OutputString()
def _usage_total_tokens(payload: dict) -> int | None: def _usage_total_tokens(payload: dict) -> int | None:
usage = payload.get("usage") usage = payload.get("usage")
if not isinstance(usage, dict): if not isinstance(usage, dict):
@@ -2183,6 +2221,13 @@ def _tracker_log(
update_console_key: str | None = None, update_console_key: str | None = None,
**fields: Any, **fields: Any,
) -> None: ) -> None:
log_level = {
"debug": 10,
"info": 20,
"warn": 30,
"warning": 30,
"error": 40,
}.get(level.lower(), 20)
event = { event = {
"ts": time.time(), "ts": time.time(),
"level": level, "level": level,
@@ -2209,9 +2254,10 @@ def _tracker_log(
server.console_events.append(event) server.console_events.append(event)
else: else:
server.console_events.append(event) server.console_events.append(event)
if stdout:
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items()) extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
suffix = f" {extras}" if extras else "" suffix = f" {extras}" if extras else ""
tracker_logger().log(log_level, f"{message}{suffix}")
if stdout:
print(f"[tracker] {level}: {message}{suffix}", flush=True) print(f"[tracker] {level}: {message}{suffix}", flush=True)
@@ -2264,10 +2310,14 @@ def _request_proxy_cancel(server: "_TrackerHTTPServer", request_id: str) -> bool
return True return True
def _set_upstream_read_timeout(upstream: Any, timeout: float) -> None: def _upstream_socket(upstream: Any) -> Any | None:
fp = getattr(upstream, "fp", None) fp = getattr(upstream, "fp", None)
raw = getattr(fp, "raw", None) if fp is not None else None raw = getattr(fp, "raw", None) if fp is not None else None
sock = getattr(raw, "_sock", None) if raw is not None else None return getattr(raw, "_sock", None) if raw is not None else None
def _set_upstream_read_timeout(upstream: Any, timeout: float | None) -> None:
sock = _upstream_socket(upstream)
if sock is not None: if sock is not None:
sock.settimeout(timeout) sock.settimeout(timeout)
@@ -2415,11 +2465,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args): # noqa: suppress request logs in tests def log_message(self, fmt, *args): # noqa: suppress request logs in tests
pass pass
def _send_json(self, status: int, data: dict) -> None: def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
body = json.dumps(data).encode() body = json.dumps(data).encode()
self.send_response(status) self.send_response(status)
self.send_header("Content-Type", "application/json") self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body))) self.send_header("Content-Length", str(len(body)))
for name, value in (headers or {}).items():
self.send_header(name, value)
self.end_headers() self.end_headers()
try: try:
self.wfile.write(body) self.wfile.write(body)
@@ -2436,7 +2488,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
inference and wallet binding only, never operator endpoints. inference and wallet binding only, never operator endpoints.
""" """
server: _TrackerHTTPServer = self.server # type: ignore[assignment] server: _TrackerHTTPServer = self.server # type: ignore[assignment]
token = _api_key_from_headers(self.headers) token = _session_token_from_headers(self.headers)
if not token: if not token:
return None, None return None, None
if is_validator_token(token, server.validator_service_token): if is_validator_token(token, server.validator_service_token):
@@ -3327,6 +3379,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
upstream = upstream_result[0] upstream = upstream_result[0]
with proxy_ctx.upstream_lock: with proxy_ctx.upstream_lock:
proxy_ctx.upstream = upstream proxy_ctx.upstream = upstream
upstream_sock = _upstream_socket(upstream)
if upstream_sock is not None:
_set_upstream_read_timeout(upstream, None)
else:
_set_upstream_read_timeout(upstream, 0.5) _set_upstream_read_timeout(upstream, 0.5)
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url) _tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
except urllib.error.HTTPError as exc: except urllib.error.HTTPError as exc:
@@ -3387,6 +3443,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
while True: while True:
if proxy_ctx.cancel_event.is_set(): if proxy_ctx.cancel_event.is_set():
break break
if upstream_sock is not None:
readable, _, _ = select.select([upstream_sock], [], [], 0.5)
if not readable:
continue
try: try:
line = upstream.readline() line = upstream.readline()
except TimeoutError: except TimeoutError:
@@ -4324,7 +4384,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
server: _TrackerHTTPServer = self.server # type: ignore[assignment] server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if server.accounts is None: if server.accounts is None:
return None return None
return server.accounts.session_account(_api_key_from_headers(self.headers)) return server.accounts.session_account(_session_token_from_headers(self.headers))
def _require_accounts(self) -> "AccountStore | None": def _require_accounts(self) -> "AccountStore | None":
server: _TrackerHTTPServer = self.server # type: ignore[assignment] server: _TrackerHTTPServer = self.server # type: ignore[assignment]
@@ -4364,7 +4424,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
f"[tracker] account registered: {account.get('email') or account.get('wallet')} " f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
f"role={account['role']}", flush=True, f"role={account['role']}", flush=True,
) )
self._send_json(200, {"account": account, "session_token": token, "api_key": api_key}) self._send_json(
200,
{"account": account, "session_token": token, "api_key": api_key},
headers={"Set-Cookie": _session_cookie_header(token)},
)
def _handle_auth_login(self): def _handle_auth_login(self):
accounts = self._require_accounts() accounts = self._require_accounts()
@@ -4379,14 +4443,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(401, {"error": "invalid credentials"}) self._send_json(401, {"error": "invalid credentials"})
return return
token = accounts.create_session(account["account_id"]) token = accounts.create_session(account["account_id"])
self._send_json(200, {"account": account, "session_token": token}) self._send_json(
200,
{"account": account, "session_token": token},
headers={"Set-Cookie": _session_cookie_header(token)},
)
def _handle_auth_logout(self): def _handle_auth_logout(self):
accounts = self._require_accounts() accounts = self._require_accounts()
if accounts is None: if accounts is None:
return return
accounts.destroy_session(_api_key_from_headers(self.headers)) accounts.destroy_session(_session_token_from_headers(self.headers))
self._send_json(200, {"ok": True}) self._send_json(200, {"ok": True}, headers={"Set-Cookie": _session_cookie_header(None)})
def _handle_account_me(self): def _handle_account_me(self):
"""Balance, usage, and API keys for the logged-in account.""" """Balance, usage, and API keys for the logged-in account."""

View File

@@ -5,6 +5,7 @@ register/login/logout, per-account balance and usage, API-key lifecycle
(revoked keys rejected by the OpenAI proxy), and the admin listing. (revoked keys rejected by the OpenAI proxy), and the admin listing.
""" """
import http.cookies
import json import json
import urllib.error import urllib.error
import urllib.request import urllib.request
@@ -68,6 +69,17 @@ def test_sessions_resolve_and_destroy():
assert store.session_account("bogus") is None assert store.session_account("bogus") is None
def test_sessions_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="cookie@example.com", password="secret-123")
token = store.create_session(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.session_account(token)["account_id"] == account["account_id"]
def test_api_key_lifecycle(): def test_api_key_lifecycle():
store = AccountStore() store = AccountStore()
account = store.register(email="k@example.com", password="secret-123") account = store.register(email="k@example.com", password="secret-123")
@@ -156,6 +168,59 @@ def test_register_login_and_account_view(account_tracker):
assert me["usage"]["requests"] == 0 assert me["usage"]["requests"] == 0
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
accounts_db = str(tmp_path / "accounts.db")
tracker = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_call(f"{url}/v1/auth/register", "POST",
{"email": "cookie-http@example.com", "password": "secret-123"})
req = urllib.request.Request(
f"{url}/v1/auth/login",
data=json.dumps({
"identifier": "cookie-http@example.com",
"password": "secret-123",
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req) as r:
assert json.loads(r.read())["session_token"]
cookie_header = r.headers["Set-Cookie"]
finally:
tracker.stop()
cookie = http.cookies.SimpleCookie(cookie_header)
session_cookie = cookie["meshnet_session"].OutputString()
restarted = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
restarted_port = restarted.start()
restarted_url = f"http://127.0.0.1:{restarted_port}"
try:
req = urllib.request.Request(
f"{restarted_url}/v1/account",
headers={"Cookie": session_cookie},
method="GET",
)
with urllib.request.urlopen(req) as r:
me = json.loads(r.read())
finally:
restarted.stop()
assert me["account"]["email"] == "cookie-http@example.com"
def test_bad_credentials_and_missing_session_are_401(account_tracker): def test_bad_credentials_and_missing_session_are_401(account_tracker):
url, _ = account_tracker url, _ = account_tracker
_call(f"{url}/v1/auth/register", "POST", _call(f"{url}/v1/auth/register", "POST",

View File

@@ -14,7 +14,7 @@ PANELS = [
"Node pending payouts", "Settlement history", "Node pending payouts", "Settlement history",
"Strikes / bans / forfeitures", "Model usage", "Call wall", "Strikes / bans / forfeitures", "Model usage", "Call wall",
"Usage summary", "Node throughput", "Request history", "Usage summary", "Node throughput", "Request history",
'id="tab-chat"', "Chat / inference",
"Console output", "Console output",
] ]
@@ -33,6 +33,21 @@ def test_dashboard_served_with_all_panels():
tracker.stop() tracker.stop()
def test_dashboard_chat_uses_streaming_fetch():
tracker = TrackerServer(billing=BillingLedger())
port = tracker.start()
try:
html = urllib.request.urlopen(
f"http://127.0.0.1:{port}/dashboard"
).read().decode()
finally:
tracker.stop()
assert "stream: true" in html
assert ".body.getReader()" in html
assert '=== "[DONE]"' in html
def test_dashboard_served_by_follower(): def test_dashboard_served_by_follower():
"""A tracker that is not the leader (unreachable peers → never elected) """A tracker that is not the leader (unreachable peers → never elected)
still serves the dashboard from its own replicated state.""" still serves the dashboard from its own replicated state."""

View File

@@ -142,6 +142,21 @@ def _send_chat_request(gateway_url: str, prompt: str) -> dict:
return json.loads(r.read()) return json.loads(r.read())
def _send_streaming_chat_request(gateway_url: str, prompt: str):
data = json.dumps({
"model": GPT2_MODEL,
"messages": [{"role": "user", "content": prompt}],
"stream": True,
}).encode()
req = urllib.request.Request(
f"{gateway_url}/v1/chat/completions",
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
return urllib.request.urlopen(req)
def test_all_responses_valid_openai_format(tracker_node_setup): def test_all_responses_valid_openai_format(tracker_node_setup):
"""Ten requests via gateway all return valid OpenAI chat completion format.""" """Ten requests via gateway all return valid OpenAI chat completion format."""
gateway_url, _, _ = tracker_node_setup gateway_url, _, _ = tracker_node_setup
@@ -155,6 +170,30 @@ def test_all_responses_valid_openai_format(tracker_node_setup):
assert isinstance(message.get("content"), str), f"request {i}: content must be a string" assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
def test_streaming_head_worker_response_is_not_buffered_with_content_length(tracker_node_setup):
"""Gateway must relay head-worker SSE as a live stream, not a buffered JSON-sized body."""
gateway_url, _, _ = tracker_node_setup
with _send_streaming_chat_request(gateway_url, "stream through head worker") as resp:
assert resp.status == 200
assert "text/event-stream" in resp.headers["Content-Type"]
assert "Content-Length" not in resp.headers
data_lines = []
while len(data_lines) < 4:
line = resp.readline().decode().strip()
if line.startswith("data: "):
data_lines.append(line)
if line == "data: [DONE]":
break
assert data_lines[-1] == "data: [DONE]"
content = "".join(
json.loads(line[6:])["choices"][0].get("delta", {}).get("content", "")
for line in data_lines[:-1]
)
assert "head-worker" in content
def test_both_tracker_nodes_receive_load(tracker_node_setup): def test_both_tracker_nodes_receive_load(tracker_node_setup):
"""Both head workers handle at least one request each out of ten.""" """Both head workers handle at least one request each out of ten."""
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup

View File

@@ -0,0 +1,67 @@
import logging
import sys
from meshnet_tracker.logging_setup import configure_tracker_file_logging, tracker_logger
def test_tracker_file_logging_writes_separate_level_files(tmp_path):
original_stdout = sys.stdout
original_stderr = sys.stderr
try:
log_dir = configure_tracker_file_logging(tmp_path, tee_stdio=False)
logger = tracker_logger()
logger.info("info-event")
logger.warning("warning-event")
logger.error("error-event")
for handler in logger.handlers:
handler.flush()
assert (log_dir / "info.log").read_text().count("info-event") == 1
assert "warning-event" not in (log_dir / "info.log").read_text()
assert "error-event" not in (log_dir / "info.log").read_text()
assert "warning-event" in (log_dir / "warning.log").read_text()
assert "info-event" not in (log_dir / "warning.log").read_text()
assert "error-event" not in (log_dir / "warning.log").read_text()
assert "error-event" in (log_dir / "error.log").read_text()
assert "info-event" not in (log_dir / "error.log").read_text()
assert "warning-event" not in (log_dir / "error.log").read_text()
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
def test_tracker_file_logging_tees_stdio_and_rotates(tmp_path):
original_stdout = sys.stdout
original_stderr = sys.stderr
try:
log_dir = configure_tracker_file_logging(
tmp_path,
max_bytes=120,
backup_count=1,
)
print("stdout goes to info", flush=True)
print("stderr goes to error", file=sys.stderr, flush=True)
for handler in tracker_logger().handlers:
handler.flush()
assert "stdout goes to info" in (log_dir / "info.log").read_text()
assert "stderr goes to error" in (log_dir / "error.log").read_text()
for index in range(12):
tracker_logger().info("rotating-info-line-%02d", index)
for handler in tracker_logger().handlers:
handler.flush()
assert (log_dir / "info.log.1").exists()
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
logger = tracker_logger()
for handler in logger.handlers:
handler.close()
logger.handlers.clear()
logger.setLevel(logging.NOTSET)

View File

@@ -502,6 +502,75 @@ def test_tracker_logs_stream_progress_before_request_completes():
node_thread.join(timeout=1.0) node_thread.join(timeout=1.0)
def test_tracker_stream_survives_idle_gap_between_sse_chunks():
first_chunk_sent = threading.Event()
class IdleStreamingChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, format, *args):
pass
def do_POST(self):
if self.path != "/v1/chat/completions":
self.send_response(404)
self.end_headers()
return
self.rfile.read(int(self.headers.get("Content-Length", 0)))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
self.end_headers()
first = json.dumps({
"choices": [{"delta": {"content": "hello"}}],
}).encode()
second = json.dumps({
"choices": [{"delta": {"content": " world"}}],
}).encode()
self.wfile.write(b"data: " + first + b"\n\n")
self.wfile.flush()
first_chunk_sent.set()
time.sleep(1.0)
self.wfile.write(b"data: " + second + b"\n\n")
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker_port = tracker.start()
response = None
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
"model": "idle-stream-model", "num_layers": 1,
"shard_start": 0, "shard_end": 0,
"hardware_profile": {}, "score": 1.0},
)
req = urllib.request.Request(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
data=json.dumps({
"model": "idle-stream-model",
"stream": True,
"messages": [{"role": "user", "content": "hi"}],
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
response = urllib.request.urlopen(req, timeout=3.0)
assert response.readline().startswith(b"data:")
assert first_chunk_sent.wait(timeout=1.0)
remaining = response.read().splitlines()
assert b"data: [DONE]" in remaining
finally:
if response is not None:
response.close()
tracker.stop()
node.shutdown()
node.server_close()
node_thread.join(timeout=1.0)
def test_tracker_dashboard_can_cancel_inflight_proxy(): def test_tracker_dashboard_can_cancel_inflight_proxy():
chunk_sent = threading.Event() chunk_sent = threading.Event()
release = threading.Event() release = threading.Event()