# Conflicts:
#	packages/tracker/meshnet_tracker/cli.py
#	packages/tracker/meshnet_tracker/dashboard.html
#	packages/tracker/meshnet_tracker/server.py
#	tests/test_dashboard.py
This commit is contained in:
Dobromir Popov
2026-07-08 16:14:24 +02:00
12 changed files with 6759 additions and 6249 deletions

3
.gitignore vendored
View File

@@ -23,3 +23,6 @@ dist/
# Local tracker/node sqlite databases (never commit runtime state)
*.sqlite
*.sqlite3
logs/tracker/error.log
logs/tracker/info.log
logs/tracker/warning.log

View File

@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, completion)
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
"""Forward a raw request body to a head worker and stream the response back."""
"""Forward a raw request body to a head worker and relay SSE without buffering."""
target_url = f"{url}/v1/chat/completions"
req = urllib.request.Request(
target_url,
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
try:
with urllib.request.urlopen(req, timeout=30.0) as r:
content_type = r.headers.get("Content-Type", "application/json")
if "text/event-stream" in content_type:
self.send_response(r.status)
self.send_header("Content-Type", content_type)
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.end_headers()
while True:
line = r.readline()
if not line:
break
self.wfile.write(line)
self.wfile.flush()
return
resp_body = r.read()
status = r.status
except urllib.error.HTTPError as exc:

View File

@@ -8,7 +8,8 @@ regular user.
Mutations are append-only events with unique ids — the same replication
model as ``BillingLedger`` — so accounts and API keys converge across the
tracker hive via gossip, and every dashboard can serve registration/login.
Sessions are deliberately local to each tracker (bearer tokens in memory).
Sessions are local to each tracker and persisted so dashboard cookies survive
tracker restarts.
"""
from __future__ import annotations
@@ -115,6 +116,8 @@ class AccountStore:
"account_id": account_id,
"expires": time.time() + SESSION_TTL,
}
self._dirty = True
self.save_to_db()
return token
def session_account(self, token: str | None) -> dict | None:
@@ -134,7 +137,9 @@ class AccountStore:
if not token:
return
with self._lock:
self._sessions.pop(token, None)
if self._sessions.pop(token, None) is not None:
self._dirty = True
self.save_to_db()
# ---- API keys ----
@@ -271,6 +276,10 @@ class AccountStore:
"CREATE TABLE IF NOT EXISTS account_events "
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
)
con.execute(
"CREATE TABLE IF NOT EXISTS account_sessions "
"(token TEXT PRIMARY KEY, account_id TEXT NOT NULL, expires REAL NOT NULL)"
)
con.commit()
con.close()
@@ -279,6 +288,10 @@ class AccountStore:
rows = con.execute(
"SELECT payload FROM account_events ORDER BY ts, event_id"
).fetchall()
session_rows = con.execute(
"SELECT token, account_id, expires FROM account_sessions WHERE expires >= ?",
(time.time(),),
).fetchall()
con.close()
with self._lock:
for (payload,) in rows:
@@ -288,6 +301,11 @@ class AccountStore:
continue
if event.get("id") not in self._seen_event_ids:
self._apply_locked(event)
self._sessions = {
token: {"account_id": account_id, "expires": float(expires)}
for token, account_id, expires in session_rows
if account_id in self._accounts
}
self._dirty = False
def save_to_db(self) -> None:
@@ -297,11 +315,21 @@ class AccountStore:
if not self._dirty:
return
events = list(self._event_log)
sessions = [
(token, session["account_id"], float(session["expires"]))
for token, session in self._sessions.items()
if session["expires"] >= time.time()
]
self._dirty = False
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
con.executemany(
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
)
con.execute("DELETE FROM account_sessions")
con.executemany(
"INSERT INTO account_sessions (token, account_id, expires) VALUES (?, ?, ?)",
sessions,
)
con.commit()
con.close()

View File

@@ -9,6 +9,12 @@ from pathlib import Path
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
from .billing import DEFAULT_BILLING_DB_PATH
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
from .logging_setup import (
DEFAULT_LOG_BACKUP_COUNT,
DEFAULT_LOG_DIR,
DEFAULT_LOG_MAX_BYTES,
configure_tracker_file_logging,
)
from .routing_stats import RoutingConfig
from .server import (
DEFAULT_CALLER_CREDIT_USDT,
@@ -302,6 +308,34 @@ def main() -> None:
metavar="SECONDS",
help="Half-life for decaying route throughput observations (default 600)",
)
common.add_argument(
"--log-dir",
default=DEFAULT_LOG_DIR,
metavar="PATH",
help=(
"Directory for rotating tracker logs "
f"(default: {DEFAULT_LOG_DIR}; files: info.log, warning.log, error.log)"
),
)
common.add_argument(
"--log-max-bytes",
type=int,
default=DEFAULT_LOG_MAX_BYTES,
metavar="BYTES",
help=f"Rotate each tracker log file after this many bytes (default: {DEFAULT_LOG_MAX_BYTES})",
)
common.add_argument(
"--log-backup-count",
type=int,
default=DEFAULT_LOG_BACKUP_COUNT,
metavar="N",
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
)
common.add_argument(
"--no-file-logs",
action="store_true",
help="Disable rotating tracker log files and only write to the terminal",
)
parser = argparse.ArgumentParser(
prog="meshnet-tracker",
@@ -315,6 +349,13 @@ def main() -> None:
args = parser.parse_args()
if args.command in {None, "start"}:
if not args.no_file_logs:
log_dir = configure_tracker_file_logging(
args.log_dir,
max_bytes=args.log_max_bytes,
backup_count=args.log_backup_count,
)
print(f"meshnet-tracker logs: {log_dir}", flush=True)
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
treasury = None

View File

@@ -234,6 +234,7 @@
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
<section data-tab="chat" class="wide chat-section">
<h2 style="display:none">Chat / inference</h2>
<div class="chat-app">
<aside class="chat-sidebar">
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
@@ -283,7 +284,7 @@ async function fetchJson(path) {
const headers = {};
const token = localStorage.getItem("meshnet_session");
if (token) headers["Authorization"] = "Bearer " + token;
const r = await fetch(path, { headers });
const r = await fetch(path, { headers, credentials: "same-origin" });
if (!r.ok) return null;
return await r.json();
} catch { return null; }
@@ -1109,6 +1110,7 @@ async function apiCall(path, method, body, bearerToken) {
const r = await fetch(path, {
method: method || "GET",
headers,
credentials: "same-origin",
body: body ? JSON.stringify(body) : undefined,
});
const data = await r.json().catch(() => ({}));
@@ -1368,10 +1370,11 @@ async function sendChat() {
promptEl.value = "";
promptEl.style.height = "auto";
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
chatHistory.push(assistantMessage);
renderChatHistory();
persistActiveChatSession();
renderChatStatus("sending request…");
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
let tokens = 0;
let usage = null;
const started = Date.now();
@@ -1382,6 +1385,7 @@ async function sendChat() {
const resp = await fetch("/v1/chat/completions", {
method: "POST",
headers,
credentials: "same-origin",
body: JSON.stringify(body),
signal: chatAbortController.signal,
});
@@ -1396,12 +1400,10 @@ async function sendChat() {
const contentType = resp.headers.get("Content-Type") || "";
if (!resp.body || !contentType.includes("text/event-stream")) {
const data = await resp.json();
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
assistantMessage.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
usage = data.usage || null;
tokens = (usage && usage.completion_tokens) || 0;
} else {
chatHistory.push(assistantMsg);
renderChatHistory();
const reader = resp.body.getReader();
const decoder = new TextDecoder();
let buffered = "";
@@ -1425,30 +1427,31 @@ async function sendChat() {
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
const piece = delta && delta.content;
if (piece) {
assistantMsg.content += piece;
assistantMessage.content += piece;
tokens += 1;
updateStreamingChatBubble(assistantMsg.content);
updateStreamingChatBubble(assistantMessage.content);
const secs = Math.max((Date.now() - started) / 1000, 0.001);
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
}
}
}
}
finalizeAssistantMessage(assistantMsg);
finalizeAssistantMessage(assistantMessage);
renderChatStatus(usage
? `done: ${usage.total_tokens ?? "?"} tokens`
: `done: ${tokens} tokens`);
} catch (err) {
if (err && err.name === "AbortError") {
if (assistantMsg.content) {
finalizeAssistantMessage(assistantMsg);
if (assistantMessage.content) {
finalizeAssistantMessage(assistantMessage);
renderChatStatus(`stopped after ${tokens} tokens`);
} else {
removeAssistantMessage(assistantMsg);
removeAssistantMessage(assistantMessage);
persistActiveChatSession();
renderChatStatus("stopped");
}
} else {
removeAssistantMessage(assistantMsg);
removeAssistantMessage(assistantMessage);
const error = (err && err.message) || "request failed";
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
renderChatHistory();
@@ -1557,7 +1560,7 @@ renderChatModels();
renderChatHistory();
renderChatAuthHint();
setInterval(refresh, 4000);
setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000);
setInterval(() => { if (sessionToken || isLoggedIn) renderAccountPanel(); }, 8000);
</script>
</body>
</html>

View File

@@ -0,0 +1,99 @@
"""Rotating file logging for the tracker CLI."""
from __future__ import annotations
import logging
import sys
from logging.handlers import RotatingFileHandler
from pathlib import Path
from typing import TextIO
DEFAULT_LOG_DIR = "logs/tracker"
DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
DEFAULT_LOG_BACKUP_COUNT = 5
TRACKER_LOGGER_NAME = "meshnet.tracker"
class _ExactLevelFilter(logging.Filter):
def __init__(self, level: int) -> None:
super().__init__()
self._level = level
def filter(self, record: logging.LogRecord) -> bool:
return record.levelno == self._level
class _TeeStream:
def __init__(self, stream: TextIO, logger: logging.Logger, level: int) -> None:
self._stream = stream
self._logger = logger
self._level = level
self._buffer = ""
def write(self, text: str) -> int:
self._stream.write(text)
self._stream.flush()
self._buffer += text
while "\n" in self._buffer:
line, self._buffer = self._buffer.split("\n", 1)
line = line.rstrip()
if line:
self._logger.log(self._level, line)
return len(text)
def flush(self) -> None:
self._stream.flush()
line = self._buffer.rstrip()
if line:
self._logger.log(self._level, line)
self._buffer = ""
def isatty(self) -> bool:
return self._stream.isatty()
def _make_handler(path: Path, level: int, *, max_bytes: int, backup_count: int) -> RotatingFileHandler:
handler = RotatingFileHandler(
path,
maxBytes=max_bytes,
backupCount=backup_count,
encoding="utf-8",
)
handler.setLevel(level)
handler.addFilter(_ExactLevelFilter(level))
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
return handler
def configure_tracker_file_logging(
log_dir: str | Path = DEFAULT_LOG_DIR,
*,
max_bytes: int = DEFAULT_LOG_MAX_BYTES,
backup_count: int = DEFAULT_LOG_BACKUP_COUNT,
tee_stdio: bool = True,
) -> Path:
"""Configure rotatable info/warning/error log files and return the directory."""
path = Path(log_dir).expanduser()
path.mkdir(parents=True, exist_ok=True)
logger = logging.getLogger(TRACKER_LOGGER_NAME)
logger.setLevel(logging.INFO)
logger.propagate = False
logger.handlers.clear()
logger.addHandler(_make_handler(path / "info.log", logging.INFO, max_bytes=max_bytes, backup_count=backup_count))
logger.addHandler(_make_handler(path / "warning.log", logging.WARNING, max_bytes=max_bytes, backup_count=backup_count))
logger.addHandler(_make_handler(path / "error.log", logging.ERROR, max_bytes=max_bytes, backup_count=backup_count))
if tee_stdio:
if not isinstance(sys.stdout, _TeeStream):
sys.stdout = _TeeStream(sys.stdout, logger, logging.INFO) # type: ignore[assignment]
if not isinstance(sys.stderr, _TeeStream):
sys.stderr = _TeeStream(sys.stderr, logger, logging.ERROR) # type: ignore[assignment]
return path
def tracker_logger() -> logging.Logger:
return logging.getLogger(TRACKER_LOGGER_NAME)

File diff suppressed because it is too large Load Diff

View File

@@ -5,6 +5,7 @@ register/login/logout, per-account balance and usage, API-key lifecycle
(revoked keys rejected by the OpenAI proxy), and the admin listing.
"""
import http.cookies
import json
import urllib.error
import urllib.request
@@ -68,6 +69,17 @@ def test_sessions_resolve_and_destroy():
assert store.session_account("bogus") is None
def test_sessions_persist_across_restart(tmp_path):
db = str(tmp_path / "accounts.db")
store = AccountStore(db_path=db)
account = store.register(email="cookie@example.com", password="secret-123")
token = store.create_session(account["account_id"])
store.save_to_db()
reloaded = AccountStore(db_path=db)
assert reloaded.session_account(token)["account_id"] == account["account_id"]
def test_api_key_lifecycle():
store = AccountStore()
account = store.register(email="k@example.com", password="secret-123")
@@ -156,6 +168,59 @@ def test_register_login_and_account_view(account_tracker):
assert me["usage"]["requests"] == 0
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
accounts_db = str(tmp_path / "accounts.db")
tracker = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_call(f"{url}/v1/auth/register", "POST",
{"email": "cookie-http@example.com", "password": "secret-123"})
req = urllib.request.Request(
f"{url}/v1/auth/login",
data=json.dumps({
"identifier": "cookie-http@example.com",
"password": "secret-123",
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req) as r:
assert json.loads(r.read())["session_token"]
cookie_header = r.headers["Set-Cookie"]
finally:
tracker.stop()
cookie = http.cookies.SimpleCookie(cookie_header)
session_cookie = cookie["meshnet_session"].OutputString()
restarted = TrackerServer(
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
accounts_db=accounts_db,
starting_credit=0.0,
devnet_topup_amount=0.0,
)
restarted_port = restarted.start()
restarted_url = f"http://127.0.0.1:{restarted_port}"
try:
req = urllib.request.Request(
f"{restarted_url}/v1/account",
headers={"Cookie": session_cookie},
method="GET",
)
with urllib.request.urlopen(req) as r:
me = json.loads(r.read())
finally:
restarted.stop()
assert me["account"]["email"] == "cookie-http@example.com"
def test_bad_credentials_and_missing_session_are_401(account_tracker):
url, _ = account_tracker
_call(f"{url}/v1/auth/register", "POST",

View File

@@ -1,138 +1,153 @@
"""US-035: tracker web dashboard — served from any tracker, embedded asset."""
import json
import time
import urllib.request
from meshnet_contracts import LocalSolanaContracts
from meshnet_tracker.accounts import AccountStore
from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer
PANELS = [
"Tracker hive", "Nodes &amp; coverage", "Client balances",
"Node pending payouts", "Settlement history",
"Strikes / bans / forfeitures", "Model usage", "Call wall",
"Usage summary", "Node throughput", "Request history",
'id="tab-chat"',
"Console output",
]
def test_dashboard_served_with_all_panels():
tracker = TrackerServer(billing=BillingLedger())
port = tracker.start()
try:
html = urllib.request.urlopen(
f"http://127.0.0.1:{port}/dashboard"
).read().decode()
for panel in PANELS:
assert panel in html
assert "<script>" in html # polling client embedded, no build step
finally:
tracker.stop()
def test_dashboard_served_by_follower():
"""A tracker that is not the leader (unreachable peers → never elected)
still serves the dashboard from its own replicated state."""
tracker = TrackerServer(
billing=BillingLedger(),
cluster_peers=["http://127.0.0.1:1", "http://127.0.0.1:2"],
)
port = tracker.start()
try:
response = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard")
assert response.status == 200
assert "meshnet tracker" in response.read().decode()
finally:
tracker.stop()
def test_registry_wallets_endpoint():
contracts = LocalSolanaContracts()
contracts.registry.submit_stake("wallet-a", 100)
contracts.registry.record_strike("wallet-a")
accounts = AccountStore()
admin = accounts.register(email="admin@example.com", password="admin-pass-123")
session = accounts.create_session(admin["account_id"])
tracker = TrackerServer(contracts=contracts, accounts=accounts)
port = tracker.start()
try:
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/registry/wallets",
headers={"Authorization": f"Bearer {session}"},
)
data = json.loads(urllib.request.urlopen(req).read())
assert data["wallets"]["wallet-a"]["strike_count"] == 1
assert data["wallets"]["wallet-a"]["banned"] is False
finally:
tracker.stop()
def test_console_endpoint_exposes_tracker_events():
tracker = TrackerServer()
port = tracker.start()
try:
body = json.dumps({
"endpoint": "http://127.0.0.1:9001",
"model": "stub-model",
"shard_start": 0,
"shard_end": 3,
"hardware_profile": {},
}).encode()
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=body,
headers={"Content-Type": "application/json"},
method="POST",
)
urllib.request.urlopen(req).read()
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
finally:
tracker.stop()
assert any(event["message"] == "node registered" for event in data["events"])
def test_console_node_lifecycle_events_include_model_health():
tracker = TrackerServer(heartbeat_timeout=0.05)
port = tracker.start()
try:
body = json.dumps({
"endpoint": "http://127.0.0.1:9002",
"model": "console-health-test",
"hf_repo": "example/console-health-test",
"num_layers": 4,
"shard_start": 0,
"shard_end": 1,
"hardware_profile": {},
}).encode()
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=body,
headers={"Content-Type": "application/json"},
method="POST",
)
urllib.request.urlopen(req).read()
registered = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
registered_event = next(
event for event in registered["events"]
if event["message"] == "node registered"
)
assert registered_event["fields"]["model_health"]["served_model_copies"] == 0.5
assert registered_event["fields"]["model_health"]["coverage_percentage"] == 50.0
time.sleep(0.06)
urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map").read()
expired = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
expired_event = next(
event for event in expired["events"]
if event["message"] == "node expired"
)
assert expired_event["fields"]["model_health"]["served_model_copies"] == 0.0
assert expired_event["fields"]["model_health"]["coverage_percentage"] == 0.0
finally:
tracker.stop()
"""US-035: tracker web dashboard — served from any tracker, embedded asset."""
import json
import time
import urllib.request
from meshnet_contracts import LocalSolanaContracts
from meshnet_tracker.accounts import AccountStore
from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer
PANELS = [
"Tracker hive", "Nodes &amp; coverage", "Client balances",
"Node pending payouts", "Settlement history",
"Strikes / bans / forfeitures", "Model usage", "Call wall",
"Usage summary", "Node throughput", "Request history",
"Chat / inference",
"Console output",
]
def test_dashboard_served_with_all_panels():
tracker = TrackerServer(billing=BillingLedger())
port = tracker.start()
try:
html = urllib.request.urlopen(
f"http://127.0.0.1:{port}/dashboard"
).read().decode()
for panel in PANELS:
assert panel in html
assert "<script>" in html # polling client embedded, no build step
finally:
tracker.stop()
def test_dashboard_chat_uses_streaming_fetch():
tracker = TrackerServer(billing=BillingLedger())
port = tracker.start()
try:
html = urllib.request.urlopen(
f"http://127.0.0.1:{port}/dashboard"
).read().decode()
finally:
tracker.stop()
assert "stream: true" in html
assert ".body.getReader()" in html
assert '=== "[DONE]"' in html
def test_dashboard_served_by_follower():
"""A tracker that is not the leader (unreachable peers → never elected)
still serves the dashboard from its own replicated state."""
tracker = TrackerServer(
billing=BillingLedger(),
cluster_peers=["http://127.0.0.1:1", "http://127.0.0.1:2"],
)
port = tracker.start()
try:
response = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard")
assert response.status == 200
assert "meshnet tracker" in response.read().decode()
finally:
tracker.stop()
def test_registry_wallets_endpoint():
contracts = LocalSolanaContracts()
contracts.registry.submit_stake("wallet-a", 100)
contracts.registry.record_strike("wallet-a")
accounts = AccountStore()
admin = accounts.register(email="admin@example.com", password="admin-pass-123")
session = accounts.create_session(admin["account_id"])
tracker = TrackerServer(contracts=contracts, accounts=accounts)
port = tracker.start()
try:
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/registry/wallets",
headers={"Authorization": f"Bearer {session}"},
)
data = json.loads(urllib.request.urlopen(req).read())
assert data["wallets"]["wallet-a"]["strike_count"] == 1
assert data["wallets"]["wallet-a"]["banned"] is False
finally:
tracker.stop()
def test_console_endpoint_exposes_tracker_events():
tracker = TrackerServer()
port = tracker.start()
try:
body = json.dumps({
"endpoint": "http://127.0.0.1:9001",
"model": "stub-model",
"shard_start": 0,
"shard_end": 3,
"hardware_profile": {},
}).encode()
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=body,
headers={"Content-Type": "application/json"},
method="POST",
)
urllib.request.urlopen(req).read()
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
finally:
tracker.stop()
assert any(event["message"] == "node registered" for event in data["events"])
def test_console_node_lifecycle_events_include_model_health():
tracker = TrackerServer(heartbeat_timeout=0.05)
port = tracker.start()
try:
body = json.dumps({
"endpoint": "http://127.0.0.1:9002",
"model": "console-health-test",
"hf_repo": "example/console-health-test",
"num_layers": 4,
"shard_start": 0,
"shard_end": 1,
"hardware_profile": {},
}).encode()
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=body,
headers={"Content-Type": "application/json"},
method="POST",
)
urllib.request.urlopen(req).read()
registered = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
registered_event = next(
event for event in registered["events"]
if event["message"] == "node registered"
)
assert registered_event["fields"]["model_health"]["served_model_copies"] == 0.5
assert registered_event["fields"]["model_health"]["coverage_percentage"] == 50.0
time.sleep(0.06)
urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map").read()
expired = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
expired_event = next(
event for event in expired["events"]
if event["message"] == "node expired"
)
assert expired_event["fields"]["model_health"]["served_model_copies"] == 0.0
assert expired_event["fields"]["model_health"]["coverage_percentage"] == 0.0
finally:
tracker.stop()

View File

@@ -142,6 +142,21 @@ def _send_chat_request(gateway_url: str, prompt: str) -> dict:
return json.loads(r.read())
def _send_streaming_chat_request(gateway_url: str, prompt: str):
data = json.dumps({
"model": GPT2_MODEL,
"messages": [{"role": "user", "content": prompt}],
"stream": True,
}).encode()
req = urllib.request.Request(
f"{gateway_url}/v1/chat/completions",
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
return urllib.request.urlopen(req)
def test_all_responses_valid_openai_format(tracker_node_setup):
"""Ten requests via gateway all return valid OpenAI chat completion format."""
gateway_url, _, _ = tracker_node_setup
@@ -155,6 +170,30 @@ def test_all_responses_valid_openai_format(tracker_node_setup):
assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
def test_streaming_head_worker_response_is_not_buffered_with_content_length(tracker_node_setup):
"""Gateway must relay head-worker SSE as a live stream, not a buffered JSON-sized body."""
gateway_url, _, _ = tracker_node_setup
with _send_streaming_chat_request(gateway_url, "stream through head worker") as resp:
assert resp.status == 200
assert "text/event-stream" in resp.headers["Content-Type"]
assert "Content-Length" not in resp.headers
data_lines = []
while len(data_lines) < 4:
line = resp.readline().decode().strip()
if line.startswith("data: "):
data_lines.append(line)
if line == "data: [DONE]":
break
assert data_lines[-1] == "data: [DONE]"
content = "".join(
json.loads(line[6:])["choices"][0].get("delta", {}).get("content", "")
for line in data_lines[:-1]
)
assert "head-worker" in content
def test_both_tracker_nodes_receive_load(tracker_node_setup):
"""Both head workers handle at least one request each out of ten."""
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup

View File

@@ -0,0 +1,67 @@
import logging
import sys
from meshnet_tracker.logging_setup import configure_tracker_file_logging, tracker_logger
def test_tracker_file_logging_writes_separate_level_files(tmp_path):
original_stdout = sys.stdout
original_stderr = sys.stderr
try:
log_dir = configure_tracker_file_logging(tmp_path, tee_stdio=False)
logger = tracker_logger()
logger.info("info-event")
logger.warning("warning-event")
logger.error("error-event")
for handler in logger.handlers:
handler.flush()
assert (log_dir / "info.log").read_text().count("info-event") == 1
assert "warning-event" not in (log_dir / "info.log").read_text()
assert "error-event" not in (log_dir / "info.log").read_text()
assert "warning-event" in (log_dir / "warning.log").read_text()
assert "info-event" not in (log_dir / "warning.log").read_text()
assert "error-event" not in (log_dir / "warning.log").read_text()
assert "error-event" in (log_dir / "error.log").read_text()
assert "info-event" not in (log_dir / "error.log").read_text()
assert "warning-event" not in (log_dir / "error.log").read_text()
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
def test_tracker_file_logging_tees_stdio_and_rotates(tmp_path):
original_stdout = sys.stdout
original_stderr = sys.stderr
try:
log_dir = configure_tracker_file_logging(
tmp_path,
max_bytes=120,
backup_count=1,
)
print("stdout goes to info", flush=True)
print("stderr goes to error", file=sys.stderr, flush=True)
for handler in tracker_logger().handlers:
handler.flush()
assert "stdout goes to info" in (log_dir / "info.log").read_text()
assert "stderr goes to error" in (log_dir / "error.log").read_text()
for index in range(12):
tracker_logger().info("rotating-info-line-%02d", index)
for handler in tracker_logger().handlers:
handler.flush()
assert (log_dir / "info.log.1").exists()
finally:
sys.stdout = original_stdout
sys.stderr = original_stderr
logger = tracker_logger()
for handler in logger.handlers:
handler.close()
logger.handlers.clear()
logger.setLevel(logging.NOTSET)

View File

@@ -502,6 +502,75 @@ def test_tracker_logs_stream_progress_before_request_completes():
node_thread.join(timeout=1.0)
def test_tracker_stream_survives_idle_gap_between_sse_chunks():
first_chunk_sent = threading.Event()
class IdleStreamingChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, format, *args):
pass
def do_POST(self):
if self.path != "/v1/chat/completions":
self.send_response(404)
self.end_headers()
return
self.rfile.read(int(self.headers.get("Content-Length", 0)))
self.send_response(200)
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
self.end_headers()
first = json.dumps({
"choices": [{"delta": {"content": "hello"}}],
}).encode()
second = json.dumps({
"choices": [{"delta": {"content": " world"}}],
}).encode()
self.wfile.write(b"data: " + first + b"\n\n")
self.wfile.flush()
first_chunk_sent.set()
time.sleep(1.0)
self.wfile.write(b"data: " + second + b"\n\n")
self.wfile.write(b"data: [DONE]\n\n")
self.wfile.flush()
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker_port = tracker.start()
response = None
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
"model": "idle-stream-model", "num_layers": 1,
"shard_start": 0, "shard_end": 0,
"hardware_profile": {}, "score": 1.0},
)
req = urllib.request.Request(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
data=json.dumps({
"model": "idle-stream-model",
"stream": True,
"messages": [{"role": "user", "content": "hi"}],
}).encode(),
headers={"Content-Type": "application/json"},
method="POST",
)
response = urllib.request.urlopen(req, timeout=3.0)
assert response.readline().startswith(b"data:")
assert first_chunk_sent.wait(timeout=1.0)
remaining = response.read().splitlines()
assert b"data: [DONE]" in remaining
finally:
if response is not None:
response.close()
tracker.stop()
node.shutdown()
node.server_close()
node_thread.join(timeout=1.0)
def test_tracker_dashboard_can_cancel_inflight_proxy():
chunk_sent = threading.Event()
release = threading.Event()