Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai
# Conflicts: # packages/tracker/meshnet_tracker/cli.py # packages/tracker/meshnet_tracker/dashboard.html # packages/tracker/meshnet_tracker/server.py # tests/test_dashboard.py
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -23,3 +23,6 @@ dist/
|
||||
# Local tracker/node sqlite databases (never commit runtime state)
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
logs/tracker/error.log
|
||||
logs/tracker/info.log
|
||||
logs/tracker/warning.log
|
||||
|
||||
@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._send_json(200, completion)
|
||||
|
||||
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
|
||||
"""Forward a raw request body to a head worker and stream the response back."""
|
||||
"""Forward a raw request body to a head worker and relay SSE without buffering."""
|
||||
target_url = f"{url}/v1/chat/completions"
|
||||
req = urllib.request.Request(
|
||||
target_url,
|
||||
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30.0) as r:
|
||||
content_type = r.headers.get("Content-Type", "application/json")
|
||||
if "text/event-stream" in content_type:
|
||||
self.send_response(r.status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("X-Accel-Buffering", "no")
|
||||
self.end_headers()
|
||||
while True:
|
||||
line = r.readline()
|
||||
if not line:
|
||||
break
|
||||
self.wfile.write(line)
|
||||
self.wfile.flush()
|
||||
return
|
||||
resp_body = r.read()
|
||||
status = r.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
|
||||
@@ -8,7 +8,8 @@ regular user.
|
||||
Mutations are append-only events with unique ids — the same replication
|
||||
model as ``BillingLedger`` — so accounts and API keys converge across the
|
||||
tracker hive via gossip, and every dashboard can serve registration/login.
|
||||
Sessions are deliberately local to each tracker (bearer tokens in memory).
|
||||
Sessions are local to each tracker and persisted so dashboard cookies survive
|
||||
tracker restarts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -115,6 +116,8 @@ class AccountStore:
|
||||
"account_id": account_id,
|
||||
"expires": time.time() + SESSION_TTL,
|
||||
}
|
||||
self._dirty = True
|
||||
self.save_to_db()
|
||||
return token
|
||||
|
||||
def session_account(self, token: str | None) -> dict | None:
|
||||
@@ -134,7 +137,9 @@ class AccountStore:
|
||||
if not token:
|
||||
return
|
||||
with self._lock:
|
||||
self._sessions.pop(token, None)
|
||||
if self._sessions.pop(token, None) is not None:
|
||||
self._dirty = True
|
||||
self.save_to_db()
|
||||
|
||||
# ---- API keys ----
|
||||
|
||||
@@ -271,6 +276,10 @@ class AccountStore:
|
||||
"CREATE TABLE IF NOT EXISTS account_events "
|
||||
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
|
||||
)
|
||||
con.execute(
|
||||
"CREATE TABLE IF NOT EXISTS account_sessions "
|
||||
"(token TEXT PRIMARY KEY, account_id TEXT NOT NULL, expires REAL NOT NULL)"
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
@@ -279,6 +288,10 @@ class AccountStore:
|
||||
rows = con.execute(
|
||||
"SELECT payload FROM account_events ORDER BY ts, event_id"
|
||||
).fetchall()
|
||||
session_rows = con.execute(
|
||||
"SELECT token, account_id, expires FROM account_sessions WHERE expires >= ?",
|
||||
(time.time(),),
|
||||
).fetchall()
|
||||
con.close()
|
||||
with self._lock:
|
||||
for (payload,) in rows:
|
||||
@@ -288,6 +301,11 @@ class AccountStore:
|
||||
continue
|
||||
if event.get("id") not in self._seen_event_ids:
|
||||
self._apply_locked(event)
|
||||
self._sessions = {
|
||||
token: {"account_id": account_id, "expires": float(expires)}
|
||||
for token, account_id, expires in session_rows
|
||||
if account_id in self._accounts
|
||||
}
|
||||
self._dirty = False
|
||||
|
||||
def save_to_db(self) -> None:
|
||||
@@ -297,11 +315,21 @@ class AccountStore:
|
||||
if not self._dirty:
|
||||
return
|
||||
events = list(self._event_log)
|
||||
sessions = [
|
||||
(token, session["account_id"], float(session["expires"]))
|
||||
for token, session in self._sessions.items()
|
||||
if session["expires"] >= time.time()
|
||||
]
|
||||
self._dirty = False
|
||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||
con.executemany(
|
||||
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
|
||||
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
|
||||
)
|
||||
con.execute("DELETE FROM account_sessions")
|
||||
con.executemany(
|
||||
"INSERT INTO account_sessions (token, account_id, expires) VALUES (?, ?, ?)",
|
||||
sessions,
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
|
||||
@@ -9,6 +9,12 @@ from pathlib import Path
|
||||
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
|
||||
from .billing import DEFAULT_BILLING_DB_PATH
|
||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
|
||||
from .logging_setup import (
|
||||
DEFAULT_LOG_BACKUP_COUNT,
|
||||
DEFAULT_LOG_DIR,
|
||||
DEFAULT_LOG_MAX_BYTES,
|
||||
configure_tracker_file_logging,
|
||||
)
|
||||
from .routing_stats import RoutingConfig
|
||||
from .server import (
|
||||
DEFAULT_CALLER_CREDIT_USDT,
|
||||
@@ -302,6 +308,34 @@ def main() -> None:
|
||||
metavar="SECONDS",
|
||||
help="Half-life for decaying route throughput observations (default 600)",
|
||||
)
|
||||
common.add_argument(
|
||||
"--log-dir",
|
||||
default=DEFAULT_LOG_DIR,
|
||||
metavar="PATH",
|
||||
help=(
|
||||
"Directory for rotating tracker logs "
|
||||
f"(default: {DEFAULT_LOG_DIR}; files: info.log, warning.log, error.log)"
|
||||
),
|
||||
)
|
||||
common.add_argument(
|
||||
"--log-max-bytes",
|
||||
type=int,
|
||||
default=DEFAULT_LOG_MAX_BYTES,
|
||||
metavar="BYTES",
|
||||
help=f"Rotate each tracker log file after this many bytes (default: {DEFAULT_LOG_MAX_BYTES})",
|
||||
)
|
||||
common.add_argument(
|
||||
"--log-backup-count",
|
||||
type=int,
|
||||
default=DEFAULT_LOG_BACKUP_COUNT,
|
||||
metavar="N",
|
||||
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
|
||||
)
|
||||
common.add_argument(
|
||||
"--no-file-logs",
|
||||
action="store_true",
|
||||
help="Disable rotating tracker log files and only write to the terminal",
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="meshnet-tracker",
|
||||
@@ -315,6 +349,13 @@ def main() -> None:
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.command in {None, "start"}:
|
||||
if not args.no_file_logs:
|
||||
log_dir = configure_tracker_file_logging(
|
||||
args.log_dir,
|
||||
max_bytes=args.log_max_bytes,
|
||||
backup_count=args.log_backup_count,
|
||||
)
|
||||
print(f"meshnet-tracker logs: {log_dir}", flush=True)
|
||||
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
|
||||
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
|
||||
treasury = None
|
||||
|
||||
@@ -234,6 +234,7 @@
|
||||
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
||||
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
||||
<section data-tab="chat" class="wide chat-section">
|
||||
<h2 style="display:none">Chat / inference</h2>
|
||||
<div class="chat-app">
|
||||
<aside class="chat-sidebar">
|
||||
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
|
||||
@@ -283,7 +284,7 @@ async function fetchJson(path) {
|
||||
const headers = {};
|
||||
const token = localStorage.getItem("meshnet_session");
|
||||
if (token) headers["Authorization"] = "Bearer " + token;
|
||||
const r = await fetch(path, { headers });
|
||||
const r = await fetch(path, { headers, credentials: "same-origin" });
|
||||
if (!r.ok) return null;
|
||||
return await r.json();
|
||||
} catch { return null; }
|
||||
@@ -1109,6 +1110,7 @@ async function apiCall(path, method, body, bearerToken) {
|
||||
const r = await fetch(path, {
|
||||
method: method || "GET",
|
||||
headers,
|
||||
credentials: "same-origin",
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
const data = await r.json().catch(() => ({}));
|
||||
@@ -1368,10 +1370,11 @@ async function sendChat() {
|
||||
promptEl.value = "";
|
||||
promptEl.style.height = "auto";
|
||||
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
||||
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
||||
chatHistory.push(assistantMessage);
|
||||
renderChatHistory();
|
||||
persistActiveChatSession();
|
||||
renderChatStatus("sending request…");
|
||||
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
||||
let tokens = 0;
|
||||
let usage = null;
|
||||
const started = Date.now();
|
||||
@@ -1382,6 +1385,7 @@ async function sendChat() {
|
||||
const resp = await fetch("/v1/chat/completions", {
|
||||
method: "POST",
|
||||
headers,
|
||||
credentials: "same-origin",
|
||||
body: JSON.stringify(body),
|
||||
signal: chatAbortController.signal,
|
||||
});
|
||||
@@ -1396,12 +1400,10 @@ async function sendChat() {
|
||||
const contentType = resp.headers.get("Content-Type") || "";
|
||||
if (!resp.body || !contentType.includes("text/event-stream")) {
|
||||
const data = await resp.json();
|
||||
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
||||
assistantMessage.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
||||
usage = data.usage || null;
|
||||
tokens = (usage && usage.completion_tokens) || 0;
|
||||
} else {
|
||||
chatHistory.push(assistantMsg);
|
||||
renderChatHistory();
|
||||
const reader = resp.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffered = "";
|
||||
@@ -1425,30 +1427,31 @@ async function sendChat() {
|
||||
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
||||
const piece = delta && delta.content;
|
||||
if (piece) {
|
||||
assistantMsg.content += piece;
|
||||
assistantMessage.content += piece;
|
||||
tokens += 1;
|
||||
updateStreamingChatBubble(assistantMsg.content);
|
||||
updateStreamingChatBubble(assistantMessage.content);
|
||||
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
||||
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
finalizeAssistantMessage(assistantMsg);
|
||||
finalizeAssistantMessage(assistantMessage);
|
||||
renderChatStatus(usage
|
||||
? `done: ${usage.total_tokens ?? "?"} tokens`
|
||||
: `done: ${tokens} tokens`);
|
||||
} catch (err) {
|
||||
if (err && err.name === "AbortError") {
|
||||
if (assistantMsg.content) {
|
||||
finalizeAssistantMessage(assistantMsg);
|
||||
if (assistantMessage.content) {
|
||||
finalizeAssistantMessage(assistantMessage);
|
||||
renderChatStatus(`stopped after ${tokens} tokens`);
|
||||
} else {
|
||||
removeAssistantMessage(assistantMsg);
|
||||
removeAssistantMessage(assistantMessage);
|
||||
persistActiveChatSession();
|
||||
renderChatStatus("stopped");
|
||||
}
|
||||
} else {
|
||||
removeAssistantMessage(assistantMsg);
|
||||
removeAssistantMessage(assistantMessage);
|
||||
const error = (err && err.message) || "request failed";
|
||||
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
||||
renderChatHistory();
|
||||
@@ -1557,7 +1560,7 @@ renderChatModels();
|
||||
renderChatHistory();
|
||||
renderChatAuthHint();
|
||||
setInterval(refresh, 4000);
|
||||
setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000);
|
||||
setInterval(() => { if (sessionToken || isLoggedIn) renderAccountPanel(); }, 8000);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
99
packages/tracker/meshnet_tracker/logging_setup.py
Normal file
99
packages/tracker/meshnet_tracker/logging_setup.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""Rotating file logging for the tracker CLI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
from typing import TextIO
|
||||
|
||||
|
||||
DEFAULT_LOG_DIR = "logs/tracker"
|
||||
DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
|
||||
DEFAULT_LOG_BACKUP_COUNT = 5
|
||||
TRACKER_LOGGER_NAME = "meshnet.tracker"
|
||||
|
||||
|
||||
class _ExactLevelFilter(logging.Filter):
|
||||
def __init__(self, level: int) -> None:
|
||||
super().__init__()
|
||||
self._level = level
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
return record.levelno == self._level
|
||||
|
||||
|
||||
class _TeeStream:
|
||||
def __init__(self, stream: TextIO, logger: logging.Logger, level: int) -> None:
|
||||
self._stream = stream
|
||||
self._logger = logger
|
||||
self._level = level
|
||||
self._buffer = ""
|
||||
|
||||
def write(self, text: str) -> int:
|
||||
self._stream.write(text)
|
||||
self._stream.flush()
|
||||
self._buffer += text
|
||||
while "\n" in self._buffer:
|
||||
line, self._buffer = self._buffer.split("\n", 1)
|
||||
line = line.rstrip()
|
||||
if line:
|
||||
self._logger.log(self._level, line)
|
||||
return len(text)
|
||||
|
||||
def flush(self) -> None:
|
||||
self._stream.flush()
|
||||
line = self._buffer.rstrip()
|
||||
if line:
|
||||
self._logger.log(self._level, line)
|
||||
self._buffer = ""
|
||||
|
||||
def isatty(self) -> bool:
|
||||
return self._stream.isatty()
|
||||
|
||||
|
||||
def _make_handler(path: Path, level: int, *, max_bytes: int, backup_count: int) -> RotatingFileHandler:
|
||||
handler = RotatingFileHandler(
|
||||
path,
|
||||
maxBytes=max_bytes,
|
||||
backupCount=backup_count,
|
||||
encoding="utf-8",
|
||||
)
|
||||
handler.setLevel(level)
|
||||
handler.addFilter(_ExactLevelFilter(level))
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||
return handler
|
||||
|
||||
|
||||
def configure_tracker_file_logging(
|
||||
log_dir: str | Path = DEFAULT_LOG_DIR,
|
||||
*,
|
||||
max_bytes: int = DEFAULT_LOG_MAX_BYTES,
|
||||
backup_count: int = DEFAULT_LOG_BACKUP_COUNT,
|
||||
tee_stdio: bool = True,
|
||||
) -> Path:
|
||||
"""Configure rotatable info/warning/error log files and return the directory."""
|
||||
|
||||
path = Path(log_dir).expanduser()
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger = logging.getLogger(TRACKER_LOGGER_NAME)
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.propagate = False
|
||||
logger.handlers.clear()
|
||||
logger.addHandler(_make_handler(path / "info.log", logging.INFO, max_bytes=max_bytes, backup_count=backup_count))
|
||||
logger.addHandler(_make_handler(path / "warning.log", logging.WARNING, max_bytes=max_bytes, backup_count=backup_count))
|
||||
logger.addHandler(_make_handler(path / "error.log", logging.ERROR, max_bytes=max_bytes, backup_count=backup_count))
|
||||
|
||||
if tee_stdio:
|
||||
if not isinstance(sys.stdout, _TeeStream):
|
||||
sys.stdout = _TeeStream(sys.stdout, logger, logging.INFO) # type: ignore[assignment]
|
||||
if not isinstance(sys.stderr, _TeeStream):
|
||||
sys.stderr = _TeeStream(sys.stderr, logger, logging.ERROR) # type: ignore[assignment]
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def tracker_logger() -> logging.Logger:
|
||||
return logging.getLogger(TRACKER_LOGGER_NAME)
|
||||
@@ -23,12 +23,14 @@ HTTP API contract:
|
||||
Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}}
|
||||
"""
|
||||
|
||||
import http.cookies
|
||||
import http.server
|
||||
import hashlib
|
||||
import itertools
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import select
|
||||
import socketserver
|
||||
import sqlite3
|
||||
import tarfile
|
||||
@@ -50,6 +52,7 @@ from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
|
||||
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||||
from .gossip import NodeGossip
|
||||
from .logging_setup import tracker_logger
|
||||
from .routing_stats import (
|
||||
RouteCandidate,
|
||||
RouteStatsStore,
|
||||
@@ -64,6 +67,7 @@ from .raft import RaftNode
|
||||
|
||||
_CONSOLE_LIMIT = 300
|
||||
_PROXY_PROGRESS_LOG_INTERVAL = 5.0
|
||||
_SESSION_COOKIE_NAME = "meshnet_session"
|
||||
|
||||
|
||||
def _preset_price_keys(name: str, preset: dict) -> set[str]:
|
||||
@@ -1487,7 +1491,9 @@ def _relay_http_request_frames(
|
||||
*,
|
||||
cancel_event: threading.Event | None = None,
|
||||
ws_holder: list[Any] | None = None,
|
||||
ws_lock: threading.Lock | None = None,
|
||||
# Quoted: threading.Lock is a factory function (not a class) before
|
||||
# Python 3.13, so an unquoted `| None` union crashes at import time.
|
||||
ws_lock: "threading.Lock | None" = None,
|
||||
):
|
||||
"""Send an HTTP-shaped request through a relay RPC WebSocket, yielding
|
||||
response frames until a terminal one (US-036).
|
||||
@@ -2015,6 +2021,38 @@ def _api_key_from_headers(headers) -> str | None:
|
||||
return auth.strip() or None
|
||||
|
||||
|
||||
def _session_token_from_headers(headers) -> str | None:
|
||||
token = _api_key_from_headers(headers)
|
||||
if token:
|
||||
return token
|
||||
cookie_header = headers.get("Cookie")
|
||||
if not cookie_header:
|
||||
return None
|
||||
cookie = http.cookies.SimpleCookie()
|
||||
try:
|
||||
cookie.load(cookie_header)
|
||||
except http.cookies.CookieError:
|
||||
return None
|
||||
morsel = cookie.get(_SESSION_COOKIE_NAME)
|
||||
if morsel is None:
|
||||
return None
|
||||
return morsel.value.strip() or None
|
||||
|
||||
|
||||
def _session_cookie_header(token: str | None) -> str:
|
||||
cookie = http.cookies.SimpleCookie()
|
||||
cookie[_SESSION_COOKIE_NAME] = token or ""
|
||||
morsel = cookie[_SESSION_COOKIE_NAME]
|
||||
morsel["path"] = "/"
|
||||
morsel["httponly"] = True
|
||||
morsel["samesite"] = "Lax"
|
||||
if token:
|
||||
morsel["max-age"] = str(int(7 * 86400))
|
||||
else:
|
||||
morsel["max-age"] = "0"
|
||||
return morsel.OutputString()
|
||||
|
||||
|
||||
def _usage_total_tokens(payload: dict) -> int | None:
|
||||
usage = payload.get("usage")
|
||||
if not isinstance(usage, dict):
|
||||
@@ -2183,6 +2221,13 @@ def _tracker_log(
|
||||
update_console_key: str | None = None,
|
||||
**fields: Any,
|
||||
) -> None:
|
||||
log_level = {
|
||||
"debug": 10,
|
||||
"info": 20,
|
||||
"warn": 30,
|
||||
"warning": 30,
|
||||
"error": 40,
|
||||
}.get(level.lower(), 20)
|
||||
event = {
|
||||
"ts": time.time(),
|
||||
"level": level,
|
||||
@@ -2209,9 +2254,10 @@ def _tracker_log(
|
||||
server.console_events.append(event)
|
||||
else:
|
||||
server.console_events.append(event)
|
||||
if stdout:
|
||||
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
|
||||
suffix = f" {extras}" if extras else ""
|
||||
tracker_logger().log(log_level, f"{message}{suffix}")
|
||||
if stdout:
|
||||
print(f"[tracker] {level}: {message}{suffix}", flush=True)
|
||||
|
||||
|
||||
@@ -2264,10 +2310,14 @@ def _request_proxy_cancel(server: "_TrackerHTTPServer", request_id: str) -> bool
|
||||
return True
|
||||
|
||||
|
||||
def _set_upstream_read_timeout(upstream: Any, timeout: float) -> None:
|
||||
def _upstream_socket(upstream: Any) -> Any | None:
|
||||
fp = getattr(upstream, "fp", None)
|
||||
raw = getattr(fp, "raw", None) if fp is not None else None
|
||||
sock = getattr(raw, "_sock", None) if raw is not None else None
|
||||
return getattr(raw, "_sock", None) if raw is not None else None
|
||||
|
||||
|
||||
def _set_upstream_read_timeout(upstream: Any, timeout: float | None) -> None:
|
||||
sock = _upstream_socket(upstream)
|
||||
if sock is not None:
|
||||
sock.settimeout(timeout)
|
||||
|
||||
@@ -2415,11 +2465,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
||||
pass
|
||||
|
||||
def _send_json(self, status: int, data: dict) -> None:
|
||||
def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
|
||||
body = json.dumps(data).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
for name, value in (headers or {}).items():
|
||||
self.send_header(name, value)
|
||||
self.end_headers()
|
||||
try:
|
||||
self.wfile.write(body)
|
||||
@@ -2436,7 +2488,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
inference and wallet binding only, never operator endpoints.
|
||||
"""
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
token = _api_key_from_headers(self.headers)
|
||||
token = _session_token_from_headers(self.headers)
|
||||
if not token:
|
||||
return None, None
|
||||
if is_validator_token(token, server.validator_service_token):
|
||||
@@ -3327,6 +3379,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
upstream = upstream_result[0]
|
||||
with proxy_ctx.upstream_lock:
|
||||
proxy_ctx.upstream = upstream
|
||||
upstream_sock = _upstream_socket(upstream)
|
||||
if upstream_sock is not None:
|
||||
_set_upstream_read_timeout(upstream, None)
|
||||
else:
|
||||
_set_upstream_read_timeout(upstream, 0.5)
|
||||
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
|
||||
except urllib.error.HTTPError as exc:
|
||||
@@ -3387,6 +3443,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
while True:
|
||||
if proxy_ctx.cancel_event.is_set():
|
||||
break
|
||||
if upstream_sock is not None:
|
||||
readable, _, _ = select.select([upstream_sock], [], [], 0.5)
|
||||
if not readable:
|
||||
continue
|
||||
try:
|
||||
line = upstream.readline()
|
||||
except TimeoutError:
|
||||
@@ -4324,7 +4384,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
if server.accounts is None:
|
||||
return None
|
||||
return server.accounts.session_account(_api_key_from_headers(self.headers))
|
||||
return server.accounts.session_account(_session_token_from_headers(self.headers))
|
||||
|
||||
def _require_accounts(self) -> "AccountStore | None":
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
@@ -4364,7 +4424,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
|
||||
f"role={account['role']}", flush=True,
|
||||
)
|
||||
self._send_json(200, {"account": account, "session_token": token, "api_key": api_key})
|
||||
self._send_json(
|
||||
200,
|
||||
{"account": account, "session_token": token, "api_key": api_key},
|
||||
headers={"Set-Cookie": _session_cookie_header(token)},
|
||||
)
|
||||
|
||||
def _handle_auth_login(self):
|
||||
accounts = self._require_accounts()
|
||||
@@ -4379,14 +4443,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._send_json(401, {"error": "invalid credentials"})
|
||||
return
|
||||
token = accounts.create_session(account["account_id"])
|
||||
self._send_json(200, {"account": account, "session_token": token})
|
||||
self._send_json(
|
||||
200,
|
||||
{"account": account, "session_token": token},
|
||||
headers={"Set-Cookie": _session_cookie_header(token)},
|
||||
)
|
||||
|
||||
def _handle_auth_logout(self):
|
||||
accounts = self._require_accounts()
|
||||
if accounts is None:
|
||||
return
|
||||
accounts.destroy_session(_api_key_from_headers(self.headers))
|
||||
self._send_json(200, {"ok": True})
|
||||
accounts.destroy_session(_session_token_from_headers(self.headers))
|
||||
self._send_json(200, {"ok": True}, headers={"Set-Cookie": _session_cookie_header(None)})
|
||||
|
||||
def _handle_account_me(self):
|
||||
"""Balance, usage, and API keys for the logged-in account."""
|
||||
|
||||
@@ -5,6 +5,7 @@ register/login/logout, per-account balance and usage, API-key lifecycle
|
||||
(revoked keys rejected by the OpenAI proxy), and the admin listing.
|
||||
"""
|
||||
|
||||
import http.cookies
|
||||
import json
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
@@ -68,6 +69,17 @@ def test_sessions_resolve_and_destroy():
|
||||
assert store.session_account("bogus") is None
|
||||
|
||||
|
||||
def test_sessions_persist_across_restart(tmp_path):
|
||||
db = str(tmp_path / "accounts.db")
|
||||
store = AccountStore(db_path=db)
|
||||
account = store.register(email="cookie@example.com", password="secret-123")
|
||||
token = store.create_session(account["account_id"])
|
||||
store.save_to_db()
|
||||
|
||||
reloaded = AccountStore(db_path=db)
|
||||
assert reloaded.session_account(token)["account_id"] == account["account_id"]
|
||||
|
||||
|
||||
def test_api_key_lifecycle():
|
||||
store = AccountStore()
|
||||
account = store.register(email="k@example.com", password="secret-123")
|
||||
@@ -156,6 +168,59 @@ def test_register_login_and_account_view(account_tracker):
|
||||
assert me["usage"]["requests"] == 0
|
||||
|
||||
|
||||
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
|
||||
accounts_db = str(tmp_path / "accounts.db")
|
||||
tracker = TrackerServer(
|
||||
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
|
||||
accounts_db=accounts_db,
|
||||
starting_credit=0.0,
|
||||
devnet_topup_amount=0.0,
|
||||
)
|
||||
port = tracker.start()
|
||||
url = f"http://127.0.0.1:{port}"
|
||||
try:
|
||||
_call(f"{url}/v1/auth/register", "POST",
|
||||
{"email": "cookie-http@example.com", "password": "secret-123"})
|
||||
req = urllib.request.Request(
|
||||
f"{url}/v1/auth/login",
|
||||
data=json.dumps({
|
||||
"identifier": "cookie-http@example.com",
|
||||
"password": "secret-123",
|
||||
}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
assert json.loads(r.read())["session_token"]
|
||||
cookie_header = r.headers["Set-Cookie"]
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
cookie = http.cookies.SimpleCookie(cookie_header)
|
||||
session_cookie = cookie["meshnet_session"].OutputString()
|
||||
|
||||
restarted = TrackerServer(
|
||||
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
|
||||
accounts_db=accounts_db,
|
||||
starting_credit=0.0,
|
||||
devnet_topup_amount=0.0,
|
||||
)
|
||||
restarted_port = restarted.start()
|
||||
restarted_url = f"http://127.0.0.1:{restarted_port}"
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
f"{restarted_url}/v1/account",
|
||||
headers={"Cookie": session_cookie},
|
||||
method="GET",
|
||||
)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
me = json.loads(r.read())
|
||||
finally:
|
||||
restarted.stop()
|
||||
|
||||
assert me["account"]["email"] == "cookie-http@example.com"
|
||||
|
||||
|
||||
def test_bad_credentials_and_missing_session_are_401(account_tracker):
|
||||
url, _ = account_tracker
|
||||
_call(f"{url}/v1/auth/register", "POST",
|
||||
|
||||
@@ -14,7 +14,7 @@ PANELS = [
|
||||
"Node pending payouts", "Settlement history",
|
||||
"Strikes / bans / forfeitures", "Model usage", "Call wall",
|
||||
"Usage summary", "Node throughput", "Request history",
|
||||
'id="tab-chat"',
|
||||
"Chat / inference",
|
||||
"Console output",
|
||||
]
|
||||
|
||||
@@ -33,6 +33,21 @@ def test_dashboard_served_with_all_panels():
|
||||
tracker.stop()
|
||||
|
||||
|
||||
def test_dashboard_chat_uses_streaming_fetch():
|
||||
tracker = TrackerServer(billing=BillingLedger())
|
||||
port = tracker.start()
|
||||
try:
|
||||
html = urllib.request.urlopen(
|
||||
f"http://127.0.0.1:{port}/dashboard"
|
||||
).read().decode()
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
assert "stream: true" in html
|
||||
assert ".body.getReader()" in html
|
||||
assert '=== "[DONE]"' in html
|
||||
|
||||
|
||||
def test_dashboard_served_by_follower():
|
||||
"""A tracker that is not the leader (unreachable peers → never elected)
|
||||
still serves the dashboard from its own replicated state."""
|
||||
|
||||
@@ -142,6 +142,21 @@ def _send_chat_request(gateway_url: str, prompt: str) -> dict:
|
||||
return json.loads(r.read())
|
||||
|
||||
|
||||
def _send_streaming_chat_request(gateway_url: str, prompt: str):
|
||||
data = json.dumps({
|
||||
"model": GPT2_MODEL,
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"stream": True,
|
||||
}).encode()
|
||||
req = urllib.request.Request(
|
||||
f"{gateway_url}/v1/chat/completions",
|
||||
data=data,
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
return urllib.request.urlopen(req)
|
||||
|
||||
|
||||
def test_all_responses_valid_openai_format(tracker_node_setup):
|
||||
"""Ten requests via gateway all return valid OpenAI chat completion format."""
|
||||
gateway_url, _, _ = tracker_node_setup
|
||||
@@ -155,6 +170,30 @@ def test_all_responses_valid_openai_format(tracker_node_setup):
|
||||
assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
|
||||
|
||||
|
||||
def test_streaming_head_worker_response_is_not_buffered_with_content_length(tracker_node_setup):
|
||||
"""Gateway must relay head-worker SSE as a live stream, not a buffered JSON-sized body."""
|
||||
gateway_url, _, _ = tracker_node_setup
|
||||
|
||||
with _send_streaming_chat_request(gateway_url, "stream through head worker") as resp:
|
||||
assert resp.status == 200
|
||||
assert "text/event-stream" in resp.headers["Content-Type"]
|
||||
assert "Content-Length" not in resp.headers
|
||||
data_lines = []
|
||||
while len(data_lines) < 4:
|
||||
line = resp.readline().decode().strip()
|
||||
if line.startswith("data: "):
|
||||
data_lines.append(line)
|
||||
if line == "data: [DONE]":
|
||||
break
|
||||
|
||||
assert data_lines[-1] == "data: [DONE]"
|
||||
content = "".join(
|
||||
json.loads(line[6:])["choices"][0].get("delta", {}).get("content", "")
|
||||
for line in data_lines[:-1]
|
||||
)
|
||||
assert "head-worker" in content
|
||||
|
||||
|
||||
def test_both_tracker_nodes_receive_load(tracker_node_setup):
|
||||
"""Both head workers handle at least one request each out of ten."""
|
||||
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup
|
||||
|
||||
67
tests/test_tracker_logging.py
Normal file
67
tests/test_tracker_logging.py
Normal file
@@ -0,0 +1,67 @@
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from meshnet_tracker.logging_setup import configure_tracker_file_logging, tracker_logger
|
||||
|
||||
|
||||
def test_tracker_file_logging_writes_separate_level_files(tmp_path):
|
||||
original_stdout = sys.stdout
|
||||
original_stderr = sys.stderr
|
||||
try:
|
||||
log_dir = configure_tracker_file_logging(tmp_path, tee_stdio=False)
|
||||
logger = tracker_logger()
|
||||
|
||||
logger.info("info-event")
|
||||
logger.warning("warning-event")
|
||||
logger.error("error-event")
|
||||
for handler in logger.handlers:
|
||||
handler.flush()
|
||||
|
||||
assert (log_dir / "info.log").read_text().count("info-event") == 1
|
||||
assert "warning-event" not in (log_dir / "info.log").read_text()
|
||||
assert "error-event" not in (log_dir / "info.log").read_text()
|
||||
|
||||
assert "warning-event" in (log_dir / "warning.log").read_text()
|
||||
assert "info-event" not in (log_dir / "warning.log").read_text()
|
||||
assert "error-event" not in (log_dir / "warning.log").read_text()
|
||||
|
||||
assert "error-event" in (log_dir / "error.log").read_text()
|
||||
assert "info-event" not in (log_dir / "error.log").read_text()
|
||||
assert "warning-event" not in (log_dir / "error.log").read_text()
|
||||
finally:
|
||||
sys.stdout = original_stdout
|
||||
sys.stderr = original_stderr
|
||||
|
||||
|
||||
def test_tracker_file_logging_tees_stdio_and_rotates(tmp_path):
|
||||
original_stdout = sys.stdout
|
||||
original_stderr = sys.stderr
|
||||
try:
|
||||
log_dir = configure_tracker_file_logging(
|
||||
tmp_path,
|
||||
max_bytes=120,
|
||||
backup_count=1,
|
||||
)
|
||||
|
||||
print("stdout goes to info", flush=True)
|
||||
print("stderr goes to error", file=sys.stderr, flush=True)
|
||||
for handler in tracker_logger().handlers:
|
||||
handler.flush()
|
||||
|
||||
assert "stdout goes to info" in (log_dir / "info.log").read_text()
|
||||
assert "stderr goes to error" in (log_dir / "error.log").read_text()
|
||||
|
||||
for index in range(12):
|
||||
tracker_logger().info("rotating-info-line-%02d", index)
|
||||
for handler in tracker_logger().handlers:
|
||||
handler.flush()
|
||||
|
||||
assert (log_dir / "info.log.1").exists()
|
||||
finally:
|
||||
sys.stdout = original_stdout
|
||||
sys.stderr = original_stderr
|
||||
logger = tracker_logger()
|
||||
for handler in logger.handlers:
|
||||
handler.close()
|
||||
logger.handlers.clear()
|
||||
logger.setLevel(logging.NOTSET)
|
||||
@@ -502,6 +502,75 @@ def test_tracker_logs_stream_progress_before_request_completes():
|
||||
node_thread.join(timeout=1.0)
|
||||
|
||||
|
||||
def test_tracker_stream_survives_idle_gap_between_sse_chunks():
|
||||
first_chunk_sent = threading.Event()
|
||||
|
||||
class IdleStreamingChatHandler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
if self.path != "/v1/chat/completions":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
||||
self.end_headers()
|
||||
first = json.dumps({
|
||||
"choices": [{"delta": {"content": "hello"}}],
|
||||
}).encode()
|
||||
second = json.dumps({
|
||||
"choices": [{"delta": {"content": " world"}}],
|
||||
}).encode()
|
||||
self.wfile.write(b"data: " + first + b"\n\n")
|
||||
self.wfile.flush()
|
||||
first_chunk_sent.set()
|
||||
time.sleep(1.0)
|
||||
self.wfile.write(b"data: " + second + b"\n\n")
|
||||
self.wfile.write(b"data: [DONE]\n\n")
|
||||
self.wfile.flush()
|
||||
|
||||
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
|
||||
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
|
||||
node_thread.start()
|
||||
tracker = TrackerServer(heartbeat_timeout=60.0)
|
||||
tracker_port = tracker.start()
|
||||
response = None
|
||||
try:
|
||||
_post_json(
|
||||
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
|
||||
"model": "idle-stream-model", "num_layers": 1,
|
||||
"shard_start": 0, "shard_end": 0,
|
||||
"hardware_profile": {}, "score": 1.0},
|
||||
)
|
||||
req = urllib.request.Request(
|
||||
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
|
||||
data=json.dumps({
|
||||
"model": "idle-stream-model",
|
||||
"stream": True,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}).encode(),
|
||||
headers={"Content-Type": "application/json"},
|
||||
method="POST",
|
||||
)
|
||||
response = urllib.request.urlopen(req, timeout=3.0)
|
||||
assert response.readline().startswith(b"data:")
|
||||
assert first_chunk_sent.wait(timeout=1.0)
|
||||
|
||||
remaining = response.read().splitlines()
|
||||
assert b"data: [DONE]" in remaining
|
||||
finally:
|
||||
if response is not None:
|
||||
response.close()
|
||||
tracker.stop()
|
||||
node.shutdown()
|
||||
node.server_close()
|
||||
node_thread.join(timeout=1.0)
|
||||
|
||||
|
||||
def test_tracker_dashboard_can_cancel_inflight_proxy():
|
||||
chunk_sent = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
Reference in New Issue
Block a user