Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai
# Conflicts: # packages/tracker/meshnet_tracker/cli.py # packages/tracker/meshnet_tracker/dashboard.html # packages/tracker/meshnet_tracker/server.py # tests/test_dashboard.py
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -23,3 +23,6 @@ dist/
|
|||||||
# Local tracker/node sqlite databases (never commit runtime state)
|
# Local tracker/node sqlite databases (never commit runtime state)
|
||||||
*.sqlite
|
*.sqlite
|
||||||
*.sqlite3
|
*.sqlite3
|
||||||
|
logs/tracker/error.log
|
||||||
|
logs/tracker/info.log
|
||||||
|
logs/tracker/warning.log
|
||||||
|
|||||||
@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._send_json(200, completion)
|
self._send_json(200, completion)
|
||||||
|
|
||||||
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
|
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
|
||||||
"""Forward a raw request body to a head worker and stream the response back."""
|
"""Forward a raw request body to a head worker and relay SSE without buffering."""
|
||||||
target_url = f"{url}/v1/chat/completions"
|
target_url = f"{url}/v1/chat/completions"
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
target_url,
|
target_url,
|
||||||
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=30.0) as r:
|
with urllib.request.urlopen(req, timeout=30.0) as r:
|
||||||
content_type = r.headers.get("Content-Type", "application/json")
|
content_type = r.headers.get("Content-Type", "application/json")
|
||||||
|
if "text/event-stream" in content_type:
|
||||||
|
self.send_response(r.status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Cache-Control", "no-cache")
|
||||||
|
self.send_header("X-Accel-Buffering", "no")
|
||||||
|
self.end_headers()
|
||||||
|
while True:
|
||||||
|
line = r.readline()
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
self.wfile.write(line)
|
||||||
|
self.wfile.flush()
|
||||||
|
return
|
||||||
resp_body = r.read()
|
resp_body = r.read()
|
||||||
status = r.status
|
status = r.status
|
||||||
except urllib.error.HTTPError as exc:
|
except urllib.error.HTTPError as exc:
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ regular user.
|
|||||||
Mutations are append-only events with unique ids — the same replication
|
Mutations are append-only events with unique ids — the same replication
|
||||||
model as ``BillingLedger`` — so accounts and API keys converge across the
|
model as ``BillingLedger`` — so accounts and API keys converge across the
|
||||||
tracker hive via gossip, and every dashboard can serve registration/login.
|
tracker hive via gossip, and every dashboard can serve registration/login.
|
||||||
Sessions are deliberately local to each tracker (bearer tokens in memory).
|
Sessions are local to each tracker and persisted so dashboard cookies survive
|
||||||
|
tracker restarts.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -115,6 +116,8 @@ class AccountStore:
|
|||||||
"account_id": account_id,
|
"account_id": account_id,
|
||||||
"expires": time.time() + SESSION_TTL,
|
"expires": time.time() + SESSION_TTL,
|
||||||
}
|
}
|
||||||
|
self._dirty = True
|
||||||
|
self.save_to_db()
|
||||||
return token
|
return token
|
||||||
|
|
||||||
def session_account(self, token: str | None) -> dict | None:
|
def session_account(self, token: str | None) -> dict | None:
|
||||||
@@ -134,7 +137,9 @@ class AccountStore:
|
|||||||
if not token:
|
if not token:
|
||||||
return
|
return
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._sessions.pop(token, None)
|
if self._sessions.pop(token, None) is not None:
|
||||||
|
self._dirty = True
|
||||||
|
self.save_to_db()
|
||||||
|
|
||||||
# ---- API keys ----
|
# ---- API keys ----
|
||||||
|
|
||||||
@@ -271,6 +276,10 @@ class AccountStore:
|
|||||||
"CREATE TABLE IF NOT EXISTS account_events "
|
"CREATE TABLE IF NOT EXISTS account_events "
|
||||||
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
|
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
|
||||||
)
|
)
|
||||||
|
con.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS account_sessions "
|
||||||
|
"(token TEXT PRIMARY KEY, account_id TEXT NOT NULL, expires REAL NOT NULL)"
|
||||||
|
)
|
||||||
con.commit()
|
con.commit()
|
||||||
con.close()
|
con.close()
|
||||||
|
|
||||||
@@ -279,6 +288,10 @@ class AccountStore:
|
|||||||
rows = con.execute(
|
rows = con.execute(
|
||||||
"SELECT payload FROM account_events ORDER BY ts, event_id"
|
"SELECT payload FROM account_events ORDER BY ts, event_id"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
session_rows = con.execute(
|
||||||
|
"SELECT token, account_id, expires FROM account_sessions WHERE expires >= ?",
|
||||||
|
(time.time(),),
|
||||||
|
).fetchall()
|
||||||
con.close()
|
con.close()
|
||||||
with self._lock:
|
with self._lock:
|
||||||
for (payload,) in rows:
|
for (payload,) in rows:
|
||||||
@@ -288,6 +301,11 @@ class AccountStore:
|
|||||||
continue
|
continue
|
||||||
if event.get("id") not in self._seen_event_ids:
|
if event.get("id") not in self._seen_event_ids:
|
||||||
self._apply_locked(event)
|
self._apply_locked(event)
|
||||||
|
self._sessions = {
|
||||||
|
token: {"account_id": account_id, "expires": float(expires)}
|
||||||
|
for token, account_id, expires in session_rows
|
||||||
|
if account_id in self._accounts
|
||||||
|
}
|
||||||
self._dirty = False
|
self._dirty = False
|
||||||
|
|
||||||
def save_to_db(self) -> None:
|
def save_to_db(self) -> None:
|
||||||
@@ -297,11 +315,21 @@ class AccountStore:
|
|||||||
if not self._dirty:
|
if not self._dirty:
|
||||||
return
|
return
|
||||||
events = list(self._event_log)
|
events = list(self._event_log)
|
||||||
|
sessions = [
|
||||||
|
(token, session["account_id"], float(session["expires"]))
|
||||||
|
for token, session in self._sessions.items()
|
||||||
|
if session["expires"] >= time.time()
|
||||||
|
]
|
||||||
self._dirty = False
|
self._dirty = False
|
||||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||||
con.executemany(
|
con.executemany(
|
||||||
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
|
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
|
||||||
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
|
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
|
||||||
)
|
)
|
||||||
|
con.execute("DELETE FROM account_sessions")
|
||||||
|
con.executemany(
|
||||||
|
"INSERT INTO account_sessions (token, account_id, expires) VALUES (?, ?, ?)",
|
||||||
|
sessions,
|
||||||
|
)
|
||||||
con.commit()
|
con.commit()
|
||||||
con.close()
|
con.close()
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ from pathlib import Path
|
|||||||
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
|
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
|
||||||
from .billing import DEFAULT_BILLING_DB_PATH
|
from .billing import DEFAULT_BILLING_DB_PATH
|
||||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
|
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
|
||||||
|
from .logging_setup import (
|
||||||
|
DEFAULT_LOG_BACKUP_COUNT,
|
||||||
|
DEFAULT_LOG_DIR,
|
||||||
|
DEFAULT_LOG_MAX_BYTES,
|
||||||
|
configure_tracker_file_logging,
|
||||||
|
)
|
||||||
from .routing_stats import RoutingConfig
|
from .routing_stats import RoutingConfig
|
||||||
from .server import (
|
from .server import (
|
||||||
DEFAULT_CALLER_CREDIT_USDT,
|
DEFAULT_CALLER_CREDIT_USDT,
|
||||||
@@ -302,6 +308,34 @@ def main() -> None:
|
|||||||
metavar="SECONDS",
|
metavar="SECONDS",
|
||||||
help="Half-life for decaying route throughput observations (default 600)",
|
help="Half-life for decaying route throughput observations (default 600)",
|
||||||
)
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--log-dir",
|
||||||
|
default=DEFAULT_LOG_DIR,
|
||||||
|
metavar="PATH",
|
||||||
|
help=(
|
||||||
|
"Directory for rotating tracker logs "
|
||||||
|
f"(default: {DEFAULT_LOG_DIR}; files: info.log, warning.log, error.log)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--log-max-bytes",
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_LOG_MAX_BYTES,
|
||||||
|
metavar="BYTES",
|
||||||
|
help=f"Rotate each tracker log file after this many bytes (default: {DEFAULT_LOG_MAX_BYTES})",
|
||||||
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--log-backup-count",
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_LOG_BACKUP_COUNT,
|
||||||
|
metavar="N",
|
||||||
|
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
|
||||||
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--no-file-logs",
|
||||||
|
action="store_true",
|
||||||
|
help="Disable rotating tracker log files and only write to the terminal",
|
||||||
|
)
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
prog="meshnet-tracker",
|
prog="meshnet-tracker",
|
||||||
@@ -315,6 +349,13 @@ def main() -> None:
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.command in {None, "start"}:
|
if args.command in {None, "start"}:
|
||||||
|
if not args.no_file_logs:
|
||||||
|
log_dir = configure_tracker_file_logging(
|
||||||
|
args.log_dir,
|
||||||
|
max_bytes=args.log_max_bytes,
|
||||||
|
backup_count=args.log_backup_count,
|
||||||
|
)
|
||||||
|
print(f"meshnet-tracker logs: {log_dir}", flush=True)
|
||||||
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
|
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
|
||||||
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
|
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
|
||||||
treasury = None
|
treasury = None
|
||||||
|
|||||||
@@ -234,6 +234,7 @@
|
|||||||
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
||||||
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
||||||
<section data-tab="chat" class="wide chat-section">
|
<section data-tab="chat" class="wide chat-section">
|
||||||
|
<h2 style="display:none">Chat / inference</h2>
|
||||||
<div class="chat-app">
|
<div class="chat-app">
|
||||||
<aside class="chat-sidebar">
|
<aside class="chat-sidebar">
|
||||||
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
|
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
|
||||||
@@ -283,7 +284,7 @@ async function fetchJson(path) {
|
|||||||
const headers = {};
|
const headers = {};
|
||||||
const token = localStorage.getItem("meshnet_session");
|
const token = localStorage.getItem("meshnet_session");
|
||||||
if (token) headers["Authorization"] = "Bearer " + token;
|
if (token) headers["Authorization"] = "Bearer " + token;
|
||||||
const r = await fetch(path, { headers });
|
const r = await fetch(path, { headers, credentials: "same-origin" });
|
||||||
if (!r.ok) return null;
|
if (!r.ok) return null;
|
||||||
return await r.json();
|
return await r.json();
|
||||||
} catch { return null; }
|
} catch { return null; }
|
||||||
@@ -1109,6 +1110,7 @@ async function apiCall(path, method, body, bearerToken) {
|
|||||||
const r = await fetch(path, {
|
const r = await fetch(path, {
|
||||||
method: method || "GET",
|
method: method || "GET",
|
||||||
headers,
|
headers,
|
||||||
|
credentials: "same-origin",
|
||||||
body: body ? JSON.stringify(body) : undefined,
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
});
|
});
|
||||||
const data = await r.json().catch(() => ({}));
|
const data = await r.json().catch(() => ({}));
|
||||||
@@ -1368,10 +1370,11 @@ async function sendChat() {
|
|||||||
promptEl.value = "";
|
promptEl.value = "";
|
||||||
promptEl.style.height = "auto";
|
promptEl.style.height = "auto";
|
||||||
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
||||||
|
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
||||||
|
chatHistory.push(assistantMessage);
|
||||||
renderChatHistory();
|
renderChatHistory();
|
||||||
persistActiveChatSession();
|
persistActiveChatSession();
|
||||||
renderChatStatus("sending request…");
|
renderChatStatus("sending request…");
|
||||||
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
|
||||||
let tokens = 0;
|
let tokens = 0;
|
||||||
let usage = null;
|
let usage = null;
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
@@ -1382,6 +1385,7 @@ async function sendChat() {
|
|||||||
const resp = await fetch("/v1/chat/completions", {
|
const resp = await fetch("/v1/chat/completions", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers,
|
headers,
|
||||||
|
credentials: "same-origin",
|
||||||
body: JSON.stringify(body),
|
body: JSON.stringify(body),
|
||||||
signal: chatAbortController.signal,
|
signal: chatAbortController.signal,
|
||||||
});
|
});
|
||||||
@@ -1396,12 +1400,10 @@ async function sendChat() {
|
|||||||
const contentType = resp.headers.get("Content-Type") || "";
|
const contentType = resp.headers.get("Content-Type") || "";
|
||||||
if (!resp.body || !contentType.includes("text/event-stream")) {
|
if (!resp.body || !contentType.includes("text/event-stream")) {
|
||||||
const data = await resp.json();
|
const data = await resp.json();
|
||||||
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
assistantMessage.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
||||||
usage = data.usage || null;
|
usage = data.usage || null;
|
||||||
tokens = (usage && usage.completion_tokens) || 0;
|
tokens = (usage && usage.completion_tokens) || 0;
|
||||||
} else {
|
} else {
|
||||||
chatHistory.push(assistantMsg);
|
|
||||||
renderChatHistory();
|
|
||||||
const reader = resp.body.getReader();
|
const reader = resp.body.getReader();
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
let buffered = "";
|
let buffered = "";
|
||||||
@@ -1425,30 +1427,31 @@ async function sendChat() {
|
|||||||
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
||||||
const piece = delta && delta.content;
|
const piece = delta && delta.content;
|
||||||
if (piece) {
|
if (piece) {
|
||||||
assistantMsg.content += piece;
|
assistantMessage.content += piece;
|
||||||
tokens += 1;
|
tokens += 1;
|
||||||
updateStreamingChatBubble(assistantMsg.content);
|
updateStreamingChatBubble(assistantMessage.content);
|
||||||
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
||||||
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finalizeAssistantMessage(assistantMsg);
|
finalizeAssistantMessage(assistantMessage);
|
||||||
renderChatStatus(usage
|
renderChatStatus(usage
|
||||||
? `done: ${usage.total_tokens ?? "?"} tokens`
|
? `done: ${usage.total_tokens ?? "?"} tokens`
|
||||||
: `done: ${tokens} tokens`);
|
: `done: ${tokens} tokens`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err && err.name === "AbortError") {
|
if (err && err.name === "AbortError") {
|
||||||
if (assistantMsg.content) {
|
if (assistantMessage.content) {
|
||||||
finalizeAssistantMessage(assistantMsg);
|
finalizeAssistantMessage(assistantMessage);
|
||||||
renderChatStatus(`stopped after ${tokens} tokens`);
|
renderChatStatus(`stopped after ${tokens} tokens`);
|
||||||
} else {
|
} else {
|
||||||
removeAssistantMessage(assistantMsg);
|
removeAssistantMessage(assistantMessage);
|
||||||
|
persistActiveChatSession();
|
||||||
renderChatStatus("stopped");
|
renderChatStatus("stopped");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
removeAssistantMessage(assistantMsg);
|
removeAssistantMessage(assistantMessage);
|
||||||
const error = (err && err.message) || "request failed";
|
const error = (err && err.message) || "request failed";
|
||||||
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
||||||
renderChatHistory();
|
renderChatHistory();
|
||||||
@@ -1557,7 +1560,7 @@ renderChatModels();
|
|||||||
renderChatHistory();
|
renderChatHistory();
|
||||||
renderChatAuthHint();
|
renderChatAuthHint();
|
||||||
setInterval(refresh, 4000);
|
setInterval(refresh, 4000);
|
||||||
setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000);
|
setInterval(() => { if (sessionToken || isLoggedIn) renderAccountPanel(); }, 8000);
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
99
packages/tracker/meshnet_tracker/logging_setup.py
Normal file
99
packages/tracker/meshnet_tracker/logging_setup.py
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
"""Rotating file logging for the tracker CLI."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TextIO
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_LOG_DIR = "logs/tracker"
|
||||||
|
DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
|
||||||
|
DEFAULT_LOG_BACKUP_COUNT = 5
|
||||||
|
TRACKER_LOGGER_NAME = "meshnet.tracker"
|
||||||
|
|
||||||
|
|
||||||
|
class _ExactLevelFilter(logging.Filter):
|
||||||
|
def __init__(self, level: int) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self._level = level
|
||||||
|
|
||||||
|
def filter(self, record: logging.LogRecord) -> bool:
|
||||||
|
return record.levelno == self._level
|
||||||
|
|
||||||
|
|
||||||
|
class _TeeStream:
|
||||||
|
def __init__(self, stream: TextIO, logger: logging.Logger, level: int) -> None:
|
||||||
|
self._stream = stream
|
||||||
|
self._logger = logger
|
||||||
|
self._level = level
|
||||||
|
self._buffer = ""
|
||||||
|
|
||||||
|
def write(self, text: str) -> int:
|
||||||
|
self._stream.write(text)
|
||||||
|
self._stream.flush()
|
||||||
|
self._buffer += text
|
||||||
|
while "\n" in self._buffer:
|
||||||
|
line, self._buffer = self._buffer.split("\n", 1)
|
||||||
|
line = line.rstrip()
|
||||||
|
if line:
|
||||||
|
self._logger.log(self._level, line)
|
||||||
|
return len(text)
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self._stream.flush()
|
||||||
|
line = self._buffer.rstrip()
|
||||||
|
if line:
|
||||||
|
self._logger.log(self._level, line)
|
||||||
|
self._buffer = ""
|
||||||
|
|
||||||
|
def isatty(self) -> bool:
|
||||||
|
return self._stream.isatty()
|
||||||
|
|
||||||
|
|
||||||
|
def _make_handler(path: Path, level: int, *, max_bytes: int, backup_count: int) -> RotatingFileHandler:
|
||||||
|
handler = RotatingFileHandler(
|
||||||
|
path,
|
||||||
|
maxBytes=max_bytes,
|
||||||
|
backupCount=backup_count,
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
handler.setLevel(level)
|
||||||
|
handler.addFilter(_ExactLevelFilter(level))
|
||||||
|
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||||
|
return handler
|
||||||
|
|
||||||
|
|
||||||
|
def configure_tracker_file_logging(
|
||||||
|
log_dir: str | Path = DEFAULT_LOG_DIR,
|
||||||
|
*,
|
||||||
|
max_bytes: int = DEFAULT_LOG_MAX_BYTES,
|
||||||
|
backup_count: int = DEFAULT_LOG_BACKUP_COUNT,
|
||||||
|
tee_stdio: bool = True,
|
||||||
|
) -> Path:
|
||||||
|
"""Configure rotatable info/warning/error log files and return the directory."""
|
||||||
|
|
||||||
|
path = Path(log_dir).expanduser()
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
logger = logging.getLogger(TRACKER_LOGGER_NAME)
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
logger.propagate = False
|
||||||
|
logger.handlers.clear()
|
||||||
|
logger.addHandler(_make_handler(path / "info.log", logging.INFO, max_bytes=max_bytes, backup_count=backup_count))
|
||||||
|
logger.addHandler(_make_handler(path / "warning.log", logging.WARNING, max_bytes=max_bytes, backup_count=backup_count))
|
||||||
|
logger.addHandler(_make_handler(path / "error.log", logging.ERROR, max_bytes=max_bytes, backup_count=backup_count))
|
||||||
|
|
||||||
|
if tee_stdio:
|
||||||
|
if not isinstance(sys.stdout, _TeeStream):
|
||||||
|
sys.stdout = _TeeStream(sys.stdout, logger, logging.INFO) # type: ignore[assignment]
|
||||||
|
if not isinstance(sys.stderr, _TeeStream):
|
||||||
|
sys.stderr = _TeeStream(sys.stderr, logger, logging.ERROR) # type: ignore[assignment]
|
||||||
|
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def tracker_logger() -> logging.Logger:
|
||||||
|
return logging.getLogger(TRACKER_LOGGER_NAME)
|
||||||
@@ -23,12 +23,14 @@ HTTP API contract:
|
|||||||
Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}}
|
Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}}
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import http.cookies
|
||||||
import http.server
|
import http.server
|
||||||
import hashlib
|
import hashlib
|
||||||
import itertools
|
import itertools
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
|
import select
|
||||||
import socketserver
|
import socketserver
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import tarfile
|
import tarfile
|
||||||
@@ -50,6 +52,7 @@ from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
|
|||||||
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
||||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||||||
from .gossip import NodeGossip
|
from .gossip import NodeGossip
|
||||||
|
from .logging_setup import tracker_logger
|
||||||
from .routing_stats import (
|
from .routing_stats import (
|
||||||
RouteCandidate,
|
RouteCandidate,
|
||||||
RouteStatsStore,
|
RouteStatsStore,
|
||||||
@@ -64,6 +67,7 @@ from .raft import RaftNode
|
|||||||
|
|
||||||
_CONSOLE_LIMIT = 300
|
_CONSOLE_LIMIT = 300
|
||||||
_PROXY_PROGRESS_LOG_INTERVAL = 5.0
|
_PROXY_PROGRESS_LOG_INTERVAL = 5.0
|
||||||
|
_SESSION_COOKIE_NAME = "meshnet_session"
|
||||||
|
|
||||||
|
|
||||||
def _preset_price_keys(name: str, preset: dict) -> set[str]:
|
def _preset_price_keys(name: str, preset: dict) -> set[str]:
|
||||||
@@ -1487,7 +1491,9 @@ def _relay_http_request_frames(
|
|||||||
*,
|
*,
|
||||||
cancel_event: threading.Event | None = None,
|
cancel_event: threading.Event | None = None,
|
||||||
ws_holder: list[Any] | None = None,
|
ws_holder: list[Any] | None = None,
|
||||||
ws_lock: threading.Lock | None = None,
|
# Quoted: threading.Lock is a factory function (not a class) before
|
||||||
|
# Python 3.13, so an unquoted `| None` union crashes at import time.
|
||||||
|
ws_lock: "threading.Lock | None" = None,
|
||||||
):
|
):
|
||||||
"""Send an HTTP-shaped request through a relay RPC WebSocket, yielding
|
"""Send an HTTP-shaped request through a relay RPC WebSocket, yielding
|
||||||
response frames until a terminal one (US-036).
|
response frames until a terminal one (US-036).
|
||||||
@@ -2015,6 +2021,38 @@ def _api_key_from_headers(headers) -> str | None:
|
|||||||
return auth.strip() or None
|
return auth.strip() or None
|
||||||
|
|
||||||
|
|
||||||
|
def _session_token_from_headers(headers) -> str | None:
|
||||||
|
token = _api_key_from_headers(headers)
|
||||||
|
if token:
|
||||||
|
return token
|
||||||
|
cookie_header = headers.get("Cookie")
|
||||||
|
if not cookie_header:
|
||||||
|
return None
|
||||||
|
cookie = http.cookies.SimpleCookie()
|
||||||
|
try:
|
||||||
|
cookie.load(cookie_header)
|
||||||
|
except http.cookies.CookieError:
|
||||||
|
return None
|
||||||
|
morsel = cookie.get(_SESSION_COOKIE_NAME)
|
||||||
|
if morsel is None:
|
||||||
|
return None
|
||||||
|
return morsel.value.strip() or None
|
||||||
|
|
||||||
|
|
||||||
|
def _session_cookie_header(token: str | None) -> str:
|
||||||
|
cookie = http.cookies.SimpleCookie()
|
||||||
|
cookie[_SESSION_COOKIE_NAME] = token or ""
|
||||||
|
morsel = cookie[_SESSION_COOKIE_NAME]
|
||||||
|
morsel["path"] = "/"
|
||||||
|
morsel["httponly"] = True
|
||||||
|
morsel["samesite"] = "Lax"
|
||||||
|
if token:
|
||||||
|
morsel["max-age"] = str(int(7 * 86400))
|
||||||
|
else:
|
||||||
|
morsel["max-age"] = "0"
|
||||||
|
return morsel.OutputString()
|
||||||
|
|
||||||
|
|
||||||
def _usage_total_tokens(payload: dict) -> int | None:
|
def _usage_total_tokens(payload: dict) -> int | None:
|
||||||
usage = payload.get("usage")
|
usage = payload.get("usage")
|
||||||
if not isinstance(usage, dict):
|
if not isinstance(usage, dict):
|
||||||
@@ -2183,6 +2221,13 @@ def _tracker_log(
|
|||||||
update_console_key: str | None = None,
|
update_console_key: str | None = None,
|
||||||
**fields: Any,
|
**fields: Any,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
log_level = {
|
||||||
|
"debug": 10,
|
||||||
|
"info": 20,
|
||||||
|
"warn": 30,
|
||||||
|
"warning": 30,
|
||||||
|
"error": 40,
|
||||||
|
}.get(level.lower(), 20)
|
||||||
event = {
|
event = {
|
||||||
"ts": time.time(),
|
"ts": time.time(),
|
||||||
"level": level,
|
"level": level,
|
||||||
@@ -2209,9 +2254,10 @@ def _tracker_log(
|
|||||||
server.console_events.append(event)
|
server.console_events.append(event)
|
||||||
else:
|
else:
|
||||||
server.console_events.append(event)
|
server.console_events.append(event)
|
||||||
if stdout:
|
|
||||||
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
|
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
|
||||||
suffix = f" {extras}" if extras else ""
|
suffix = f" {extras}" if extras else ""
|
||||||
|
tracker_logger().log(log_level, f"{message}{suffix}")
|
||||||
|
if stdout:
|
||||||
print(f"[tracker] {level}: {message}{suffix}", flush=True)
|
print(f"[tracker] {level}: {message}{suffix}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
@@ -2264,10 +2310,14 @@ def _request_proxy_cancel(server: "_TrackerHTTPServer", request_id: str) -> bool
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def _set_upstream_read_timeout(upstream: Any, timeout: float) -> None:
|
def _upstream_socket(upstream: Any) -> Any | None:
|
||||||
fp = getattr(upstream, "fp", None)
|
fp = getattr(upstream, "fp", None)
|
||||||
raw = getattr(fp, "raw", None) if fp is not None else None
|
raw = getattr(fp, "raw", None) if fp is not None else None
|
||||||
sock = getattr(raw, "_sock", None) if raw is not None else None
|
return getattr(raw, "_sock", None) if raw is not None else None
|
||||||
|
|
||||||
|
|
||||||
|
def _set_upstream_read_timeout(upstream: Any, timeout: float | None) -> None:
|
||||||
|
sock = _upstream_socket(upstream)
|
||||||
if sock is not None:
|
if sock is not None:
|
||||||
sock.settimeout(timeout)
|
sock.settimeout(timeout)
|
||||||
|
|
||||||
@@ -2415,11 +2465,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def _send_json(self, status: int, data: dict) -> None:
|
def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
|
||||||
body = json.dumps(data).encode()
|
body = json.dumps(data).encode()
|
||||||
self.send_response(status)
|
self.send_response(status)
|
||||||
self.send_header("Content-Type", "application/json")
|
self.send_header("Content-Type", "application/json")
|
||||||
self.send_header("Content-Length", str(len(body)))
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
for name, value in (headers or {}).items():
|
||||||
|
self.send_header(name, value)
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
try:
|
try:
|
||||||
self.wfile.write(body)
|
self.wfile.write(body)
|
||||||
@@ -2436,7 +2488,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
inference and wallet binding only, never operator endpoints.
|
inference and wallet binding only, never operator endpoints.
|
||||||
"""
|
"""
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
token = _api_key_from_headers(self.headers)
|
token = _session_token_from_headers(self.headers)
|
||||||
if not token:
|
if not token:
|
||||||
return None, None
|
return None, None
|
||||||
if is_validator_token(token, server.validator_service_token):
|
if is_validator_token(token, server.validator_service_token):
|
||||||
@@ -3327,6 +3379,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
upstream = upstream_result[0]
|
upstream = upstream_result[0]
|
||||||
with proxy_ctx.upstream_lock:
|
with proxy_ctx.upstream_lock:
|
||||||
proxy_ctx.upstream = upstream
|
proxy_ctx.upstream = upstream
|
||||||
|
upstream_sock = _upstream_socket(upstream)
|
||||||
|
if upstream_sock is not None:
|
||||||
|
_set_upstream_read_timeout(upstream, None)
|
||||||
|
else:
|
||||||
_set_upstream_read_timeout(upstream, 0.5)
|
_set_upstream_read_timeout(upstream, 0.5)
|
||||||
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
|
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
|
||||||
except urllib.error.HTTPError as exc:
|
except urllib.error.HTTPError as exc:
|
||||||
@@ -3387,6 +3443,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
while True:
|
while True:
|
||||||
if proxy_ctx.cancel_event.is_set():
|
if proxy_ctx.cancel_event.is_set():
|
||||||
break
|
break
|
||||||
|
if upstream_sock is not None:
|
||||||
|
readable, _, _ = select.select([upstream_sock], [], [], 0.5)
|
||||||
|
if not readable:
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
line = upstream.readline()
|
line = upstream.readline()
|
||||||
except TimeoutError:
|
except TimeoutError:
|
||||||
@@ -4324,7 +4384,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
if server.accounts is None:
|
if server.accounts is None:
|
||||||
return None
|
return None
|
||||||
return server.accounts.session_account(_api_key_from_headers(self.headers))
|
return server.accounts.session_account(_session_token_from_headers(self.headers))
|
||||||
|
|
||||||
def _require_accounts(self) -> "AccountStore | None":
|
def _require_accounts(self) -> "AccountStore | None":
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
@@ -4364,7 +4424,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
|
f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
|
||||||
f"role={account['role']}", flush=True,
|
f"role={account['role']}", flush=True,
|
||||||
)
|
)
|
||||||
self._send_json(200, {"account": account, "session_token": token, "api_key": api_key})
|
self._send_json(
|
||||||
|
200,
|
||||||
|
{"account": account, "session_token": token, "api_key": api_key},
|
||||||
|
headers={"Set-Cookie": _session_cookie_header(token)},
|
||||||
|
)
|
||||||
|
|
||||||
def _handle_auth_login(self):
|
def _handle_auth_login(self):
|
||||||
accounts = self._require_accounts()
|
accounts = self._require_accounts()
|
||||||
@@ -4379,14 +4443,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._send_json(401, {"error": "invalid credentials"})
|
self._send_json(401, {"error": "invalid credentials"})
|
||||||
return
|
return
|
||||||
token = accounts.create_session(account["account_id"])
|
token = accounts.create_session(account["account_id"])
|
||||||
self._send_json(200, {"account": account, "session_token": token})
|
self._send_json(
|
||||||
|
200,
|
||||||
|
{"account": account, "session_token": token},
|
||||||
|
headers={"Set-Cookie": _session_cookie_header(token)},
|
||||||
|
)
|
||||||
|
|
||||||
def _handle_auth_logout(self):
|
def _handle_auth_logout(self):
|
||||||
accounts = self._require_accounts()
|
accounts = self._require_accounts()
|
||||||
if accounts is None:
|
if accounts is None:
|
||||||
return
|
return
|
||||||
accounts.destroy_session(_api_key_from_headers(self.headers))
|
accounts.destroy_session(_session_token_from_headers(self.headers))
|
||||||
self._send_json(200, {"ok": True})
|
self._send_json(200, {"ok": True}, headers={"Set-Cookie": _session_cookie_header(None)})
|
||||||
|
|
||||||
def _handle_account_me(self):
|
def _handle_account_me(self):
|
||||||
"""Balance, usage, and API keys for the logged-in account."""
|
"""Balance, usage, and API keys for the logged-in account."""
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ register/login/logout, per-account balance and usage, API-key lifecycle
|
|||||||
(revoked keys rejected by the OpenAI proxy), and the admin listing.
|
(revoked keys rejected by the OpenAI proxy), and the admin listing.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import http.cookies
|
||||||
import json
|
import json
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
@@ -68,6 +69,17 @@ def test_sessions_resolve_and_destroy():
|
|||||||
assert store.session_account("bogus") is None
|
assert store.session_account("bogus") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_sessions_persist_across_restart(tmp_path):
|
||||||
|
db = str(tmp_path / "accounts.db")
|
||||||
|
store = AccountStore(db_path=db)
|
||||||
|
account = store.register(email="cookie@example.com", password="secret-123")
|
||||||
|
token = store.create_session(account["account_id"])
|
||||||
|
store.save_to_db()
|
||||||
|
|
||||||
|
reloaded = AccountStore(db_path=db)
|
||||||
|
assert reloaded.session_account(token)["account_id"] == account["account_id"]
|
||||||
|
|
||||||
|
|
||||||
def test_api_key_lifecycle():
|
def test_api_key_lifecycle():
|
||||||
store = AccountStore()
|
store = AccountStore()
|
||||||
account = store.register(email="k@example.com", password="secret-123")
|
account = store.register(email="k@example.com", password="secret-123")
|
||||||
@@ -156,6 +168,59 @@ def test_register_login_and_account_view(account_tracker):
|
|||||||
assert me["usage"]["requests"] == 0
|
assert me["usage"]["requests"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
|
||||||
|
accounts_db = str(tmp_path / "accounts.db")
|
||||||
|
tracker = TrackerServer(
|
||||||
|
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
|
||||||
|
accounts_db=accounts_db,
|
||||||
|
starting_credit=0.0,
|
||||||
|
devnet_topup_amount=0.0,
|
||||||
|
)
|
||||||
|
port = tracker.start()
|
||||||
|
url = f"http://127.0.0.1:{port}"
|
||||||
|
try:
|
||||||
|
_call(f"{url}/v1/auth/register", "POST",
|
||||||
|
{"email": "cookie-http@example.com", "password": "secret-123"})
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{url}/v1/auth/login",
|
||||||
|
data=json.dumps({
|
||||||
|
"identifier": "cookie-http@example.com",
|
||||||
|
"password": "secret-123",
|
||||||
|
}).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req) as r:
|
||||||
|
assert json.loads(r.read())["session_token"]
|
||||||
|
cookie_header = r.headers["Set-Cookie"]
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
cookie = http.cookies.SimpleCookie(cookie_header)
|
||||||
|
session_cookie = cookie["meshnet_session"].OutputString()
|
||||||
|
|
||||||
|
restarted = TrackerServer(
|
||||||
|
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
|
||||||
|
accounts_db=accounts_db,
|
||||||
|
starting_credit=0.0,
|
||||||
|
devnet_topup_amount=0.0,
|
||||||
|
)
|
||||||
|
restarted_port = restarted.start()
|
||||||
|
restarted_url = f"http://127.0.0.1:{restarted_port}"
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{restarted_url}/v1/account",
|
||||||
|
headers={"Cookie": session_cookie},
|
||||||
|
method="GET",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req) as r:
|
||||||
|
me = json.loads(r.read())
|
||||||
|
finally:
|
||||||
|
restarted.stop()
|
||||||
|
|
||||||
|
assert me["account"]["email"] == "cookie-http@example.com"
|
||||||
|
|
||||||
|
|
||||||
def test_bad_credentials_and_missing_session_are_401(account_tracker):
|
def test_bad_credentials_and_missing_session_are_401(account_tracker):
|
||||||
url, _ = account_tracker
|
url, _ = account_tracker
|
||||||
_call(f"{url}/v1/auth/register", "POST",
|
_call(f"{url}/v1/auth/register", "POST",
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ PANELS = [
|
|||||||
"Node pending payouts", "Settlement history",
|
"Node pending payouts", "Settlement history",
|
||||||
"Strikes / bans / forfeitures", "Model usage", "Call wall",
|
"Strikes / bans / forfeitures", "Model usage", "Call wall",
|
||||||
"Usage summary", "Node throughput", "Request history",
|
"Usage summary", "Node throughput", "Request history",
|
||||||
'id="tab-chat"',
|
"Chat / inference",
|
||||||
"Console output",
|
"Console output",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -33,6 +33,21 @@ def test_dashboard_served_with_all_panels():
|
|||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_dashboard_chat_uses_streaming_fetch():
|
||||||
|
tracker = TrackerServer(billing=BillingLedger())
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
html = urllib.request.urlopen(
|
||||||
|
f"http://127.0.0.1:{port}/dashboard"
|
||||||
|
).read().decode()
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert "stream: true" in html
|
||||||
|
assert ".body.getReader()" in html
|
||||||
|
assert '=== "[DONE]"' in html
|
||||||
|
|
||||||
|
|
||||||
def test_dashboard_served_by_follower():
|
def test_dashboard_served_by_follower():
|
||||||
"""A tracker that is not the leader (unreachable peers → never elected)
|
"""A tracker that is not the leader (unreachable peers → never elected)
|
||||||
still serves the dashboard from its own replicated state."""
|
still serves the dashboard from its own replicated state."""
|
||||||
|
|||||||
@@ -142,6 +142,21 @@ def _send_chat_request(gateway_url: str, prompt: str) -> dict:
|
|||||||
return json.loads(r.read())
|
return json.loads(r.read())
|
||||||
|
|
||||||
|
|
||||||
|
def _send_streaming_chat_request(gateway_url: str, prompt: str):
|
||||||
|
data = json.dumps({
|
||||||
|
"model": GPT2_MODEL,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"stream": True,
|
||||||
|
}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{gateway_url}/v1/chat/completions",
|
||||||
|
data=data,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
return urllib.request.urlopen(req)
|
||||||
|
|
||||||
|
|
||||||
def test_all_responses_valid_openai_format(tracker_node_setup):
|
def test_all_responses_valid_openai_format(tracker_node_setup):
|
||||||
"""Ten requests via gateway all return valid OpenAI chat completion format."""
|
"""Ten requests via gateway all return valid OpenAI chat completion format."""
|
||||||
gateway_url, _, _ = tracker_node_setup
|
gateway_url, _, _ = tracker_node_setup
|
||||||
@@ -155,6 +170,30 @@ def test_all_responses_valid_openai_format(tracker_node_setup):
|
|||||||
assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
|
assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
|
||||||
|
|
||||||
|
|
||||||
|
def test_streaming_head_worker_response_is_not_buffered_with_content_length(tracker_node_setup):
|
||||||
|
"""Gateway must relay head-worker SSE as a live stream, not a buffered JSON-sized body."""
|
||||||
|
gateway_url, _, _ = tracker_node_setup
|
||||||
|
|
||||||
|
with _send_streaming_chat_request(gateway_url, "stream through head worker") as resp:
|
||||||
|
assert resp.status == 200
|
||||||
|
assert "text/event-stream" in resp.headers["Content-Type"]
|
||||||
|
assert "Content-Length" not in resp.headers
|
||||||
|
data_lines = []
|
||||||
|
while len(data_lines) < 4:
|
||||||
|
line = resp.readline().decode().strip()
|
||||||
|
if line.startswith("data: "):
|
||||||
|
data_lines.append(line)
|
||||||
|
if line == "data: [DONE]":
|
||||||
|
break
|
||||||
|
|
||||||
|
assert data_lines[-1] == "data: [DONE]"
|
||||||
|
content = "".join(
|
||||||
|
json.loads(line[6:])["choices"][0].get("delta", {}).get("content", "")
|
||||||
|
for line in data_lines[:-1]
|
||||||
|
)
|
||||||
|
assert "head-worker" in content
|
||||||
|
|
||||||
|
|
||||||
def test_both_tracker_nodes_receive_load(tracker_node_setup):
|
def test_both_tracker_nodes_receive_load(tracker_node_setup):
|
||||||
"""Both head workers handle at least one request each out of ten."""
|
"""Both head workers handle at least one request each out of ten."""
|
||||||
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup
|
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup
|
||||||
|
|||||||
67
tests/test_tracker_logging.py
Normal file
67
tests/test_tracker_logging.py
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from meshnet_tracker.logging_setup import configure_tracker_file_logging, tracker_logger
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_file_logging_writes_separate_level_files(tmp_path):
|
||||||
|
original_stdout = sys.stdout
|
||||||
|
original_stderr = sys.stderr
|
||||||
|
try:
|
||||||
|
log_dir = configure_tracker_file_logging(tmp_path, tee_stdio=False)
|
||||||
|
logger = tracker_logger()
|
||||||
|
|
||||||
|
logger.info("info-event")
|
||||||
|
logger.warning("warning-event")
|
||||||
|
logger.error("error-event")
|
||||||
|
for handler in logger.handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
assert (log_dir / "info.log").read_text().count("info-event") == 1
|
||||||
|
assert "warning-event" not in (log_dir / "info.log").read_text()
|
||||||
|
assert "error-event" not in (log_dir / "info.log").read_text()
|
||||||
|
|
||||||
|
assert "warning-event" in (log_dir / "warning.log").read_text()
|
||||||
|
assert "info-event" not in (log_dir / "warning.log").read_text()
|
||||||
|
assert "error-event" not in (log_dir / "warning.log").read_text()
|
||||||
|
|
||||||
|
assert "error-event" in (log_dir / "error.log").read_text()
|
||||||
|
assert "info-event" not in (log_dir / "error.log").read_text()
|
||||||
|
assert "warning-event" not in (log_dir / "error.log").read_text()
|
||||||
|
finally:
|
||||||
|
sys.stdout = original_stdout
|
||||||
|
sys.stderr = original_stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_file_logging_tees_stdio_and_rotates(tmp_path):
|
||||||
|
original_stdout = sys.stdout
|
||||||
|
original_stderr = sys.stderr
|
||||||
|
try:
|
||||||
|
log_dir = configure_tracker_file_logging(
|
||||||
|
tmp_path,
|
||||||
|
max_bytes=120,
|
||||||
|
backup_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
print("stdout goes to info", flush=True)
|
||||||
|
print("stderr goes to error", file=sys.stderr, flush=True)
|
||||||
|
for handler in tracker_logger().handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
assert "stdout goes to info" in (log_dir / "info.log").read_text()
|
||||||
|
assert "stderr goes to error" in (log_dir / "error.log").read_text()
|
||||||
|
|
||||||
|
for index in range(12):
|
||||||
|
tracker_logger().info("rotating-info-line-%02d", index)
|
||||||
|
for handler in tracker_logger().handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
assert (log_dir / "info.log.1").exists()
|
||||||
|
finally:
|
||||||
|
sys.stdout = original_stdout
|
||||||
|
sys.stderr = original_stderr
|
||||||
|
logger = tracker_logger()
|
||||||
|
for handler in logger.handlers:
|
||||||
|
handler.close()
|
||||||
|
logger.handlers.clear()
|
||||||
|
logger.setLevel(logging.NOTSET)
|
||||||
@@ -502,6 +502,75 @@ def test_tracker_logs_stream_progress_before_request_completes():
|
|||||||
node_thread.join(timeout=1.0)
|
node_thread.join(timeout=1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_stream_survives_idle_gap_between_sse_chunks():
|
||||||
|
first_chunk_sent = threading.Event()
|
||||||
|
|
||||||
|
class IdleStreamingChatHandler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
if self.path != "/v1/chat/completions":
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
return
|
||||||
|
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
||||||
|
self.end_headers()
|
||||||
|
first = json.dumps({
|
||||||
|
"choices": [{"delta": {"content": "hello"}}],
|
||||||
|
}).encode()
|
||||||
|
second = json.dumps({
|
||||||
|
"choices": [{"delta": {"content": " world"}}],
|
||||||
|
}).encode()
|
||||||
|
self.wfile.write(b"data: " + first + b"\n\n")
|
||||||
|
self.wfile.flush()
|
||||||
|
first_chunk_sent.set()
|
||||||
|
time.sleep(1.0)
|
||||||
|
self.wfile.write(b"data: " + second + b"\n\n")
|
||||||
|
self.wfile.write(b"data: [DONE]\n\n")
|
||||||
|
self.wfile.flush()
|
||||||
|
|
||||||
|
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
|
||||||
|
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
|
||||||
|
node_thread.start()
|
||||||
|
tracker = TrackerServer(heartbeat_timeout=60.0)
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
response = None
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
|
||||||
|
"model": "idle-stream-model", "num_layers": 1,
|
||||||
|
"shard_start": 0, "shard_end": 0,
|
||||||
|
"hardware_profile": {}, "score": 1.0},
|
||||||
|
)
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
|
||||||
|
data=json.dumps({
|
||||||
|
"model": "idle-stream-model",
|
||||||
|
"stream": True,
|
||||||
|
"messages": [{"role": "user", "content": "hi"}],
|
||||||
|
}).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
response = urllib.request.urlopen(req, timeout=3.0)
|
||||||
|
assert response.readline().startswith(b"data:")
|
||||||
|
assert first_chunk_sent.wait(timeout=1.0)
|
||||||
|
|
||||||
|
remaining = response.read().splitlines()
|
||||||
|
assert b"data: [DONE]" in remaining
|
||||||
|
finally:
|
||||||
|
if response is not None:
|
||||||
|
response.close()
|
||||||
|
tracker.stop()
|
||||||
|
node.shutdown()
|
||||||
|
node.server_close()
|
||||||
|
node_thread.join(timeout=1.0)
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_dashboard_can_cancel_inflight_proxy():
|
def test_tracker_dashboard_can_cancel_inflight_proxy():
|
||||||
chunk_sent = threading.Event()
|
chunk_sent = threading.Event()
|
||||||
release = threading.Event()
|
release = threading.Event()
|
||||||
|
|||||||
Reference in New Issue
Block a user