Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai
# Conflicts: # packages/tracker/meshnet_tracker/cli.py # packages/tracker/meshnet_tracker/dashboard.html # packages/tracker/meshnet_tracker/server.py # tests/test_dashboard.py
This commit is contained in:
3
.gitignore
vendored
3
.gitignore
vendored
@@ -23,3 +23,6 @@ dist/
|
|||||||
# Local tracker/node sqlite databases (never commit runtime state)
|
# Local tracker/node sqlite databases (never commit runtime state)
|
||||||
*.sqlite
|
*.sqlite
|
||||||
*.sqlite3
|
*.sqlite3
|
||||||
|
logs/tracker/error.log
|
||||||
|
logs/tracker/info.log
|
||||||
|
logs/tracker/warning.log
|
||||||
|
|||||||
@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._send_json(200, completion)
|
self._send_json(200, completion)
|
||||||
|
|
||||||
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
|
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
|
||||||
"""Forward a raw request body to a head worker and stream the response back."""
|
"""Forward a raw request body to a head worker and relay SSE without buffering."""
|
||||||
target_url = f"{url}/v1/chat/completions"
|
target_url = f"{url}/v1/chat/completions"
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
target_url,
|
target_url,
|
||||||
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(req, timeout=30.0) as r:
|
with urllib.request.urlopen(req, timeout=30.0) as r:
|
||||||
content_type = r.headers.get("Content-Type", "application/json")
|
content_type = r.headers.get("Content-Type", "application/json")
|
||||||
|
if "text/event-stream" in content_type:
|
||||||
|
self.send_response(r.status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Cache-Control", "no-cache")
|
||||||
|
self.send_header("X-Accel-Buffering", "no")
|
||||||
|
self.end_headers()
|
||||||
|
while True:
|
||||||
|
line = r.readline()
|
||||||
|
if not line:
|
||||||
|
break
|
||||||
|
self.wfile.write(line)
|
||||||
|
self.wfile.flush()
|
||||||
|
return
|
||||||
resp_body = r.read()
|
resp_body = r.read()
|
||||||
status = r.status
|
status = r.status
|
||||||
except urllib.error.HTTPError as exc:
|
except urllib.error.HTTPError as exc:
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ regular user.
|
|||||||
Mutations are append-only events with unique ids — the same replication
|
Mutations are append-only events with unique ids — the same replication
|
||||||
model as ``BillingLedger`` — so accounts and API keys converge across the
|
model as ``BillingLedger`` — so accounts and API keys converge across the
|
||||||
tracker hive via gossip, and every dashboard can serve registration/login.
|
tracker hive via gossip, and every dashboard can serve registration/login.
|
||||||
Sessions are deliberately local to each tracker (bearer tokens in memory).
|
Sessions are local to each tracker and persisted so dashboard cookies survive
|
||||||
|
tracker restarts.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -115,6 +116,8 @@ class AccountStore:
|
|||||||
"account_id": account_id,
|
"account_id": account_id,
|
||||||
"expires": time.time() + SESSION_TTL,
|
"expires": time.time() + SESSION_TTL,
|
||||||
}
|
}
|
||||||
|
self._dirty = True
|
||||||
|
self.save_to_db()
|
||||||
return token
|
return token
|
||||||
|
|
||||||
def session_account(self, token: str | None) -> dict | None:
|
def session_account(self, token: str | None) -> dict | None:
|
||||||
@@ -134,7 +137,9 @@ class AccountStore:
|
|||||||
if not token:
|
if not token:
|
||||||
return
|
return
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._sessions.pop(token, None)
|
if self._sessions.pop(token, None) is not None:
|
||||||
|
self._dirty = True
|
||||||
|
self.save_to_db()
|
||||||
|
|
||||||
# ---- API keys ----
|
# ---- API keys ----
|
||||||
|
|
||||||
@@ -271,6 +276,10 @@ class AccountStore:
|
|||||||
"CREATE TABLE IF NOT EXISTS account_events "
|
"CREATE TABLE IF NOT EXISTS account_events "
|
||||||
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
|
"(event_id TEXT PRIMARY KEY, payload TEXT NOT NULL, ts REAL NOT NULL)"
|
||||||
)
|
)
|
||||||
|
con.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS account_sessions "
|
||||||
|
"(token TEXT PRIMARY KEY, account_id TEXT NOT NULL, expires REAL NOT NULL)"
|
||||||
|
)
|
||||||
con.commit()
|
con.commit()
|
||||||
con.close()
|
con.close()
|
||||||
|
|
||||||
@@ -279,6 +288,10 @@ class AccountStore:
|
|||||||
rows = con.execute(
|
rows = con.execute(
|
||||||
"SELECT payload FROM account_events ORDER BY ts, event_id"
|
"SELECT payload FROM account_events ORDER BY ts, event_id"
|
||||||
).fetchall()
|
).fetchall()
|
||||||
|
session_rows = con.execute(
|
||||||
|
"SELECT token, account_id, expires FROM account_sessions WHERE expires >= ?",
|
||||||
|
(time.time(),),
|
||||||
|
).fetchall()
|
||||||
con.close()
|
con.close()
|
||||||
with self._lock:
|
with self._lock:
|
||||||
for (payload,) in rows:
|
for (payload,) in rows:
|
||||||
@@ -288,6 +301,11 @@ class AccountStore:
|
|||||||
continue
|
continue
|
||||||
if event.get("id") not in self._seen_event_ids:
|
if event.get("id") not in self._seen_event_ids:
|
||||||
self._apply_locked(event)
|
self._apply_locked(event)
|
||||||
|
self._sessions = {
|
||||||
|
token: {"account_id": account_id, "expires": float(expires)}
|
||||||
|
for token, account_id, expires in session_rows
|
||||||
|
if account_id in self._accounts
|
||||||
|
}
|
||||||
self._dirty = False
|
self._dirty = False
|
||||||
|
|
||||||
def save_to_db(self) -> None:
|
def save_to_db(self) -> None:
|
||||||
@@ -297,11 +315,21 @@ class AccountStore:
|
|||||||
if not self._dirty:
|
if not self._dirty:
|
||||||
return
|
return
|
||||||
events = list(self._event_log)
|
events = list(self._event_log)
|
||||||
|
sessions = [
|
||||||
|
(token, session["account_id"], float(session["expires"]))
|
||||||
|
for token, session in self._sessions.items()
|
||||||
|
if session["expires"] >= time.time()
|
||||||
|
]
|
||||||
self._dirty = False
|
self._dirty = False
|
||||||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||||
con.executemany(
|
con.executemany(
|
||||||
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
|
"INSERT OR IGNORE INTO account_events (event_id, payload, ts) VALUES (?, ?, ?)",
|
||||||
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
|
[(e["id"], json.dumps(e), float(e.get("ts", 0.0))) for e in events],
|
||||||
)
|
)
|
||||||
|
con.execute("DELETE FROM account_sessions")
|
||||||
|
con.executemany(
|
||||||
|
"INSERT INTO account_sessions (token, account_id, expires) VALUES (?, ?, ?)",
|
||||||
|
sessions,
|
||||||
|
)
|
||||||
con.commit()
|
con.commit()
|
||||||
con.close()
|
con.close()
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ from pathlib import Path
|
|||||||
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
|
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
|
||||||
from .billing import DEFAULT_BILLING_DB_PATH
|
from .billing import DEFAULT_BILLING_DB_PATH
|
||||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
|
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
|
||||||
|
from .logging_setup import (
|
||||||
|
DEFAULT_LOG_BACKUP_COUNT,
|
||||||
|
DEFAULT_LOG_DIR,
|
||||||
|
DEFAULT_LOG_MAX_BYTES,
|
||||||
|
configure_tracker_file_logging,
|
||||||
|
)
|
||||||
from .routing_stats import RoutingConfig
|
from .routing_stats import RoutingConfig
|
||||||
from .server import (
|
from .server import (
|
||||||
DEFAULT_CALLER_CREDIT_USDT,
|
DEFAULT_CALLER_CREDIT_USDT,
|
||||||
@@ -302,6 +308,34 @@ def main() -> None:
|
|||||||
metavar="SECONDS",
|
metavar="SECONDS",
|
||||||
help="Half-life for decaying route throughput observations (default 600)",
|
help="Half-life for decaying route throughput observations (default 600)",
|
||||||
)
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--log-dir",
|
||||||
|
default=DEFAULT_LOG_DIR,
|
||||||
|
metavar="PATH",
|
||||||
|
help=(
|
||||||
|
"Directory for rotating tracker logs "
|
||||||
|
f"(default: {DEFAULT_LOG_DIR}; files: info.log, warning.log, error.log)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--log-max-bytes",
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_LOG_MAX_BYTES,
|
||||||
|
metavar="BYTES",
|
||||||
|
help=f"Rotate each tracker log file after this many bytes (default: {DEFAULT_LOG_MAX_BYTES})",
|
||||||
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--log-backup-count",
|
||||||
|
type=int,
|
||||||
|
default=DEFAULT_LOG_BACKUP_COUNT,
|
||||||
|
metavar="N",
|
||||||
|
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
|
||||||
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--no-file-logs",
|
||||||
|
action="store_true",
|
||||||
|
help="Disable rotating tracker log files and only write to the terminal",
|
||||||
|
)
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
prog="meshnet-tracker",
|
prog="meshnet-tracker",
|
||||||
@@ -315,6 +349,13 @@ def main() -> None:
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
if args.command in {None, "start"}:
|
if args.command in {None, "start"}:
|
||||||
|
if not args.no_file_logs:
|
||||||
|
log_dir = configure_tracker_file_logging(
|
||||||
|
args.log_dir,
|
||||||
|
max_bytes=args.log_max_bytes,
|
||||||
|
backup_count=args.log_backup_count,
|
||||||
|
)
|
||||||
|
print(f"meshnet-tracker logs: {log_dir}", flush=True)
|
||||||
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
|
cluster_peers = [u.strip() for u in args.cluster_peers.split(",") if u.strip()]
|
||||||
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
|
relay_url = args.relay_url or derive_relay_url_from_public_tracker_url(args.self_url)
|
||||||
treasury = None
|
treasury = None
|
||||||
|
|||||||
@@ -234,6 +234,7 @@
|
|||||||
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
||||||
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
||||||
<section data-tab="chat" class="wide chat-section">
|
<section data-tab="chat" class="wide chat-section">
|
||||||
|
<h2 style="display:none">Chat / inference</h2>
|
||||||
<div class="chat-app">
|
<div class="chat-app">
|
||||||
<aside class="chat-sidebar">
|
<aside class="chat-sidebar">
|
||||||
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
|
<button type="button" class="chat-new-btn" onclick="createNewChatSession()">+ New chat</button>
|
||||||
@@ -283,7 +284,7 @@ async function fetchJson(path) {
|
|||||||
const headers = {};
|
const headers = {};
|
||||||
const token = localStorage.getItem("meshnet_session");
|
const token = localStorage.getItem("meshnet_session");
|
||||||
if (token) headers["Authorization"] = "Bearer " + token;
|
if (token) headers["Authorization"] = "Bearer " + token;
|
||||||
const r = await fetch(path, { headers });
|
const r = await fetch(path, { headers, credentials: "same-origin" });
|
||||||
if (!r.ok) return null;
|
if (!r.ok) return null;
|
||||||
return await r.json();
|
return await r.json();
|
||||||
} catch { return null; }
|
} catch { return null; }
|
||||||
@@ -1109,6 +1110,7 @@ async function apiCall(path, method, body, bearerToken) {
|
|||||||
const r = await fetch(path, {
|
const r = await fetch(path, {
|
||||||
method: method || "GET",
|
method: method || "GET",
|
||||||
headers,
|
headers,
|
||||||
|
credentials: "same-origin",
|
||||||
body: body ? JSON.stringify(body) : undefined,
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
});
|
});
|
||||||
const data = await r.json().catch(() => ({}));
|
const data = await r.json().catch(() => ({}));
|
||||||
@@ -1368,10 +1370,11 @@ async function sendChat() {
|
|||||||
promptEl.value = "";
|
promptEl.value = "";
|
||||||
promptEl.style.height = "auto";
|
promptEl.style.height = "auto";
|
||||||
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
chatHistory.push({ role: "user", content: prompt, model: selectedChatModel });
|
||||||
|
const assistantMessage = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
||||||
|
chatHistory.push(assistantMessage);
|
||||||
renderChatHistory();
|
renderChatHistory();
|
||||||
persistActiveChatSession();
|
persistActiveChatSession();
|
||||||
renderChatStatus("sending request…");
|
renderChatStatus("sending request…");
|
||||||
const assistantMsg = { role: "assistant", content: "", model: selectedChatModel, streaming: true };
|
|
||||||
let tokens = 0;
|
let tokens = 0;
|
||||||
let usage = null;
|
let usage = null;
|
||||||
const started = Date.now();
|
const started = Date.now();
|
||||||
@@ -1382,6 +1385,7 @@ async function sendChat() {
|
|||||||
const resp = await fetch("/v1/chat/completions", {
|
const resp = await fetch("/v1/chat/completions", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers,
|
headers,
|
||||||
|
credentials: "same-origin",
|
||||||
body: JSON.stringify(body),
|
body: JSON.stringify(body),
|
||||||
signal: chatAbortController.signal,
|
signal: chatAbortController.signal,
|
||||||
});
|
});
|
||||||
@@ -1396,12 +1400,10 @@ async function sendChat() {
|
|||||||
const contentType = resp.headers.get("Content-Type") || "";
|
const contentType = resp.headers.get("Content-Type") || "";
|
||||||
if (!resp.body || !contentType.includes("text/event-stream")) {
|
if (!resp.body || !contentType.includes("text/event-stream")) {
|
||||||
const data = await resp.json();
|
const data = await resp.json();
|
||||||
assistantMsg.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
assistantMessage.content = (data.choices && data.choices[0] && data.choices[0].message && data.choices[0].message.content) || "";
|
||||||
usage = data.usage || null;
|
usage = data.usage || null;
|
||||||
tokens = (usage && usage.completion_tokens) || 0;
|
tokens = (usage && usage.completion_tokens) || 0;
|
||||||
} else {
|
} else {
|
||||||
chatHistory.push(assistantMsg);
|
|
||||||
renderChatHistory();
|
|
||||||
const reader = resp.body.getReader();
|
const reader = resp.body.getReader();
|
||||||
const decoder = new TextDecoder();
|
const decoder = new TextDecoder();
|
||||||
let buffered = "";
|
let buffered = "";
|
||||||
@@ -1425,30 +1427,31 @@ async function sendChat() {
|
|||||||
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
const delta = chunk.choices && chunk.choices[0] && chunk.choices[0].delta;
|
||||||
const piece = delta && delta.content;
|
const piece = delta && delta.content;
|
||||||
if (piece) {
|
if (piece) {
|
||||||
assistantMsg.content += piece;
|
assistantMessage.content += piece;
|
||||||
tokens += 1;
|
tokens += 1;
|
||||||
updateStreamingChatBubble(assistantMsg.content);
|
updateStreamingChatBubble(assistantMessage.content);
|
||||||
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
const secs = Math.max((Date.now() - started) / 1000, 0.001);
|
||||||
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
renderChatStatus(`generating… ${tokens} tokens · ${(tokens / secs).toFixed(1)} tok/s`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finalizeAssistantMessage(assistantMsg);
|
finalizeAssistantMessage(assistantMessage);
|
||||||
renderChatStatus(usage
|
renderChatStatus(usage
|
||||||
? `done: ${usage.total_tokens ?? "?"} tokens`
|
? `done: ${usage.total_tokens ?? "?"} tokens`
|
||||||
: `done: ${tokens} tokens`);
|
: `done: ${tokens} tokens`);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (err && err.name === "AbortError") {
|
if (err && err.name === "AbortError") {
|
||||||
if (assistantMsg.content) {
|
if (assistantMessage.content) {
|
||||||
finalizeAssistantMessage(assistantMsg);
|
finalizeAssistantMessage(assistantMessage);
|
||||||
renderChatStatus(`stopped after ${tokens} tokens`);
|
renderChatStatus(`stopped after ${tokens} tokens`);
|
||||||
} else {
|
} else {
|
||||||
removeAssistantMessage(assistantMsg);
|
removeAssistantMessage(assistantMessage);
|
||||||
|
persistActiveChatSession();
|
||||||
renderChatStatus("stopped");
|
renderChatStatus("stopped");
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
removeAssistantMessage(assistantMsg);
|
removeAssistantMessage(assistantMessage);
|
||||||
const error = (err && err.message) || "request failed";
|
const error = (err && err.message) || "request failed";
|
||||||
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
chatHistory.push({ role: "error", content: error, model: selectedChatModel });
|
||||||
renderChatHistory();
|
renderChatHistory();
|
||||||
@@ -1557,7 +1560,7 @@ renderChatModels();
|
|||||||
renderChatHistory();
|
renderChatHistory();
|
||||||
renderChatAuthHint();
|
renderChatAuthHint();
|
||||||
setInterval(refresh, 4000);
|
setInterval(refresh, 4000);
|
||||||
setInterval(() => { if (sessionToken) renderAccountPanel(); }, 8000);
|
setInterval(() => { if (sessionToken || isLoggedIn) renderAccountPanel(); }, 8000);
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
99
packages/tracker/meshnet_tracker/logging_setup.py
Normal file
99
packages/tracker/meshnet_tracker/logging_setup.py
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
"""Rotating file logging for the tracker CLI."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
from logging.handlers import RotatingFileHandler
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import TextIO
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_LOG_DIR = "logs/tracker"
|
||||||
|
DEFAULT_LOG_MAX_BYTES = 10 * 1024 * 1024
|
||||||
|
DEFAULT_LOG_BACKUP_COUNT = 5
|
||||||
|
TRACKER_LOGGER_NAME = "meshnet.tracker"
|
||||||
|
|
||||||
|
|
||||||
|
class _ExactLevelFilter(logging.Filter):
|
||||||
|
def __init__(self, level: int) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self._level = level
|
||||||
|
|
||||||
|
def filter(self, record: logging.LogRecord) -> bool:
|
||||||
|
return record.levelno == self._level
|
||||||
|
|
||||||
|
|
||||||
|
class _TeeStream:
|
||||||
|
def __init__(self, stream: TextIO, logger: logging.Logger, level: int) -> None:
|
||||||
|
self._stream = stream
|
||||||
|
self._logger = logger
|
||||||
|
self._level = level
|
||||||
|
self._buffer = ""
|
||||||
|
|
||||||
|
def write(self, text: str) -> int:
|
||||||
|
self._stream.write(text)
|
||||||
|
self._stream.flush()
|
||||||
|
self._buffer += text
|
||||||
|
while "\n" in self._buffer:
|
||||||
|
line, self._buffer = self._buffer.split("\n", 1)
|
||||||
|
line = line.rstrip()
|
||||||
|
if line:
|
||||||
|
self._logger.log(self._level, line)
|
||||||
|
return len(text)
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self._stream.flush()
|
||||||
|
line = self._buffer.rstrip()
|
||||||
|
if line:
|
||||||
|
self._logger.log(self._level, line)
|
||||||
|
self._buffer = ""
|
||||||
|
|
||||||
|
def isatty(self) -> bool:
|
||||||
|
return self._stream.isatty()
|
||||||
|
|
||||||
|
|
||||||
|
def _make_handler(path: Path, level: int, *, max_bytes: int, backup_count: int) -> RotatingFileHandler:
|
||||||
|
handler = RotatingFileHandler(
|
||||||
|
path,
|
||||||
|
maxBytes=max_bytes,
|
||||||
|
backupCount=backup_count,
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
handler.setLevel(level)
|
||||||
|
handler.addFilter(_ExactLevelFilter(level))
|
||||||
|
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
||||||
|
return handler
|
||||||
|
|
||||||
|
|
||||||
|
def configure_tracker_file_logging(
|
||||||
|
log_dir: str | Path = DEFAULT_LOG_DIR,
|
||||||
|
*,
|
||||||
|
max_bytes: int = DEFAULT_LOG_MAX_BYTES,
|
||||||
|
backup_count: int = DEFAULT_LOG_BACKUP_COUNT,
|
||||||
|
tee_stdio: bool = True,
|
||||||
|
) -> Path:
|
||||||
|
"""Configure rotatable info/warning/error log files and return the directory."""
|
||||||
|
|
||||||
|
path = Path(log_dir).expanduser()
|
||||||
|
path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
logger = logging.getLogger(TRACKER_LOGGER_NAME)
|
||||||
|
logger.setLevel(logging.INFO)
|
||||||
|
logger.propagate = False
|
||||||
|
logger.handlers.clear()
|
||||||
|
logger.addHandler(_make_handler(path / "info.log", logging.INFO, max_bytes=max_bytes, backup_count=backup_count))
|
||||||
|
logger.addHandler(_make_handler(path / "warning.log", logging.WARNING, max_bytes=max_bytes, backup_count=backup_count))
|
||||||
|
logger.addHandler(_make_handler(path / "error.log", logging.ERROR, max_bytes=max_bytes, backup_count=backup_count))
|
||||||
|
|
||||||
|
if tee_stdio:
|
||||||
|
if not isinstance(sys.stdout, _TeeStream):
|
||||||
|
sys.stdout = _TeeStream(sys.stdout, logger, logging.INFO) # type: ignore[assignment]
|
||||||
|
if not isinstance(sys.stderr, _TeeStream):
|
||||||
|
sys.stderr = _TeeStream(sys.stderr, logger, logging.ERROR) # type: ignore[assignment]
|
||||||
|
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def tracker_logger() -> logging.Logger:
|
||||||
|
return logging.getLogger(TRACKER_LOGGER_NAME)
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -5,6 +5,7 @@ register/login/logout, per-account balance and usage, API-key lifecycle
|
|||||||
(revoked keys rejected by the OpenAI proxy), and the admin listing.
|
(revoked keys rejected by the OpenAI proxy), and the admin listing.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import http.cookies
|
||||||
import json
|
import json
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
@@ -68,6 +69,17 @@ def test_sessions_resolve_and_destroy():
|
|||||||
assert store.session_account("bogus") is None
|
assert store.session_account("bogus") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_sessions_persist_across_restart(tmp_path):
|
||||||
|
db = str(tmp_path / "accounts.db")
|
||||||
|
store = AccountStore(db_path=db)
|
||||||
|
account = store.register(email="cookie@example.com", password="secret-123")
|
||||||
|
token = store.create_session(account["account_id"])
|
||||||
|
store.save_to_db()
|
||||||
|
|
||||||
|
reloaded = AccountStore(db_path=db)
|
||||||
|
assert reloaded.session_account(token)["account_id"] == account["account_id"]
|
||||||
|
|
||||||
|
|
||||||
def test_api_key_lifecycle():
|
def test_api_key_lifecycle():
|
||||||
store = AccountStore()
|
store = AccountStore()
|
||||||
account = store.register(email="k@example.com", password="secret-123")
|
account = store.register(email="k@example.com", password="secret-123")
|
||||||
@@ -156,6 +168,59 @@ def test_register_login_and_account_view(account_tracker):
|
|||||||
assert me["usage"]["requests"] == 0
|
assert me["usage"]["requests"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_sets_cookie_and_cookie_auth_survives_tracker_restart(tmp_path):
|
||||||
|
accounts_db = str(tmp_path / "accounts.db")
|
||||||
|
tracker = TrackerServer(
|
||||||
|
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
|
||||||
|
accounts_db=accounts_db,
|
||||||
|
starting_credit=0.0,
|
||||||
|
devnet_topup_amount=0.0,
|
||||||
|
)
|
||||||
|
port = tracker.start()
|
||||||
|
url = f"http://127.0.0.1:{port}"
|
||||||
|
try:
|
||||||
|
_call(f"{url}/v1/auth/register", "POST",
|
||||||
|
{"email": "cookie-http@example.com", "password": "secret-123"})
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{url}/v1/auth/login",
|
||||||
|
data=json.dumps({
|
||||||
|
"identifier": "cookie-http@example.com",
|
||||||
|
"password": "secret-123",
|
||||||
|
}).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req) as r:
|
||||||
|
assert json.loads(r.read())["session_token"]
|
||||||
|
cookie_header = r.headers["Set-Cookie"]
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
cookie = http.cookies.SimpleCookie(cookie_header)
|
||||||
|
session_cookie = cookie["meshnet_session"].OutputString()
|
||||||
|
|
||||||
|
restarted = TrackerServer(
|
||||||
|
billing=BillingLedger(starting_credit=0.0, default_price_per_1k=0.02),
|
||||||
|
accounts_db=accounts_db,
|
||||||
|
starting_credit=0.0,
|
||||||
|
devnet_topup_amount=0.0,
|
||||||
|
)
|
||||||
|
restarted_port = restarted.start()
|
||||||
|
restarted_url = f"http://127.0.0.1:{restarted_port}"
|
||||||
|
try:
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{restarted_url}/v1/account",
|
||||||
|
headers={"Cookie": session_cookie},
|
||||||
|
method="GET",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(req) as r:
|
||||||
|
me = json.loads(r.read())
|
||||||
|
finally:
|
||||||
|
restarted.stop()
|
||||||
|
|
||||||
|
assert me["account"]["email"] == "cookie-http@example.com"
|
||||||
|
|
||||||
|
|
||||||
def test_bad_credentials_and_missing_session_are_401(account_tracker):
|
def test_bad_credentials_and_missing_session_are_401(account_tracker):
|
||||||
url, _ = account_tracker
|
url, _ = account_tracker
|
||||||
_call(f"{url}/v1/auth/register", "POST",
|
_call(f"{url}/v1/auth/register", "POST",
|
||||||
|
|||||||
@@ -1,138 +1,153 @@
|
|||||||
"""US-035: tracker web dashboard — served from any tracker, embedded asset."""
|
"""US-035: tracker web dashboard — served from any tracker, embedded asset."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
|
||||||
from meshnet_contracts import LocalSolanaContracts
|
from meshnet_contracts import LocalSolanaContracts
|
||||||
from meshnet_tracker.accounts import AccountStore
|
from meshnet_tracker.accounts import AccountStore
|
||||||
from meshnet_tracker.billing import BillingLedger
|
from meshnet_tracker.billing import BillingLedger
|
||||||
from meshnet_tracker.server import TrackerServer
|
from meshnet_tracker.server import TrackerServer
|
||||||
|
|
||||||
PANELS = [
|
PANELS = [
|
||||||
"Tracker hive", "Nodes & coverage", "Client balances",
|
"Tracker hive", "Nodes & coverage", "Client balances",
|
||||||
"Node pending payouts", "Settlement history",
|
"Node pending payouts", "Settlement history",
|
||||||
"Strikes / bans / forfeitures", "Model usage", "Call wall",
|
"Strikes / bans / forfeitures", "Model usage", "Call wall",
|
||||||
"Usage summary", "Node throughput", "Request history",
|
"Usage summary", "Node throughput", "Request history",
|
||||||
'id="tab-chat"',
|
"Chat / inference",
|
||||||
"Console output",
|
"Console output",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_dashboard_served_with_all_panels():
|
def test_dashboard_served_with_all_panels():
|
||||||
tracker = TrackerServer(billing=BillingLedger())
|
tracker = TrackerServer(billing=BillingLedger())
|
||||||
port = tracker.start()
|
port = tracker.start()
|
||||||
try:
|
try:
|
||||||
html = urllib.request.urlopen(
|
html = urllib.request.urlopen(
|
||||||
f"http://127.0.0.1:{port}/dashboard"
|
f"http://127.0.0.1:{port}/dashboard"
|
||||||
).read().decode()
|
).read().decode()
|
||||||
for panel in PANELS:
|
for panel in PANELS:
|
||||||
assert panel in html
|
assert panel in html
|
||||||
assert "<script>" in html # polling client embedded, no build step
|
assert "<script>" in html # polling client embedded, no build step
|
||||||
finally:
|
finally:
|
||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
def test_dashboard_served_by_follower():
|
def test_dashboard_chat_uses_streaming_fetch():
|
||||||
"""A tracker that is not the leader (unreachable peers → never elected)
|
tracker = TrackerServer(billing=BillingLedger())
|
||||||
still serves the dashboard from its own replicated state."""
|
port = tracker.start()
|
||||||
tracker = TrackerServer(
|
try:
|
||||||
billing=BillingLedger(),
|
html = urllib.request.urlopen(
|
||||||
cluster_peers=["http://127.0.0.1:1", "http://127.0.0.1:2"],
|
f"http://127.0.0.1:{port}/dashboard"
|
||||||
)
|
).read().decode()
|
||||||
port = tracker.start()
|
finally:
|
||||||
try:
|
tracker.stop()
|
||||||
response = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard")
|
|
||||||
assert response.status == 200
|
assert "stream: true" in html
|
||||||
assert "meshnet tracker" in response.read().decode()
|
assert ".body.getReader()" in html
|
||||||
finally:
|
assert '=== "[DONE]"' in html
|
||||||
tracker.stop()
|
|
||||||
|
|
||||||
|
def test_dashboard_served_by_follower():
|
||||||
def test_registry_wallets_endpoint():
|
"""A tracker that is not the leader (unreachable peers → never elected)
|
||||||
contracts = LocalSolanaContracts()
|
still serves the dashboard from its own replicated state."""
|
||||||
contracts.registry.submit_stake("wallet-a", 100)
|
tracker = TrackerServer(
|
||||||
contracts.registry.record_strike("wallet-a")
|
billing=BillingLedger(),
|
||||||
accounts = AccountStore()
|
cluster_peers=["http://127.0.0.1:1", "http://127.0.0.1:2"],
|
||||||
admin = accounts.register(email="admin@example.com", password="admin-pass-123")
|
)
|
||||||
session = accounts.create_session(admin["account_id"])
|
port = tracker.start()
|
||||||
tracker = TrackerServer(contracts=contracts, accounts=accounts)
|
try:
|
||||||
port = tracker.start()
|
response = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard")
|
||||||
try:
|
assert response.status == 200
|
||||||
req = urllib.request.Request(
|
assert "meshnet tracker" in response.read().decode()
|
||||||
f"http://127.0.0.1:{port}/v1/registry/wallets",
|
finally:
|
||||||
headers={"Authorization": f"Bearer {session}"},
|
tracker.stop()
|
||||||
)
|
|
||||||
data = json.loads(urllib.request.urlopen(req).read())
|
|
||||||
assert data["wallets"]["wallet-a"]["strike_count"] == 1
|
def test_registry_wallets_endpoint():
|
||||||
assert data["wallets"]["wallet-a"]["banned"] is False
|
contracts = LocalSolanaContracts()
|
||||||
finally:
|
contracts.registry.submit_stake("wallet-a", 100)
|
||||||
tracker.stop()
|
contracts.registry.record_strike("wallet-a")
|
||||||
|
accounts = AccountStore()
|
||||||
|
admin = accounts.register(email="admin@example.com", password="admin-pass-123")
|
||||||
def test_console_endpoint_exposes_tracker_events():
|
session = accounts.create_session(admin["account_id"])
|
||||||
tracker = TrackerServer()
|
tracker = TrackerServer(contracts=contracts, accounts=accounts)
|
||||||
port = tracker.start()
|
port = tracker.start()
|
||||||
try:
|
try:
|
||||||
body = json.dumps({
|
req = urllib.request.Request(
|
||||||
"endpoint": "http://127.0.0.1:9001",
|
f"http://127.0.0.1:{port}/v1/registry/wallets",
|
||||||
"model": "stub-model",
|
headers={"Authorization": f"Bearer {session}"},
|
||||||
"shard_start": 0,
|
)
|
||||||
"shard_end": 3,
|
data = json.loads(urllib.request.urlopen(req).read())
|
||||||
"hardware_profile": {},
|
assert data["wallets"]["wallet-a"]["strike_count"] == 1
|
||||||
}).encode()
|
assert data["wallets"]["wallet-a"]["banned"] is False
|
||||||
req = urllib.request.Request(
|
finally:
|
||||||
f"http://127.0.0.1:{port}/v1/nodes/register",
|
tracker.stop()
|
||||||
data=body,
|
|
||||||
headers={"Content-Type": "application/json"},
|
|
||||||
method="POST",
|
def test_console_endpoint_exposes_tracker_events():
|
||||||
)
|
tracker = TrackerServer()
|
||||||
urllib.request.urlopen(req).read()
|
port = tracker.start()
|
||||||
|
try:
|
||||||
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
body = json.dumps({
|
||||||
finally:
|
"endpoint": "http://127.0.0.1:9001",
|
||||||
tracker.stop()
|
"model": "stub-model",
|
||||||
|
"shard_start": 0,
|
||||||
assert any(event["message"] == "node registered" for event in data["events"])
|
"shard_end": 3,
|
||||||
|
"hardware_profile": {},
|
||||||
|
}).encode()
|
||||||
def test_console_node_lifecycle_events_include_model_health():
|
req = urllib.request.Request(
|
||||||
tracker = TrackerServer(heartbeat_timeout=0.05)
|
f"http://127.0.0.1:{port}/v1/nodes/register",
|
||||||
port = tracker.start()
|
data=body,
|
||||||
try:
|
headers={"Content-Type": "application/json"},
|
||||||
body = json.dumps({
|
method="POST",
|
||||||
"endpoint": "http://127.0.0.1:9002",
|
)
|
||||||
"model": "console-health-test",
|
urllib.request.urlopen(req).read()
|
||||||
"hf_repo": "example/console-health-test",
|
|
||||||
"num_layers": 4,
|
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
||||||
"shard_start": 0,
|
finally:
|
||||||
"shard_end": 1,
|
tracker.stop()
|
||||||
"hardware_profile": {},
|
|
||||||
}).encode()
|
assert any(event["message"] == "node registered" for event in data["events"])
|
||||||
req = urllib.request.Request(
|
|
||||||
f"http://127.0.0.1:{port}/v1/nodes/register",
|
|
||||||
data=body,
|
def test_console_node_lifecycle_events_include_model_health():
|
||||||
headers={"Content-Type": "application/json"},
|
tracker = TrackerServer(heartbeat_timeout=0.05)
|
||||||
method="POST",
|
port = tracker.start()
|
||||||
)
|
try:
|
||||||
urllib.request.urlopen(req).read()
|
body = json.dumps({
|
||||||
|
"endpoint": "http://127.0.0.1:9002",
|
||||||
registered = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
"model": "console-health-test",
|
||||||
registered_event = next(
|
"hf_repo": "example/console-health-test",
|
||||||
event for event in registered["events"]
|
"num_layers": 4,
|
||||||
if event["message"] == "node registered"
|
"shard_start": 0,
|
||||||
)
|
"shard_end": 1,
|
||||||
assert registered_event["fields"]["model_health"]["served_model_copies"] == 0.5
|
"hardware_profile": {},
|
||||||
assert registered_event["fields"]["model_health"]["coverage_percentage"] == 50.0
|
}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
time.sleep(0.06)
|
f"http://127.0.0.1:{port}/v1/nodes/register",
|
||||||
urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map").read()
|
data=body,
|
||||||
expired = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
headers={"Content-Type": "application/json"},
|
||||||
expired_event = next(
|
method="POST",
|
||||||
event for event in expired["events"]
|
)
|
||||||
if event["message"] == "node expired"
|
urllib.request.urlopen(req).read()
|
||||||
)
|
|
||||||
assert expired_event["fields"]["model_health"]["served_model_copies"] == 0.0
|
registered = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
||||||
assert expired_event["fields"]["model_health"]["coverage_percentage"] == 0.0
|
registered_event = next(
|
||||||
finally:
|
event for event in registered["events"]
|
||||||
tracker.stop()
|
if event["message"] == "node registered"
|
||||||
|
)
|
||||||
|
assert registered_event["fields"]["model_health"]["served_model_copies"] == 0.5
|
||||||
|
assert registered_event["fields"]["model_health"]["coverage_percentage"] == 50.0
|
||||||
|
|
||||||
|
time.sleep(0.06)
|
||||||
|
urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/network/map").read()
|
||||||
|
expired = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
||||||
|
expired_event = next(
|
||||||
|
event for event in expired["events"]
|
||||||
|
if event["message"] == "node expired"
|
||||||
|
)
|
||||||
|
assert expired_event["fields"]["model_health"]["served_model_copies"] == 0.0
|
||||||
|
assert expired_event["fields"]["model_health"]["coverage_percentage"] == 0.0
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|||||||
@@ -142,6 +142,21 @@ def _send_chat_request(gateway_url: str, prompt: str) -> dict:
|
|||||||
return json.loads(r.read())
|
return json.loads(r.read())
|
||||||
|
|
||||||
|
|
||||||
|
def _send_streaming_chat_request(gateway_url: str, prompt: str):
|
||||||
|
data = json.dumps({
|
||||||
|
"model": GPT2_MODEL,
|
||||||
|
"messages": [{"role": "user", "content": prompt}],
|
||||||
|
"stream": True,
|
||||||
|
}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"{gateway_url}/v1/chat/completions",
|
||||||
|
data=data,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
return urllib.request.urlopen(req)
|
||||||
|
|
||||||
|
|
||||||
def test_all_responses_valid_openai_format(tracker_node_setup):
|
def test_all_responses_valid_openai_format(tracker_node_setup):
|
||||||
"""Ten requests via gateway all return valid OpenAI chat completion format."""
|
"""Ten requests via gateway all return valid OpenAI chat completion format."""
|
||||||
gateway_url, _, _ = tracker_node_setup
|
gateway_url, _, _ = tracker_node_setup
|
||||||
@@ -155,6 +170,30 @@ def test_all_responses_valid_openai_format(tracker_node_setup):
|
|||||||
assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
|
assert isinstance(message.get("content"), str), f"request {i}: content must be a string"
|
||||||
|
|
||||||
|
|
||||||
|
def test_streaming_head_worker_response_is_not_buffered_with_content_length(tracker_node_setup):
|
||||||
|
"""Gateway must relay head-worker SSE as a live stream, not a buffered JSON-sized body."""
|
||||||
|
gateway_url, _, _ = tracker_node_setup
|
||||||
|
|
||||||
|
with _send_streaming_chat_request(gateway_url, "stream through head worker") as resp:
|
||||||
|
assert resp.status == 200
|
||||||
|
assert "text/event-stream" in resp.headers["Content-Type"]
|
||||||
|
assert "Content-Length" not in resp.headers
|
||||||
|
data_lines = []
|
||||||
|
while len(data_lines) < 4:
|
||||||
|
line = resp.readline().decode().strip()
|
||||||
|
if line.startswith("data: "):
|
||||||
|
data_lines.append(line)
|
||||||
|
if line == "data: [DONE]":
|
||||||
|
break
|
||||||
|
|
||||||
|
assert data_lines[-1] == "data: [DONE]"
|
||||||
|
content = "".join(
|
||||||
|
json.loads(line[6:])["choices"][0].get("delta", {}).get("content", "")
|
||||||
|
for line in data_lines[:-1]
|
||||||
|
)
|
||||||
|
assert "head-worker" in content
|
||||||
|
|
||||||
|
|
||||||
def test_both_tracker_nodes_receive_load(tracker_node_setup):
|
def test_both_tracker_nodes_receive_load(tracker_node_setup):
|
||||||
"""Both head workers handle at least one request each out of ten."""
|
"""Both head workers handle at least one request each out of ten."""
|
||||||
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup
|
gateway_url, tracker_node_a, tracker_node_b = tracker_node_setup
|
||||||
|
|||||||
67
tests/test_tracker_logging.py
Normal file
67
tests/test_tracker_logging.py
Normal file
@@ -0,0 +1,67 @@
|
|||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from meshnet_tracker.logging_setup import configure_tracker_file_logging, tracker_logger
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_file_logging_writes_separate_level_files(tmp_path):
|
||||||
|
original_stdout = sys.stdout
|
||||||
|
original_stderr = sys.stderr
|
||||||
|
try:
|
||||||
|
log_dir = configure_tracker_file_logging(tmp_path, tee_stdio=False)
|
||||||
|
logger = tracker_logger()
|
||||||
|
|
||||||
|
logger.info("info-event")
|
||||||
|
logger.warning("warning-event")
|
||||||
|
logger.error("error-event")
|
||||||
|
for handler in logger.handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
assert (log_dir / "info.log").read_text().count("info-event") == 1
|
||||||
|
assert "warning-event" not in (log_dir / "info.log").read_text()
|
||||||
|
assert "error-event" not in (log_dir / "info.log").read_text()
|
||||||
|
|
||||||
|
assert "warning-event" in (log_dir / "warning.log").read_text()
|
||||||
|
assert "info-event" not in (log_dir / "warning.log").read_text()
|
||||||
|
assert "error-event" not in (log_dir / "warning.log").read_text()
|
||||||
|
|
||||||
|
assert "error-event" in (log_dir / "error.log").read_text()
|
||||||
|
assert "info-event" not in (log_dir / "error.log").read_text()
|
||||||
|
assert "warning-event" not in (log_dir / "error.log").read_text()
|
||||||
|
finally:
|
||||||
|
sys.stdout = original_stdout
|
||||||
|
sys.stderr = original_stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_file_logging_tees_stdio_and_rotates(tmp_path):
|
||||||
|
original_stdout = sys.stdout
|
||||||
|
original_stderr = sys.stderr
|
||||||
|
try:
|
||||||
|
log_dir = configure_tracker_file_logging(
|
||||||
|
tmp_path,
|
||||||
|
max_bytes=120,
|
||||||
|
backup_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
print("stdout goes to info", flush=True)
|
||||||
|
print("stderr goes to error", file=sys.stderr, flush=True)
|
||||||
|
for handler in tracker_logger().handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
assert "stdout goes to info" in (log_dir / "info.log").read_text()
|
||||||
|
assert "stderr goes to error" in (log_dir / "error.log").read_text()
|
||||||
|
|
||||||
|
for index in range(12):
|
||||||
|
tracker_logger().info("rotating-info-line-%02d", index)
|
||||||
|
for handler in tracker_logger().handlers:
|
||||||
|
handler.flush()
|
||||||
|
|
||||||
|
assert (log_dir / "info.log.1").exists()
|
||||||
|
finally:
|
||||||
|
sys.stdout = original_stdout
|
||||||
|
sys.stderr = original_stderr
|
||||||
|
logger = tracker_logger()
|
||||||
|
for handler in logger.handlers:
|
||||||
|
handler.close()
|
||||||
|
logger.handlers.clear()
|
||||||
|
logger.setLevel(logging.NOTSET)
|
||||||
@@ -502,6 +502,75 @@ def test_tracker_logs_stream_progress_before_request_completes():
|
|||||||
node_thread.join(timeout=1.0)
|
node_thread.join(timeout=1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_stream_survives_idle_gap_between_sse_chunks():
|
||||||
|
first_chunk_sent = threading.Event()
|
||||||
|
|
||||||
|
class IdleStreamingChatHandler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
if self.path != "/v1/chat/completions":
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
return
|
||||||
|
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
||||||
|
self.end_headers()
|
||||||
|
first = json.dumps({
|
||||||
|
"choices": [{"delta": {"content": "hello"}}],
|
||||||
|
}).encode()
|
||||||
|
second = json.dumps({
|
||||||
|
"choices": [{"delta": {"content": " world"}}],
|
||||||
|
}).encode()
|
||||||
|
self.wfile.write(b"data: " + first + b"\n\n")
|
||||||
|
self.wfile.flush()
|
||||||
|
first_chunk_sent.set()
|
||||||
|
time.sleep(1.0)
|
||||||
|
self.wfile.write(b"data: " + second + b"\n\n")
|
||||||
|
self.wfile.write(b"data: [DONE]\n\n")
|
||||||
|
self.wfile.flush()
|
||||||
|
|
||||||
|
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
|
||||||
|
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
|
||||||
|
node_thread.start()
|
||||||
|
tracker = TrackerServer(heartbeat_timeout=60.0)
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
response = None
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
|
||||||
|
"model": "idle-stream-model", "num_layers": 1,
|
||||||
|
"shard_start": 0, "shard_end": 0,
|
||||||
|
"hardware_profile": {}, "score": 1.0},
|
||||||
|
)
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
|
||||||
|
data=json.dumps({
|
||||||
|
"model": "idle-stream-model",
|
||||||
|
"stream": True,
|
||||||
|
"messages": [{"role": "user", "content": "hi"}],
|
||||||
|
}).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
response = urllib.request.urlopen(req, timeout=3.0)
|
||||||
|
assert response.readline().startswith(b"data:")
|
||||||
|
assert first_chunk_sent.wait(timeout=1.0)
|
||||||
|
|
||||||
|
remaining = response.read().splitlines()
|
||||||
|
assert b"data: [DONE]" in remaining
|
||||||
|
finally:
|
||||||
|
if response is not None:
|
||||||
|
response.close()
|
||||||
|
tracker.stop()
|
||||||
|
node.shutdown()
|
||||||
|
node.server_close()
|
||||||
|
node_thread.join(timeout=1.0)
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_dashboard_can_cancel_inflight_proxy():
|
def test_tracker_dashboard_can_cancel_inflight_proxy():
|
||||||
chunk_sent = threading.Event()
|
chunk_sent = threading.Event()
|
||||||
release = threading.Event()
|
release = threading.Event()
|
||||||
|
|||||||
Reference in New Issue
Block a user