popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-07 15:27:34 +03:00
parent 5e89bba78f 339577a26c
commit ca49675f50
9 changed files with 732 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.claude/memory/project-status.md
View File

@@ -35,3 +35,10 @@ Historical handoff note: `/mnt/c/Users/popov/Downloads/neuron-tai-alpha-handoff-
- Run: `meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct`
- Known: tracker registration fails with `http://` — must use `https://`
- pynvml deprecation warning is harmless (use nvidia-ml-py to silence it)
## Model cache layout (2026-07-07)
- Node downloads now cache files directly under `<download_dir>/<model>/`, not `<model>/layers_<start>-<end>/`, so a wider cached layer assignment can satisfy a later narrower assignment without duplicate shard folders.
- Downloader checks tracker-advertised `files` + `file_sizes` before peer/HF download; complete local files return immediately and preserve any extra files already in the model folder.
- Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures.
- Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`.
- Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events.

64
packages/node/meshnet_node/downloader.py
View File

@@ -1,6 +1,6 @@
"""Shard downloader — fetches model shards from peers or HuggingFace Hub.
Cache layout: ~/.cache/meshnet/shards/<model>/layers_<start>-<end>/
Cache layout: ~/.cache/meshnet/shards/<model>/
For "stub-model" (no HF repo), a placeholder JSON file is written so the
test suite never touches the network.
@@ -106,9 +106,7 @@ def _download_shard_from_peer(
_safe_extract_shard(archive_path, extract_dir)
if compute_shard_checksum(extract_dir) != checksum:
return False
if shard_dir.exists():
shutil.rmtree(shard_dir)
shutil.move(str(extract_dir), str(shard_dir))
_merge_tree(extract_dir, shard_dir)
return True
except Exception:
return False
@@ -217,6 +215,38 @@ def _reuse_local_file(expected: int, dest: Path, final: Path) -> bool:
return False
def _valid_source_rel_files(source: dict) -> list[str]:
return [
rel for rel in (source.get("files") or [])
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
]
def _source_files_cached(source: dict, shard_dir: Path) -> bool:
rel_files = _valid_source_rel_files(source)
if not rel_files:
return False
sizes = source.get("file_sizes")
if not isinstance(sizes, dict):
return False
for rel in rel_files:
expected = sizes.get(rel)
path = shard_dir / rel
if not isinstance(expected, int) or not path.exists() or path.stat().st_size != expected:
return False
return True
def _merge_tree(src: Path, dest: Path) -> None:
dest.mkdir(parents=True, exist_ok=True)
for path in sorted(p for p in src.rglob("*") if p.is_file()):
target = dest / path.relative_to(src)
target.parent.mkdir(parents=True, exist_ok=True)
if target.exists():
target.unlink()
shutil.move(str(path), str(target))
def _download_source_files(
source: dict,
shard_dir: Path,
@@ -231,10 +261,7 @@ def _download_source_files(
the download is retried or the node is restarted.
"""
url = source.get("url")
rel_files = [
rel for rel in (source.get("files") or [])
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
]
rel_files = _valid_source_rel_files(source)
if not isinstance(url, str) or not url or not rel_files:
return None
sizes = source.get("file_sizes")
@@ -283,9 +310,11 @@ def _download_source_files(
time.sleep(1.0 * attempt)
finally:
tracker_bar.close()
if shard_dir.exists():
shutil.rmtree(shard_dir)
shutil.move(str(partial_dir), str(shard_dir))
_merge_tree(partial_dir, shard_dir)
try:
partial_dir.rmdir()
except OSError:
pass
return shard_dir
@@ -331,9 +360,7 @@ def _download_model_source(
if progress:
print(f" {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
_safe_extract_shard(archive_path, extract_dir)
if shard_dir.exists():
shutil.rmtree(shard_dir)
shutil.move(str(extract_dir), str(shard_dir))
_merge_tree(extract_dir, shard_dir)
return shard_dir
except Exception as exc:
if progress:
@@ -431,10 +458,17 @@ def download_shard(
the test suite hermetic while the real download path is exercised by
passing a non-stub *hf_repo*.
"""
shard_dir = cache_dir / model / f"layers_{shard_start}-{shard_end}"
shard_dir = cache_dir / model
if progress:
print(f" Target location: {shard_dir}", flush=True)
for source in model_sources or []:
label = str(source.get("type") or "model-source")
if _source_files_cached(source, shard_dir):
if progress:
print(f" [{label}] requested files already cached at {shard_dir}", flush=True)
return shard_dir
for peer in peers or []:
if progress:
print(f" Trying peer shard download from {peer.get('endpoint')} ...", flush=True)

17
packages/node/meshnet_node/model_catalog.py
View File

@@ -64,6 +64,17 @@ def _load_model_metadata() -> dict[str, dict]:
_MODEL_METADATA = _load_model_metadata()
def _local_model_path(hf_repo: str, cache_dir: Path | None) -> Path | None:
if cache_dir is None:
return None
if (cache_dir / "config.json").exists():
return cache_dir
candidate = cache_dir / hf_repo.split("/")[-1]
if (candidate / "config.json").exists():
return candidate
return None
CURATED_MODELS: list[ModelPreset] = [
ModelPreset(
name="Qwen2.5-0.5B-Instruct",
@@ -215,9 +226,11 @@ def model_metadata_for(
try:
from transformers import AutoConfig # type: ignore[import]
local_model = _local_model_path(hf_repo, cache_dir)
load_source = str(local_model) if local_model is not None else hf_repo
cfg = AutoConfig.from_pretrained(
hf_repo,
cache_dir=str(cache_dir) if cache_dir is not None else None,
load_source,
cache_dir=str(cache_dir) if cache_dir is not None and local_model is None else None,
)
# Composite configs (VLM/MoE) nest decoder fields in text_config.
text_cfg = getattr(cfg, "text_config", None) or cfg

151
packages/node/meshnet_node/startup.py
View File

@@ -26,6 +26,86 @@ from .wallet import load_or_create_wallet
_DEFAULT_BYTES_PER_LAYER = 30 * 1024 * 1024
def _downloaded_model_inventory(
model: str,
shard_start: int,
shard_end: int,
shard_path: Path,
hf_repo: str | None = None,
model_sources: list[dict] | None = None,
) -> list[dict]:
"""Return a cheap local inventory record without reading model file contents."""
file_count = 0
total_bytes = 0
existing_rel_files: set[str] = set()
if shard_path.exists():
for path in shard_path.rglob("*"):
if not path.is_file():
continue
file_count += 1
try:
existing_rel_files.add(path.relative_to(shard_path).as_posix())
except ValueError:
pass
try:
total_bytes += path.stat().st_size
except OSError:
pass
record = {
"model": model,
"shard_start": shard_start,
"shard_end": shard_end,
"path": str(shard_path),
"file_count": file_count,
"total_bytes": total_bytes,
}
if hf_repo is not None:
record["hf_repo"] = hf_repo
expected_files: set[str] = set()
file_sizes: dict[str, int] = {}
for source in model_sources or []:
for rel in source.get("full_files") or source.get("files") or []:
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts:
expected_files.add(rel)
sizes = source.get("file_sizes")
if isinstance(sizes, dict):
for rel, size in sizes.items():
if isinstance(rel, str) and isinstance(size, int):
file_sizes[rel] = size
if expected_files:
expected_bytes = sum(file_sizes.get(rel, 0) for rel in expected_files)
local_expected_files = existing_rel_files & expected_files
local_expected_bytes = sum(file_sizes.get(rel, 0) for rel in local_expected_files)
record["expected_file_count"] = len(expected_files)
record["local_expected_file_count"] = len(local_expected_files)
record["expected_bytes"] = expected_bytes
record["local_expected_bytes"] = local_expected_bytes
record["local_model_percentage"] = (
round((local_expected_bytes / expected_bytes) * 100, 4)
if expected_bytes > 0
else 0.0
)
return [record]
def _registration_shard_checksum(model: str, shard_path: Path) -> str | None:
"""Only checksum tiny stub shards; real model folders are too large to hash at startup."""
if model != "stub-model":
return None
return compute_shard_checksum(shard_path)
def _model_cache_path(model_id: str, cache_dir: Path | None) -> Path | None:
if cache_dir is None:
return None
if (cache_dir / "config.json").exists():
return cache_dir
candidate = cache_dir / model_id.split("/")[-1]
if candidate.exists():
return candidate
return None
def _memory_budget(device: str, vram_mb: int, ram_mb: int, shared_vram_mb: int = 0) -> tuple[int, str]:
"""Return the capacity budget in MB and whether it came from VRAM or RAM."""
if device == "cuda" and vram_mb > 0:
@@ -502,6 +582,7 @@ def run_startup(
if model_id: # treat "" the same as None — no explicit model given
user_pinned_shard = shard_start is not None or shard_end is not None
full_sources: list[dict] = []
# Auto-detect shard range from model config if not explicitly provided
if shard_start is None or shard_end is None:
try:
@@ -581,6 +662,7 @@ def run_startup(
_attach_relay_bridge(node, relay_bridge)
# Register with tracker so other nodes can auto-join this model.
total_layers = getattr(getattr(node, "backend", None), "total_layers", None)
model_cache_path = _model_cache_path(model_id, cache_dir)
reg_payload = {
"endpoint": endpoint,
"model": model_id.split("/")[-1],
@@ -595,6 +677,18 @@ def run_startup(
"tracker_mode": (shard_start == 0),
"managed_assignment": not user_pinned_shard,
"model_metadata": model_metadata_for(model_id, total_layers, cache_dir=cache_dir),
"downloaded_models": (
_downloaded_model_inventory(
model_id.split("/")[-1],
shard_start,
shard_end,
model_cache_path,
hf_repo=model_id,
model_sources=full_sources,
)
if model_cache_path is not None
else []
),
**registration_capabilities,
**relay_fields,
}
@@ -693,6 +787,7 @@ def run_startup(
relay_url=relay_url,
)
_attach_relay_bridge(node, relay_bridge)
model_cache_path = _model_cache_path(assigned_hf_repo, cache_dir)
auto_reg_payload = {
"endpoint": endpoint,
"model": assigned_hf_repo.split("/")[-1],
@@ -707,6 +802,18 @@ def run_startup(
"tracker_mode": (assigned_shard_start == 0),
"managed_assignment": True,
"model_metadata": model_metadata_for(assigned_hf_repo, assigned_num_layers, cache_dir=cache_dir),
"downloaded_models": (
_downloaded_model_inventory(
assigned_hf_repo.split("/")[-1],
assigned_shard_start,
assigned_shard_end,
model_cache_path,
hf_repo=assigned_hf_repo,
model_sources=full_sources,
)
if model_cache_path is not None
else []
),
**registration_capabilities,
**relay_fields,
}
@@ -776,7 +883,15 @@ def run_startup(
if model_sources:
dl_kwargs["model_sources"] = model_sources
shard_path = download_shard(assigned_model, shard_start, shard_end, **dl_kwargs)
shard_checksum = compute_shard_checksum(shard_path)
shard_checksum = _registration_shard_checksum(assigned_model, shard_path)
downloaded_models = _downloaded_model_inventory(
assigned_model,
shard_start,
shard_end,
shard_path,
hf_repo=hf_repo,
model_sources=model_sources,
)
print(f" Cached at: {shard_path}", flush=True)
# 5. Start HTTP server
@@ -790,6 +905,7 @@ def run_startup(
model=assigned_model,
shard_path=shard_path,
)
_node_start_time = time.monotonic()
actual_port = node.start()
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
endpoint = f"http://{public_host}:{actual_port}"
@@ -805,24 +921,27 @@ def run_startup(
# 6. Register with tracker
print("Registering with tracker...", flush=True)
reg_payload = {
"endpoint": endpoint,
"model": assigned_model,
"shard_start": shard_start,
"shard_end": shard_end,
"shard_checksum": shard_checksum,
"downloaded_models": downloaded_models,
"hardware_profile": hw,
"wallet_address": address,
"score": 1.0,
**registration_capabilities,
**relay_fields,
}
try:
reg_resp = _post_json(
f"{tracker_url}/v1/nodes/register",
{
"endpoint": endpoint,
"model": assigned_model,
"shard_start": shard_start,
"shard_end": shard_end,
"shard_checksum": shard_checksum,
"hardware_profile": hw,
"wallet_address": address,
"score": 1.0,
**registration_capabilities,
**relay_fields,
},
reg_payload,
)
node_id = str(reg_resp["node_id"])
setattr(node, "tracker_node_id", node_id)
_start_heartbeat(tracker_url, node_id, reg_payload, node_ref=node, start_time=_node_start_time)
except Exception:
node.stop()
raise
@@ -855,9 +974,11 @@ def _detect_num_layers(model_id: str, cache_dir: Path | None = None) -> int | No
from .model_catalog import layers_from_config
local_model = _model_cache_path(model_id, cache_dir)
load_source = str(local_model) if local_model is not None else model_id
cfg = AutoConfig.from_pretrained(
model_id,
cache_dir=str(cache_dir) if cache_dir is not None else None,
load_source,
cache_dir=str(cache_dir) if cache_dir is not None and local_model is None else None,
)
layers = layers_from_config(cfg)
if layers is None:

31
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -44,6 +44,17 @@
.tabs { display:flex; gap:10px; margin-bottom:8px; }
.tabs a { color:var(--dim); cursor:pointer; }
.tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
.wide { grid-column:1 / -1; }
.console {
background:var(--bg); border:1px solid var(--border); border-radius:6px;
min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
white-space:pre-wrap; word-break:break-word; font-size:11px;
}
.console-line { padding:1px 0; border-bottom:1px solid #161b22; }
.console-time { color:var(--dim); }
.console-level-info { color:var(--accent); }
.console-level-warn { color:var(--warn); }
.console-level-error { color:var(--bad); }
</style>
</head>
<body>
@@ -63,6 +74,7 @@
<section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
</main>
<script>
"use strict";
@@ -212,6 +224,21 @@ function renderThroughput(stats) {
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
}
function renderConsole(data) {
const events = (data && data.events) || [];
if (!events.length) {
$("console").innerHTML = '<div class="empty">no console events</div>';
return;
}
$("console").innerHTML = events.slice(-120).map(e => {
const level = String(e.level || "info");
const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` +
`<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`;
}).join("");
}
// ---- account panel (registration / login / balance / usage / API keys) ----
let sessionToken = localStorage.getItem("meshnet_session") || null;
@@ -366,13 +393,14 @@ async function renderAdminPanel() {
async function refresh() {
$("self-url").textContent = location.host;
const [raft, map, summary, settlements, wallets, stats] = await Promise.all([
const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
fetchJson("/v1/raft/status"),
fetchJson("/v1/network/map"),
fetchJson("/v1/billing/summary"),
fetchJson("/v1/billing/settlements"),
fetchJson("/v1/registry/wallets"),
fetchJson("/v1/stats"),
fetchJson("/v1/console"),
]);
renderHive(raft);
renderNodes(map);
@@ -381,6 +409,7 @@ async function refresh() {
renderFraud(wallets, summary);
renderStats(stats);
renderThroughput(stats);
renderConsole(consoleData);
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
}
refresh();

188
packages/tracker/meshnet_tracker/server.py
View File

@@ -34,6 +34,7 @@ import time
import urllib.parse
import urllib.request
import uuid
from collections import deque
from importlib.resources import files
from pathlib import Path
from typing import Any
@@ -49,6 +50,9 @@ from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode
_CONSOLE_LIMIT = 300
def _preset_price_keys(name: str, preset: dict) -> set[str]:
"""All model strings a client may bill under for one preset.
@@ -529,7 +533,7 @@ class _StatsCollector:
class _NodeEntry:
__slots__ = (
"node_id", "endpoint", "shard_start", "shard_end",
"model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "hardware_profile", "wallet_address",
"model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "downloaded_models", "hardware_profile", "wallet_address",
"score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
"model_tokens_per_sec",
@@ -565,6 +569,7 @@ class _NodeEntry:
hf_repo: str | None = None,
num_layers: int | None = None,
model_metadata: dict | None = None,
downloaded_models: list[dict] | None = None,
relay_addr: str | None = None,
cert_fingerprint: str | None = None,
peer_id: str | None = None,
@@ -590,6 +595,7 @@ class _NodeEntry:
self.hf_repo = hf_repo
self.num_layers = num_layers
self.model_metadata = dict(model_metadata or {})
self.downloaded_models = [dict(item) for item in (downloaded_models or []) if isinstance(item, dict)]
self.relay_addr = relay_addr
self.cert_fingerprint = cert_fingerprint
self.peer_id = peer_id
@@ -1247,10 +1253,15 @@ def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
]
for node_id in expired_ids:
entry = server.registry.pop(node_id)
print(
f"[tracker] node expired: {node_id} {entry.endpoint} "
f"(no heartbeat for >{server.heartbeat_timeout:.0f}s)",
flush=True,
_tracker_log(
server,
"warn",
"node expired",
node_id=node_id,
endpoint=entry.endpoint,
model=entry.model,
shard=f"{entry.shard_start}-{entry.shard_end}",
heartbeat_timeout_seconds=server.heartbeat_timeout,
)
if expired_ids:
_rebalance_all_locked(server)
@@ -1617,6 +1628,24 @@ def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -
return None
def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
event = {
"ts": time.time(),
"level": level,
"message": message,
"fields": {
key: value
for key, value in fields.items()
if value is not None
},
}
with server.console_lock:
server.console_events.append(event)
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
suffix = f" {extras}" if extras else ""
print(f"[tracker] {level}: {message}{suffix}", flush=True)
def _node_id_for_registration(
endpoint: str,
model: str,
@@ -1700,6 +1729,8 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
self.toploc_backend = toploc_backend
self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
self.models_dir = models_dir
self.console_events = deque(maxlen=_CONSOLE_LIMIT)
self.console_lock = threading.Lock()
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
@@ -1887,6 +1918,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._handle_raft_status()
elif parsed.path == "/v1/stats":
self._handle_stats()
elif parsed.path == "/v1/console":
self._handle_console()
elif parsed.path == "/v1/billing/summary":
self._handle_billing_summary()
elif parsed.path == "/v1/billing/settlements":
@@ -2118,6 +2151,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"hf_repo": node.hf_repo,
"num_layers": node.num_layers,
"model_metadata": dict(node.model_metadata),
"downloaded_models": [dict(item) for item in node.downloaded_models],
"shard_start": node.shard_start,
"shard_end": node.shard_end,
"tracker_mode": node.tracker_mode,
@@ -2264,6 +2298,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
]
if not candidates:
with server.lock:
registered = [
{
"node_id": n.node_id,
"model": n.model,
"hf_repo": n.hf_repo,
"shard": f"{n.shard_start}-{n.shard_end}",
"tracker_mode": n.tracker_mode,
}
for n in server.registry.values()
]
_tracker_log(
server,
"warn",
"no nodes available for model",
model=model,
registered_nodes=registered,
)
self._send_json(503, {"error": {
"message": f"no nodes available for model {model!r}",
"type": "service_unavailable",
@@ -2296,6 +2348,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else:
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
if route_error:
_tracker_log(
server,
"warn",
"route unavailable",
model=model,
route_model=route_model,
error=route_error,
candidate_count=len(all_nodes),
)
self._send_json(503, {"error": {
"message": route_error,
"type": "service_unavailable",
@@ -2322,11 +2383,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
for n in route_nodes
)
print(
f"[tracker] proxy route {request_id}: model={model!r} "
f"head={node.node_id}@{node.endpoint} downstream={downstream_urls} "
f"route={route_debug or '<empty>'}",
flush=True,
_tracker_log(
server,
"info",
"proxy route selected",
request_id=request_id,
model=model,
head_node_id=node.node_id,
head_endpoint=node.endpoint,
downstream=downstream_urls,
route=route_debug or "<empty>",
)
req = urllib.request.Request(
@@ -2350,10 +2416,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
}
if node.relay_addr:
print(
f"[tracker] proxy via relay {request_id}: {node.relay_addr} "
f"(direct endpoint {target_url})",
flush=True,
_tracker_log(
server,
"info",
"proxy via relay",
request_id=request_id,
relay_addr=node.relay_addr,
direct_endpoint=target_url,
)
started = time.monotonic()
frames = _relay_http_request_frames(
@@ -2388,16 +2457,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
input_tokens=in_tokens, output_tokens=out_tokens,
)
return
print(
f"[tracker] relay proxy failed {request_id}: {node.relay_addr}; "
f"trying direct {target_url}",
flush=True,
_tracker_log(
server,
"warn",
"relay proxy failed, trying direct",
request_id=request_id,
relay_addr=node.relay_addr,
direct_endpoint=target_url,
)
try:
started = time.monotonic()
upstream = urllib.request.urlopen(req, timeout=300.0)
print(f"[tracker] proxy connected {request_id}: {target_url}", flush=True)
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
except urllib.error.HTTPError as exc:
# Relay error status + body from node
err_body = exc.read()
@@ -2412,13 +2484,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return
except Exception as exc:
if node.relay_addr:
print(
f"[tracker] direct proxy failed {request_id}: {target_url}: {exc}; "
f"relay already attempted for {node.relay_addr}",
flush=True,
_tracker_log(
server,
"error",
"direct proxy failed after relay",
request_id=request_id,
target_url=target_url,
relay_addr=node.relay_addr,
error=repr(exc),
)
else:
print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True)
_tracker_log(
server,
"error",
"proxy failed",
request_id=request_id,
target_url=target_url,
error=repr(exc),
)
self._send_json(503, {"error": {
"message": f"upstream node unreachable: {exc}",
"type": "service_unavailable",
@@ -2467,9 +2550,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Non-streaming: buffer and relay
resp_body = upstream.read()
elapsed = time.monotonic() - started
print(
f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
flush=True,
_tracker_log(
server,
"info",
"proxy complete",
request_id=request_id,
target_url=target_url,
bytes=len(resp_body),
)
observed_output = ""
try:
@@ -2812,6 +2899,15 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model_metadata, dict):
self._send_json(400, {"error": "model_metadata must be an object"})
return
downloaded_models = body.get("downloaded_models", [])
if downloaded_models is None:
downloaded_models = []
if not (
isinstance(downloaded_models, list)
and all(isinstance(item, dict) for item in downloaded_models)
):
self._send_json(400, {"error": "downloaded_models must be an array of objects"})
return
relay_addr = body.get("relay_addr") or None
cert_fingerprint = body.get("cert_fingerprint") or None
peer_id = body.get("peer_id") or None
@@ -2845,6 +2941,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
hf_repo=hf_repo,
num_layers=num_layers,
model_metadata=model_metadata,
downloaded_models=downloaded_models,
relay_addr=relay_addr,
cert_fingerprint=cert_fingerprint,
peer_id=peer_id,
@@ -2858,10 +2955,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
]
for eid in stale_ids:
old = server.registry.pop(eid)
print(
f"[tracker] node re-registered: replaced {eid} with {node_id}"
f" {old.endpoint}",
flush=True,
_tracker_log(
server,
"info",
"node re-registered",
old_node_id=eid,
node_id=node_id,
endpoint=old.endpoint,
)
server.registry[node_id] = entry
if entry.managed_assignment:
@@ -2873,6 +2973,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if assignment_directive is not None:
entry.pending_directives.clear()
_tracker_log(
server,
"info",
"node registered",
node_id=node_id,
endpoint=entry.endpoint,
model=entry.model,
hf_repo=entry.hf_repo,
shard=f"{entry.shard_start}-{entry.shard_end}",
tracker_mode=entry.tracker_mode,
)
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
repo_info = f" [{hf_repo}]" if hf_repo else ""
budget_bytes, budget_source = _node_memory_budget_bytes(entry)
@@ -2913,6 +3025,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._purge_expired_nodes()
entry = server.registry.get(node_id)
if entry is None:
_tracker_log(server, "warn", "heartbeat for unknown node", node_id=node_id)
self._send_json(404, {"error": "node not found"})
return
entry.last_heartbeat = time.monotonic()
@@ -3041,6 +3154,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
except BrokenPipeError:
pass
def _handle_console(self):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
with server.console_lock:
events = [dict(event) for event in server.console_events]
self._send_json(200, {"events": events})
def _handle_registry_wallets(self):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if not self._require_role("admin"):
@@ -4195,6 +4314,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"hf_repo": e.hf_repo,
"num_layers": e.num_layers,
"model_metadata": dict(e.model_metadata),
"downloaded_models": [dict(item) for item in e.downloaded_models],
"shard_checksum": e.shard_checksum,
"score": e.score,
}
@@ -4260,6 +4380,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"shard_start": e.shard_start,
"shard_end": e.shard_end,
"model": e.model,
"downloaded_models": [dict(item) for item in e.downloaded_models],
"shard_checksum": e.shard_checksum,
"score": e.score,
}
@@ -4681,6 +4802,11 @@ class TrackerServer:
hf_repo=payload.get("hf_repo"),
num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
downloaded_models=(
payload.get("downloaded_models")
if isinstance(payload.get("downloaded_models"), list)
else None
),
)
with self._lock:
self._registry[node_id] = entry

27
tests/test_dashboard.py
View File

@@ -12,6 +12,7 @@ PANELS = [
"Tracker hive", "Nodes &amp; coverage", "Client balances",
"Node pending payouts", "Settlement history",
"Strikes / bans / forfeitures", "Model usage", "Node throughput",
"Console output",
]
@@ -64,3 +65,29 @@ def test_registry_wallets_endpoint():
assert data["wallets"]["wallet-a"]["banned"] is False
finally:
tracker.stop()
def test_console_endpoint_exposes_tracker_events():
tracker = TrackerServer()
port = tracker.start()
try:
body = json.dumps({
"endpoint": "http://127.0.0.1:9001",
"model": "stub-model",
"shard_start": 0,
"shard_end": 3,
"hardware_profile": {},
}).encode()
req = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/nodes/register",
data=body,
headers={"Content-Type": "application/json"},
method="POST",
)
urllib.request.urlopen(req).read()
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
finally:
tracker.stop()
assert any(event["message"] == "node registered" for event in data["events"])

211
tests/test_node_startup.py
View File

@@ -438,7 +438,7 @@ def test_download_shard_uses_huggingface_when_repo_is_assigned(tmp_path, monkeyp
progress=False,
)
assert shard_dir == tmp_path / "tiny-llama" / "layers_0-3"
assert shard_dir == tmp_path / "tiny-llama"
assert calls == [{
"repo_id": "org/tiny-llama-shards",
"cache_dir": str(tmp_path),
@@ -446,6 +446,54 @@ def test_download_shard_uses_huggingface_when_repo_is_assigned(tmp_path, monkeyp
}]
def test_download_shard_reuses_model_cache_for_narrower_layer_range(
tmp_path,
monkeypatch,
):
"""A wider cached shard satisfies a later narrower assignment for the same model."""
cache_dir = tmp_path / "cache"
model_dir = cache_dir / "tiny-llama"
model_dir.mkdir(parents=True)
(model_dir / "config.json").write_bytes(b"{}")
(model_dir / "model-00001-of-00002.safetensors").write_bytes(b"a" * 3)
(model_dir / "model-00002-of-00002.safetensors").write_bytes(b"b" * 5)
def unexpected_urlopen(*args, **kwargs):
raise AssertionError("cached files should avoid tracker download")
def unexpected_snapshot_download(*args, **kwargs):
raise AssertionError("cached files should avoid HuggingFace download")
monkeypatch.setattr(urllib.request, "urlopen", unexpected_urlopen)
monkeypatch.setitem(
sys.modules,
"huggingface_hub",
types.SimpleNamespace(snapshot_download=unexpected_snapshot_download),
)
shard_dir = download_shard(
"tiny-llama",
0,
1,
cache_dir=cache_dir,
hf_repo="org/tiny-llama-shards",
model_sources=[{
"type": "tracker",
"url": "http://tracker/v1/model-files/download?model=tiny-llama",
"files": ["config.json", "model-00001-of-00002.safetensors"],
"file_sizes": {
"config.json": 2,
"model-00001-of-00002.safetensors": 3,
},
}],
peers=[{"endpoint": "http://peer", "checksum": "unused"}],
progress=False,
)
assert shard_dir == model_dir
assert (model_dir / "model-00002-of-00002.safetensors").read_bytes() == b"b" * 5
def test_download_shard_prefers_tracker_model_source_over_huggingface(
tmp_path,
monkeypatch,
@@ -1368,12 +1416,7 @@ def test_full_startup_sequence(tmp_path):
tracker.stop()
def test_second_node_downloads_same_shard_from_peer_without_huggingface(
tmp_path,
monkeypatch,
capsys,
):
"""Node A downloads from HF stub; node B downloads same assignment from node A."""
def test_preset_model_startup_starts_heartbeat(tmp_path, monkeypatch):
import meshnet_node.startup as startup_mod
monkeypatch.setattr(
@@ -1381,6 +1424,53 @@ def test_second_node_downloads_same_shard_from_peer_without_huggingface(
"detect_hardware",
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
)
heartbeat_calls = []
monkeypatch.setattr(
startup_mod,
"_start_heartbeat",
lambda *args, **kwargs: heartbeat_calls.append((args, kwargs)),
)
tracker = TrackerServer(model_presets={"stub-model": {"layers_start": 0, "layers_end": 15}})
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
try:
node = run_startup(
tracker_url=tracker_url,
model="stub-model",
wallet_path=tmp_path / "wallet.json",
cache_dir=tmp_path / "shards",
)
try:
assert len(heartbeat_calls) == 1
args, kwargs = heartbeat_calls[0]
assert args[0] == tracker_url
assert args[2]["model"] == "stub-model"
assert kwargs["node_ref"] is node
finally:
node.stop()
finally:
tracker.stop()
def test_real_model_startup_registers_downloaded_inventory_without_checksum(
tmp_path,
monkeypatch,
capsys,
):
"""Real model folders are reported as inventory without hashing their contents."""
import meshnet_node.startup as startup_mod
monkeypatch.setattr(
startup_mod,
"detect_hardware",
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
)
monkeypatch.setattr(
startup_mod,
"compute_shard_checksum",
lambda _path: (_ for _ in ()).throw(AssertionError("real model startup must not hash model files")),
)
hf_calls = []
def fake_snapshot_download(repo_id, cache_dir, local_dir):
@@ -1404,36 +1494,75 @@ def test_second_node_downloads_same_shard_from_peer_without_huggingface(
})
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
nodes = []
try:
node_a = run_startup(
node = run_startup(
tracker_url=tracker_url,
model="tiny-llama",
wallet_path=tmp_path / "wallet-a.json",
cache_dir=tmp_path / "node-a-shards",
wallet_path=tmp_path / "wallet.json",
cache_dir=tmp_path / "node-shards",
)
nodes.append(node_a)
assert len(hf_calls) == 1
node_b = run_startup(
tracker_url=tracker_url,
model="tiny-llama",
wallet_path=tmp_path / "wallet-b.json",
cache_dir=tmp_path / "node-b-shards",
)
nodes.append(node_b)
assert len(hf_calls) == 1
assert (tmp_path / "node-b-shards" / "tiny-llama" / "layers_0-15" / "weights.json").exists()
output = capsys.readouterr().out
assert "download source: HuggingFace" in output
assert "download source: peer" in output
finally:
for node in reversed(nodes):
try:
assert len(hf_calls) == 1
assert (tmp_path / "node-shards" / "tiny-llama" / "weights.json").exists()
output = capsys.readouterr().out
assert "Cached at:" in output
network_map = _get_json(f"{tracker_url}/v1/network/map")
registered = network_map["nodes"][0]
assert registered["downloaded_models"] == [{
"model": "tiny-llama",
"shard_start": 0,
"shard_end": 15,
"path": str(tmp_path / "node-shards" / "tiny-llama"),
"file_count": 1,
"total_bytes": (tmp_path / "node-shards" / "tiny-llama" / "weights.json").stat().st_size,
"hf_repo": "org/tiny-llama-shards",
}]
finally:
node.stop()
finally:
tracker.stop()
def test_downloaded_model_inventory_reports_local_model_percentage(tmp_path):
import meshnet_node.startup as startup_mod
model_dir = tmp_path / "models" / "tiny-llama"
model_dir.mkdir(parents=True)
(model_dir / "config.json").write_bytes(b"{}")
(model_dir / "weights-a.safetensors").write_bytes(b"a" * 3)
inventory = startup_mod._downloaded_model_inventory(
"tiny-llama",
0,
1,
model_dir,
hf_repo="org/tiny-llama",
model_sources=[{
"full_files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
"file_sizes": {
"config.json": 2,
"weights-a.safetensors": 3,
"weights-b.safetensors": 5,
},
}],
)
assert inventory == [{
"model": "tiny-llama",
"shard_start": 0,
"shard_end": 1,
"path": str(model_dir),
"file_count": 2,
"total_bytes": 5,
"hf_repo": "org/tiny-llama",
"expected_file_count": 3,
"local_expected_file_count": 2,
"expected_bytes": 10,
"local_expected_bytes": 5,
"local_model_percentage": 50.0,
}]
def test_network_assign_gap_found_field():
"""network/assign sets gap_found=True when a real gap exists, False when fully covered."""
import json as _json
@@ -1626,6 +1755,30 @@ def test_startup_cpu_fallback(tmp_path, monkeypatch):
# --------------------------------------------------- layer detection (US: composite configs)
def test_detect_num_layers_prefers_flattened_local_model_config(tmp_path, monkeypatch):
import meshnet_node.startup as startup_mod
model_dir = tmp_path / "Qwen3.6-35B-A3B"
model_dir.mkdir()
(model_dir / "config.json").write_text("{}")
calls = []
class AutoConfigStub:
@staticmethod
def from_pretrained(model_id, cache_dir=None):
calls.append({"model_id": model_id, "cache_dir": cache_dir})
return types.SimpleNamespace(num_hidden_layers=37)
monkeypatch.setitem(
sys.modules,
"transformers",
types.SimpleNamespace(AutoConfig=AutoConfigStub),
)
assert startup_mod._detect_num_layers("unsloth/Qwen3.6-35B-A3B", cache_dir=tmp_path) == 37
assert calls == [{"model_id": str(model_dir), "cache_dir": None}]
def test_layers_from_config_top_level():
from meshnet_node.model_catalog import layers_from_config

32
tests/test_tracker_routing.py
View File

@@ -75,6 +75,13 @@ def test_tracker_exposes_registered_model_metadata():
"activated_parameters": "32B",
"context_length": 256000,
},
"downloaded_models": [{
"model": "Kimi-K2.7-Code",
"hf_repo": "unsloth/Kimi-K2.7-Code",
"shard_start": 0,
"shard_end": 60,
"local_model_percentage": 100.0,
}],
},
)
@@ -91,6 +98,7 @@ def test_tracker_exposes_registered_model_metadata():
registered = network_map["nodes"][0]
assert registered["num_layers"] == 61
assert registered["model_metadata"]["context_length"] == 256000
assert registered["downloaded_models"][0]["local_model_percentage"] == 100.0
def test_tracker_lists_recommended_kimi_before_nodes_register():
@@ -975,6 +983,30 @@ def test_tracker_route_rejects_non_extending_overlap():
tracker.stop()
def test_tracker_console_records_model_not_available():
tracker = TrackerServer()
tracker_port = tracker.start()
try:
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
{"model": "missing-model", "messages": [{"role": "user", "content": "hi"}]},
)
raise AssertionError("Expected 503")
except urllib.error.HTTPError as exc:
assert exc.code == 503
console = _get_json(f"http://127.0.0.1:{tracker_port}/v1/console")
finally:
tracker.stop()
assert any(
event["message"] == "no nodes available for model"
and event["fields"]["model"] == "missing-model"
for event in console["events"]
)
def test_tracker_registration_rejects_invalid_payload():
"""Registration errors return a defined JSON 400 response."""
tracker = TrackerServer()