popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit '47b243cd98fd94da7918cacf5725373b099208e5' into ralph/distributed-gguf-runtime

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-15 23:04:52 +02:00
parent 1fe31ef38d 47b243cd98
commit 254627629b
12 changed files with 620 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.claude/memory/project-status.md
View File

@@ -8,6 +8,11 @@ metadata:
# Project Status (2026-07-13) # Project Status (2026-07-13)
## Selected-node model placement (2026-07-14)
- Admin Model placement now opens a node selector for load and release; the control-plane accepts optional `node_id` and targets only that registry assignment. Multi-model serving remains supported through `ADD_SHARD` and `max_loaded_shards`.
- Total node pool resource values are rendered from `/v1/network/map`'s `node.capacity` contract. Route selection remains assignment/capability/throughput/queue based; capacity is used for placement and falls back to tracker defaults only if a node truly omits it.
## Distributed inference performance (2026-07-14) ## Distributed inference performance (2026-07-14)
`DIP-001` is done in `.scratch/distributed-inference-performance/`: the deterministic two-node Route Session stub benchmark covers direct/relay plus cached/stateless prefill and decode. Its JSON and concise summary explicitly attribute model execution, activation encode/decode, compression, connection setup, relay queueing, local HTTP forwarding, and end-to-end seam latency. `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` passed (7); the fixture assertion checks output-token identity and connection attempts. `DIP-001` is done in `.scratch/distributed-inference-performance/`: the deterministic two-node Route Session stub benchmark covers direct/relay plus cached/stateless prefill and decode. Its JSON and concise summary explicitly attribute model execution, activation encode/decode, compression, connection setup, relay queueing, local HTTP forwarding, and end-to-end seam latency. `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` passed (7); the fixture assertion checks output-token identity and connection attempts.

7
_DEV_NOTES.md
View File

@@ -16,12 +16,9 @@
.\.venv\Scripts\meshnet-node.exe start http://192.168.0.179:8081 --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 .\.venv\Scripts\meshnet-node.exe start http://192.168.0.179:8081 --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20
.\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20
we .\.venv\Scripts\meshnet-node.exe start ` we .\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct
--tracker http://192.168.0.179:8081 `
--model Qwen/Qwen2.5-0.5B-Instruct `
--advertise-host 192.168.0.20
# trackers: # trackers:
https://meshnet.2.d-popov.com https://meshnet.2.d-popov.com
https://ai.neuron.d-popov.com https://ai.neuron.d-popov.com

24
packages/node/meshnet_node/hardware.py
View File

@@ -2,6 +2,7 @@
import json import json
import os import os
import shutil
import subprocess import subprocess
import time import time
@@ -183,6 +184,17 @@ def with_forced_cpu(hw: dict) -> dict:
return forced return forced
def _with_model_drive(profile: dict) -> dict:
"""Attach free space for the default model cache drive to tracker diagnostics."""
try:
cache_root = os.path.expanduser("~/.cache/meshnet/shards")
profile["model_drive_free_bytes"] = shutil.disk_usage(os.path.expanduser("~")).free
profile["model_drive_path"] = cache_root
except OSError:
pass
return profile
def detect_hardware() -> dict: def detect_hardware() -> dict:
"""Detect GPU model and available VRAM. Returns hardware profile dict.""" """Detect GPU model and available VRAM. Returns hardware profile dict."""
ram_mb = _detect_ram_mb() ram_mb = _detect_ram_mb()
@@ -208,23 +220,23 @@ def detect_hardware() -> dict:
} }
if torch_gpu is not None and torch_gpu.get("gcn_arch"): if torch_gpu is not None and torch_gpu.get("gcn_arch"):
profile["gcn_arch"] = torch_gpu["gcn_arch"] profile["gcn_arch"] = torch_gpu["gcn_arch"]
return profile return _with_model_drive(profile)
except ImportError: except ImportError:
pass pass
torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb) torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb)
if torch_inventory is not None: if torch_inventory is not None:
return torch_inventory return _with_model_drive(torch_inventory)
nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb) nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb)
if nvidia_gpu is not None: if nvidia_gpu is not None:
return nvidia_gpu return _with_model_drive(nvidia_gpu)
windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb) windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb)
if windows_gpu is not None: if windows_gpu is not None:
return windows_gpu return _with_model_drive(windows_gpu)
return { return _with_model_drive({
"device": "cpu", "device": "cpu",
"gpu_name": None, "gpu_name": None,
"vram_mb": 0, "vram_mb": 0,
@@ -232,7 +244,7 @@ def detect_hardware() -> dict:
"shared_vram_mb": 0, "shared_vram_mb": 0,
"ram_mb": ram_mb, "ram_mb": ram_mb,
"cuda_available": False, "cuda_available": False,
} })
def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]: def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]:

148
packages/node/meshnet_node/startup.py
View File

@@ -12,7 +12,7 @@ import urllib.error
import urllib.parse import urllib.parse
import urllib.request import urllib.request
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any, Callable
from .admission import ( from .admission import (
AdmissionRequirement, AdmissionRequirement,
@@ -420,6 +420,7 @@ def _start_heartbeat(
interval: float = _HEARTBEAT_INTERVAL_IDLE, interval: float = _HEARTBEAT_INTERVAL_IDLE,
node_ref: Any | None = None, node_ref: Any | None = None,
start_time: float | None = None, start_time: float | None = None,
refresh_capability: Callable[[dict], dict | None] | None = None,
) -> threading.Thread: ) -> threading.Thread:
"""Daemon thread: sends heartbeats and re-registers automatically after tracker restarts. """Daemon thread: sends heartbeats and re-registers automatically after tracker restarts.
@@ -431,6 +432,7 @@ def _start_heartbeat(
which is logged for now (hot-reload implemented in US-026). which is logged for now (hot-reload implemented in US-026).
""" """
_start_time = start_time or time.monotonic() _start_time = start_time or time.monotonic()
completed_directives: list[dict] = []
def _current_requests_snapshot() -> list[dict]: def _current_requests_snapshot() -> list[dict]:
if node_ref is None: if node_ref is None:
@@ -455,6 +457,8 @@ def _start_heartbeat(
current_requests = _current_requests_snapshot() current_requests = _current_requests_snapshot()
if current_requests: if current_requests:
stats["current_requests"] = current_requests stats["current_requests"] = current_requests
if completed_directives:
stats["completed_directives"] = list(completed_directives)
return stats return stats
def _sleep_interval() -> float: def _sleep_interval() -> float:
@@ -462,9 +466,26 @@ def _start_heartbeat(
return _HEARTBEAT_INTERVAL_BUSY return _HEARTBEAT_INTERVAL_BUSY
return interval return interval
def _refresh_proof(payload: dict) -> None:
"""Re-prove the current shard so a re-registration never presents an aged proof.
The tracker refuses proofs older than its freshness budget: re-sending the
startup-time report after an outage would re-register the node unroutable.
"""
if refresh_capability is None or "capability_report" not in payload:
return
try:
fresh = refresh_capability(payload)
except Exception as exc:
print(f" [node] WARNING: capability re-validation failed: {exc}", flush=True)
return
if fresh:
payload["capability_report"] = fresh
def _reregister() -> bool: def _reregister() -> bool:
nonlocal node_id nonlocal node_id
try: try:
_refresh_proof(register_payload)
resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload) resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload)
node_id = resp.get("node_id", node_id) node_id = resp.get("node_id", node_id)
if node_ref is not None: if node_ref is not None:
@@ -486,6 +507,7 @@ def _start_heartbeat(
"managed_assignment": True, "managed_assignment": True,
} }
try: try:
_refresh_proof(extra_payload)
reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", extra_payload) reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", extra_payload)
print( print(
f" [node] registered additional model — node ID: {reg_resp.get('node_id')}", f" [node] registered additional model — node ID: {reg_resp.get('node_id')}",
@@ -494,21 +516,26 @@ def _start_heartbeat(
except Exception as exc: except Exception as exc:
print(f" [node] WARNING: additional model registration failed: {exc}", flush=True) print(f" [node] WARNING: additional model registration failed: {exc}", flush=True)
def _apply_directives(directives: list[dict]) -> None: def _apply_directives(directives: list[dict]) -> dict | None:
if not directives: if not directives:
return return None
if node_ref is None or not hasattr(node_ref, "apply_tracker_directives"): if node_ref is None or not hasattr(node_ref, "apply_tracker_directives"):
print(f" [node] tracker directives received: {directives}", flush=True) print(f" [node] tracker directives received: {directives}", flush=True)
return return None
try: try:
applied = node_ref.apply_tracker_directives(directives) applied = node_ref.apply_tracker_directives(directives)
except Exception as exc: except Exception as exc:
print(f" [node] WARNING: failed to apply tracker directives: {exc}", flush=True) print(f" [node] WARNING: failed to apply tracker directives: {exc}", flush=True)
return return None
if applied: if applied:
completed_directives.append(dict(applied))
if applied.get("action") == "ADD_SHARD": if applied.get("action") == "ADD_SHARD":
_register_additional_assignment(applied) _register_additional_assignment(applied)
return return applied
if applied.get("action") in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
# A release has no replacement range. It is not a failed
# heartbeat and must not re-register the released assignment.
return applied
model_id = applied.get("model", register_payload.get("hf_repo") or register_payload.get("model")) model_id = applied.get("model", register_payload.get("hf_repo") or register_payload.get("model"))
register_payload["model"] = str(model_id).split("/")[-1] register_payload["model"] = str(model_id).split("/")[-1]
register_payload["hf_repo"] = model_id register_payload["hf_repo"] = model_id
@@ -516,6 +543,7 @@ def _start_heartbeat(
register_payload["shard_end"] = applied["shard_end"] register_payload["shard_end"] = applied["shard_end"]
register_payload["quantization"] = applied.get("quantization", register_payload.get("quantization")) register_payload["quantization"] = applied.get("quantization", register_payload.get("quantization"))
register_payload["tracker_mode"] = bool(applied.get("tracker_mode", False)) register_payload["tracker_mode"] = bool(applied.get("tracker_mode", False))
return applied
def _loop() -> None: def _loop() -> None:
nonlocal node_id nonlocal node_id
@@ -543,7 +571,10 @@ def _start_heartbeat(
continue continue
try: try:
resp = _post_json(hb_url, _get_stats()) heartbeat = _get_stats()
resp = _post_json(hb_url, heartbeat)
if heartbeat.get("completed_directives"):
completed_directives.clear()
_apply_directives(resp.get("directives", [])) _apply_directives(resp.get("directives", []))
new_asgn = resp.get("new_assignment") new_asgn = resp.get("new_assignment")
if new_asgn: if new_asgn:
@@ -580,6 +611,7 @@ def _register_with_tracker(
reg_payload: dict, reg_payload: dict,
node: Any, node: Any,
start_time: float, start_time: float,
refresh_capability: Callable[[dict], dict | None] | None = None,
) -> str | None: ) -> str | None:
"""Register with the tracker, or start background retries when it is unreachable.""" """Register with the tracker, or start background retries when it is unreachable."""
try: try:
@@ -587,7 +619,14 @@ def _register_with_tracker(
tracker_node_id = str(reg_resp.get("node_id") or "?") tracker_node_id = str(reg_resp.get("node_id") or "?")
setattr(node, "tracker_node_id", tracker_node_id) setattr(node, "tracker_node_id", tracker_node_id)
print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True) print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True)
_start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=start_time) _start_heartbeat(
tracker_url,
tracker_node_id,
reg_payload,
node_ref=node,
start_time=start_time,
refresh_capability=refresh_capability,
)
return tracker_node_id return tracker_node_id
except Exception as exc: except Exception as exc:
setattr(node, "tracker_node_id", None) setattr(node, "tracker_node_id", None)
@@ -599,6 +638,7 @@ def _register_with_tracker(
reg_payload, reg_payload,
node_ref=node, node_ref=node,
start_time=start_time, start_time=start_time,
refresh_capability=refresh_capability,
) )
return None return None
@@ -748,6 +788,54 @@ def _admit_capability(
return report return report
def _capability_refresher(
node: Any,
*,
manifest: RecipeManifest,
recipe: Recipe,
detected_device: str,
cache_dir: Path | None,
force_cpu: bool,
validator: CapabilityValidator | None = None,
) -> Callable[[dict], dict | None]:
"""A fresh proof for what the node serves *now*, run at re-registration time.
The startup proof ages past the tracker's freshness budget, and directives
can move the node to a shard the startup proof never covered — so every
re-registration re-proves against the currently loaded backend rather than
replaying the report captured at boot.
"""
def refresh(payload: dict) -> dict | None:
target_model = payload.get("hf_repo") or payload.get("model")
backend = None
accessor = getattr(node, "backend_for", None)
if callable(accessor) and target_model:
backend = accessor(str(target_model))
if backend is None:
backend = getattr(node, "backend", None)
if backend is None:
return None
context = CapabilityContext(
backend=backend,
selection=DoctorSelection(
model_id=str(getattr(backend, "model_id", target_model)),
shard_start=int(getattr(backend, "shard_start", 0) or 0),
shard_end=int(getattr(backend, "shard_end", 0) or 0),
quantization=str(getattr(backend, "quantization", None) or "auto"),
cache_dir=cache_dir,
force_cpu=force_cpu,
),
recipe=recipe,
manifest=manifest,
device=_capability_device(backend, detected_device),
)
report = (validator or probe_capability)(context)
setattr(node, "capability_report", report)
return report.to_dict()
return refresh
def run_startup( def run_startup(
tracker_url: str, tracker_url: str,
port: int = 0, port: int = 0,
@@ -1079,6 +1167,15 @@ def run_startup(
} }
tracker_node_id = _register_with_tracker( tracker_node_id = _register_with_tracker(
tracker_url, reg_payload, node, _node_start_time, tracker_url, reg_payload, node, _node_start_time,
refresh_capability=_capability_refresher(
node,
manifest=manifest,
recipe=recipe,
detected_device=device,
cache_dir=cache_dir,
force_cpu=force_cpu,
validator=capability_validator,
),
) )
print( print(
@@ -1268,6 +1365,15 @@ def run_startup(
} }
tracker_node_id = _register_with_tracker( tracker_node_id = _register_with_tracker(
tracker_url, auto_reg_payload, node, _node_start_time, tracker_url, auto_reg_payload, node, _node_start_time,
refresh_capability=_capability_refresher(
node,
manifest=manifest,
recipe=recipe,
detected_device=device,
cache_dir=cache_dir,
force_cpu=force_cpu,
validator=capability_validator,
),
) )
shard_label = _format_shard_label( shard_label = _format_shard_label(
proof_shard.start, proof_shard.start,
@@ -1477,6 +1583,15 @@ def run_startup(
} }
tracker_node_id = _register_with_tracker( tracker_node_id = _register_with_tracker(
tracker_url, reg_payload, node, _node_start_time, tracker_url, reg_payload, node, _node_start_time,
refresh_capability=_capability_refresher(
node,
manifest=manifest,
recipe=recipe,
detected_device=device,
cache_dir=cache_dir,
force_cpu=force_cpu,
validator=capability_validator,
),
) )
print( print(
f"\n{'=' * 32}\n" f"\n{'=' * 32}\n"
@@ -1564,7 +1679,22 @@ def run_startup(
) )
node_id = str(reg_resp["node_id"]) node_id = str(reg_resp["node_id"])
setattr(node, "tracker_node_id", node_id) setattr(node, "tracker_node_id", node_id)
_start_heartbeat(tracker_url, node_id, reg_payload, node_ref=node, start_time=_node_start_time) _start_heartbeat(
tracker_url,
node_id,
reg_payload,
node_ref=node,
start_time=_node_start_time,
refresh_capability=_capability_refresher(
node,
manifest=manifest,
recipe=recipe,
detected_device=device,
cache_dir=shard_path,
force_cpu=force_cpu,
validator=capability_validator,
),
)
except Exception: except Exception:
node.stop() node.stop()
raise raise

24
packages/node/meshnet_node/torch_server.py
View File

@@ -1543,8 +1543,32 @@ class TorchNodeServer:
def loaded_model_ids(self) -> list[str]: def loaded_model_ids(self) -> list[str]:
return list(self._backends.keys()) return list(self._backends.keys())
def backend_for(self, model_id: str) -> TorchModelShard | None:
"""The loaded backend serving `model_id` — full repo id or short name."""
backend = self._backends.get(model_id)
if backend is not None:
return backend
short = model_id.split("/")[-1].lower()
for key, candidate in self._backends.items():
if key.split("/")[-1].lower() == short:
return candidate
return None
def apply_tracker_directives(self, directives: list[dict]) -> dict | None: def apply_tracker_directives(self, directives: list[dict]) -> dict | None:
"""Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more).""" """Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more)."""
drop_all_directive = next(
(directive for directive in reversed(directives) if directive.get("action") == "DROP_ALL_SHARDS"),
None,
)
if drop_all_directive is not None:
self._backends.clear()
self._backend = None
self._tracker_mode = False
if self._server is not None:
self._server.backends = {}
self._server.backend = None
self._server.tracker_mode = False
return {"action": "DROP_ALL_SHARDS"}
drop_directive = next( drop_directive = next(
(directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"), (directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"),
None, None,

125
packages/tracker/meshnet_tracker/dashboard.html
View File

@@ -44,12 +44,15 @@
.empty { color:var(--dim); font-style:italic; } .empty { color:var(--dim); font-style:italic; }
.pill { display:inline-block; padding:0 7px; border-radius:9px; .pill { display:inline-block; padding:0 7px; border-radius:9px;
border:1px solid var(--border); font-size:11px; } border:1px solid var(--border); font-size:11px; }
input, button { font:inherit; color:var(--fg); background:var(--bg); input, button, select { font:inherit; color:var(--fg); background:var(--bg);
border:1px solid var(--border); border-radius:6px; padding:5px 8px; } border:1px solid var(--border); border-radius:6px; padding:5px 8px; }
input { width:100%; margin-bottom:6px; } input { width:100%; margin-bottom:6px; }
button { cursor:pointer; color:var(--accent); } button { cursor:pointer; color:var(--accent); }
button:hover { border-color:var(--accent); } button:hover { border-color:var(--accent); }
button.small { font-size:11px; padding:1px 7px; } button.small { font-size:11px; padding:1px 7px; }
dialog { color:var(--fg); background:var(--panel); border:1px solid var(--border); border-radius:8px; min-width:min(420px,calc(100vw - 32px)); }
dialog::backdrop { background:rgba(0,0,0,.55); }
.placement-dialog-actions { display:flex; justify-content:flex-end; gap:8px; margin-top:12px; }
.form-row { display:flex; gap:8px; } .form-row { display:flex; gap:8px; }
.form-row button { white-space:nowrap; } .form-row button { white-space:nowrap; }
.error-msg { color:var(--bad); font-size:12px; min-height:16px; } .error-msg { color:var(--bad); font-size:12px; min-height:16px; }
@@ -212,7 +215,7 @@
.chat-compose button:disabled { opacity:.45; cursor:not-allowed; } .chat-compose button:disabled { opacity:.45; cursor:not-allowed; }
.console { .console {
background:var(--bg); border:1px solid var(--border); border-radius:6px; background:var(--bg); border:1px solid var(--border); border-radius:6px;
min-height:160px; max-height:280px; overflow:auto; padding:7px 9px; min-height:160px; max-height:520px; overflow-y:auto; overflow-x:auto; padding:7px 9px;
white-space:pre-wrap; word-break:break-word; font-size:11px; white-space:pre-wrap; word-break:break-word; font-size:11px;
} }
.console-line { padding:1px 0; border-bottom:1px solid #161b22; } .console-line { padding:1px 0; border-bottom:1px solid #161b22; }
@@ -296,6 +299,7 @@
<section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section> <section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
<section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section> <section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
<section data-tab="admin" class="wide"><h2>Model placement</h2><div id="admin-model-placement-status" class="dim">Choose a model to load or release.</div><div id="admin-model-placement" class="empty">admin login required</div></section> <section data-tab="admin" class="wide"><h2>Model placement</h2><div id="admin-model-placement-status" class="dim">Choose a model to load or release.</div><div id="admin-model-placement" class="empty">admin login required</div></section>
<section data-tab="admin" class="wide"><h2>Total node pool</h2><div id="admin-node-pool" class="empty">admin login required</div></section>
<section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section> <section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
<section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section> <section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section>
<section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section> <section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
@@ -323,6 +327,16 @@
<div id="testing-log" class="console empty">no test output yet</div> <div id="testing-log" class="console empty">no test output yet</div>
</section> </section>
</main> </main>
<dialog id="model-placement-dialog">
<form method="dialog">
<div id="model-placement-dialog-title"></div>
<label for="model-placement-node">Node</label>
<select id="model-placement-node"></select>
<label id="model-placement-replace" style="display:none"><input type="checkbox" id="model-placement-replace-confirm"> Unload the currently loaded model before loading this one</label>
<div id="model-placement-replace-error" class="bad" style="display:none"></div>
<div class="placement-dialog-actions"><button value="cancel">Cancel</button><button type="button" id="model-placement-confirm">Confirm</button></div>
</form>
</dialog>
<script> <script>
"use strict"; "use strict";
const $ = id => document.getElementById(id); const $ = id => document.getElementById(id);
@@ -1082,6 +1096,7 @@ function renderBillingUsage(records) {
} }
let consoleClearedAt = 0; let consoleClearedAt = 0;
const CONSOLE_MAX_LINES = 1000;
function clearConsole() { function clearConsole() {
consoleClearedAt = Date.now() / 1000; consoleClearedAt = Date.now() / 1000;
@@ -1095,7 +1110,7 @@ function renderConsole(data) {
$("console").innerHTML = '<div class="empty">no console events</div>'; $("console").innerHTML = '<div class="empty">no console events</div>';
return; return;
} }
$("console").innerHTML = events.slice(-120).map(e => { $("console").innerHTML = events.slice(-CONSOLE_MAX_LINES).map(e => {
const level = String(e.level || "info"); const level = String(e.level || "info");
const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info"; const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : ""; const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
@@ -1789,7 +1804,7 @@ async function requestSelectedModelLoad() {
if (!selectedChatModel) return; if (!selectedChatModel) return;
const button = $("request-model-load"); const button = $("request-model-load");
if (button) button.disabled = true; if (button) button.disabled = true;
const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel }); const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel, force: isAdmin });
if (button) button.disabled = false; if (button) button.disabled = false;
if (!result.ok) { if (!result.ok) {
alert(result.data.error || "model load request failed"); alert(result.data.error || "model load request failed");
@@ -1799,27 +1814,73 @@ async function requestSelectedModelLoad() {
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`; $("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
} }
async function requestAdminModelLoad(model) { async function requestAdminModelLoad(model, nodeId, replacing) {
const result = await apiCall("/v1/models/load", "POST", { model, force: true }); const result = await apiCall("/v1/models/load", "POST", { model, node_id: nodeId, force: replacing });
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true); if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
const assignment = result.data.assignment || {}; const assignment = result.data.assignment || {};
showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`); showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
await refreshActiveTab(true); await refreshActiveTab(true);
} }
async function releaseAdminModel(model) { async function releaseAdminModel(model, nodeId) {
const result = await apiCall("/v1/models/release", "POST", { model }); const result = await apiCall("/v1/models/release", "POST", { model, node_id: nodeId });
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model release request failed", true); if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model release request failed", true);
showAdminModelPlacementStatus(`Release queued for ${result.data.released || 0} node(s) serving ${model}.`); showAdminModelPlacementStatus(`Release queued for ${result.data.released || 0} node(s) serving ${model}.`);
await refreshActiveTab(true); await refreshActiveTab(true);
} }
async function releaseAllNodeModels(nodeId) {
if (!confirm("Unload every model from this node?")) return;
const result = await apiCall("/v1/nodes/release-all", "POST", { node_id: nodeId });
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "node unload failed", true);
showAdminModelPlacementStatus(`Unload queued for ${short(nodeId)}.`);
await refreshActiveTab(true);
}
function showAdminModelPlacementStatus(message, isError) { function showAdminModelPlacementStatus(message, isError) {
const status = $("admin-model-placement-status"); const status = $("admin-model-placement-status");
status.textContent = message; status.textContent = message;
status.className = isError ? "bad" : "ok"; status.className = isError ? "bad" : "ok";
} }
function gib(bytes) { return bytes == null ? "not reported" : `${(Number(bytes) / 1073741824).toFixed(1)} GiB`; }
function renderAdminNodePool(map) {
const groups = {};
for (const node of (map && map.nodes) || []) {
const account = node.wallet_address || "unbound account";
(groups[account] = groups[account] || []).push(node);
}
let html = "";
for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) {
html += `<div style="margin-top:10px"><b>${esc(short(account, 20))}</b> <span class="dim">${nodes.length} node(s)</span></div>`;
html += table(["node", "assignment", "state / slots", "model RAM", "RAM", "GPU / VRAM", "model drive", "action"], nodes.map(node => {
const hw = node.hardware_profile || {};
const cap = node.capacity || {};
// The network map keeps reported resource capacity under `capacity`.
node.ram_bytes = cap.ram_bytes ?? node.ram_bytes;
node.vram_bytes = cap.vram_bytes ?? node.vram_bytes;
const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
const row = [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`),
esc(gib(cap.loaded_model_bytes)),
esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))),
esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))];
return row.concat([
node.shard_start == null ? '<span class="dim">empty</span>' :
`<button class="small" data-admin-node-release="${esc(node.node_id)}">unload all</button>`,
]);
}));
}
$("admin-node-pool").innerHTML = html || '<div class="empty">no nodes registered</div>';
}
$("admin-node-pool").addEventListener("click", event => {
const unload = event.target.closest("[data-admin-node-release]");
if (unload) void releaseAllNodeModels(unload.dataset.adminNodeRelease);
});
function renderAdminModelPlacement(models, map) { function renderAdminModelPlacement(models, map) {
const nodes = (map && map.nodes) || []; const nodes = (map && map.nodes) || [];
const rows = ((models && models.data) || []).map(model => { const rows = ((models && models.data) || []).map(model => {
@@ -1827,7 +1888,8 @@ function renderAdminModelPlacement(models, map) {
const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length; const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length;
const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) || const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) ||
(node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length; (node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length;
const actions = `<button class="small" data-admin-model-load="${esc(model.id)}">load</button> ` + const loadable = model.id !== "stub-model";
const actions = `<button class="small" data-admin-model-load="${esc(model.id)}"${loadable ? "" : " disabled"}>load</button> ` +
`<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`; `<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`;
return [esc(model.name || model.id), String(serving), String(downloaded), actions]; return [esc(model.name || model.id), String(serving), String(downloaded), actions];
}); });
@@ -1839,10 +1901,49 @@ function renderAdminModelPlacement(models, map) {
$("admin-model-placement").addEventListener("click", event => { $("admin-model-placement").addEventListener("click", event => {
const load = event.target.closest("[data-admin-model-load]"); const load = event.target.closest("[data-admin-model-load]");
const release = event.target.closest("[data-admin-model-release]"); const release = event.target.closest("[data-admin-model-release]");
if (load) void requestAdminModelLoad(load.dataset.adminModelLoad); if (load) void chooseModelPlacementNode("load", load.dataset.adminModelLoad);
if (release) void releaseAdminModel(release.dataset.adminModelRelease); if (release) void chooseModelPlacementNode("release", release.dataset.adminModelRelease);
}); });
function chooseModelPlacementNode(action, model) {
const dialog = $("model-placement-dialog");
const select = $("model-placement-node");
const targetAlias = modelAliasKey(model);
const nodes = (lastNetworkMap?.nodes || []).filter(node => action === "load" ||
modelAliasKey(node.model) === targetAlias || modelAliasKey(node.hf_repo) === targetAlias);
if (!nodes.length) return showAdminModelPlacementStatus(`No node can ${action} ${model}.`, true);
$("model-placement-dialog-title").textContent = `${action === "load" ? "Load" : "Release"} ${model} on a node`;
select.innerHTML = nodes.map(node => `<option value="${esc(node.node_id)}">${esc(short(node.friendly_name || node.node_id, 20))} — ${esc(node.hf_repo || node.model || "unassigned")}</option>`).join("");
const replace = $("model-placement-replace");
const replaceConfirm = $("model-placement-replace-confirm");
const replaceError = $("model-placement-replace-error");
const confirmButton = $("model-placement-confirm");
const selectedNode = () => nodes.find(node => node.node_id === select.value);
const updateReplacementWarning = () => {
const node = selectedNode();
const occupied = action === "load" && node && node.shard_start != null && node.shard_end != null &&
modelAliasKey(node.hf_repo || node.model) !== targetAlias;
replace.style.display = occupied ? "" : "none";
replaceConfirm.checked = false;
replaceError.style.display = "none";
};
select.onchange = updateReplacementWarning;
updateReplacementWarning();
dialog.onclose = null;
confirmButton.onclick = () => {
const replacing = replace.style.display !== "none";
if (replacing && !replaceConfirm.checked) {
replaceError.textContent = "Tick the box to confirm that this will unload the current model.";
replaceError.style.display = "";
return;
}
dialog.close("confirm");
if (action === "load") void requestAdminModelLoad(model, select.value, replacing);
else void releaseAdminModel(model, select.value);
};
dialog.showModal();
}
function chatAuthToken() { function chatAuthToken() {
if (accountApiKeys.length) return accountApiKeys[0]; if (accountApiKeys.length) return accountApiKeys[0];
return null; return null;
@@ -2485,11 +2586,13 @@ async function fetchAdminTab() {
if (isAdmin) fetches.push(apiCall("/v1/admin/accounts")); if (isAdmin) fetches.push(apiCall("/v1/admin/accounts"));
const results = await Promise.all(fetches); const results = await Promise.all(fetches);
const [raft, consoleData, summary, wallets, models, map, adminResp] = results; const [raft, consoleData, summary, wallets, models, map, adminResp] = results;
if (map) lastNetworkMap = map;
renderIfChanged("hive", raft, renderHive); renderIfChanged("hive", raft, renderHive);
renderIfChanged("console", consoleData, renderConsole); renderIfChanged("console", consoleData, renderConsole);
renderIfChanged("billing-summary", summary, data => renderBilling(data)); renderIfChanged("billing-summary", summary, data => renderBilling(data));
renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary)); renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary));
renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map)); renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map));
renderIfChanged("admin-node-pool", map, renderAdminNodePool);
if (adminResp && adminResp.ok) { if (adminResp && adminResp.ok) {
renderIfChanged("admin", adminResp.data.accounts || [], accounts => { renderIfChanged("admin", adminResp.data.accounts || [], accounts => {
const rows = accounts.map(a => { const rows = accounts.map(a => {

132
packages/tracker/meshnet_tracker/server.py
View File

@@ -87,7 +87,7 @@ from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode from .raft import RaftNode
_CONSOLE_LIMIT = 300 _CONSOLE_LIMIT = 1000
_PROXY_PROGRESS_LOG_INTERVAL = 5.0 _PROXY_PROGRESS_LOG_INTERVAL = 5.0
_SESSION_COOKIE_NAME = "meshnet_session" _SESSION_COOKIE_NAME = "meshnet_session"
@@ -1121,12 +1121,15 @@ def _registration_quantization(body: dict, quantizations: list[str]) -> str | No
An absent field predates the protocol adding it: it means "unknown", not An absent field predates the protocol adding it: it means "unknown", not
"unsupported", so the node keeps the best precision it advertises and stays "unsupported", so the node keeps the best precision it advertises and stays
routable. Anything the node states explicitly is taken at its word -- a null, routable. An explicit "auto" means the same thing — the node's CLI default
a non-string, or an unsupported name leaves it with no usable precision and delegates the choice, it does not refuse one. Anything else the node states
routing excludes it. explicitly is taken at its word -- a null, a non-string, or an unsupported
name leaves it with no usable precision and routing excludes it.
""" """
if "quantization" in body: declared = body.get("quantization")
return _normalize_quantization(body["quantization"]) declared_auto = isinstance(declared, str) and declared.strip().lower() == "auto"
if "quantization" in body and not declared_auto:
return _normalize_quantization(declared)
supported = [ supported = [
normalized for value in quantizations normalized for value in quantizations
if (normalized := _normalize_quantization(value)) is not None if (normalized := _normalize_quantization(value)) is not None
@@ -1245,6 +1248,7 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
"quantization": node.quantization, "quantization": node.quantization,
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec, "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
"effective_throughput": round(_effective_throughput(node), 4), "effective_throughput": round(_effective_throughput(node), 4),
"loaded_model_bytes": _assignment_memory_bytes(node, preset),
} }
if preset is not None: if preset is not None:
summary["max_assignable_layers"] = _node_layer_capacity(node, preset) summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
@@ -1514,7 +1518,9 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
break break
def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None: def _request_model_load_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> dict | None:
"""Queue an explicitly requested model on the best available joined node.""" """Queue an explicitly requested model on the best available joined node."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key) resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None or not preset.get("hf_repo"): if preset is None or not preset.get("hf_repo"):
@@ -1530,6 +1536,8 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
continue continue
host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry] host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
placeable = [node for node in host_nodes if _has_usable_quantization(node)] placeable = [node for node in host_nodes if _has_usable_quantization(node)]
if node_id is not None:
placeable = [node for node in placeable if node.node_id == node_id]
if not placeable: if not placeable:
continue continue
anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec) anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec)
@@ -1548,21 +1556,26 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
return None return None
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None: def _force_model_load_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> dict | None:
"""Replace the fastest ready assignment after an explicit admin eviction.""" """Replace the fastest ready assignment after an explicit admin eviction."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key) resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None or not preset.get("hf_repo"): if preset is None or not preset.get("hf_repo"):
return None return None
start, end = _preset_layer_bounds(preset) start, end = _preset_layer_bounds(preset)
candidates = [node for node in server.registry.values() # An explicit admin eviction is permitted to recover a stuck/loading node
if node.status == "ready" and node.pending_new_assignment is None # and to use the preset default precision. It must only avoid a node that
and _has_usable_quantization(node)] # already has another assignment in flight.
candidates = [
node for node in server.registry.values()
if node.pending_new_assignment is None
and (node_id is None or node.node_id == node_id)
]
if not candidates: if not candidates:
return None return None
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec) node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
shard_end = min(end, start + min(_node_layer_capacity(node, preset), end - start + 1) - 1) shard_end = min(end, start + max(1, min(_node_layer_capacity(node, preset), end - start + 1)) - 1)
if shard_end < start:
return None
quantization = _node_quantization(node, preset) quantization = _node_quantization(node, preset)
directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization) directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization)
replaced = node.hf_repo or node.model replaced = node.hf_repo or node.model
@@ -1576,13 +1589,17 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
"shard_start": start, "shard_end": shard_end, "replaced_model": replaced} "shard_start": start, "shard_end": shard_end, "replaced_model": replaced}
def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int: def _release_model_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> int:
"""Queue DROP_SHARD for every served shard and remove it from routing immediately.""" """Queue DROP_SHARD for every served shard and remove it from routing immediately."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key) resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None: if preset is None:
return 0 return 0
released = 0 released = 0
for node in server.registry.values(): for node in server.registry.values():
if node_id is not None and node.node_id != node_id:
continue
if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None: if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None:
continue continue
node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16")) node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16"))
@@ -1591,6 +1608,16 @@ def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
return released return released
def _release_all_node_models_locked(server: "_TrackerHTTPServer", node_id: str) -> int:
"""Queue removal of every loaded assignment on one node."""
node = server.registry.get(node_id)
if node is None or node.shard_start is None or node.shard_end is None:
return 0
node.pending_directives.append({"action": "DROP_ALL_SHARDS"})
node.status = "loading"
return 1
def _preferred_node_quantization( def _preferred_node_quantization(
node: _NodeEntry, node: _NodeEntry,
preset: dict, preset: dict,
@@ -3109,6 +3136,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if self.path == "/v1/models/release": if self.path == "/v1/models/release":
self._handle_model_release_request() self._handle_model_release_request()
return return
if self.path == "/v1/nodes/release-all":
self._handle_node_release_all_request()
return
if self.path == "/v1/models/vote": if self.path == "/v1/models/vote":
self._handle_model_coverage_vote() self._handle_model_coverage_vote()
return return
@@ -3285,7 +3315,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
node.hf_repo or node.model node.hf_repo or node.model
for node in alive for node in alive
if node.model is not None if node.model is not None
and node.model not in server.model_presets # The same model can be registered under its HF repository while
# the catalogue exposes its short preset id. Do not emit a second
# repo-keyed entry when either node identifier resolves to a preset.
and _resolve_model_preset(
server.model_presets, node.hf_repo or node.model,
)[1] is None
and node.shard_start is not None and node.shard_start is not None
and node.shard_end is not None and node.shard_end is not None
and node.num_layers is not None and node.num_layers is not None
@@ -3386,6 +3421,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"endpoint": node.endpoint, "endpoint": node.endpoint,
"relay_addr": node.relay_addr, "relay_addr": node.relay_addr,
"peer_id": node.peer_id, "peer_id": node.peer_id,
"wallet_address": node.wallet_address,
"hardware_profile": dict(node.hardware_profile),
"ram_bytes": node.ram_bytes,
"vram_bytes": node.vram_bytes,
"max_loaded_shards": node.max_loaded_shards,
} }
for node in tracker_nodes for node in tracker_nodes
], ],
@@ -3409,12 +3449,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
memory_pool = _memory_pool_map(server) memory_pool = _memory_pool_map(server)
def capacity_for(node: _NodeEntry) -> dict: def capacity_for(node: _NodeEntry) -> dict:
preset = None return _node_capacity_summary(node, _preset_for_node(server, node))
if node.model:
preset = server.model_presets.get(node.model)
if preset is None and node.hf_repo and node.num_layers:
preset = _hf_rebalance_preset([node])
return _node_capacity_summary(node, preset)
def throughput_for(node: _NodeEntry) -> dict: def throughput_for(node: _NodeEntry) -> dict:
if server.stats is None: if server.stats is None:
@@ -4832,6 +4867,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
entry.uptime_seconds = float(body["uptime_seconds"]) entry.uptime_seconds = float(body["uptime_seconds"])
if "status" in body and body["status"] in ("ready", "loading"): if "status" in body and body["status"] in ("ready", "loading"):
entry.status = body["status"] entry.status = body["status"]
completed_directives = body.get("completed_directives", [])
if isinstance(completed_directives, list):
for directive in completed_directives:
if not isinstance(directive, dict) or directive.get("action") not in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
continue
# A node has confirmed the release. Stop advertising its
# old route immediately so the dashboard and routing state
# agree with the runtime.
entry.model = "stub-model"
entry.hf_repo = None
entry.shard_start = None
entry.shard_end = None
entry.tracker_mode = False
entry.status = "ready"
if "friendly_name" in body: if "friendly_name" in body:
try: try:
entry.friendly_name = _normalize_friendly_name(body.get("friendly_name")) entry.friendly_name = _normalize_friendly_name(body.get("friendly_name"))
@@ -4903,11 +4952,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model, str) or not model.strip(): if not isinstance(model, str) or not model.strip():
self._send_json(400, {"error": "model is required"}) self._send_json(400, {"error": "model is required"})
return return
node_id = body.get("node_id")
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
_resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None or str(preset.get("hf_repo") or "").strip().lower() == "stub-model":
self._send_json(400, {"error": "stub-model is a local test backend and cannot be loaded onto a node"})
return
with server.lock: with server.lock:
self._purge_expired_nodes() self._purge_expired_nodes()
assignment = _request_model_load_locked(server, model) assignment = _request_model_load_locked(server, model, node_id)
if assignment is None and body.get("force") is True: if assignment is None and body.get("force") is True:
assignment = _force_model_load_locked(server, model) assignment = _force_model_load_locked(server, model, node_id)
if assignment is None: if assignment is None:
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"}) self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
return return
@@ -4924,14 +4981,39 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model, str) or not model.strip(): if not isinstance(model, str) or not model.strip():
self._send_json(400, {"error": "model is required"}) self._send_json(400, {"error": "model is required"})
return return
node_id = body.get("node_id")
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock: with server.lock:
self._purge_expired_nodes() self._purge_expired_nodes()
released = _release_model_locked(server, model) released = _release_model_locked(server, model, node_id)
if not released: if not released:
self._send_json(404, {"error": "no served shards found for model"}) self._send_json(404, {"error": "no served shards found for model"})
return return
self._send_json(202, {"status": "release_queued", "released": released}) self._send_json(202, {"status": "release_queued", "released": released})
def _handle_node_release_all_request(self):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if not self._require_role("admin", "validator"):
return
body = self._read_json_body()
if body is None:
return
node_id = body.get("node_id")
if not isinstance(node_id, str) or not node_id:
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock:
self._purge_expired_nodes()
released = _release_all_node_models_locked(server, node_id)
if not released:
self._send_json(404, {"error": "no loaded models found for node"})
return
self._send_json(202, {
"status": "release_queued", "released": released, "node_id": node_id,
})
def _handle_model_coverage_vote(self): def _handle_model_coverage_vote(self):
"""Record a rolling wish-list signal for an unavailable precision.""" """Record a rolling wish-list signal for an unavailable precision."""
server: _TrackerHTTPServer = self.server # type: ignore[assignment] server: _TrackerHTTPServer = self.server # type: ignore[assignment]

22
tests/test_dashboard.py
View File

@@ -44,6 +44,9 @@ def test_dashboard_served_with_all_panels():
assert ".wide { grid-column:span 2; }" in html assert ".wide { grid-column:span 2; }" in html
assert 'onclick="clearConsole()"' in html assert 'onclick="clearConsole()"' in html
assert "let consoleClearedAt = 0;" in html assert "let consoleClearedAt = 0;" in html
assert "max-height:520px; overflow-y:auto; overflow-x:auto;" in html
assert "const CONSOLE_MAX_LINES = 1000;" in html
assert "events.slice(-CONSOLE_MAX_LINES)" in html
finally: finally:
tracker.stop() tracker.stop()
@@ -114,6 +117,25 @@ def test_dashboard_exposes_admin_model_inventory_and_release_controls():
assert 'data-admin-model-load=' in html assert 'data-admin-model-load=' in html
assert 'data-admin-model-release=' in html assert 'data-admin-model-release=' in html
assert "admin-model-placement-status" in html assert "admin-model-placement-status" in html
assert 'id="admin-node-pool"' in html
assert "renderAdminNodePool" in html
assert "model drive" in html
# RAM and VRAM live in the network-map capacity object, not at node top level.
assert "node.ram_bytes = cap.ram_bytes" in html
assert "node.vram_bytes = cap.vram_bytes" in html
assert 'id="model-placement-dialog"' in html
assert "chooseModelPlacementNode" in html
assert "node_id: nodeId" in html
assert "modelAliasKey(node.model)" in html
assert 'id="model-placement-replace"' in html
assert 'id="model-placement-confirm"' in html
assert 'id="model-placement-replace-error"' in html
assert "force: replacing" in html
assert "Tick the box to confirm" in html
assert "releaseAllNodeModels" in html
assert '"/v1/nodes/release-all"' in html
assert "model RAM" in html
assert "loaded_model_bytes" in html
def test_network_map_includes_node_friendly_name(): def test_network_map_includes_node_friendly_name():

38
tests/test_dynamic_routing.py
View File

@@ -370,11 +370,20 @@ def test_admin_can_replace_a_served_model_and_release_it():
headers = {"Content-Type": "application/json", "Authorization": "Bearer test-admin"} headers = {"Content-Type": "application/json", "Authorization": "Bearer test-admin"}
load = urllib.request.Request( load = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/models/load", f"http://127.0.0.1:{port}/v1/models/load",
data=json.dumps({"model": "qwen2.5-0.5b-instruct", "force": True}).encode(), data=json.dumps({
"model": "qwen2.5-0.5b-instruct",
"node_id": node["node_id"],
"force": True,
}).encode(),
headers=headers, method="POST") headers=headers, method="POST")
with urllib.request.urlopen(load) as response: with urllib.request.urlopen(load) as response:
assert json.loads(response.read())["assignment"]["node_id"] == node["node_id"] assert json.loads(response.read())["assignment"]["node_id"] == node["node_id"]
heartbeat = _post_json(f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {}) heartbeat = _post_json(f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
_post_json(
f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat",
{"completed_directives": [{"action": "DROP_SHARD", "model": "Qwen/Qwen2.5-0.5B-Instruct"}]},
)
network = _get_json(f"http://127.0.0.1:{port}/v1/network/map")
assert heartbeat["directives"][0]["action"] == "LOAD_SHARD" assert heartbeat["directives"][0]["action"] == "LOAD_SHARD"
release = urllib.request.Request( release = urllib.request.Request(
f"http://127.0.0.1:{port}/v1/models/release", f"http://127.0.0.1:{port}/v1/models/release",
@@ -386,6 +395,33 @@ def test_admin_can_replace_a_served_model_and_release_it():
tracker.stop() tracker.stop()
assert heartbeat["directives"][0]["action"] == "DROP_SHARD" assert heartbeat["directives"][0]["action"] == "DROP_SHARD"
released_node = next(item for item in network["nodes"] if item["node_id"] == node["node_id"])
assert released_node["shard_start"] is None
assert released_node["shard_end"] is None
def test_models_list_does_not_duplicate_a_preset_registered_by_hf_repo():
"""A preset and its canonical repository are one selectable model."""
tracker = TrackerServer(enable_billing=False)
port = tracker.start()
try:
_post_json(
f"http://127.0.0.1:{port}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:9913",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"num_layers": 24,
"shard_start": 0,
"shard_end": 23,
},
)
models = _get_json(f"http://127.0.0.1:{port}/v1/models")["data"]
finally:
tracker.stop()
assert [model["id"] for model in models].count("qwen2.5-0.5b-instruct") == 1
assert not any(model["id"] == "Qwen/Qwen2.5-0.5B-Instruct" for model in models)
def test_endpoint_key_distinguishes_same_port_different_hosts(): def test_endpoint_key_distinguishes_same_port_different_hosts():

67
tests/test_node_admission.py
View File

@@ -385,6 +385,73 @@ def test_a_stale_report_cannot_be_reused_to_register(startup_env):
assert startup_env == [] assert startup_env == []
# ---------------------------------------------------------------------------
# Re-registration: the proof presented is fresh, never the one captured at boot
# ---------------------------------------------------------------------------
def test_run_startup_hands_the_heartbeat_a_refresher_for_the_current_shard(startup_env, monkeypatch):
"The tracker refuses aged proofs, so the heartbeat must be able to re-prove what the node serves now.\n\nTags: node, admission, startup"
import meshnet_node.startup as startup_mod
captured: dict = {}
monkeypatch.setattr(
startup_mod, "_start_heartbeat", lambda *a, **kw: captured.update(kw)
)
_start(capability_validator=capability_stub())
refresh = captured.get("refresh_capability")
assert callable(refresh), "run_startup no longer wires a capability refresher"
fresh = refresh({"hf_repo": MODEL, "model": MODEL.split("/")[-1]})
assert fresh is not None
assert fresh["model"]["model_id"] == MODEL
assert (fresh["shard"]["start"], fresh["shard"]["end"]) == (0, 23)
assert fresh["validated_at"] > time.time() - 60
def test_a_reregistration_presents_a_refreshed_proof(monkeypatch):
"Replaying the boot-time report after an outage re-registers the node unroutable; the re-register path must present a fresh proof.\n\nTags: node, admission, startup"
import json
import meshnet_node.startup as startup_mod
model = "acme/refresh-model-7b"
boot_report = {"validated_at": 1.0, "marker": "boot"}
fresh_report = {"validated_at": 2.0, "marker": "fresh"}
posted: list[dict] = []
def _record(url, payload, timeout=10.0):
if url.endswith("/v1/nodes/register") and payload.get("hf_repo") == model:
posted.append(json.loads(json.dumps(payload)))
return {"node_id": "node-refresh"}
raise SystemExit # first heartbeat POST ends the daemon loop
monkeypatch.setattr(startup_mod, "_post_json", _record)
payload = {
"hf_repo": model,
"model": model.split("/")[-1],
"capability_report": dict(boot_report),
}
thread = startup_mod._start_heartbeat(
"http://tracker.invalid",
startup_mod._PENDING_NODE_ID, # forces a re-registration on the first tick
payload,
interval=0.02,
refresh_capability=lambda _payload: dict(fresh_report),
)
# The loop must be dead before this test returns: once monkeypatch restores
# `_post_json`, a surviving thread would re-register through whatever the
# *next* test patches in and corrupt its call counts.
thread.join(timeout=5.0)
assert not thread.is_alive(), "the heartbeat loop outlived the test"
assert posted, "the heartbeat never re-registered"
assert posted[0]["capability_report"] == fresh_report
def test_a_matching_passing_report_registers_and_travels_with_the_payload(startup_env): def test_a_matching_passing_report_registers_and_travels_with_the_payload(startup_env):
"Registration carries the proof for exactly the model/shard/recipe it advertises.\n\nTags: node, admission, startup" "Registration carries the proof for exactly the model/shard/recipe it advertises.\n\nTags: node, admission, startup"
node = _start() # production validator against a working fake backend node = _start() # production validator against a working fake backend

41
tests/test_node_startup.py
View File

@@ -287,6 +287,47 @@ def test_configure_torch_threads_applies_explicit_settings(monkeypatch):
assert active == {"torch_threads": 12, "torch_interop_threads": 2} assert active == {"torch_threads": 12, "torch_interop_threads": 2}
def test_heartbeat_applies_release_without_reregistering(monkeypatch):
"""DROP_SHARD has no replacement range and must not look like an outage."""
import meshnet_node.startup as startup_mod
released = threading.Event()
requests: list[tuple[str, dict]] = []
class FakeNode:
def apply_tracker_directives(self, directives):
assert directives == [{"action": "DROP_SHARD", "model": "Qwen/Qwen2.5-0.5B-Instruct"}]
return {"action": "DROP_SHARD", "model": "Qwen/Qwen2.5-0.5B-Instruct"}
def fake_post(url, payload, timeout=10.0):
requests.append((url, dict(payload)))
released.set()
return {"directives": [{"action": "DROP_SHARD", "model": "Qwen/Qwen2.5-0.5B-Instruct"}]}
sleep_calls = 0
def one_heartbeat(_seconds):
nonlocal sleep_calls
sleep_calls += 1
if sleep_calls > 1:
raise SystemExit
monkeypatch.setattr(startup_mod, "_post_json", fake_post)
monkeypatch.setattr(startup_mod.time, "sleep", one_heartbeat)
payload = {
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"shard_start": 0,
"shard_end": 23,
}
startup_mod._start_heartbeat("http://tracker", "node-1", payload, interval=0, node_ref=FakeNode())
assert released.wait(1), "heartbeat did not receive the queued release"
assert len(requests) == 1, "release must not trigger a re-registration"
assert payload["shard_start"] == 0
assert payload["shard_end"] == 23
def test_benchmark_throughput_is_registered_in_payload(monkeypatch, tmp_path): def test_benchmark_throughput_is_registered_in_payload(monkeypatch, tmp_path):
"benchmark_tokens_per_sec from the benchmark is included in the tracker registration.\n\nTags: node, performance, startup" "benchmark_tokens_per_sec from the benchmark is included in the tracker registration.\n\nTags: node, performance, startup"
import meshnet_node.startup as startup_mod import meshnet_node.startup as startup_mod

44
tests/test_tracker_routing.py
View File

@@ -2869,6 +2869,43 @@ def test_same_endpoint_can_register_multiple_models():
tracker.stop() tracker.stop()
def test_explicit_model_placement_targets_only_the_selected_node():
"An admin can add and release a model on one chosen multi-model node.\n\nTags: http, routing, tracker"
from meshnet_tracker.server import _release_model_locked, _request_model_load_locked
tracker = _tracker(model_presets={
"model-a": {"total_layers": 4, "bytes_per_layer": {"bfloat16": 1_000}, "hf_repo": "org/ModelA"},
"model-b": {"total_layers": 4, "bytes_per_layer": {"bfloat16": 1_000}, "hf_repo": "org/ModelB"},
})
tracker_port = tracker.start()
try:
registrations = []
for port in (9062, 9063):
registrations.append(_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{port}", "model": "model-a", "hf_repo": "org/ModelA",
"num_layers": 4, "shard_start": 0, "shard_end": 3, "max_loaded_shards": 2,
"vram_bytes": 20_000, "ram_bytes": 20_000, "quantizations": ["bfloat16"],
"benchmark_tokens_per_sec": 1.0, "hardware_profile": {}, "score": 1.0},
))
selected, other = (item["node_id"] for item in registrations)
with tracker._lock:
assignment = _request_model_load_locked(tracker._server, "model-b", selected) # type: ignore[arg-type]
assert assignment is not None
assert assignment["node_id"] == selected
assert tracker._registry[selected].pending_new_assignment is not None
assert tracker._registry[other].pending_new_assignment is None
with tracker._lock:
released = _release_model_locked(tracker._server, "model-a", selected) # type: ignore[arg-type]
assert released == 1
assert len(tracker._registry[selected].pending_directives) == 2
assert tracker._registry[other].pending_directives == []
finally:
tracker.stop()
def test_scale_demanded_models_queues_add_shard_on_spare_host(): def test_scale_demanded_models_queues_add_shard_on_spare_host():
"Scale demanded models queues add shard on spare host\n\nTags: http, routing, tracker" "Scale demanded models queues add shard on spare host\n\nTags: http, routing, tracker"
tracker = _tracker(model_presets={ tracker = _tracker(model_presets={
@@ -3005,6 +3042,13 @@ def test_a_node_declaring_an_unsupported_quantization_is_never_routed():
assert status == 503 assert status == 503
def test_a_node_declaring_auto_quantization_serves_a_default_precision_request():
"'auto' is the CLI default that delegates the choice — it is not a refusal, so the node must resolve to its best advertised precision and route.\n\nTags: http, routing, tracker"
status, response = _proxy_chat_status(POLICY_COMPAT, quantization="auto")
assert status == 200
assert response["choices"][0]["message"]["content"] == "ok"
def test_a_node_declaring_a_null_quantization_is_never_routed(): def test_a_node_declaring_a_null_quantization_is_never_routed():
"An explicit null states 'no usable precision' -- only an absent field is legacy.\n\nTags: http, routing, tracker" "An explicit null states 'no usable precision' -- only an absent field is legacy.\n\nTags: http, routing, tracker"
node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler) node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)