From 8cb00e951f6ec11f74e40ec9179f1f5ddfd3d304 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 16:11:18 +0200 Subject: [PATCH 1/8] feat: show admin node pool capacity --- packages/node/meshnet_node/hardware.py | 24 +++++++++++---- .../tracker/meshnet_tracker/dashboard.html | 29 ++++++++++++++++++- packages/tracker/meshnet_tracker/server.py | 5 ++++ tests/test_dashboard.py | 3 ++ 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/packages/node/meshnet_node/hardware.py b/packages/node/meshnet_node/hardware.py index 7c97a06..9bffa9a 100644 --- a/packages/node/meshnet_node/hardware.py +++ b/packages/node/meshnet_node/hardware.py @@ -2,6 +2,7 @@ import json import os +import shutil import subprocess import time @@ -183,6 +184,17 @@ def with_forced_cpu(hw: dict) -> dict: return forced +def _with_model_drive(profile: dict) -> dict: + """Attach free space for the default model cache drive to tracker diagnostics.""" + try: + cache_root = os.path.expanduser("~/.cache/meshnet/shards") + profile["model_drive_free_bytes"] = shutil.disk_usage(os.path.expanduser("~")).free + profile["model_drive_path"] = cache_root + except OSError: + pass + return profile + + def detect_hardware() -> dict: """Detect GPU model and available VRAM. Returns hardware profile dict.""" ram_mb = _detect_ram_mb() @@ -208,23 +220,23 @@ def detect_hardware() -> dict: } if torch_gpu is not None and torch_gpu.get("gcn_arch"): profile["gcn_arch"] = torch_gpu["gcn_arch"] - return profile + return _with_model_drive(profile) except ImportError: pass torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb) if torch_inventory is not None: - return torch_inventory + return _with_model_drive(torch_inventory) nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb) if nvidia_gpu is not None: - return nvidia_gpu + return _with_model_drive(nvidia_gpu) windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb) if windows_gpu is not None: - return windows_gpu + return _with_model_drive(windows_gpu) - return { + return _with_model_drive({ "device": "cpu", "gpu_name": None, "vram_mb": 0, @@ -232,7 +244,7 @@ def detect_hardware() -> dict: "shared_vram_mb": 0, "ram_mb": ram_mb, "cuda_available": False, - } + }) def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]: diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index bbddc6c..ef4b282 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -296,6 +296,7 @@

Settlement history

admin login required

Tracker hive

loading…

Model placement

Choose a model to load or release.
admin login required
+

Total node pool

admin login required

All accounts (admin)

Strikes / bans / forfeitures

admin login required

Client balances

admin login required
@@ -1789,7 +1790,7 @@ async function requestSelectedModelLoad() { if (!selectedChatModel) return; const button = $("request-model-load"); if (button) button.disabled = true; - const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel }); + const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel, force: isAdmin }); if (button) button.disabled = false; if (!result.ok) { alert(result.data.error || "model load request failed"); @@ -1820,6 +1821,31 @@ function showAdminModelPlacementStatus(message, isError) { status.className = isError ? "bad" : "ok"; } +function gib(bytes) { return bytes == null ? "not reported" : `${(Number(bytes) / 1073741824).toFixed(1)} GiB`; } + +function renderAdminNodePool(map) { + const groups = {}; + for (const node of (map && map.nodes) || []) { + const account = node.wallet_address || "unbound account"; + (groups[account] = groups[account] || []).push(node); + } + let html = ""; + for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) { + html += `
${esc(short(account, 20))} ${nodes.length} node(s)
`; + html += table(["node", "assignment", "state / slots", "RAM", "GPU / VRAM", "model drive"], nodes.map(node => { + const hw = node.hardware_profile || {}; + const cap = node.capacity || {}; + const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes; + const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only"); + return [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"), + esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`), + esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))), + esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))]; + })); + } + $("admin-node-pool").innerHTML = html || '
no nodes registered
'; +} + function renderAdminModelPlacement(models, map) { const nodes = (map && map.nodes) || []; const rows = ((models && models.data) || []).map(model => { @@ -2490,6 +2516,7 @@ async function fetchAdminTab() { renderIfChanged("billing-summary", summary, data => renderBilling(data)); renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary)); renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map)); + renderIfChanged("admin-node-pool", map, renderAdminNodePool); if (adminResp && adminResp.ok) { renderIfChanged("admin", adminResp.data.accounts || [], accounts => { const rows = accounts.map(a => { diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 32c7239..1ea2144 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -3366,6 +3366,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): "endpoint": node.endpoint, "relay_addr": node.relay_addr, "peer_id": node.peer_id, + "wallet_address": node.wallet_address, + "hardware_profile": dict(node.hardware_profile), + "ram_bytes": node.ram_bytes, + "vram_bytes": node.vram_bytes, + "max_loaded_shards": node.max_loaded_shards, } for node in tracker_nodes ], diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 9492b12..d3c1234 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -114,6 +114,9 @@ def test_dashboard_exposes_admin_model_inventory_and_release_controls(): assert 'data-admin-model-load=' in html assert 'data-admin-model-release=' in html assert "admin-model-placement-status" in html + assert 'id="admin-node-pool"' in html + assert "renderAdminNodePool" in html + assert "model drive" in html def test_network_map_includes_node_friendly_name(): From 21e6c861470682a06be6919be32a6c7beac34fb6 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 16:37:42 +0200 Subject: [PATCH 2/8] fix: let admin placement recover joined nodes --- packages/tracker/meshnet_tracker/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 1ea2144..149ff3d 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -1534,15 +1534,15 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di if preset is None or not preset.get("hf_repo"): return None start, end = _preset_layer_bounds(preset) + # An explicit admin eviction is permitted to recover a stuck/loading node + # and to use the preset default precision. It must only avoid a node that + # already has another assignment in flight. candidates = [node for node in server.registry.values() - if node.status == "ready" and node.pending_new_assignment is None - and _has_usable_quantization(node)] + if node.pending_new_assignment is None] if not candidates: return None node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec) - shard_end = min(end, start + min(_node_layer_capacity(node, preset), end - start + 1) - 1) - if shard_end < start: - return None + shard_end = min(end, start + max(1, min(_node_layer_capacity(node, preset), end - start + 1)) - 1) quantization = _node_quantization(node, preset) directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization) replaced = node.hf_repo or node.model From b661590ac7bf0a142cdf1c68a637a1dd2c1233ca Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 17:47:20 +0200 Subject: [PATCH 3/8] log window bigger --- packages/tracker/meshnet_tracker/dashboard.html | 5 +++-- packages/tracker/meshnet_tracker/server.py | 2 +- tests/test_dashboard.py | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index ef4b282..8f4df39 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -212,7 +212,7 @@ .chat-compose button:disabled { opacity:.45; cursor:not-allowed; } .console { background:var(--bg); border:1px solid var(--border); border-radius:6px; - min-height:160px; max-height:280px; overflow:auto; padding:7px 9px; + min-height:160px; max-height:520px; overflow-y:auto; overflow-x:auto; padding:7px 9px; white-space:pre-wrap; word-break:break-word; font-size:11px; } .console-line { padding:1px 0; border-bottom:1px solid #161b22; } @@ -1083,6 +1083,7 @@ function renderBillingUsage(records) { } let consoleClearedAt = 0; +const CONSOLE_MAX_LINES = 1000; function clearConsole() { consoleClearedAt = Date.now() / 1000; @@ -1096,7 +1097,7 @@ function renderConsole(data) { $("console").innerHTML = '
no console events
'; return; } - $("console").innerHTML = events.slice(-120).map(e => { + $("console").innerHTML = events.slice(-CONSOLE_MAX_LINES).map(e => { const level = String(e.level || "info"); const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info"; const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : ""; diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 149ff3d..cbb3663 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -86,7 +86,7 @@ from .model_files import files_for_layer_range, snapshot_dir_for_repo from .raft import RaftNode -_CONSOLE_LIMIT = 300 +_CONSOLE_LIMIT = 1000 _PROXY_PROGRESS_LOG_INTERVAL = 5.0 _SESSION_COOKIE_NAME = "meshnet_session" diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index d3c1234..f387d21 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -44,6 +44,9 @@ def test_dashboard_served_with_all_panels(): assert ".wide { grid-column:span 2; }" in html assert 'onclick="clearConsole()"' in html assert "let consoleClearedAt = 0;" in html + assert "max-height:520px; overflow-y:auto; overflow-x:auto;" in html + assert "const CONSOLE_MAX_LINES = 1000;" in html + assert "events.slice(-CONSOLE_MAX_LINES)" in html finally: tracker.stop() From ba7c656364367afc32305d9f4a7c01dc3a05d219 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 20:33:02 +0200 Subject: [PATCH 4/8] node metrics --- .claude/memory/project-status.md | 5 +++ _DEV_NOTES.md | 7 +-- .../tracker/meshnet_tracker/dashboard.html | 45 ++++++++++++++++--- packages/tracker/meshnet_tracker/server.py | 26 ++++++++--- tests/test_dashboard.py | 6 +++ tests/test_tracker_routing.py | 37 +++++++++++++++ 6 files changed, 109 insertions(+), 17 deletions(-) diff --git a/.claude/memory/project-status.md b/.claude/memory/project-status.md index 3fb0fc7..adbf7e7 100644 --- a/.claude/memory/project-status.md +++ b/.claude/memory/project-status.md @@ -8,6 +8,11 @@ metadata: # Project Status (2026-07-13) +## Selected-node model placement (2026-07-14) + +- Admin Model placement now opens a node selector for load and release; the control-plane accepts optional `node_id` and targets only that registry assignment. Multi-model serving remains supported through `ADD_SHARD` and `max_loaded_shards`. +- Total node pool resource values are rendered from `/v1/network/map`'s `node.capacity` contract. Route selection remains assignment/capability/throughput/queue based; capacity is used for placement and falls back to tracker defaults only if a node truly omits it. + ## Distributed inference performance (2026-07-14) `DIP-001` is done in `.scratch/distributed-inference-performance/`: the deterministic two-node Route Session stub benchmark covers direct/relay plus cached/stateless prefill and decode. Its JSON and concise summary explicitly attribute model execution, activation encode/decode, compression, connection setup, relay queueing, local HTTP forwarding, and end-to-end seam latency. `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` passed (7); the fixture assertion checks output-token identity and connection attempts. diff --git a/_DEV_NOTES.md b/_DEV_NOTES.md index 28f7987..acb8dc4 100644 --- a/_DEV_NOTES.md +++ b/_DEV_NOTES.md @@ -16,12 +16,9 @@ .\.venv\Scripts\meshnet-node.exe start http://192.168.0.179:8081 --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 - .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 + .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 - we .\.venv\Scripts\meshnet-node.exe start ` - --tracker http://192.168.0.179:8081 ` - --model Qwen/Qwen2.5-0.5B-Instruct ` - --advertise-host 192.168.0.20 + we .\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct # trackers: https://meshnet.2.d-popov.com https://ai.neuron.d-popov.com diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index 8f4df39..279d61a 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -44,12 +44,15 @@ .empty { color:var(--dim); font-style:italic; } .pill { display:inline-block; padding:0 7px; border-radius:9px; border:1px solid var(--border); font-size:11px; } - input, button { font:inherit; color:var(--fg); background:var(--bg); + input, button, select { font:inherit; color:var(--fg); background:var(--bg); border:1px solid var(--border); border-radius:6px; padding:5px 8px; } input { width:100%; margin-bottom:6px; } button { cursor:pointer; color:var(--accent); } button:hover { border-color:var(--accent); } button.small { font-size:11px; padding:1px 7px; } + dialog { color:var(--fg); background:var(--panel); border:1px solid var(--border); border-radius:8px; min-width:min(420px,calc(100vw - 32px)); } + dialog::backdrop { background:rgba(0,0,0,.55); } + .placement-dialog-actions { display:flex; justify-content:flex-end; gap:8px; margin-top:12px; } .form-row { display:flex; gap:8px; } .form-row button { white-space:nowrap; } .error-msg { color:var(--bad); font-size:12px; min-height:16px; } @@ -324,6 +327,14 @@
no test output yet
+ +
+
+ + +
+
+