feat: show admin node pool capacity
This commit is contained in:
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
@@ -183,6 +184,17 @@ def with_forced_cpu(hw: dict) -> dict:
|
||||
return forced
|
||||
|
||||
|
||||
def _with_model_drive(profile: dict) -> dict:
|
||||
"""Attach free space for the default model cache drive to tracker diagnostics."""
|
||||
try:
|
||||
cache_root = os.path.expanduser("~/.cache/meshnet/shards")
|
||||
profile["model_drive_free_bytes"] = shutil.disk_usage(os.path.expanduser("~")).free
|
||||
profile["model_drive_path"] = cache_root
|
||||
except OSError:
|
||||
pass
|
||||
return profile
|
||||
|
||||
|
||||
def detect_hardware() -> dict:
|
||||
"""Detect GPU model and available VRAM. Returns hardware profile dict."""
|
||||
ram_mb = _detect_ram_mb()
|
||||
@@ -208,23 +220,23 @@ def detect_hardware() -> dict:
|
||||
}
|
||||
if torch_gpu is not None and torch_gpu.get("gcn_arch"):
|
||||
profile["gcn_arch"] = torch_gpu["gcn_arch"]
|
||||
return profile
|
||||
return _with_model_drive(profile)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb)
|
||||
if torch_inventory is not None:
|
||||
return torch_inventory
|
||||
return _with_model_drive(torch_inventory)
|
||||
|
||||
nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb)
|
||||
if nvidia_gpu is not None:
|
||||
return nvidia_gpu
|
||||
return _with_model_drive(nvidia_gpu)
|
||||
|
||||
windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb)
|
||||
if windows_gpu is not None:
|
||||
return windows_gpu
|
||||
return _with_model_drive(windows_gpu)
|
||||
|
||||
return {
|
||||
return _with_model_drive({
|
||||
"device": "cpu",
|
||||
"gpu_name": None,
|
||||
"vram_mb": 0,
|
||||
@@ -232,7 +244,7 @@ def detect_hardware() -> dict:
|
||||
"shared_vram_mb": 0,
|
||||
"ram_mb": ram_mb,
|
||||
"cuda_available": False,
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]:
|
||||
|
||||
@@ -296,6 +296,7 @@
|
||||
<section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
|
||||
<section data-tab="admin" class="wide"><h2>Model placement</h2><div id="admin-model-placement-status" class="dim">Choose a model to load or release.</div><div id="admin-model-placement" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin" class="wide"><h2>Total node pool</h2><div id="admin-node-pool" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
|
||||
<section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
|
||||
@@ -1789,7 +1790,7 @@ async function requestSelectedModelLoad() {
|
||||
if (!selectedChatModel) return;
|
||||
const button = $("request-model-load");
|
||||
if (button) button.disabled = true;
|
||||
const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel });
|
||||
const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel, force: isAdmin });
|
||||
if (button) button.disabled = false;
|
||||
if (!result.ok) {
|
||||
alert(result.data.error || "model load request failed");
|
||||
@@ -1820,6 +1821,31 @@ function showAdminModelPlacementStatus(message, isError) {
|
||||
status.className = isError ? "bad" : "ok";
|
||||
}
|
||||
|
||||
function gib(bytes) { return bytes == null ? "not reported" : `${(Number(bytes) / 1073741824).toFixed(1)} GiB`; }
|
||||
|
||||
function renderAdminNodePool(map) {
|
||||
const groups = {};
|
||||
for (const node of (map && map.nodes) || []) {
|
||||
const account = node.wallet_address || "unbound account";
|
||||
(groups[account] = groups[account] || []).push(node);
|
||||
}
|
||||
let html = "";
|
||||
for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) {
|
||||
html += `<div style="margin-top:10px"><b>${esc(short(account, 20))}</b> <span class="dim">${nodes.length} node(s)</span></div>`;
|
||||
html += table(["node", "assignment", "state / slots", "RAM", "GPU / VRAM", "model drive"], nodes.map(node => {
|
||||
const hw = node.hardware_profile || {};
|
||||
const cap = node.capacity || {};
|
||||
const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
|
||||
const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
|
||||
return [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
|
||||
esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`),
|
||||
esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))),
|
||||
esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))];
|
||||
}));
|
||||
}
|
||||
$("admin-node-pool").innerHTML = html || '<div class="empty">no nodes registered</div>';
|
||||
}
|
||||
|
||||
function renderAdminModelPlacement(models, map) {
|
||||
const nodes = (map && map.nodes) || [];
|
||||
const rows = ((models && models.data) || []).map(model => {
|
||||
@@ -2490,6 +2516,7 @@ async function fetchAdminTab() {
|
||||
renderIfChanged("billing-summary", summary, data => renderBilling(data));
|
||||
renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary));
|
||||
renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map));
|
||||
renderIfChanged("admin-node-pool", map, renderAdminNodePool);
|
||||
if (adminResp && adminResp.ok) {
|
||||
renderIfChanged("admin", adminResp.data.accounts || [], accounts => {
|
||||
const rows = accounts.map(a => {
|
||||
|
||||
@@ -3366,6 +3366,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
"endpoint": node.endpoint,
|
||||
"relay_addr": node.relay_addr,
|
||||
"peer_id": node.peer_id,
|
||||
"wallet_address": node.wallet_address,
|
||||
"hardware_profile": dict(node.hardware_profile),
|
||||
"ram_bytes": node.ram_bytes,
|
||||
"vram_bytes": node.vram_bytes,
|
||||
"max_loaded_shards": node.max_loaded_shards,
|
||||
}
|
||||
for node in tracker_nodes
|
||||
],
|
||||
|
||||
@@ -114,6 +114,9 @@ def test_dashboard_exposes_admin_model_inventory_and_release_controls():
|
||||
assert 'data-admin-model-load=' in html
|
||||
assert 'data-admin-model-release=' in html
|
||||
assert "admin-model-placement-status" in html
|
||||
assert 'id="admin-node-pool"' in html
|
||||
assert "renderAdminNodePool" in html
|
||||
assert "model drive" in html
|
||||
|
||||
|
||||
def test_network_map_includes_node_friendly_name():
|
||||
|
||||
Reference in New Issue
Block a user