Compare commits
1 Commits
master
...
ralph/deep
| Author | SHA1 | Date | |
|---|---|---|---|
|
f102be1098 |
@@ -8,10 +8,6 @@ metadata:
|
||||
|
||||
# Project Status (2026-07-13)
|
||||
|
||||
## Distributed inference performance (2026-07-14)
|
||||
|
||||
`DIP-001` is done in `.scratch/distributed-inference-performance/`: the deterministic two-node Route Session stub benchmark covers direct/relay plus cached/stateless prefill and decode. Its JSON and concise summary explicitly attribute model execution, activation encode/decode, compression, connection setup, relay queueing, local HTTP forwarding, and end-to-end seam latency. `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` passed (7); the fixture assertion checks output-token identity and connection attempts.
|
||||
|
||||
> Doc reconciliation 2026-07-13: `docs/prd.json` tracks US-001…US-050 (048 memory budget, 049 mainnet pilot, 050 Qwen demand placement). ADRs 0025–0026 added (TAI phase B/C, assignment ownership).
|
||||
|
||||
All 35 user stories in docs/prd.json are done (35/35), including the reward-system arc US-030…US-035 completed 2026-07-02:
|
||||
|
||||
@@ -12,10 +12,4 @@ Provide an opt-in, admin-only tracker Dashboard Testing tab that dynamically dis
|
||||
- One active run.
|
||||
- Real inference stays separately environment-gated and excluded from default suites.
|
||||
|
||||
## Operator workflow
|
||||
|
||||
See [`docs/dev/dashboard-test-runner.md`](../../docs/dev/dashboard-test-runner.md)
|
||||
for launch configuration, default safe suites vs the gated real-inference suite,
|
||||
and required environment variables.
|
||||
|
||||
See `prd.json` for executable Ralph user stories and acceptance criteria.
|
||||
|
||||
@@ -51,16 +51,15 @@
|
||||
"uv run pytest tests/test_dashboard.py tests/test_dynamic_routing.py -q passes."
|
||||
],
|
||||
"priority": 3,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Do not reintroduce --enable-test-runner without implementing its CLI argument in US-001.",
|
||||
"dependsOn": [
|
||||
"US-001",
|
||||
"US-002"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"updatedAt": "2026-07-12T01:58:06.286Z"
|
||||
"updatedAt": "2026-07-11T17:02:30.520Z"
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
Status: done (2026-07-14)
|
||||
Status: ready-for-agent
|
||||
|
||||
# 01 — Baseline and profiling harness
|
||||
|
||||
@@ -12,15 +12,16 @@ sizes and connection counts without requiring a real model or external host.
|
||||
|
||||
## Acceptance criteria
|
||||
|
||||
- [x] The harness runs a fixed prompt and fixed generated-token count through a
|
||||
- [ ] The harness runs a fixed prompt and fixed generated-token count through a
|
||||
two-node route in direct and relay modes.
|
||||
- [x] It reports p50/p95 per-token latency, per-hop latency, payload bytes,
|
||||
- [ ] It reports p50/p95 per-token latency, per-hop latency, payload bytes,
|
||||
compression ratio, connection attempts, and queue wait.
|
||||
- [x] It distinguishes prefill from decode and cached from stateless mode.
|
||||
- [x] It emits machine-readable JSON suitable for CI artifacts and a concise
|
||||
- [ ] It distinguishes prefill from decode and cached from stateless mode.
|
||||
- [ ] It emits machine-readable JSON suitable for CI artifacts and a concise
|
||||
human-readable summary.
|
||||
- [x] A test fixture can assert connection attempts and output token identity.
|
||||
- [ ] A test fixture can assert connection attempts and output token identity.
|
||||
|
||||
## Blocked by
|
||||
|
||||
None - completed. Verified with `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` (7 passed).
|
||||
None - can start immediately.
|
||||
|
||||
|
||||
@@ -15,10 +15,9 @@
|
||||
"Can assert connection count and output token identity"
|
||||
],
|
||||
"priority": 1,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md",
|
||||
"dependsOn": [],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": []
|
||||
},
|
||||
{
|
||||
"id": "DIP-002",
|
||||
@@ -32,12 +31,9 @@
|
||||
"Tests cover binary, JSON, timeout, disconnect, cancellation, and cleanup"
|
||||
],
|
||||
"priority": 2,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md",
|
||||
"dependsOn": [
|
||||
"DIP-001"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-001"]
|
||||
},
|
||||
{
|
||||
"id": "DIP-003",
|
||||
@@ -51,12 +47,9 @@
|
||||
"Benchmark shows healthy-session connection count independent of token count"
|
||||
],
|
||||
"priority": 3,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/03-http-keepalive.md",
|
||||
"dependsOn": [
|
||||
"DIP-001"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-001"]
|
||||
},
|
||||
{
|
||||
"id": "DIP-004",
|
||||
@@ -70,12 +63,9 @@
|
||||
"Tests verify cadence and cleanup"
|
||||
],
|
||||
"priority": 4,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/04-seam-telemetry.md",
|
||||
"dependsOn": [
|
||||
"DIP-001"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-001"]
|
||||
},
|
||||
{
|
||||
"id": "DIP-005",
|
||||
@@ -89,12 +79,9 @@
|
||||
"Tests cover compressible, incompressible, threshold, malformed, and legacy bodies"
|
||||
],
|
||||
"priority": 5,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/05-adaptive-compression.md",
|
||||
"dependsOn": [
|
||||
"DIP-001"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-001"]
|
||||
},
|
||||
{
|
||||
"id": "DIP-006",
|
||||
@@ -108,12 +95,9 @@
|
||||
"Wire and token-output regression tests pass"
|
||||
],
|
||||
"priority": 6,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/06-activation-framing-copies.md",
|
||||
"dependsOn": [
|
||||
"DIP-001"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-001"]
|
||||
},
|
||||
{
|
||||
"id": "DIP-007",
|
||||
@@ -127,13 +111,9 @@
|
||||
"Tests cover chunking, slow consumers, failure, and legacy peers"
|
||||
],
|
||||
"priority": 7,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/07-prefill-backpressure.md",
|
||||
"dependsOn": [
|
||||
"DIP-001",
|
||||
"DIP-004"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-001", "DIP-004"]
|
||||
},
|
||||
{
|
||||
"id": "DIP-008",
|
||||
@@ -147,20 +127,9 @@
|
||||
"Gate verifies token identity, session stability, and resource cleanup"
|
||||
],
|
||||
"priority": 8,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md",
|
||||
"dependsOn": [
|
||||
"DIP-002",
|
||||
"DIP-003",
|
||||
"DIP-004",
|
||||
"DIP-005",
|
||||
"DIP-006",
|
||||
"DIP-007"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
"dependsOn": ["DIP-002", "DIP-003", "DIP-004", "DIP-005", "DIP-006", "DIP-007"]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"updatedAt": "2026-07-12T02:35:28.752Z"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -35,12 +35,11 @@
|
||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||
],
|
||||
"priority": 2,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/node-capability-admission/issues/02-doctor-real-forward.md",
|
||||
"dependsOn": [
|
||||
"NCA-001"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NCA-003",
|
||||
@@ -55,13 +54,12 @@
|
||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||
],
|
||||
"priority": 3,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md",
|
||||
"dependsOn": [
|
||||
"NCA-001",
|
||||
"NCA-002"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NCA-004",
|
||||
@@ -78,13 +76,12 @@
|
||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||
],
|
||||
"priority": 4,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md",
|
||||
"dependsOn": [
|
||||
"NCA-001",
|
||||
"NCA-003"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NCA-005",
|
||||
@@ -99,16 +96,15 @@
|
||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||
],
|
||||
"priority": 5,
|
||||
"passes": true,
|
||||
"passes": false,
|
||||
"notes": "Source issue: .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md",
|
||||
"dependsOn": [
|
||||
"NCA-002",
|
||||
"NCA-004"
|
||||
],
|
||||
"completionNotes": "Completed by agent"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"updatedAt": "2026-07-12T01:54:03.030Z"
|
||||
"updatedAt": "2026-07-11T19:16:52.768Z"
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
@@ -184,17 +183,6 @@ def with_forced_cpu(hw: dict) -> dict:
|
||||
return forced
|
||||
|
||||
|
||||
def _with_model_drive(profile: dict) -> dict:
|
||||
"""Attach free space for the default model cache drive to tracker diagnostics."""
|
||||
try:
|
||||
cache_root = os.path.expanduser("~/.cache/meshnet/shards")
|
||||
profile["model_drive_free_bytes"] = shutil.disk_usage(os.path.expanduser("~")).free
|
||||
profile["model_drive_path"] = cache_root
|
||||
except OSError:
|
||||
pass
|
||||
return profile
|
||||
|
||||
|
||||
def detect_hardware() -> dict:
|
||||
"""Detect GPU model and available VRAM. Returns hardware profile dict."""
|
||||
ram_mb = _detect_ram_mb()
|
||||
@@ -220,23 +208,23 @@ def detect_hardware() -> dict:
|
||||
}
|
||||
if torch_gpu is not None and torch_gpu.get("gcn_arch"):
|
||||
profile["gcn_arch"] = torch_gpu["gcn_arch"]
|
||||
return _with_model_drive(profile)
|
||||
return profile
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb)
|
||||
if torch_inventory is not None:
|
||||
return _with_model_drive(torch_inventory)
|
||||
return torch_inventory
|
||||
|
||||
nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb)
|
||||
if nvidia_gpu is not None:
|
||||
return _with_model_drive(nvidia_gpu)
|
||||
return nvidia_gpu
|
||||
|
||||
windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb)
|
||||
if windows_gpu is not None:
|
||||
return _with_model_drive(windows_gpu)
|
||||
return windows_gpu
|
||||
|
||||
return _with_model_drive({
|
||||
return {
|
||||
"device": "cpu",
|
||||
"gpu_name": None,
|
||||
"vram_mb": 0,
|
||||
@@ -244,7 +232,7 @@ def detect_hardware() -> dict:
|
||||
"shared_vram_mb": 0,
|
||||
"ram_mb": ram_mb,
|
||||
"cuda_available": False,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]:
|
||||
|
||||
@@ -44,7 +44,6 @@ class SeamSample:
|
||||
cache_mode: CacheMode
|
||||
model_ms: float
|
||||
encode_ms: float
|
||||
activation_decode_ms: float
|
||||
framing_ms: float
|
||||
metadata_ms: float
|
||||
copy_allocation_ms: float
|
||||
@@ -53,7 +52,6 @@ class SeamSample:
|
||||
decompression_ms: float
|
||||
connection_setup_ms: float
|
||||
queue_wait_ms: float
|
||||
local_http_forwarding_ms: float
|
||||
transport_ms: float
|
||||
seam_latency_ms: float
|
||||
payload_bytes: int
|
||||
@@ -122,10 +120,6 @@ def _summary(samples: list[SeamSample]) -> dict[str, float | int]:
|
||||
"compression_cpu_ms": round(
|
||||
sum(sample.compression_ms + sample.decompression_ms for sample in samples), 4
|
||||
),
|
||||
"model_execution_ms": round(sum(sample.model_ms for sample in samples), 4),
|
||||
"activation_encoding_ms": round(sum(sample.encode_ms for sample in samples), 4),
|
||||
"activation_decoding_ms": round(sum(sample.activation_decode_ms for sample in samples), 4),
|
||||
"local_http_forwarding_ms": round(sum(sample.local_http_forwarding_ms for sample in samples), 4),
|
||||
"peak_buffered_bytes": max((sample.copy_allocation_bytes for sample in samples), default=0),
|
||||
}
|
||||
|
||||
@@ -165,7 +159,6 @@ class _StubTransport:
|
||||
queue_wait_ms = 0.0 if self.mode == "direct" else 0.18 + (0.05 if token_index is not None and token_index % 2 else 0.0)
|
||||
model_ms = 1.6 if phase == "prefill" else 0.45
|
||||
encode_ms = 0.16 if phase == "prefill" else 0.06
|
||||
activation_decode_ms = 0.055 if phase == "prefill" else 0.02
|
||||
# Keep framing/metadata/copy costs explicit rather than hiding them in
|
||||
# serialization or transport time. The stub owns one binary frame and
|
||||
# one response body per hop; no base64 body is modeled.
|
||||
@@ -175,26 +168,20 @@ class _StubTransport:
|
||||
copy_allocation_bytes = wire_bytes + payload_bytes
|
||||
compression_ms = 0.09 if self.scenario.compression else 0.0
|
||||
decompression_ms = 0.07 if self.scenario.compression else 0.0
|
||||
# Both routes finish by forwarding the decoded activation to the local
|
||||
# tail-node HTTP handler; relay adds its own queue before that hop.
|
||||
local_http_forwarding_ms = 0.11 if self.mode == "direct" else 0.16
|
||||
transport_ms = (0.32 if self.mode == "direct" else 0.61) + wire_bytes / 100_000
|
||||
seam_latency_ms = round(
|
||||
model_ms + encode_ms + activation_decode_ms + framing_ms + metadata_ms + copy_allocation_ms
|
||||
+ compression_ms + decompression_ms + connection_setup_ms + queue_wait_ms + transport_ms
|
||||
+ local_http_forwarding_ms,
|
||||
model_ms + encode_ms + framing_ms + metadata_ms + copy_allocation_ms
|
||||
+ compression_ms + decompression_ms + connection_setup_ms + queue_wait_ms + transport_ms,
|
||||
4,
|
||||
)
|
||||
return SeamSample(
|
||||
phase=phase, token_index=token_index, session_id=self.session_id,
|
||||
activation_id=f"benchmark-activation-{self._activation_count}", seam="head->tail", mode=self.mode,
|
||||
cache_mode=self.cache_mode, model_ms=model_ms, encode_ms=encode_ms,
|
||||
activation_decode_ms=activation_decode_ms,
|
||||
framing_ms=framing_ms, metadata_ms=metadata_ms,
|
||||
copy_allocation_ms=copy_allocation_ms, copy_allocation_bytes=copy_allocation_bytes,
|
||||
compression_ms=compression_ms, decompression_ms=decompression_ms,
|
||||
connection_setup_ms=connection_setup_ms, queue_wait_ms=queue_wait_ms,
|
||||
local_http_forwarding_ms=local_http_forwarding_ms,
|
||||
transport_ms=round(transport_ms, 4), seam_latency_ms=seam_latency_ms,
|
||||
payload_bytes=payload_bytes, wire_bytes=wire_bytes,
|
||||
compression_ratio=round(payload_bytes / wire_bytes, 4), connection_attempted=connection_attempted,
|
||||
@@ -342,10 +329,9 @@ def run_real_model_lan_benchmark(url: str, *, model: str, timeout: float = 120.0
|
||||
sample = SeamSample(
|
||||
phase="decode", token_index=0, session_id=session_id, activation_id="lan-activation-1",
|
||||
seam="head->tail", mode="direct", cache_mode="cached", model_ms=0.0, encode_ms=0.0,
|
||||
activation_decode_ms=0.0,
|
||||
framing_ms=0.0, metadata_ms=0.0, copy_allocation_ms=0.0, copy_allocation_bytes=0,
|
||||
compression_ms=0.0, decompression_ms=0.0, connection_setup_ms=elapsed_ms,
|
||||
queue_wait_ms=0.0, local_http_forwarding_ms=0.0, transport_ms=elapsed_ms, seam_latency_ms=elapsed_ms,
|
||||
queue_wait_ms=0.0, transport_ms=elapsed_ms, seam_latency_ms=elapsed_ms,
|
||||
payload_bytes=len(body), wire_bytes=len(body) + len(response_body), compression_ratio=1.0,
|
||||
connection_attempted=True,
|
||||
)
|
||||
@@ -368,10 +354,6 @@ def format_summary(report: dict) -> str:
|
||||
f"{decode['tokens_per_sec']:.1f} tok/s; {decode['bytes_per_token']:.0f} B/tok; "
|
||||
f"seam {seam['payload_bytes']}/{seam['wire_bytes']} B "
|
||||
f"({seam['compression_ratio']:.2f}x); connections {run['connections']['attempts']}; "
|
||||
f"model/encode/decode {decode['model_execution_ms']:.2f}/"
|
||||
f"{decode['activation_encoding_ms']:.2f}/{decode['activation_decoding_ms']:.2f} ms; "
|
||||
f"compression {decode['compression_cpu_ms']:.2f} ms; "
|
||||
f"HTTP {decode['local_http_forwarding_ms']:.2f} ms; "
|
||||
f"queue p95 {decode['p95_queue_wait_ms']:.2f} ms"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -1545,26 +1545,6 @@ class TorchNodeServer:
|
||||
|
||||
def apply_tracker_directives(self, directives: list[dict]) -> dict | None:
|
||||
"""Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more)."""
|
||||
drop_directive = next(
|
||||
(directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"),
|
||||
None,
|
||||
)
|
||||
if drop_directive is not None:
|
||||
model_id = str(drop_directive.get("model") or "")
|
||||
removed = self._backends.pop(model_id, None)
|
||||
if removed is None:
|
||||
return None
|
||||
if self._backends:
|
||||
self._backend = next(iter(self._backends.values()))
|
||||
self._tracker_mode = self._backend.shard_start == 0
|
||||
else:
|
||||
self._backend = None
|
||||
self._tracker_mode = False
|
||||
if self._server is not None:
|
||||
self._server.backends = dict(self._backends)
|
||||
self._server.backend = self._backend
|
||||
self._server.tracker_mode = self._tracker_mode
|
||||
return {"action": "DROP_SHARD", "model": model_id}
|
||||
add_directive = next(
|
||||
(directive for directive in reversed(directives) if directive.get("action") == "ADD_SHARD"),
|
||||
None,
|
||||
@@ -1594,8 +1574,6 @@ class TorchNodeServer:
|
||||
flush=True,
|
||||
)
|
||||
try:
|
||||
if replacing:
|
||||
self._backends.clear()
|
||||
new_backend = _load_backend(model_id, shard_start, shard_end, quantization, self._cache_dir)
|
||||
except TypeError:
|
||||
new_backend = _load_backend(model_id, shard_start, shard_end, quantization)
|
||||
|
||||
@@ -22,9 +22,8 @@
|
||||
border-bottom:1px solid var(--border); flex-shrink:0; }
|
||||
header h1 { font-size:16px; margin:0; color:var(--accent); }
|
||||
header .meta { color:var(--dim); font-size:12px; }
|
||||
main { display:grid; grid-template-columns:1fr;
|
||||
main { display:grid; grid-template-columns:repeat(auto-fit,minmax(340px,1fr));
|
||||
gap:14px; padding:14px 20px; }
|
||||
main > section { width:100%; min-width:0; }
|
||||
body.chat-tab-active main {
|
||||
flex:1; min-height:0; display:flex; flex-direction:column;
|
||||
padding:0; gap:0; overflow:hidden;
|
||||
@@ -72,12 +71,6 @@
|
||||
background:transparent; color:var(--dim); padding:5px 0 8px; }
|
||||
.dashboard-tabs button.active { color:var(--accent); border-bottom-color:var(--accent); }
|
||||
.wide { grid-column:1 / -1; }
|
||||
/* Compact status cards fan out on desktop; tables remain readable at half width. */
|
||||
@media (min-width:900px) {
|
||||
main { grid-template-columns:repeat(4,minmax(0,1fr)); }
|
||||
main > section { grid-column:span 1; }
|
||||
.wide { grid-column:span 2; }
|
||||
}
|
||||
section[hidden] { display:none !important; }
|
||||
section.chat-section {
|
||||
padding:0; border:0; border-radius:0; background:var(--bg); min-height:0;
|
||||
@@ -295,8 +288,6 @@
|
||||
<section data-tab="billing" data-admin-only><h2>Node pending payouts</h2><div id="pending" class="empty">admin login required</div></section>
|
||||
<section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
|
||||
<section data-tab="admin" class="wide"><h2>Model placement</h2><div id="admin-model-placement-status" class="dim">Choose a model to load or release.</div><div id="admin-model-placement" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin" class="wide"><h2>Total node pool</h2><div id="admin-node-pool" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
|
||||
<section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section>
|
||||
<section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
|
||||
@@ -1790,7 +1781,7 @@ async function requestSelectedModelLoad() {
|
||||
if (!selectedChatModel) return;
|
||||
const button = $("request-model-load");
|
||||
if (button) button.disabled = true;
|
||||
const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel, force: isAdmin });
|
||||
const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel });
|
||||
if (button) button.disabled = false;
|
||||
if (!result.ok) {
|
||||
alert(result.data.error || "model load request failed");
|
||||
@@ -1800,75 +1791,6 @@ async function requestSelectedModelLoad() {
|
||||
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
|
||||
}
|
||||
|
||||
async function requestAdminModelLoad(model) {
|
||||
const result = await apiCall("/v1/models/load", "POST", { model, force: true });
|
||||
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
|
||||
const assignment = result.data.assignment || {};
|
||||
showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
|
||||
await refreshActiveTab(true);
|
||||
}
|
||||
|
||||
async function releaseAdminModel(model) {
|
||||
const result = await apiCall("/v1/models/release", "POST", { model });
|
||||
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model release request failed", true);
|
||||
showAdminModelPlacementStatus(`Release queued for ${result.data.released || 0} node(s) serving ${model}.`);
|
||||
await refreshActiveTab(true);
|
||||
}
|
||||
|
||||
function showAdminModelPlacementStatus(message, isError) {
|
||||
const status = $("admin-model-placement-status");
|
||||
status.textContent = message;
|
||||
status.className = isError ? "bad" : "ok";
|
||||
}
|
||||
|
||||
function gib(bytes) { return bytes == null ? "not reported" : `${(Number(bytes) / 1073741824).toFixed(1)} GiB`; }
|
||||
|
||||
function renderAdminNodePool(map) {
|
||||
const groups = {};
|
||||
for (const node of (map && map.nodes) || []) {
|
||||
const account = node.wallet_address || "unbound account";
|
||||
(groups[account] = groups[account] || []).push(node);
|
||||
}
|
||||
let html = "";
|
||||
for (const [account, nodes] of Object.entries(groups).sort(([a], [b]) => a.localeCompare(b))) {
|
||||
html += `<div style="margin-top:10px"><b>${esc(short(account, 20))}</b> <span class="dim">${nodes.length} node(s)</span></div>`;
|
||||
html += table(["node", "assignment", "state / slots", "RAM", "GPU / VRAM", "model drive"], nodes.map(node => {
|
||||
const hw = node.hardware_profile || {};
|
||||
const cap = node.capacity || {};
|
||||
const disk = hw.model_drive_free_bytes ?? hw.model_path_free_bytes ?? hw.disk_free_bytes;
|
||||
const gpu = hw.gpu_name || (hw.cuda_available ? "CUDA GPU" : "CPU only");
|
||||
return [nodeDisplayCell(node), esc(node.hf_repo || node.model || "unassigned"),
|
||||
esc(`${node.stats?.status || "?"} · ${cap.loaded_slots ?? "?"}/${cap.max_loaded_shards ?? node.max_loaded_shards ?? "?"} slots`),
|
||||
esc(gib(node.ram_bytes || (hw.ram_mb && hw.ram_mb * 1048576))),
|
||||
esc(`${gpu} · ${gib(node.vram_bytes || (hw.vram_mb && hw.vram_mb * 1048576))}`), esc(gib(disk))];
|
||||
}));
|
||||
}
|
||||
$("admin-node-pool").innerHTML = html || '<div class="empty">no nodes registered</div>';
|
||||
}
|
||||
|
||||
function renderAdminModelPlacement(models, map) {
|
||||
const nodes = (map && map.nodes) || [];
|
||||
const rows = ((models && models.data) || []).map(model => {
|
||||
const aliases = new Set([model.id, model.hf_repo, ...(model.aliases || [])].filter(Boolean));
|
||||
const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length;
|
||||
const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) ||
|
||||
(node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length;
|
||||
const actions = `<button class="small" data-admin-model-load="${esc(model.id)}">load</button> ` +
|
||||
`<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`;
|
||||
return [esc(model.name || model.id), String(serving), String(downloaded), actions];
|
||||
});
|
||||
$("admin-model-placement").innerHTML = rows.length
|
||||
? table(["model", "serving nodes", "downloaded on nodes", "admin action"], rows)
|
||||
: '<div class="empty">no model presets configured</div>';
|
||||
}
|
||||
|
||||
$("admin-model-placement").addEventListener("click", event => {
|
||||
const load = event.target.closest("[data-admin-model-load]");
|
||||
const release = event.target.closest("[data-admin-model-release]");
|
||||
if (load) void requestAdminModelLoad(load.dataset.adminModelLoad);
|
||||
if (release) void releaseAdminModel(release.dataset.adminModelRelease);
|
||||
});
|
||||
|
||||
function chatAuthToken() {
|
||||
if (accountApiKeys.length) return accountApiKeys[0];
|
||||
return null;
|
||||
@@ -2505,18 +2427,14 @@ async function fetchAdminTab() {
|
||||
fetchJson("/v1/console"),
|
||||
fetchJson("/v1/billing/summary"),
|
||||
fetchJson("/v1/registry/wallets"),
|
||||
fetchJson("/v1/models"),
|
||||
fetchJson("/v1/network/map"),
|
||||
];
|
||||
if (isAdmin) fetches.push(apiCall("/v1/admin/accounts"));
|
||||
const results = await Promise.all(fetches);
|
||||
const [raft, consoleData, summary, wallets, models, map, adminResp] = results;
|
||||
const [raft, consoleData, summary, wallets, adminResp] = results;
|
||||
renderIfChanged("hive", raft, renderHive);
|
||||
renderIfChanged("console", consoleData, renderConsole);
|
||||
renderIfChanged("billing-summary", summary, data => renderBilling(data));
|
||||
renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary));
|
||||
renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map));
|
||||
renderIfChanged("admin-node-pool", map, renderAdminNodePool);
|
||||
if (adminResp && adminResp.ok) {
|
||||
renderIfChanged("admin", adminResp.data.accounts || [], accounts => {
|
||||
const rows = accounts.map(a => {
|
||||
|
||||
@@ -1528,49 +1528,6 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
|
||||
return None
|
||||
|
||||
|
||||
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
|
||||
"""Replace the fastest ready assignment after an explicit admin eviction."""
|
||||
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||
if preset is None or not preset.get("hf_repo"):
|
||||
return None
|
||||
start, end = _preset_layer_bounds(preset)
|
||||
# An explicit admin eviction is permitted to recover a stuck/loading node
|
||||
# and to use the preset default precision. It must only avoid a node that
|
||||
# already has another assignment in flight.
|
||||
candidates = [node for node in server.registry.values()
|
||||
if node.pending_new_assignment is None]
|
||||
if not candidates:
|
||||
return None
|
||||
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
|
||||
shard_end = min(end, start + max(1, min(_node_layer_capacity(node, preset), end - start + 1)) - 1)
|
||||
quantization = _node_quantization(node, preset)
|
||||
directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization)
|
||||
replaced = node.hf_repo or node.model
|
||||
node.model, node.hf_repo = resolved_name, str(preset["hf_repo"])
|
||||
node.shard_start, node.shard_end, node.quantization = start, shard_end, quantization
|
||||
node.managed_assignment, node.pending_new_assignment = True, directive
|
||||
node.pending_directives.append(directive)
|
||||
_tracker_log(server, "warn", "model load forced", node_id=node.node_id,
|
||||
model=resolved_name, replaced_model=replaced, shard=f"{start}-{shard_end}")
|
||||
return {"node_id": node.node_id, "model": resolved_name, "hf_repo": preset["hf_repo"],
|
||||
"shard_start": start, "shard_end": shard_end, "replaced_model": replaced}
|
||||
|
||||
|
||||
def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
|
||||
"""Queue DROP_SHARD for every served shard and remove it from routing immediately."""
|
||||
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||
if preset is None:
|
||||
return 0
|
||||
released = 0
|
||||
for node in server.registry.values():
|
||||
if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None:
|
||||
continue
|
||||
node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16"))
|
||||
node.status = "loading"
|
||||
released += 1
|
||||
return released
|
||||
|
||||
|
||||
def _preferred_node_quantization(
|
||||
node: _NodeEntry,
|
||||
preset: dict,
|
||||
@@ -3086,9 +3043,6 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
if self.path == "/v1/models/load":
|
||||
self._handle_model_load_request()
|
||||
return
|
||||
if self.path == "/v1/models/release":
|
||||
self._handle_model_release_request()
|
||||
return
|
||||
if self.path == "/v1/models/vote":
|
||||
self._handle_model_coverage_vote()
|
||||
return
|
||||
@@ -3216,6 +3170,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
seen_ids: set[str] = set()
|
||||
for name, preset in server.model_presets.items():
|
||||
model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
|
||||
if not model_nodes and not preset.get("recommended"):
|
||||
continue
|
||||
required_start, required_end = _preset_layer_bounds(preset)
|
||||
coverage = _coverage_percentage(
|
||||
model_nodes,
|
||||
@@ -3366,11 +3322,6 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
"endpoint": node.endpoint,
|
||||
"relay_addr": node.relay_addr,
|
||||
"peer_id": node.peer_id,
|
||||
"wallet_address": node.wallet_address,
|
||||
"hardware_profile": dict(node.hardware_profile),
|
||||
"ram_bytes": node.ram_bytes,
|
||||
"vram_bytes": node.vram_bytes,
|
||||
"max_loaded_shards": node.max_loaded_shards,
|
||||
}
|
||||
for node in tracker_nodes
|
||||
],
|
||||
@@ -4883,32 +4834,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
assignment = _request_model_load_locked(server, model)
|
||||
if assignment is None and body.get("force") is True:
|
||||
assignment = _force_model_load_locked(server, model)
|
||||
if assignment is None:
|
||||
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
||||
return
|
||||
self._send_json(202, {"status": "queued", "assignment": assignment})
|
||||
|
||||
def _handle_model_release_request(self):
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
if not self._require_role("admin", "validator"):
|
||||
return
|
||||
body = self._read_json_body()
|
||||
if body is None:
|
||||
return
|
||||
model = body.get("model")
|
||||
if not isinstance(model, str) or not model.strip():
|
||||
self._send_json(400, {"error": "model is required"})
|
||||
return
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
released = _release_model_locked(server, model)
|
||||
if not released:
|
||||
self._send_json(404, {"error": "no served shards found for model"})
|
||||
return
|
||||
self._send_json(202, {"status": "release_queued", "released": released})
|
||||
|
||||
def _handle_model_coverage_vote(self):
|
||||
"""Record a rolling wish-list signal for an unavailable precision."""
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
|
||||
@@ -39,9 +39,7 @@ def test_dashboard_served_with_all_panels():
|
||||
assert "resolveModelGroup" in html
|
||||
assert "buildModelAliasMap" in html
|
||||
assert "modelAliasKey(raw)" in html
|
||||
assert "@media (min-width:900px)" in html
|
||||
assert "grid-template-columns:repeat(4,minmax(0,1fr));" in html
|
||||
assert ".wide { grid-column:span 2; }" in html
|
||||
assert "main { display:grid; grid-template-columns:repeat(auto-fit,minmax(340px,1fr));" in html
|
||||
assert 'onclick="clearConsole()"' in html
|
||||
assert "let consoleClearedAt = 0;" in html
|
||||
finally:
|
||||
@@ -102,23 +100,6 @@ def test_dashboard_allows_admin_to_request_selected_model_load():
|
||||
assert '$("request-model-load").style.display = enabled ? "" : "none"' in html
|
||||
|
||||
|
||||
def test_dashboard_exposes_admin_model_inventory_and_release_controls():
|
||||
"Admin placement controls show the full model inventory and can release capacity."
|
||||
html = _dashboard_html()
|
||||
|
||||
assert 'id="admin-model-placement"' in html
|
||||
assert "renderAdminModelPlacement" in html
|
||||
assert '"/v1/models/release"' in html
|
||||
assert "requestAdminModelLoad" in html
|
||||
assert "releaseAdminModel" in html
|
||||
assert 'data-admin-model-load=' in html
|
||||
assert 'data-admin-model-release=' in html
|
||||
assert "admin-model-placement-status" in html
|
||||
assert 'id="admin-node-pool"' in html
|
||||
assert "renderAdminNodePool" in html
|
||||
assert "model drive" in html
|
||||
|
||||
|
||||
def test_network_map_includes_node_friendly_name():
|
||||
"Network map includes node friendly name\n\nTags: dashboard, http"
|
||||
tracker = TrackerServer()
|
||||
|
||||
@@ -355,39 +355,6 @@ def test_admin_model_load_request_queues_directive_on_joined_node():
|
||||
assert heartbeat["directives"][0]["model"] == "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
|
||||
|
||||
def test_admin_can_replace_a_served_model_and_release_it():
|
||||
"Forced admin placement replaces a served shard; release queues DROP_SHARD."
|
||||
tracker = TrackerServer(enable_billing=False, validator_service_token="test-admin")
|
||||
port = tracker.start()
|
||||
try:
|
||||
node = _post_json(
|
||||
f"http://127.0.0.1:{port}/v1/nodes/register",
|
||||
{"endpoint": "http://127.0.0.1:9912", "model": "stub-model",
|
||||
"shard_start": 0, "shard_end": 3, "managed_assignment": True,
|
||||
"max_loaded_shards": 1, "memory_mb": 1,
|
||||
"hardware_profile": {"host_id": "full-host"}},
|
||||
)
|
||||
headers = {"Content-Type": "application/json", "Authorization": "Bearer test-admin"}
|
||||
load = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/v1/models/load",
|
||||
data=json.dumps({"model": "qwen2.5-0.5b-instruct", "force": True}).encode(),
|
||||
headers=headers, method="POST")
|
||||
with urllib.request.urlopen(load) as response:
|
||||
assert json.loads(response.read())["assignment"]["node_id"] == node["node_id"]
|
||||
heartbeat = _post_json(f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
|
||||
assert heartbeat["directives"][0]["action"] == "LOAD_SHARD"
|
||||
release = urllib.request.Request(
|
||||
f"http://127.0.0.1:{port}/v1/models/release",
|
||||
data=json.dumps({"model": "qwen2.5-0.5b-instruct"}).encode(), headers=headers, method="POST")
|
||||
with urllib.request.urlopen(release) as response:
|
||||
assert json.loads(response.read())["released"] == 1
|
||||
heartbeat = _post_json(f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
|
||||
finally:
|
||||
tracker.stop()
|
||||
|
||||
assert heartbeat["directives"][0]["action"] == "DROP_SHARD"
|
||||
|
||||
|
||||
def test_endpoint_key_distinguishes_same_port_different_hosts():
|
||||
"Endpoint key distinguishes same port different hosts\n\nTags: http, performance, routing, tracker"
|
||||
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
|
||||
|
||||
@@ -32,18 +32,12 @@ def test_matrix_reports_direct_relay_prefill_decode_and_machine_readable_metrics
|
||||
assert {"p50_latency_ms", "p95_latency_ms", "payload_bytes", "compression_ratio",
|
||||
"connection_attempts", "p95_queue_wait_ms"} <= set(run["phases"]["decode"])
|
||||
sample = run["samples"][0]
|
||||
assert sample["model_ms"] > 0
|
||||
assert sample["encode_ms"] > 0
|
||||
assert sample["activation_decode_ms"] > 0
|
||||
assert sample["framing_ms"] > 0
|
||||
assert sample["metadata_ms"] > 0
|
||||
assert sample["copy_allocation_ms"] > 0
|
||||
assert sample["copy_allocation_bytes"] >= sample["payload_bytes"]
|
||||
assert sample["local_http_forwarding_ms"] > 0
|
||||
assert len(run["samples"]) == 1 + len(run["output_tokens"])
|
||||
assert {"tokens_per_sec", "bytes_per_token", "compression_cpu_ms", "peak_buffered_bytes",
|
||||
"model_execution_ms", "activation_encoding_ms", "activation_decoding_ms",
|
||||
"local_http_forwarding_ms"} <= set(run["phases"]["decode"])
|
||||
assert {"tokens_per_sec", "bytes_per_token", "compression_cpu_ms", "peak_buffered_bytes"} <= set(run["phases"]["decode"])
|
||||
|
||||
|
||||
def test_cached_sessions_reuse_one_connection_and_preserve_stub_tokens():
|
||||
@@ -80,10 +74,7 @@ def test_cli_writes_json_artifact_and_human_summary(tmp_path, capsys):
|
||||
report = json.loads(output.read_text())
|
||||
assert report["schema_version"] == 1
|
||||
assert "Route Session benchmark" in capsys.readouterr().out
|
||||
summary = format_summary(report)
|
||||
assert "relay" in summary
|
||||
assert "model/encode/decode" in summary
|
||||
assert "HTTP" in summary
|
||||
assert "relay" in format_summary(report)
|
||||
|
||||
|
||||
def test_performance_gate_checks_comparison_identity_session_and_cleanup():
|
||||
|
||||
Reference in New Issue
Block a user