Compare commits
6 Commits
ralph/dist
...
temp/maste
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7b3399760e | ||
|
|
22467f145c | ||
|
|
35af1e21de | ||
|
|
348b003d6e | ||
|
|
1e64a5b2b9 | ||
|
|
e2f3ae32b8 |
@@ -12,4 +12,10 @@ Provide an opt-in, admin-only tracker Dashboard Testing tab that dynamically dis
|
|||||||
- One active run.
|
- One active run.
|
||||||
- Real inference stays separately environment-gated and excluded from default suites.
|
- Real inference stays separately environment-gated and excluded from default suites.
|
||||||
|
|
||||||
|
## Operator workflow
|
||||||
|
|
||||||
|
See [`docs/dev/dashboard-test-runner.md`](../../docs/dev/dashboard-test-runner.md)
|
||||||
|
for launch configuration, default safe suites vs the gated real-inference suite,
|
||||||
|
and required environment variables.
|
||||||
|
|
||||||
See `prd.json` for executable Ralph user stories and acceptance criteria.
|
See `prd.json` for executable Ralph user stories and acceptance criteria.
|
||||||
|
|||||||
@@ -51,15 +51,16 @@
|
|||||||
"uv run pytest tests/test_dashboard.py tests/test_dynamic_routing.py -q passes."
|
"uv run pytest tests/test_dashboard.py tests/test_dynamic_routing.py -q passes."
|
||||||
],
|
],
|
||||||
"priority": 3,
|
"priority": 3,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Do not reintroduce --enable-test-runner without implementing its CLI argument in US-001.",
|
"notes": "Do not reintroduce --enable-test-runner without implementing its CLI argument in US-001.",
|
||||||
"dependsOn": [
|
"dependsOn": [
|
||||||
"US-001",
|
"US-001",
|
||||||
"US-002"
|
"US-002"
|
||||||
]
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"updatedAt": "2026-07-11T17:02:30.520Z"
|
"updatedAt": "2026-07-12T01:58:06.286Z"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -16,8 +16,9 @@
|
|||||||
],
|
],
|
||||||
"priority": 1,
|
"priority": 1,
|
||||||
"passes": true,
|
"passes": true,
|
||||||
"notes": "Completed 2026-07-14. Deterministic direct/relay and cached/stateless stub benchmark with JSON/summary attribution; focused test suite passes (7). Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md",
|
||||||
"dependsOn": []
|
"dependsOn": [],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-002",
|
"id": "DIP-002",
|
||||||
@@ -31,9 +32,12 @@
|
|||||||
"Tests cover binary, JSON, timeout, disconnect, cancellation, and cleanup"
|
"Tests cover binary, JSON, timeout, disconnect, cancellation, and cleanup"
|
||||||
],
|
],
|
||||||
"priority": 2,
|
"priority": 2,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md",
|
||||||
"dependsOn": ["DIP-001"]
|
"dependsOn": [
|
||||||
|
"DIP-001"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-003",
|
"id": "DIP-003",
|
||||||
@@ -47,9 +51,12 @@
|
|||||||
"Benchmark shows healthy-session connection count independent of token count"
|
"Benchmark shows healthy-session connection count independent of token count"
|
||||||
],
|
],
|
||||||
"priority": 3,
|
"priority": 3,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/03-http-keepalive.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/03-http-keepalive.md",
|
||||||
"dependsOn": ["DIP-001"]
|
"dependsOn": [
|
||||||
|
"DIP-001"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-004",
|
"id": "DIP-004",
|
||||||
@@ -63,9 +70,12 @@
|
|||||||
"Tests verify cadence and cleanup"
|
"Tests verify cadence and cleanup"
|
||||||
],
|
],
|
||||||
"priority": 4,
|
"priority": 4,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/04-seam-telemetry.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/04-seam-telemetry.md",
|
||||||
"dependsOn": ["DIP-001"]
|
"dependsOn": [
|
||||||
|
"DIP-001"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-005",
|
"id": "DIP-005",
|
||||||
@@ -79,9 +89,12 @@
|
|||||||
"Tests cover compressible, incompressible, threshold, malformed, and legacy bodies"
|
"Tests cover compressible, incompressible, threshold, malformed, and legacy bodies"
|
||||||
],
|
],
|
||||||
"priority": 5,
|
"priority": 5,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/05-adaptive-compression.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/05-adaptive-compression.md",
|
||||||
"dependsOn": ["DIP-001"]
|
"dependsOn": [
|
||||||
|
"DIP-001"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-006",
|
"id": "DIP-006",
|
||||||
@@ -95,9 +108,12 @@
|
|||||||
"Wire and token-output regression tests pass"
|
"Wire and token-output regression tests pass"
|
||||||
],
|
],
|
||||||
"priority": 6,
|
"priority": 6,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/06-activation-framing-copies.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/06-activation-framing-copies.md",
|
||||||
"dependsOn": ["DIP-001"]
|
"dependsOn": [
|
||||||
|
"DIP-001"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-007",
|
"id": "DIP-007",
|
||||||
@@ -111,9 +127,13 @@
|
|||||||
"Tests cover chunking, slow consumers, failure, and legacy peers"
|
"Tests cover chunking, slow consumers, failure, and legacy peers"
|
||||||
],
|
],
|
||||||
"priority": 7,
|
"priority": 7,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/07-prefill-backpressure.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/07-prefill-backpressure.md",
|
||||||
"dependsOn": ["DIP-001", "DIP-004"]
|
"dependsOn": [
|
||||||
|
"DIP-001",
|
||||||
|
"DIP-004"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "DIP-008",
|
"id": "DIP-008",
|
||||||
@@ -127,9 +147,20 @@
|
|||||||
"Gate verifies token identity, session stability, and resource cleanup"
|
"Gate verifies token identity, session stability, and resource cleanup"
|
||||||
],
|
],
|
||||||
"priority": 8,
|
"priority": 8,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md",
|
"notes": "Source issue: .scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md",
|
||||||
"dependsOn": ["DIP-002", "DIP-003", "DIP-004", "DIP-005", "DIP-006", "DIP-007"]
|
"dependsOn": [
|
||||||
|
"DIP-002",
|
||||||
|
"DIP-003",
|
||||||
|
"DIP-004",
|
||||||
|
"DIP-005",
|
||||||
|
"DIP-006",
|
||||||
|
"DIP-007"
|
||||||
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
}
|
"metadata": {
|
||||||
|
"updatedAt": "2026-07-12T02:35:28.752Z"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -35,11 +35,12 @@
|
|||||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||||
],
|
],
|
||||||
"priority": 2,
|
"priority": 2,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/node-capability-admission/issues/02-doctor-real-forward.md",
|
"notes": "Source issue: .scratch/node-capability-admission/issues/02-doctor-real-forward.md",
|
||||||
"dependsOn": [
|
"dependsOn": [
|
||||||
"NCA-001"
|
"NCA-001"
|
||||||
]
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "NCA-003",
|
"id": "NCA-003",
|
||||||
@@ -54,12 +55,13 @@
|
|||||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||||
],
|
],
|
||||||
"priority": 3,
|
"priority": 3,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md",
|
"notes": "Source issue: .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md",
|
||||||
"dependsOn": [
|
"dependsOn": [
|
||||||
"NCA-001",
|
"NCA-001",
|
||||||
"NCA-002"
|
"NCA-002"
|
||||||
]
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "NCA-004",
|
"id": "NCA-004",
|
||||||
@@ -76,12 +78,13 @@
|
|||||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||||
],
|
],
|
||||||
"priority": 4,
|
"priority": 4,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md",
|
"notes": "Source issue: .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md",
|
||||||
"dependsOn": [
|
"dependsOn": [
|
||||||
"NCA-001",
|
"NCA-001",
|
||||||
"NCA-003"
|
"NCA-003"
|
||||||
]
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"id": "NCA-005",
|
"id": "NCA-005",
|
||||||
@@ -96,15 +99,16 @@
|
|||||||
"Full pytest passes or an exact unrelated blocker is recorded"
|
"Full pytest passes or an exact unrelated blocker is recorded"
|
||||||
],
|
],
|
||||||
"priority": 5,
|
"priority": 5,
|
||||||
"passes": false,
|
"passes": true,
|
||||||
"notes": "Source issue: .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md",
|
"notes": "Source issue: .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md",
|
||||||
"dependsOn": [
|
"dependsOn": [
|
||||||
"NCA-002",
|
"NCA-002",
|
||||||
"NCA-004"
|
"NCA-004"
|
||||||
]
|
],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"updatedAt": "2026-07-11T19:16:52.768Z"
|
"updatedAt": "2026-07-12T01:54:03.030Z"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,9 +1,16 @@
|
|||||||
# US-042 — GGUF/llama.cpp node backend
|
# US-042 — GGUF/llama.cpp node backend
|
||||||
|
|
||||||
Status: planned
|
Status: planned
|
||||||
Priority: High (whole-model GGUF shortcut; distributed path in [ADR-0024](../adr/0024-distributed-gguf-runtime.md))
|
Priority: High (unlocks DeepSeek-V4-Flash on volunteer hardware — the pool's core value)
|
||||||
Stage: Draft design
|
Stage: Draft design
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Run **DeepSeek-V4-Flash** as the first real large-model target on volunteer
|
||||||
|
hardware via GGUF/llama.cpp. This epic is no longer GLM-oriented: the initial
|
||||||
|
objective is to prove that DeepSeek-V4-Flash can load and serve correctly on
|
||||||
|
consumer/unified-memory nodes, then expand from there.
|
||||||
|
|
||||||
## Context
|
## Context
|
||||||
|
|
||||||
The node backend is transformers-only (`model_backend.py` →
|
The node backend is transformers-only (`model_backend.py` →
|
||||||
@@ -35,17 +42,7 @@ to it (single-hop route). Smallest step, no cross-node activation work, and
|
|||||||
already useful: Strix Halo 128 GB serves DeepSeek-V4-Flash IQ3_XXS (114 GB)
|
already useful: Strix Halo 128 GB serves DeepSeek-V4-Flash IQ3_XXS (114 GB)
|
||||||
via llama.cpp Vulkan today.
|
via llama.cpp Vulkan today.
|
||||||
|
|
||||||
Recommended sequencing: **C first** (US-042), then **ADR-0024 benchmark gate** (DGR-001), then distributed native worker (DGR-002+). Direction B (llama.cpp RPC) is rejected per ADR-0024.
|
Recommended sequencing: C first (small, real value), then A/B investigation.
|
||||||
|
|
||||||
## Runtime sequencing
|
|
||||||
|
|
||||||
| Stage | Track | Delivers |
|
|
||||||
|---|---|---|
|
|
||||||
| **C — Whole-model GGUF** | US-042 (this issue) | Single-hop llama.cpp, billing, relay streaming |
|
|
||||||
| **0 — Benchmark gate** | ADR-0024 DGR-001 | Safetensors vs GGUF measured contract |
|
|
||||||
| **1 — Distributed GGUF** | ADR-0024 `.scratch/distributed-gguf-runtime/` | gRPC C++ worker, layer-range GGUF |
|
|
||||||
|
|
||||||
Phase C uses the existing tracker hop path (whole model, one node). ADR-0024 direction A (layer-range GGUF + activations) merges into the native worker track after the benchmark gate — not in parallel with phase C on the same backend without an integration plan.
|
|
||||||
|
|
||||||
## Also in scope
|
## Also in scope
|
||||||
|
|
||||||
|
|||||||
@@ -1545,6 +1545,26 @@ class TorchNodeServer:
|
|||||||
|
|
||||||
def apply_tracker_directives(self, directives: list[dict]) -> dict | None:
|
def apply_tracker_directives(self, directives: list[dict]) -> dict | None:
|
||||||
"""Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more)."""
|
"""Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more)."""
|
||||||
|
drop_directive = next(
|
||||||
|
(directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if drop_directive is not None:
|
||||||
|
model_id = str(drop_directive.get("model") or "")
|
||||||
|
removed = self._backends.pop(model_id, None)
|
||||||
|
if removed is None:
|
||||||
|
return None
|
||||||
|
if self._backends:
|
||||||
|
self._backend = next(iter(self._backends.values()))
|
||||||
|
self._tracker_mode = self._backend.shard_start == 0
|
||||||
|
else:
|
||||||
|
self._backend = None
|
||||||
|
self._tracker_mode = False
|
||||||
|
if self._server is not None:
|
||||||
|
self._server.backends = dict(self._backends)
|
||||||
|
self._server.backend = self._backend
|
||||||
|
self._server.tracker_mode = self._tracker_mode
|
||||||
|
return {"action": "DROP_SHARD", "model": model_id}
|
||||||
add_directive = next(
|
add_directive = next(
|
||||||
(directive for directive in reversed(directives) if directive.get("action") == "ADD_SHARD"),
|
(directive for directive in reversed(directives) if directive.get("action") == "ADD_SHARD"),
|
||||||
None,
|
None,
|
||||||
@@ -1574,6 +1594,8 @@ class TorchNodeServer:
|
|||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
|
if replacing:
|
||||||
|
self._backends.clear()
|
||||||
new_backend = _load_backend(model_id, shard_start, shard_end, quantization, self._cache_dir)
|
new_backend = _load_backend(model_id, shard_start, shard_end, quantization, self._cache_dir)
|
||||||
except TypeError:
|
except TypeError:
|
||||||
new_backend = _load_backend(model_id, shard_start, shard_end, quantization)
|
new_backend = _load_backend(model_id, shard_start, shard_end, quantization)
|
||||||
|
|||||||
@@ -22,8 +22,9 @@
|
|||||||
border-bottom:1px solid var(--border); flex-shrink:0; }
|
border-bottom:1px solid var(--border); flex-shrink:0; }
|
||||||
header h1 { font-size:16px; margin:0; color:var(--accent); }
|
header h1 { font-size:16px; margin:0; color:var(--accent); }
|
||||||
header .meta { color:var(--dim); font-size:12px; }
|
header .meta { color:var(--dim); font-size:12px; }
|
||||||
main { display:grid; grid-template-columns:repeat(auto-fit,minmax(340px,1fr));
|
main { display:grid; grid-template-columns:1fr;
|
||||||
gap:14px; padding:14px 20px; }
|
gap:14px; padding:14px 20px; }
|
||||||
|
main > section { width:100%; min-width:0; }
|
||||||
body.chat-tab-active main {
|
body.chat-tab-active main {
|
||||||
flex:1; min-height:0; display:flex; flex-direction:column;
|
flex:1; min-height:0; display:flex; flex-direction:column;
|
||||||
padding:0; gap:0; overflow:hidden;
|
padding:0; gap:0; overflow:hidden;
|
||||||
@@ -71,6 +72,12 @@
|
|||||||
background:transparent; color:var(--dim); padding:5px 0 8px; }
|
background:transparent; color:var(--dim); padding:5px 0 8px; }
|
||||||
.dashboard-tabs button.active { color:var(--accent); border-bottom-color:var(--accent); }
|
.dashboard-tabs button.active { color:var(--accent); border-bottom-color:var(--accent); }
|
||||||
.wide { grid-column:1 / -1; }
|
.wide { grid-column:1 / -1; }
|
||||||
|
/* Compact status cards fan out on desktop; tables remain readable at half width. */
|
||||||
|
@media (min-width:900px) {
|
||||||
|
main { grid-template-columns:repeat(4,minmax(0,1fr)); }
|
||||||
|
main > section { grid-column:span 1; }
|
||||||
|
.wide { grid-column:span 2; }
|
||||||
|
}
|
||||||
section[hidden] { display:none !important; }
|
section[hidden] { display:none !important; }
|
||||||
section.chat-section {
|
section.chat-section {
|
||||||
padding:0; border:0; border-radius:0; background:var(--bg); min-height:0;
|
padding:0; border:0; border-radius:0; background:var(--bg); min-height:0;
|
||||||
@@ -288,6 +295,7 @@
|
|||||||
<section data-tab="billing" data-admin-only><h2>Node pending payouts</h2><div id="pending" class="empty">admin login required</div></section>
|
<section data-tab="billing" data-admin-only><h2>Node pending payouts</h2><div id="pending" class="empty">admin login required</div></section>
|
||||||
<section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
|
<section data-tab="billing" data-admin-only><h2>Settlement history</h2><div id="settlements" class="empty">admin login required</div></section>
|
||||||
<section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
|
<section data-tab="admin"><h2>Tracker hive</h2><div id="hive" class="empty">loading…</div></section>
|
||||||
|
<section data-tab="admin" class="wide"><h2>Model placement</h2><div id="admin-model-placement-status" class="dim">Choose a model to load or release.</div><div id="admin-model-placement" class="empty">admin login required</div></section>
|
||||||
<section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
|
<section data-tab="admin" id="admin-section"><h2>All accounts (admin)</h2><div id="admin" class="empty"></div></section>
|
||||||
<section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section>
|
<section data-tab="admin" data-admin-only><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">admin login required</div></section>
|
||||||
<section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
|
<section data-tab="admin"><h2>Client balances</h2><div id="clients" class="empty">admin login required</div></section>
|
||||||
@@ -1791,6 +1799,50 @@ async function requestSelectedModelLoad() {
|
|||||||
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
|
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function requestAdminModelLoad(model) {
|
||||||
|
const result = await apiCall("/v1/models/load", "POST", { model, force: true });
|
||||||
|
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model load request failed", true);
|
||||||
|
const assignment = result.data.assignment || {};
|
||||||
|
showAdminModelPlacementStatus(`Load queued on ${short(assignment.node_id || "node")} for ${model}.`);
|
||||||
|
await refreshActiveTab(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function releaseAdminModel(model) {
|
||||||
|
const result = await apiCall("/v1/models/release", "POST", { model });
|
||||||
|
if (!result.ok) return showAdminModelPlacementStatus(result.data.error || "model release request failed", true);
|
||||||
|
showAdminModelPlacementStatus(`Release queued for ${result.data.released || 0} node(s) serving ${model}.`);
|
||||||
|
await refreshActiveTab(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
function showAdminModelPlacementStatus(message, isError) {
|
||||||
|
const status = $("admin-model-placement-status");
|
||||||
|
status.textContent = message;
|
||||||
|
status.className = isError ? "bad" : "ok";
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderAdminModelPlacement(models, map) {
|
||||||
|
const nodes = (map && map.nodes) || [];
|
||||||
|
const rows = ((models && models.data) || []).map(model => {
|
||||||
|
const aliases = new Set([model.id, model.hf_repo, ...(model.aliases || [])].filter(Boolean));
|
||||||
|
const serving = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo)).length;
|
||||||
|
const downloaded = nodes.filter(node => aliases.has(node.model) || aliases.has(node.hf_repo) ||
|
||||||
|
(node.downloaded_models || []).some(item => aliases.has(item.model) || aliases.has(item.hf_repo))).length;
|
||||||
|
const actions = `<button class="small" data-admin-model-load="${esc(model.id)}">load</button> ` +
|
||||||
|
`<button class="small" data-admin-model-release="${esc(model.id)}"${serving ? "" : " disabled"}>release</button>`;
|
||||||
|
return [esc(model.name || model.id), String(serving), String(downloaded), actions];
|
||||||
|
});
|
||||||
|
$("admin-model-placement").innerHTML = rows.length
|
||||||
|
? table(["model", "serving nodes", "downloaded on nodes", "admin action"], rows)
|
||||||
|
: '<div class="empty">no model presets configured</div>';
|
||||||
|
}
|
||||||
|
|
||||||
|
$("admin-model-placement").addEventListener("click", event => {
|
||||||
|
const load = event.target.closest("[data-admin-model-load]");
|
||||||
|
const release = event.target.closest("[data-admin-model-release]");
|
||||||
|
if (load) void requestAdminModelLoad(load.dataset.adminModelLoad);
|
||||||
|
if (release) void releaseAdminModel(release.dataset.adminModelRelease);
|
||||||
|
});
|
||||||
|
|
||||||
function chatAuthToken() {
|
function chatAuthToken() {
|
||||||
if (accountApiKeys.length) return accountApiKeys[0];
|
if (accountApiKeys.length) return accountApiKeys[0];
|
||||||
return null;
|
return null;
|
||||||
@@ -2427,14 +2479,17 @@ async function fetchAdminTab() {
|
|||||||
fetchJson("/v1/console"),
|
fetchJson("/v1/console"),
|
||||||
fetchJson("/v1/billing/summary"),
|
fetchJson("/v1/billing/summary"),
|
||||||
fetchJson("/v1/registry/wallets"),
|
fetchJson("/v1/registry/wallets"),
|
||||||
|
fetchJson("/v1/models"),
|
||||||
|
fetchJson("/v1/network/map"),
|
||||||
];
|
];
|
||||||
if (isAdmin) fetches.push(apiCall("/v1/admin/accounts"));
|
if (isAdmin) fetches.push(apiCall("/v1/admin/accounts"));
|
||||||
const results = await Promise.all(fetches);
|
const results = await Promise.all(fetches);
|
||||||
const [raft, consoleData, summary, wallets, adminResp] = results;
|
const [raft, consoleData, summary, wallets, models, map, adminResp] = results;
|
||||||
renderIfChanged("hive", raft, renderHive);
|
renderIfChanged("hive", raft, renderHive);
|
||||||
renderIfChanged("console", consoleData, renderConsole);
|
renderIfChanged("console", consoleData, renderConsole);
|
||||||
renderIfChanged("billing-summary", summary, data => renderBilling(data));
|
renderIfChanged("billing-summary", summary, data => renderBilling(data));
|
||||||
renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary));
|
renderIfChanged("fraud", { wallets, summary }, data => renderFraud(data.wallets, data.summary));
|
||||||
|
renderIfChanged("admin-model-placement", { models, map }, data => renderAdminModelPlacement(data.models, data.map));
|
||||||
if (adminResp && adminResp.ok) {
|
if (adminResp && adminResp.ok) {
|
||||||
renderIfChanged("admin", adminResp.data.accounts || [], accounts => {
|
renderIfChanged("admin", adminResp.data.accounts || [], accounts => {
|
||||||
const rows = accounts.map(a => {
|
const rows = accounts.map(a => {
|
||||||
|
|||||||
@@ -1528,6 +1528,49 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
|
||||||
|
"""Replace the fastest ready assignment after an explicit admin eviction."""
|
||||||
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||||
|
if preset is None or not preset.get("hf_repo"):
|
||||||
|
return None
|
||||||
|
start, end = _preset_layer_bounds(preset)
|
||||||
|
candidates = [node for node in server.registry.values()
|
||||||
|
if node.status == "ready" and node.pending_new_assignment is None
|
||||||
|
and _has_usable_quantization(node)]
|
||||||
|
if not candidates:
|
||||||
|
return None
|
||||||
|
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
|
||||||
|
shard_end = min(end, start + min(_node_layer_capacity(node, preset), end - start + 1) - 1)
|
||||||
|
if shard_end < start:
|
||||||
|
return None
|
||||||
|
quantization = _node_quantization(node, preset)
|
||||||
|
directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization)
|
||||||
|
replaced = node.hf_repo or node.model
|
||||||
|
node.model, node.hf_repo = resolved_name, str(preset["hf_repo"])
|
||||||
|
node.shard_start, node.shard_end, node.quantization = start, shard_end, quantization
|
||||||
|
node.managed_assignment, node.pending_new_assignment = True, directive
|
||||||
|
node.pending_directives.append(directive)
|
||||||
|
_tracker_log(server, "warn", "model load forced", node_id=node.node_id,
|
||||||
|
model=resolved_name, replaced_model=replaced, shard=f"{start}-{shard_end}")
|
||||||
|
return {"node_id": node.node_id, "model": resolved_name, "hf_repo": preset["hf_repo"],
|
||||||
|
"shard_start": start, "shard_end": shard_end, "replaced_model": replaced}
|
||||||
|
|
||||||
|
|
||||||
|
def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
|
||||||
|
"""Queue DROP_SHARD for every served shard and remove it from routing immediately."""
|
||||||
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||||
|
if preset is None:
|
||||||
|
return 0
|
||||||
|
released = 0
|
||||||
|
for node in server.registry.values():
|
||||||
|
if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None:
|
||||||
|
continue
|
||||||
|
node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16"))
|
||||||
|
node.status = "loading"
|
||||||
|
released += 1
|
||||||
|
return released
|
||||||
|
|
||||||
|
|
||||||
def _preferred_node_quantization(
|
def _preferred_node_quantization(
|
||||||
node: _NodeEntry,
|
node: _NodeEntry,
|
||||||
preset: dict,
|
preset: dict,
|
||||||
@@ -3043,6 +3086,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
if self.path == "/v1/models/load":
|
if self.path == "/v1/models/load":
|
||||||
self._handle_model_load_request()
|
self._handle_model_load_request()
|
||||||
return
|
return
|
||||||
|
if self.path == "/v1/models/release":
|
||||||
|
self._handle_model_release_request()
|
||||||
|
return
|
||||||
if self.path == "/v1/models/vote":
|
if self.path == "/v1/models/vote":
|
||||||
self._handle_model_coverage_vote()
|
self._handle_model_coverage_vote()
|
||||||
return
|
return
|
||||||
@@ -3170,8 +3216,6 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
seen_ids: set[str] = set()
|
seen_ids: set[str] = set()
|
||||||
for name, preset in server.model_presets.items():
|
for name, preset in server.model_presets.items():
|
||||||
model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
|
model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
|
||||||
if not model_nodes and not preset.get("recommended"):
|
|
||||||
continue
|
|
||||||
required_start, required_end = _preset_layer_bounds(preset)
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
coverage = _coverage_percentage(
|
coverage = _coverage_percentage(
|
||||||
model_nodes,
|
model_nodes,
|
||||||
@@ -4834,11 +4878,32 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
assignment = _request_model_load_locked(server, model)
|
assignment = _request_model_load_locked(server, model)
|
||||||
|
if assignment is None and body.get("force") is True:
|
||||||
|
assignment = _force_model_load_locked(server, model)
|
||||||
if assignment is None:
|
if assignment is None:
|
||||||
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
||||||
return
|
return
|
||||||
self._send_json(202, {"status": "queued", "assignment": assignment})
|
self._send_json(202, {"status": "queued", "assignment": assignment})
|
||||||
|
|
||||||
|
def _handle_model_release_request(self):
|
||||||
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
if not self._require_role("admin", "validator"):
|
||||||
|
return
|
||||||
|
body = self._read_json_body()
|
||||||
|
if body is None:
|
||||||
|
return
|
||||||
|
model = body.get("model")
|
||||||
|
if not isinstance(model, str) or not model.strip():
|
||||||
|
self._send_json(400, {"error": "model is required"})
|
||||||
|
return
|
||||||
|
with server.lock:
|
||||||
|
self._purge_expired_nodes()
|
||||||
|
released = _release_model_locked(server, model)
|
||||||
|
if not released:
|
||||||
|
self._send_json(404, {"error": "no served shards found for model"})
|
||||||
|
return
|
||||||
|
self._send_json(202, {"status": "release_queued", "released": released})
|
||||||
|
|
||||||
def _handle_model_coverage_vote(self):
|
def _handle_model_coverage_vote(self):
|
||||||
"""Record a rolling wish-list signal for an unavailable precision."""
|
"""Record a rolling wish-list signal for an unavailable precision."""
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
|||||||
@@ -39,7 +39,9 @@ def test_dashboard_served_with_all_panels():
|
|||||||
assert "resolveModelGroup" in html
|
assert "resolveModelGroup" in html
|
||||||
assert "buildModelAliasMap" in html
|
assert "buildModelAliasMap" in html
|
||||||
assert "modelAliasKey(raw)" in html
|
assert "modelAliasKey(raw)" in html
|
||||||
assert "main { display:grid; grid-template-columns:repeat(auto-fit,minmax(340px,1fr));" in html
|
assert "@media (min-width:900px)" in html
|
||||||
|
assert "grid-template-columns:repeat(4,minmax(0,1fr));" in html
|
||||||
|
assert ".wide { grid-column:span 2; }" in html
|
||||||
assert 'onclick="clearConsole()"' in html
|
assert 'onclick="clearConsole()"' in html
|
||||||
assert "let consoleClearedAt = 0;" in html
|
assert "let consoleClearedAt = 0;" in html
|
||||||
finally:
|
finally:
|
||||||
@@ -100,6 +102,20 @@ def test_dashboard_allows_admin_to_request_selected_model_load():
|
|||||||
assert '$("request-model-load").style.display = enabled ? "" : "none"' in html
|
assert '$("request-model-load").style.display = enabled ? "" : "none"' in html
|
||||||
|
|
||||||
|
|
||||||
|
def test_dashboard_exposes_admin_model_inventory_and_release_controls():
|
||||||
|
"Admin placement controls show the full model inventory and can release capacity."
|
||||||
|
html = _dashboard_html()
|
||||||
|
|
||||||
|
assert 'id="admin-model-placement"' in html
|
||||||
|
assert "renderAdminModelPlacement" in html
|
||||||
|
assert '"/v1/models/release"' in html
|
||||||
|
assert "requestAdminModelLoad" in html
|
||||||
|
assert "releaseAdminModel" in html
|
||||||
|
assert 'data-admin-model-load=' in html
|
||||||
|
assert 'data-admin-model-release=' in html
|
||||||
|
assert "admin-model-placement-status" in html
|
||||||
|
|
||||||
|
|
||||||
def test_network_map_includes_node_friendly_name():
|
def test_network_map_includes_node_friendly_name():
|
||||||
"Network map includes node friendly name\n\nTags: dashboard, http"
|
"Network map includes node friendly name\n\nTags: dashboard, http"
|
||||||
tracker = TrackerServer()
|
tracker = TrackerServer()
|
||||||
|
|||||||
@@ -355,6 +355,39 @@ def test_admin_model_load_request_queues_directive_on_joined_node():
|
|||||||
assert heartbeat["directives"][0]["model"] == "Qwen/Qwen2.5-0.5B-Instruct"
|
assert heartbeat["directives"][0]["model"] == "Qwen/Qwen2.5-0.5B-Instruct"
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_can_replace_a_served_model_and_release_it():
|
||||||
|
"Forced admin placement replaces a served shard; release queues DROP_SHARD."
|
||||||
|
tracker = TrackerServer(enable_billing=False, validator_service_token="test-admin")
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
node = _post_json(
|
||||||
|
f"http://127.0.0.1:{port}/v1/nodes/register",
|
||||||
|
{"endpoint": "http://127.0.0.1:9912", "model": "stub-model",
|
||||||
|
"shard_start": 0, "shard_end": 3, "managed_assignment": True,
|
||||||
|
"max_loaded_shards": 1, "memory_mb": 1,
|
||||||
|
"hardware_profile": {"host_id": "full-host"}},
|
||||||
|
)
|
||||||
|
headers = {"Content-Type": "application/json", "Authorization": "Bearer test-admin"}
|
||||||
|
load = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{port}/v1/models/load",
|
||||||
|
data=json.dumps({"model": "qwen2.5-0.5b-instruct", "force": True}).encode(),
|
||||||
|
headers=headers, method="POST")
|
||||||
|
with urllib.request.urlopen(load) as response:
|
||||||
|
assert json.loads(response.read())["assignment"]["node_id"] == node["node_id"]
|
||||||
|
heartbeat = _post_json(f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
|
||||||
|
assert heartbeat["directives"][0]["action"] == "LOAD_SHARD"
|
||||||
|
release = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{port}/v1/models/release",
|
||||||
|
data=json.dumps({"model": "qwen2.5-0.5b-instruct"}).encode(), headers=headers, method="POST")
|
||||||
|
with urllib.request.urlopen(release) as response:
|
||||||
|
assert json.loads(response.read())["released"] == 1
|
||||||
|
heartbeat = _post_json(f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert heartbeat["directives"][0]["action"] == "DROP_SHARD"
|
||||||
|
|
||||||
|
|
||||||
def test_endpoint_key_distinguishes_same_port_different_hosts():
|
def test_endpoint_key_distinguishes_same_port_different_hosts():
|
||||||
"Endpoint key distinguishes same port different hosts\n\nTags: http, performance, routing, tracker"
|
"Endpoint key distinguishes same port different hosts\n\nTags: http, performance, routing, tracker"
|
||||||
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
|
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
|
||||||
|
|||||||
Reference in New Issue
Block a user