diff --git a/.claude/memory/project-status.md b/.claude/memory/project-status.md index 8c26a8b..adbf7e7 100644 --- a/.claude/memory/project-status.md +++ b/.claude/memory/project-status.md @@ -8,6 +8,15 @@ metadata: # Project Status (2026-07-13) +## Selected-node model placement (2026-07-14) + +- Admin Model placement now opens a node selector for load and release; the control-plane accepts optional `node_id` and targets only that registry assignment. Multi-model serving remains supported through `ADD_SHARD` and `max_loaded_shards`. +- Total node pool resource values are rendered from `/v1/network/map`'s `node.capacity` contract. Route selection remains assignment/capability/throughput/queue based; capacity is used for placement and falls back to tracker defaults only if a node truly omits it. + +## Distributed inference performance (2026-07-14) + +`DIP-001` is done in `.scratch/distributed-inference-performance/`: the deterministic two-node Route Session stub benchmark covers direct/relay plus cached/stateless prefill and decode. Its JSON and concise summary explicitly attribute model execution, activation encode/decode, compression, connection setup, relay queueing, local HTTP forwarding, and end-to-end seam latency. `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` passed (7); the fixture assertion checks output-token identity and connection attempts. + > Doc reconciliation 2026-07-13: `docs/prd.json` tracks US-001…US-050 (048 memory budget, 049 mainnet pilot, 050 Qwen demand placement). ADRs 0025–0026 added (TAI phase B/C, assignment ownership). All 35 user stories in docs/prd.json are done (35/35), including the reward-system arc US-030…US-035 completed 2026-07-02: diff --git a/.gitignore b/.gitignore index 9298b32..3c6cbc2 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ dist/ !.env.testnet .rocm-local/* .pytest-tmp/* +.cache/ # Local tracker/node sqlite databases (never commit runtime state) *.sqlite diff --git a/.scratch/dashboard-test-runner/PRD.md b/.scratch/dashboard-test-runner/PRD.md index e7782a2..ef01546 100644 --- a/.scratch/dashboard-test-runner/PRD.md +++ b/.scratch/dashboard-test-runner/PRD.md @@ -12,4 +12,10 @@ Provide an opt-in, admin-only tracker Dashboard Testing tab that dynamically dis - One active run. - Real inference stays separately environment-gated and excluded from default suites. +## Operator workflow + +See [`docs/dev/dashboard-test-runner.md`](../../docs/dev/dashboard-test-runner.md) +for launch configuration, default safe suites vs the gated real-inference suite, +and required environment variables. + See `prd.json` for executable Ralph user stories and acceptance criteria. diff --git a/.scratch/dashboard-test-runner/prd.json b/.scratch/dashboard-test-runner/prd.json index 334e875..d57977e 100644 --- a/.scratch/dashboard-test-runner/prd.json +++ b/.scratch/dashboard-test-runner/prd.json @@ -51,15 +51,16 @@ "uv run pytest tests/test_dashboard.py tests/test_dynamic_routing.py -q passes." ], "priority": 3, - "passes": false, + "passes": true, "notes": "Do not reintroduce --enable-test-runner without implementing its CLI argument in US-001.", "dependsOn": [ "US-001", "US-002" - ] + ], + "completionNotes": "Completed by agent" } ], "metadata": { - "updatedAt": "2026-07-11T17:02:30.520Z" + "updatedAt": "2026-07-12T01:58:06.286Z" } } \ No newline at end of file diff --git a/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md b/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md index 0847134..79d1170 100644 --- a/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md +++ b/.scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md @@ -1,4 +1,4 @@ -Status: ready-for-agent +Status: done (2026-07-14) # 01 — Baseline and profiling harness @@ -12,16 +12,15 @@ sizes and connection counts without requiring a real model or external host. ## Acceptance criteria -- [ ] The harness runs a fixed prompt and fixed generated-token count through a +- [x] The harness runs a fixed prompt and fixed generated-token count through a two-node route in direct and relay modes. -- [ ] It reports p50/p95 per-token latency, per-hop latency, payload bytes, +- [x] It reports p50/p95 per-token latency, per-hop latency, payload bytes, compression ratio, connection attempts, and queue wait. -- [ ] It distinguishes prefill from decode and cached from stateless mode. -- [ ] It emits machine-readable JSON suitable for CI artifacts and a concise +- [x] It distinguishes prefill from decode and cached from stateless mode. +- [x] It emits machine-readable JSON suitable for CI artifacts and a concise human-readable summary. -- [ ] A test fixture can assert connection attempts and output token identity. +- [x] A test fixture can assert connection attempts and output token identity. ## Blocked by -None - can start immediately. - +None - completed. Verified with `PYTHONPATH=packages/node pytest -q tests/test_route_session_benchmark.py` (7 passed). diff --git a/.scratch/distributed-inference-performance/prd.json b/.scratch/distributed-inference-performance/prd.json index a9000e7..9006c66 100644 --- a/.scratch/distributed-inference-performance/prd.json +++ b/.scratch/distributed-inference-performance/prd.json @@ -15,9 +15,10 @@ "Can assert connection count and output token identity" ], "priority": 1, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/01-baseline-profiling-harness.md", - "dependsOn": [] + "dependsOn": [], + "completionNotes": "Completed by agent" }, { "id": "DIP-002", @@ -31,9 +32,12 @@ "Tests cover binary, JSON, timeout, disconnect, cancellation, and cleanup" ], "priority": 2, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/02-relay-session-compatibility.md", - "dependsOn": ["DIP-001"] + "dependsOn": [ + "DIP-001" + ], + "completionNotes": "Completed by agent" }, { "id": "DIP-003", @@ -47,9 +51,12 @@ "Benchmark shows healthy-session connection count independent of token count" ], "priority": 3, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/03-http-keepalive.md", - "dependsOn": ["DIP-001"] + "dependsOn": [ + "DIP-001" + ], + "completionNotes": "Completed by agent" }, { "id": "DIP-004", @@ -63,9 +70,12 @@ "Tests verify cadence and cleanup" ], "priority": 4, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/04-seam-telemetry.md", - "dependsOn": ["DIP-001"] + "dependsOn": [ + "DIP-001" + ], + "completionNotes": "Completed by agent" }, { "id": "DIP-005", @@ -79,9 +89,12 @@ "Tests cover compressible, incompressible, threshold, malformed, and legacy bodies" ], "priority": 5, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/05-adaptive-compression.md", - "dependsOn": ["DIP-001"] + "dependsOn": [ + "DIP-001" + ], + "completionNotes": "Completed by agent" }, { "id": "DIP-006", @@ -95,9 +108,12 @@ "Wire and token-output regression tests pass" ], "priority": 6, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/06-activation-framing-copies.md", - "dependsOn": ["DIP-001"] + "dependsOn": [ + "DIP-001" + ], + "completionNotes": "Completed by agent" }, { "id": "DIP-007", @@ -111,9 +127,13 @@ "Tests cover chunking, slow consumers, failure, and legacy peers" ], "priority": 7, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/07-prefill-backpressure.md", - "dependsOn": ["DIP-001", "DIP-004"] + "dependsOn": [ + "DIP-001", + "DIP-004" + ], + "completionNotes": "Completed by agent" }, { "id": "DIP-008", @@ -127,9 +147,20 @@ "Gate verifies token identity, session stability, and resource cleanup" ], "priority": 8, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-inference-performance/issues/08-end-to-end-performance-gate.md", - "dependsOn": ["DIP-002", "DIP-003", "DIP-004", "DIP-005", "DIP-006", "DIP-007"] + "dependsOn": [ + "DIP-002", + "DIP-003", + "DIP-004", + "DIP-005", + "DIP-006", + "DIP-007" + ], + "completionNotes": "Completed by agent" } - ] -} + ], + "metadata": { + "updatedAt": "2026-07-12T02:35:28.752Z" + } +} \ No newline at end of file diff --git a/.scratch/node-capability-admission/prd.json b/.scratch/node-capability-admission/prd.json index dad87ae..1cfe28d 100644 --- a/.scratch/node-capability-admission/prd.json +++ b/.scratch/node-capability-admission/prd.json @@ -35,11 +35,12 @@ "Full pytest passes or an exact unrelated blocker is recorded" ], "priority": 2, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/node-capability-admission/issues/02-doctor-real-forward.md", "dependsOn": [ "NCA-001" - ] + ], + "completionNotes": "Completed by agent" }, { "id": "NCA-003", @@ -54,12 +55,13 @@ "Full pytest passes or an exact unrelated blocker is recorded" ], "priority": 3, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/node-capability-admission/issues/03-fail-closed-startup-admission.md", "dependsOn": [ "NCA-001", "NCA-002" - ] + ], + "completionNotes": "Completed by agent" }, { "id": "NCA-004", @@ -76,12 +78,13 @@ "Full pytest passes or an exact unrelated blocker is recorded" ], "priority": 4, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/node-capability-admission/issues/04-tracker-validated-capability-routing.md", "dependsOn": [ "NCA-001", "NCA-003" - ] + ], + "completionNotes": "Completed by agent" }, { "id": "NCA-005", @@ -96,15 +99,16 @@ "Full pytest passes or an exact unrelated blocker is recorded" ], "priority": 5, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/node-capability-admission/issues/05-docs-hardware-lane-contract.md", "dependsOn": [ "NCA-002", "NCA-004" - ] + ], + "completionNotes": "Completed by agent" } ], "metadata": { - "updatedAt": "2026-07-11T19:16:52.768Z" + "updatedAt": "2026-07-12T01:54:03.030Z" } } \ No newline at end of file diff --git a/_DEV_NOTES.md b/_DEV_NOTES.md index 28f7987..acb8dc4 100644 --- a/_DEV_NOTES.md +++ b/_DEV_NOTES.md @@ -16,12 +16,9 @@ .\.venv\Scripts\meshnet-node.exe start http://192.168.0.179:8081 --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 - .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model-id Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 + .\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20 - we .\.venv\Scripts\meshnet-node.exe start ` - --tracker http://192.168.0.179:8081 ` - --model Qwen/Qwen2.5-0.5B-Instruct ` - --advertise-host 192.168.0.20 + we .\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct # trackers: https://meshnet.2.d-popov.com https://ai.neuron.d-popov.com diff --git a/docs/issues/42-gguf-llamacpp-node-backend.md b/docs/issues/42-gguf-llamacpp-node-backend.md index dc26ac4..3a33c05 100644 --- a/docs/issues/42-gguf-llamacpp-node-backend.md +++ b/docs/issues/42-gguf-llamacpp-node-backend.md @@ -1,9 +1,16 @@ # US-042 — GGUF/llama.cpp node backend Status: planned -Priority: High (whole-model GGUF shortcut; distributed path in [ADR-0024](../adr/0024-distributed-gguf-runtime.md)) +Priority: High (unlocks DeepSeek-V4-Flash on volunteer hardware — the pool's core value) Stage: Draft design +## Goal + +Run **DeepSeek-V4-Flash** as the first real large-model target on volunteer +hardware via GGUF/llama.cpp. This epic is no longer GLM-oriented: the initial +objective is to prove that DeepSeek-V4-Flash can load and serve correctly on +consumer/unified-memory nodes, then expand from there. + ## Context The node backend is transformers-only (`model_backend.py` → @@ -35,17 +42,7 @@ to it (single-hop route). Smallest step, no cross-node activation work, and already useful: Strix Halo 128 GB serves DeepSeek-V4-Flash IQ3_XXS (114 GB) via llama.cpp Vulkan today. -Recommended sequencing: **C first** (US-042), then **ADR-0024 benchmark gate** (DGR-001), then distributed native worker (DGR-002+). Direction B (llama.cpp RPC) is rejected per ADR-0024. - -## Runtime sequencing - -| Stage | Track | Delivers | -|---|---|---| -| **C — Whole-model GGUF** | US-042 (this issue) | Single-hop llama.cpp, billing, relay streaming | -| **0 — Benchmark gate** | ADR-0024 DGR-001 | Safetensors vs GGUF measured contract | -| **1 — Distributed GGUF** | ADR-0024 `.scratch/distributed-gguf-runtime/` | gRPC C++ worker, layer-range GGUF | - -Phase C uses the existing tracker hop path (whole model, one node). ADR-0024 direction A (layer-range GGUF + activations) merges into the native worker track after the benchmark gate — not in parallel with phase C on the same backend without an integration plan. +Recommended sequencing: C first (small, real value), then A/B investigation. ## Also in scope diff --git a/packages/node/meshnet_node/hardware.py b/packages/node/meshnet_node/hardware.py index 7c97a06..9bffa9a 100644 --- a/packages/node/meshnet_node/hardware.py +++ b/packages/node/meshnet_node/hardware.py @@ -2,6 +2,7 @@ import json import os +import shutil import subprocess import time @@ -183,6 +184,17 @@ def with_forced_cpu(hw: dict) -> dict: return forced +def _with_model_drive(profile: dict) -> dict: + """Attach free space for the default model cache drive to tracker diagnostics.""" + try: + cache_root = os.path.expanduser("~/.cache/meshnet/shards") + profile["model_drive_free_bytes"] = shutil.disk_usage(os.path.expanduser("~")).free + profile["model_drive_path"] = cache_root + except OSError: + pass + return profile + + def detect_hardware() -> dict: """Detect GPU model and available VRAM. Returns hardware profile dict.""" ram_mb = _detect_ram_mb() @@ -208,23 +220,23 @@ def detect_hardware() -> dict: } if torch_gpu is not None and torch_gpu.get("gcn_arch"): profile["gcn_arch"] = torch_gpu["gcn_arch"] - return profile + return _with_model_drive(profile) except ImportError: pass torch_inventory = _gpu_inventory_profile(torch_gpu, ram_mb) if torch_inventory is not None: - return torch_inventory + return _with_model_drive(torch_inventory) nvidia_gpu = _gpu_inventory_profile(_detect_nvidia_smi_gpu_memory(), ram_mb) if nvidia_gpu is not None: - return nvidia_gpu + return _with_model_drive(nvidia_gpu) windows_gpu = _gpu_inventory_profile(_detect_windows_gpu_memory(), ram_mb) if windows_gpu is not None: - return windows_gpu + return _with_model_drive(windows_gpu) - return { + return _with_model_drive({ "device": "cpu", "gpu_name": None, "vram_mb": 0, @@ -232,7 +244,7 @@ def detect_hardware() -> dict: "shared_vram_mb": 0, "ram_mb": ram_mb, "cuda_available": False, - } + }) def benchmark_throughput_checked(device_str: str = "cpu") -> tuple[float, bool, str | None]: diff --git a/packages/node/meshnet_node/route_session_benchmark.py b/packages/node/meshnet_node/route_session_benchmark.py index f83eb5f..fdcfef4 100644 --- a/packages/node/meshnet_node/route_session_benchmark.py +++ b/packages/node/meshnet_node/route_session_benchmark.py @@ -44,6 +44,7 @@ class SeamSample: cache_mode: CacheMode model_ms: float encode_ms: float + activation_decode_ms: float framing_ms: float metadata_ms: float copy_allocation_ms: float @@ -52,6 +53,7 @@ class SeamSample: decompression_ms: float connection_setup_ms: float queue_wait_ms: float + local_http_forwarding_ms: float transport_ms: float seam_latency_ms: float payload_bytes: int @@ -120,6 +122,10 @@ def _summary(samples: list[SeamSample]) -> dict[str, float | int]: "compression_cpu_ms": round( sum(sample.compression_ms + sample.decompression_ms for sample in samples), 4 ), + "model_execution_ms": round(sum(sample.model_ms for sample in samples), 4), + "activation_encoding_ms": round(sum(sample.encode_ms for sample in samples), 4), + "activation_decoding_ms": round(sum(sample.activation_decode_ms for sample in samples), 4), + "local_http_forwarding_ms": round(sum(sample.local_http_forwarding_ms for sample in samples), 4), "peak_buffered_bytes": max((sample.copy_allocation_bytes for sample in samples), default=0), } @@ -159,6 +165,7 @@ class _StubTransport: queue_wait_ms = 0.0 if self.mode == "direct" else 0.18 + (0.05 if token_index is not None and token_index % 2 else 0.0) model_ms = 1.6 if phase == "prefill" else 0.45 encode_ms = 0.16 if phase == "prefill" else 0.06 + activation_decode_ms = 0.055 if phase == "prefill" else 0.02 # Keep framing/metadata/copy costs explicit rather than hiding them in # serialization or transport time. The stub owns one binary frame and # one response body per hop; no base64 body is modeled. @@ -168,20 +175,26 @@ class _StubTransport: copy_allocation_bytes = wire_bytes + payload_bytes compression_ms = 0.09 if self.scenario.compression else 0.0 decompression_ms = 0.07 if self.scenario.compression else 0.0 + # Both routes finish by forwarding the decoded activation to the local + # tail-node HTTP handler; relay adds its own queue before that hop. + local_http_forwarding_ms = 0.11 if self.mode == "direct" else 0.16 transport_ms = (0.32 if self.mode == "direct" else 0.61) + wire_bytes / 100_000 seam_latency_ms = round( - model_ms + encode_ms + framing_ms + metadata_ms + copy_allocation_ms - + compression_ms + decompression_ms + connection_setup_ms + queue_wait_ms + transport_ms, + model_ms + encode_ms + activation_decode_ms + framing_ms + metadata_ms + copy_allocation_ms + + compression_ms + decompression_ms + connection_setup_ms + queue_wait_ms + transport_ms + + local_http_forwarding_ms, 4, ) return SeamSample( phase=phase, token_index=token_index, session_id=self.session_id, activation_id=f"benchmark-activation-{self._activation_count}", seam="head->tail", mode=self.mode, cache_mode=self.cache_mode, model_ms=model_ms, encode_ms=encode_ms, + activation_decode_ms=activation_decode_ms, framing_ms=framing_ms, metadata_ms=metadata_ms, copy_allocation_ms=copy_allocation_ms, copy_allocation_bytes=copy_allocation_bytes, compression_ms=compression_ms, decompression_ms=decompression_ms, connection_setup_ms=connection_setup_ms, queue_wait_ms=queue_wait_ms, + local_http_forwarding_ms=local_http_forwarding_ms, transport_ms=round(transport_ms, 4), seam_latency_ms=seam_latency_ms, payload_bytes=payload_bytes, wire_bytes=wire_bytes, compression_ratio=round(payload_bytes / wire_bytes, 4), connection_attempted=connection_attempted, @@ -329,9 +342,10 @@ def run_real_model_lan_benchmark(url: str, *, model: str, timeout: float = 120.0 sample = SeamSample( phase="decode", token_index=0, session_id=session_id, activation_id="lan-activation-1", seam="head->tail", mode="direct", cache_mode="cached", model_ms=0.0, encode_ms=0.0, + activation_decode_ms=0.0, framing_ms=0.0, metadata_ms=0.0, copy_allocation_ms=0.0, copy_allocation_bytes=0, compression_ms=0.0, decompression_ms=0.0, connection_setup_ms=elapsed_ms, - queue_wait_ms=0.0, transport_ms=elapsed_ms, seam_latency_ms=elapsed_ms, + queue_wait_ms=0.0, local_http_forwarding_ms=0.0, transport_ms=elapsed_ms, seam_latency_ms=elapsed_ms, payload_bytes=len(body), wire_bytes=len(body) + len(response_body), compression_ratio=1.0, connection_attempted=True, ) @@ -354,6 +368,10 @@ def format_summary(report: dict) -> str: f"{decode['tokens_per_sec']:.1f} tok/s; {decode['bytes_per_token']:.0f} B/tok; " f"seam {seam['payload_bytes']}/{seam['wire_bytes']} B " f"({seam['compression_ratio']:.2f}x); connections {run['connections']['attempts']}; " + f"model/encode/decode {decode['model_execution_ms']:.2f}/" + f"{decode['activation_encoding_ms']:.2f}/{decode['activation_decoding_ms']:.2f} ms; " + f"compression {decode['compression_cpu_ms']:.2f} ms; " + f"HTTP {decode['local_http_forwarding_ms']:.2f} ms; " f"queue p95 {decode['p95_queue_wait_ms']:.2f} ms" ) return "\n".join(lines) diff --git a/packages/node/meshnet_node/startup.py b/packages/node/meshnet_node/startup.py index 77192f0..dd8a4df 100644 --- a/packages/node/meshnet_node/startup.py +++ b/packages/node/meshnet_node/startup.py @@ -12,7 +12,7 @@ import urllib.error import urllib.parse import urllib.request from pathlib import Path -from typing import Any +from typing import Any, Callable from .admission import ( AdmissionRequirement, @@ -419,6 +419,7 @@ def _start_heartbeat( interval: float = _HEARTBEAT_INTERVAL_IDLE, node_ref: Any | None = None, start_time: float | None = None, + refresh_capability: Callable[[dict], dict | None] | None = None, ) -> threading.Thread: """Daemon thread: sends heartbeats and re-registers automatically after tracker restarts. @@ -430,6 +431,7 @@ def _start_heartbeat( which is logged for now (hot-reload implemented in US-026). """ _start_time = start_time or time.monotonic() + completed_directives: list[dict] = [] def _current_requests_snapshot() -> list[dict]: if node_ref is None: @@ -454,6 +456,8 @@ def _start_heartbeat( current_requests = _current_requests_snapshot() if current_requests: stats["current_requests"] = current_requests + if completed_directives: + stats["completed_directives"] = list(completed_directives) return stats def _sleep_interval() -> float: @@ -461,9 +465,26 @@ def _start_heartbeat( return _HEARTBEAT_INTERVAL_BUSY return interval + def _refresh_proof(payload: dict) -> None: + """Re-prove the current shard so a re-registration never presents an aged proof. + + The tracker refuses proofs older than its freshness budget: re-sending the + startup-time report after an outage would re-register the node unroutable. + """ + if refresh_capability is None or "capability_report" not in payload: + return + try: + fresh = refresh_capability(payload) + except Exception as exc: + print(f" [node] WARNING: capability re-validation failed: {exc}", flush=True) + return + if fresh: + payload["capability_report"] = fresh + def _reregister() -> bool: nonlocal node_id try: + _refresh_proof(register_payload) resp = _post_json(f"{tracker_url}/v1/nodes/register", register_payload) node_id = resp.get("node_id", node_id) if node_ref is not None: @@ -485,6 +506,7 @@ def _start_heartbeat( "managed_assignment": True, } try: + _refresh_proof(extra_payload) reg_resp = _post_json(f"{tracker_url}/v1/nodes/register", extra_payload) print( f" [node] registered additional model — node ID: {reg_resp.get('node_id')}", @@ -493,21 +515,26 @@ def _start_heartbeat( except Exception as exc: print(f" [node] WARNING: additional model registration failed: {exc}", flush=True) - def _apply_directives(directives: list[dict]) -> None: + def _apply_directives(directives: list[dict]) -> dict | None: if not directives: - return + return None if node_ref is None or not hasattr(node_ref, "apply_tracker_directives"): print(f" [node] tracker directives received: {directives}", flush=True) - return + return None try: applied = node_ref.apply_tracker_directives(directives) except Exception as exc: print(f" [node] WARNING: failed to apply tracker directives: {exc}", flush=True) - return + return None if applied: + completed_directives.append(dict(applied)) if applied.get("action") == "ADD_SHARD": _register_additional_assignment(applied) - return + return applied + if applied.get("action") in {"DROP_SHARD", "DROP_ALL_SHARDS"}: + # A release has no replacement range. It is not a failed + # heartbeat and must not re-register the released assignment. + return applied model_id = applied.get("model", register_payload.get("hf_repo") or register_payload.get("model")) register_payload["model"] = str(model_id).split("/")[-1] register_payload["hf_repo"] = model_id @@ -515,6 +542,7 @@ def _start_heartbeat( register_payload["shard_end"] = applied["shard_end"] register_payload["quantization"] = applied.get("quantization", register_payload.get("quantization")) register_payload["tracker_mode"] = bool(applied.get("tracker_mode", False)) + return applied def _loop() -> None: nonlocal node_id @@ -542,7 +570,10 @@ def _start_heartbeat( continue try: - resp = _post_json(hb_url, _get_stats()) + heartbeat = _get_stats() + resp = _post_json(hb_url, heartbeat) + if heartbeat.get("completed_directives"): + completed_directives.clear() _apply_directives(resp.get("directives", [])) new_asgn = resp.get("new_assignment") if new_asgn: @@ -579,6 +610,7 @@ def _register_with_tracker( reg_payload: dict, node: Any, start_time: float, + refresh_capability: Callable[[dict], dict | None] | None = None, ) -> str | None: """Register with the tracker, or start background retries when it is unreachable.""" try: @@ -586,7 +618,14 @@ def _register_with_tracker( tracker_node_id = str(reg_resp.get("node_id") or "?") setattr(node, "tracker_node_id", tracker_node_id) print(f" Registered with tracker — node ID: {tracker_node_id}", flush=True) - _start_heartbeat(tracker_url, tracker_node_id, reg_payload, node_ref=node, start_time=start_time) + _start_heartbeat( + tracker_url, + tracker_node_id, + reg_payload, + node_ref=node, + start_time=start_time, + refresh_capability=refresh_capability, + ) return tracker_node_id except Exception as exc: setattr(node, "tracker_node_id", None) @@ -598,6 +637,7 @@ def _register_with_tracker( reg_payload, node_ref=node, start_time=start_time, + refresh_capability=refresh_capability, ) return None @@ -718,6 +758,54 @@ def _admit_capability( return report +def _capability_refresher( + node: Any, + *, + manifest: RecipeManifest, + recipe: Recipe, + detected_device: str, + cache_dir: Path | None, + force_cpu: bool, + validator: CapabilityValidator | None = None, +) -> Callable[[dict], dict | None]: + """A fresh proof for what the node serves *now*, run at re-registration time. + + The startup proof ages past the tracker's freshness budget, and directives + can move the node to a shard the startup proof never covered — so every + re-registration re-proves against the currently loaded backend rather than + replaying the report captured at boot. + """ + def refresh(payload: dict) -> dict | None: + target_model = payload.get("hf_repo") or payload.get("model") + backend = None + accessor = getattr(node, "backend_for", None) + if callable(accessor) and target_model: + backend = accessor(str(target_model)) + if backend is None: + backend = getattr(node, "backend", None) + if backend is None: + return None + context = CapabilityContext( + backend=backend, + selection=DoctorSelection( + model_id=str(getattr(backend, "model_id", target_model)), + shard_start=int(getattr(backend, "shard_start", 0) or 0), + shard_end=int(getattr(backend, "shard_end", 0) or 0), + quantization=str(getattr(backend, "quantization", None) or "auto"), + cache_dir=cache_dir, + force_cpu=force_cpu, + ), + recipe=recipe, + manifest=manifest, + device=_capability_device(backend, detected_device), + ) + report = (validator or probe_capability)(context) + setattr(node, "capability_report", report) + return report.to_dict() + + return refresh + + def run_startup( tracker_url: str, port: int = 0, @@ -1026,6 +1114,15 @@ def run_startup( } tracker_node_id = _register_with_tracker( tracker_url, reg_payload, node, _node_start_time, + refresh_capability=_capability_refresher( + node, + manifest=manifest, + recipe=recipe, + detected_device=device, + cache_dir=cache_dir, + force_cpu=force_cpu, + validator=capability_validator, + ), ) print( @@ -1197,6 +1294,15 @@ def run_startup( } tracker_node_id = _register_with_tracker( tracker_url, auto_reg_payload, node, _node_start_time, + refresh_capability=_capability_refresher( + node, + manifest=manifest, + recipe=recipe, + detected_device=device, + cache_dir=cache_dir, + force_cpu=force_cpu, + validator=capability_validator, + ), ) shard_label = _format_shard_label( assigned_shard_start, @@ -1389,6 +1495,15 @@ def run_startup( } tracker_node_id = _register_with_tracker( tracker_url, reg_payload, node, _node_start_time, + refresh_capability=_capability_refresher( + node, + manifest=manifest, + recipe=recipe, + detected_device=device, + cache_dir=cache_dir, + force_cpu=force_cpu, + validator=capability_validator, + ), ) print( f"\n{'=' * 32}\n" @@ -1474,7 +1589,22 @@ def run_startup( ) node_id = str(reg_resp["node_id"]) setattr(node, "tracker_node_id", node_id) - _start_heartbeat(tracker_url, node_id, reg_payload, node_ref=node, start_time=_node_start_time) + _start_heartbeat( + tracker_url, + node_id, + reg_payload, + node_ref=node, + start_time=_node_start_time, + refresh_capability=_capability_refresher( + node, + manifest=manifest, + recipe=recipe, + detected_device=device, + cache_dir=shard_path, + force_cpu=force_cpu, + validator=capability_validator, + ), + ) except Exception: node.stop() raise diff --git a/packages/node/meshnet_node/torch_server.py b/packages/node/meshnet_node/torch_server.py index e58cf6d..d464561 100644 --- a/packages/node/meshnet_node/torch_server.py +++ b/packages/node/meshnet_node/torch_server.py @@ -1543,8 +1543,52 @@ class TorchNodeServer: def loaded_model_ids(self) -> list[str]: return list(self._backends.keys()) + def backend_for(self, model_id: str) -> TorchModelShard | None: + """The loaded backend serving `model_id` — full repo id or short name.""" + backend = self._backends.get(model_id) + if backend is not None: + return backend + short = model_id.split("/")[-1].lower() + for key, candidate in self._backends.items(): + if key.split("/")[-1].lower() == short: + return candidate + return None + def apply_tracker_directives(self, directives: list[dict]) -> dict | None: """Apply tracker shard directives (LOAD_SHARD replace, ADD_SHARD load-more).""" + drop_all_directive = next( + (directive for directive in reversed(directives) if directive.get("action") == "DROP_ALL_SHARDS"), + None, + ) + if drop_all_directive is not None: + self._backends.clear() + self._backend = None + self._tracker_mode = False + if self._server is not None: + self._server.backends = {} + self._server.backend = None + self._server.tracker_mode = False + return {"action": "DROP_ALL_SHARDS"} + drop_directive = next( + (directive for directive in reversed(directives) if directive.get("action") == "DROP_SHARD"), + None, + ) + if drop_directive is not None: + model_id = str(drop_directive.get("model") or "") + removed = self._backends.pop(model_id, None) + if removed is None: + return None + if self._backends: + self._backend = next(iter(self._backends.values())) + self._tracker_mode = self._backend.shard_start == 0 + else: + self._backend = None + self._tracker_mode = False + if self._server is not None: + self._server.backends = dict(self._backends) + self._server.backend = self._backend + self._server.tracker_mode = self._tracker_mode + return {"action": "DROP_SHARD", "model": model_id} add_directive = next( (directive for directive in reversed(directives) if directive.get("action") == "ADD_SHARD"), None, @@ -1574,6 +1618,8 @@ class TorchNodeServer: flush=True, ) try: + if replacing: + self._backends.clear() new_backend = _load_backend(model_id, shard_start, shard_end, quantization, self._cache_dir) except TypeError: new_backend = _load_backend(model_id, shard_start, shard_end, quantization) diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index 03fd932..0984262 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -22,8 +22,9 @@ border-bottom:1px solid var(--border); flex-shrink:0; } header h1 { font-size:16px; margin:0; color:var(--accent); } header .meta { color:var(--dim); font-size:12px; } - main { display:grid; grid-template-columns:repeat(auto-fit,minmax(340px,1fr)); + main { display:grid; grid-template-columns:1fr; gap:14px; padding:14px 20px; } + main > section { width:100%; min-width:0; } body.chat-tab-active main { flex:1; min-height:0; display:flex; flex-direction:column; padding:0; gap:0; overflow:hidden; @@ -43,12 +44,15 @@ .empty { color:var(--dim); font-style:italic; } .pill { display:inline-block; padding:0 7px; border-radius:9px; border:1px solid var(--border); font-size:11px; } - input, button { font:inherit; color:var(--fg); background:var(--bg); + input, button, select { font:inherit; color:var(--fg); background:var(--bg); border:1px solid var(--border); border-radius:6px; padding:5px 8px; } input { width:100%; margin-bottom:6px; } button { cursor:pointer; color:var(--accent); } button:hover { border-color:var(--accent); } button.small { font-size:11px; padding:1px 7px; } + dialog { color:var(--fg); background:var(--panel); border:1px solid var(--border); border-radius:8px; min-width:min(420px,calc(100vw - 32px)); } + dialog::backdrop { background:rgba(0,0,0,.55); } + .placement-dialog-actions { display:flex; justify-content:flex-end; gap:8px; margin-top:12px; } .form-row { display:flex; gap:8px; } .form-row button { white-space:nowrap; } .error-msg { color:var(--bad); font-size:12px; min-height:16px; } @@ -71,6 +75,12 @@ background:transparent; color:var(--dim); padding:5px 0 8px; } .dashboard-tabs button.active { color:var(--accent); border-bottom-color:var(--accent); } .wide { grid-column:1 / -1; } + /* Compact status cards fan out on desktop; tables remain readable at half width. */ + @media (min-width:900px) { + main { grid-template-columns:repeat(4,minmax(0,1fr)); } + main > section { grid-column:span 1; } + .wide { grid-column:span 2; } + } section[hidden] { display:none !important; } section.chat-section { padding:0; border:0; border-radius:0; background:var(--bg); min-height:0; @@ -205,7 +215,7 @@ .chat-compose button:disabled { opacity:.45; cursor:not-allowed; } .console { background:var(--bg); border:1px solid var(--border); border-radius:6px; - min-height:160px; max-height:280px; overflow:auto; padding:7px 9px; + min-height:160px; max-height:520px; overflow-y:auto; overflow-x:auto; padding:7px 9px; white-space:pre-wrap; word-break:break-word; font-size:11px; } .console-line { padding:1px 0; border-bottom:1px solid #161b22; } @@ -288,6 +298,8 @@

Node pending payouts

admin login required

Settlement history

admin login required

Tracker hive

loading…
+

Model placement

Choose a model to load or release.
admin login required
+

Total node pool

admin login required

All accounts (admin)

Strikes / bans / forfeitures

admin login required

Client balances

admin login required
@@ -315,6 +327,16 @@
no test output yet
+ +
+
+ + + + +
+
+