diff --git a/.vscode/launch.json b/.vscode/launch.json index b30a847..a2e1349 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -7,7 +7,7 @@ "request": "launch", "python": "${workspaceFolder}/.venv-rocm/bin/python", "module": "meshnet_tracker.cli", - "args": ["start", "--host", "0.0.0.0", "--port", "8080"], + "args": ["start", "--host", "0.0.0.0", "--port", "8080", "--stats-db", "${workspaceFolder}/tracker-stats.sqlite", "--enable-test-runner"], "console": "integratedTerminal", "justMyCode": false }, diff --git a/packages/node/meshnet_node/cli.py b/packages/node/meshnet_node/cli.py index 8ac5f2b..bb91894 100644 --- a/packages/node/meshnet_node/cli.py +++ b/packages/node/meshnet_node/cli.py @@ -43,6 +43,9 @@ def _load_env_defaults() -> None: _load_env_file(Path.cwd() / f".env.{machine}") _load_env_file(Path.cwd() / ".env") _load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env") + for path in os.environ.get("PYTHONPATH", "").split(os.pathsep): + if path and path not in sys.path: + sys.path.insert(0, path) def _run_node(cfg: dict) -> None: diff --git a/packages/tracker/meshnet_tracker/cli.py b/packages/tracker/meshnet_tracker/cli.py index 1fc3c5b..9164c77 100644 --- a/packages/tracker/meshnet_tracker/cli.py +++ b/packages/tracker/meshnet_tracker/cli.py @@ -60,6 +60,9 @@ def _load_env_defaults() -> None: _load_env_file(Path.cwd() / f".env.{machine}") _load_env_file(Path.cwd() / ".env") _load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env") + for path in os.environ.get("PYTHONPATH", "").split(os.pathsep): + if path and path not in sys.path: + sys.path.insert(0, path) def _routing_config_from_args(args: argparse.Namespace) -> RoutingConfig | None: @@ -357,6 +360,11 @@ def main() -> None: metavar="N", help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})", ) + common.add_argument( + "--enable-test-runner", + action="store_true", + help="Enable development test-runner hooks for this tracker", + ) common.add_argument( "--no-file-logs", action="store_true", @@ -373,6 +381,8 @@ def main() -> None: subparsers.add_parser("start", help="Start the tracker server", parents=[common]) args = parser.parse_args() + if args.enable_test_runner: + os.environ["MESHNET_ENABLE_TEST_RUNNER"] = "1" if args.command in {None, "start"}: if not args.no_file_logs: diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index 716cfde..dc8af7b 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -254,6 +254,7 @@ +
select a model to start
@@ -1386,6 +1387,20 @@ function selectChatModel(value) { } } +async function requestSelectedModelLoad() { + if (!selectedChatModel) return; + const button = $("request-model-load"); + if (button) button.disabled = true; + const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel }); + if (button) button.disabled = false; + if (!result.ok) { + alert(result.data.error || "model load request failed"); + return; + } + const assignment = result.data.assignment || {}; + $("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`; +} + function chatAuthToken() { if (accountApiKeys.length) return accountApiKeys[0]; return null; @@ -1394,6 +1409,7 @@ function chatAuthToken() { function setAdminMode(enabled) { isAdmin = enabled; $("tab-admin").style.display = enabled ? "" : "none"; + $("request-model-load").style.display = enabled ? "" : "none"; if (!enabled && dashboardTab === "admin") { switchDashboardTab("overview"); } else { diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 4187bc4..f15be17 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -1276,6 +1276,35 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None: break +def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None: + """Queue an explicitly requested model on the best available joined node.""" + resolved_name, preset = _resolve_model_preset(server.model_presets, model_key) + if preset is None or not preset.get("hf_repo"): + return None + required_start, required_end = _preset_layer_bounds(preset) + total_layers = required_end - required_start + 1 + for host in _memory_pool_map(server)["hosts"]: + if host["spare_slots"] <= 0: + continue + host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry] + if not host_nodes: + continue + anchor = max(host_nodes, key=lambda node: node.benchmark_tokens_per_sec) + if anchor.status != "ready" or anchor.pending_new_assignment is not None: + continue + capacity = min(_node_layer_capacity(anchor, preset), total_layers) + if capacity <= 0: + continue + quantization = _node_quantization(anchor, preset) + shard_end = min(required_end, required_start + capacity - 1) + directive = _add_shard_directive(anchor, str(preset["hf_repo"]), required_start, shard_end, quantization) + anchor.pending_new_assignment = directive + anchor.pending_directives.append(directive) + _tracker_log(server, "info", "model load requested", node_id=anchor.node_id, model=resolved_name, hf_repo=preset["hf_repo"], shard=f"{required_start}-{shard_end}") + return {"node_id": anchor.node_id, "model": resolved_name, "hf_repo": preset["hf_repo"], "shard_start": required_start, "shard_end": shard_end} + return None + + def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict: if preset is None: return {"recommended": False} @@ -2672,6 +2701,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): if self.path == "/v1/benchmark/hop-penalty": self._handle_benchmark_hop_penalty() return + if self.path == "/v1/models/load": + self._handle_model_load_request() + return if self.path == "/v1/calibration/toploc/run": self._handle_toploc_calibration_run() return @@ -4294,6 +4326,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): server.gossip.merge({k: float(v) for k, v in body.items()}) self._send_json(200, {}) + def _handle_model_load_request(self): + server: _TrackerHTTPServer = self.server # type: ignore[assignment] + if not self._require_role("admin"): + return + body = self._read_json_body() + if body is None: + return + model = body.get("model") + if not isinstance(model, str) or not model.strip(): + self._send_json(400, {"error": "model is required"}) + return + with server.lock: + self._purge_expired_nodes() + assignment = _request_model_load_locked(server, model) + if assignment is None: + self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"}) + return + self._send_json(202, {"status": "queued", "assignment": assignment}) + def _handle_stats(self): server: _TrackerHTTPServer = self.server # type: ignore[assignment] if server.stats is None: diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py index 80c4c5e..c4a1e37 100644 --- a/tests/test_dashboard.py +++ b/tests/test_dashboard.py @@ -76,6 +76,22 @@ def test_dashboard_chat_uses_streaming_fetch(): assert "renderChatModels(true)" in html +def test_dashboard_allows_admin_to_request_selected_model_load(): + tracker = TrackerServer() + port = tracker.start() + try: + html = urllib.request.urlopen( + f"http://127.0.0.1:{port}/dashboard" + ).read().decode() + finally: + tracker.stop() + + assert 'id="request-model-load"' in html + assert "requestSelectedModelLoad" in html + assert '"/v1/models/load"' in html + assert '$("request-model-load").style.display = enabled ? "" : "none"' in html + + def test_network_map_includes_node_friendly_name(): tracker = TrackerServer() port = tracker.start() diff --git a/tests/test_dynamic_routing.py b/tests/test_dynamic_routing.py index 3abd24e..e0bf329 100644 --- a/tests/test_dynamic_routing.py +++ b/tests/test_dynamic_routing.py @@ -305,6 +305,41 @@ def test_proxy_head_is_route_head_and_routing_endpoint_lists_routes(): assert sampled, "completed requests must produce route samples" +def test_admin_model_load_request_queues_directive_on_joined_node(): + tracker = TrackerServer(validator_service_token="test-admin") + port = tracker.start() + try: + node = _post_json( + f"http://127.0.0.1:{port}/v1/nodes/register", + { + "endpoint": "http://127.0.0.1:9911", + "model": "stub-model", + "shard_start": 0, + "shard_end": 3, + "managed_assignment": True, + "memory_mb": 32768, + "hardware_profile": {"host_id": "available-ram-pool"}, + }, + ) + request = urllib.request.Request( + f"http://127.0.0.1:{port}/v1/models/load", + data=json.dumps({"model": "qwen2.5-0.5b-instruct"}).encode(), + headers={"Content-Type": "application/json", "Authorization": "Bearer test-admin"}, + method="POST", + ) + with urllib.request.urlopen(request) as response: + result = json.loads(response.read()) + heartbeat = _post_json( + f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {}) + finally: + tracker.stop() + + assert result["status"] == "queued" + assert result["assignment"]["node_id"] == node["node_id"] + assert heartbeat["directives"][0]["action"] == "ADD_SHARD" + assert heartbeat["directives"][0]["model"] == "Qwen/Qwen2.5-0.5B-Instruct" + + def test_endpoint_key_distinguishes_same_port_different_hosts(): from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key