tests on dash

.vscode/launch.json

@@ -7,7 +7,7 @@
       "request": "launch",
       "python": "${workspaceFolder}/.venv-rocm/bin/python",
       "module": "meshnet_tracker.cli",
-      "args": ["start", "--host", "0.0.0.0", "--port", "8080"],
+      "args": ["start", "--host", "0.0.0.0", "--port", "8080", "--stats-db", "${workspaceFolder}/tracker-stats.sqlite", "--enable-test-runner"],
       "console": "integratedTerminal",
       "justMyCode": false
     },

packages/node/meshnet_node/cli.py

@@ -43,6 +43,9 @@ def _load_env_defaults() -> None:
         _load_env_file(Path.cwd() / f".env.{machine}")
     _load_env_file(Path.cwd() / ".env")
     _load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
+    for path in os.environ.get("PYTHONPATH", "").split(os.pathsep):
+        if path and path not in sys.path:
+            sys.path.insert(0, path)
 
 
 def _run_node(cfg: dict) -> None:

packages/tracker/meshnet_tracker/cli.py

@@ -60,6 +60,9 @@ def _load_env_defaults() -> None:
         _load_env_file(Path.cwd() / f".env.{machine}")
     _load_env_file(Path.cwd() / ".env")
     _load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
+    for path in os.environ.get("PYTHONPATH", "").split(os.pathsep):
+        if path and path not in sys.path:
+            sys.path.insert(0, path)
 
 
 def _routing_config_from_args(args: argparse.Namespace) -> RoutingConfig | None:
@@ -357,6 +360,11 @@ def main() -> None:
         metavar="N",
         help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
     )
+    common.add_argument(
+        "--enable-test-runner",
+        action="store_true",
+        help="Enable development test-runner hooks for this tracker",
+    )
     common.add_argument(
         "--no-file-logs",
         action="store_true",
@@ -373,6 +381,8 @@ def main() -> None:
     subparsers.add_parser("start", help="Start the tracker server", parents=[common])
 
     args = parser.parse_args()
+    if args.enable_test_runner:
+        os.environ["MESHNET_ENABLE_TEST_RUNNER"] = "1"
 
     if args.command in {None, "start"}:
         if not args.no_file_logs:

packages/tracker/meshnet_tracker/dashboard.html

@@ -254,6 +254,7 @@
           <label>Model
             <select id="chat-model" onchange="selectChatModel(this.value)"></select>
           </label>
+          <button type="button" id="request-model-load" style="display:none" onclick="requestSelectedModelLoad()">Load on available node</button>
           <div id="chat-status" class="chat-status">select a model to start</div>
         </div>
         <div id="chat-history" class="chat-messages empty">
@@ -1386,6 +1387,20 @@ function selectChatModel(value) {
   }
 }
 
+async function requestSelectedModelLoad() {
+  if (!selectedChatModel) return;
+  const button = $("request-model-load");
+  if (button) button.disabled = true;
+  const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel });
+  if (button) button.disabled = false;
+  if (!result.ok) {
+    alert(result.data.error || "model load request failed");
+    return;
+  }
+  const assignment = result.data.assignment || {};
+  $("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
+}
+
 function chatAuthToken() {
   if (accountApiKeys.length) return accountApiKeys[0];
   return null;
@@ -1394,6 +1409,7 @@ function chatAuthToken() {
 function setAdminMode(enabled) {
   isAdmin = enabled;
   $("tab-admin").style.display = enabled ? "" : "none";
+  $("request-model-load").style.display = enabled ? "" : "none";
   if (!enabled && dashboardTab === "admin") {
     switchDashboardTab("overview");
   } else {

packages/tracker/meshnet_tracker/server.py

@@ -1276,6 +1276,35 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
             break
 
 
+def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
+    """Queue an explicitly requested model on the best available joined node."""
+    resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
+    if preset is None or not preset.get("hf_repo"):
+        return None
+    required_start, required_end = _preset_layer_bounds(preset)
+    total_layers = required_end - required_start + 1
+    for host in _memory_pool_map(server)["hosts"]:
+        if host["spare_slots"] <= 0:
+            continue
+        host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
+        if not host_nodes:
+            continue
+        anchor = max(host_nodes, key=lambda node: node.benchmark_tokens_per_sec)
+        if anchor.status != "ready" or anchor.pending_new_assignment is not None:
+            continue
+        capacity = min(_node_layer_capacity(anchor, preset), total_layers)
+        if capacity <= 0:
+            continue
+        quantization = _node_quantization(anchor, preset)
+        shard_end = min(required_end, required_start + capacity - 1)
+        directive = _add_shard_directive(anchor, str(preset["hf_repo"]), required_start, shard_end, quantization)
+        anchor.pending_new_assignment = directive
+        anchor.pending_directives.append(directive)
+        _tracker_log(server, "info", "model load requested", node_id=anchor.node_id, model=resolved_name, hf_repo=preset["hf_repo"], shard=f"{required_start}-{shard_end}")
+        return {"node_id": anchor.node_id, "model": resolved_name, "hf_repo": preset["hf_repo"], "shard_start": required_start, "shard_end": shard_end}
+    return None
+
+
 def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
     if preset is None:
         return {"recommended": False}
@@ -2672,6 +2701,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
         if self.path == "/v1/benchmark/hop-penalty":
             self._handle_benchmark_hop_penalty()
             return
+        if self.path == "/v1/models/load":
+            self._handle_model_load_request()
+            return
         if self.path == "/v1/calibration/toploc/run":
             self._handle_toploc_calibration_run()
             return
@@ -4294,6 +4326,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
             server.gossip.merge({k: float(v) for k, v in body.items()})
         self._send_json(200, {})
 
+    def _handle_model_load_request(self):
+        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
+        if not self._require_role("admin"):
+            return
+        body = self._read_json_body()
+        if body is None:
+            return
+        model = body.get("model")
+        if not isinstance(model, str) or not model.strip():
+            self._send_json(400, {"error": "model is required"})
+            return
+        with server.lock:
+            self._purge_expired_nodes()
+            assignment = _request_model_load_locked(server, model)
+        if assignment is None:
+            self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
+            return
+        self._send_json(202, {"status": "queued", "assignment": assignment})
+
     def _handle_stats(self):
         server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
         if server.stats is None:

tests/test_dashboard.py

@@ -76,6 +76,22 @@ def test_dashboard_chat_uses_streaming_fetch():
     assert "renderChatModels(true)" in html
 
 
+def test_dashboard_allows_admin_to_request_selected_model_load():
+    tracker = TrackerServer()
+    port = tracker.start()
+    try:
+        html = urllib.request.urlopen(
+            f"http://127.0.0.1:{port}/dashboard"
+        ).read().decode()
+    finally:
+        tracker.stop()
+
+    assert 'id="request-model-load"' in html
+    assert "requestSelectedModelLoad" in html
+    assert '"/v1/models/load"' in html
+    assert '$("request-model-load").style.display = enabled ? "" : "none"' in html
+
+
 def test_network_map_includes_node_friendly_name():
     tracker = TrackerServer()
     port = tracker.start()

tests/test_dynamic_routing.py

@@ -305,6 +305,41 @@ def test_proxy_head_is_route_head_and_routing_endpoint_lists_routes():
     assert sampled, "completed requests must produce route samples"
 
 
+def test_admin_model_load_request_queues_directive_on_joined_node():
+    tracker = TrackerServer(validator_service_token="test-admin")
+    port = tracker.start()
+    try:
+        node = _post_json(
+            f"http://127.0.0.1:{port}/v1/nodes/register",
+            {
+                "endpoint": "http://127.0.0.1:9911",
+                "model": "stub-model",
+                "shard_start": 0,
+                "shard_end": 3,
+                "managed_assignment": True,
+                "memory_mb": 32768,
+                "hardware_profile": {"host_id": "available-ram-pool"},
+            },
+        )
+        request = urllib.request.Request(
+            f"http://127.0.0.1:{port}/v1/models/load",
+            data=json.dumps({"model": "qwen2.5-0.5b-instruct"}).encode(),
+            headers={"Content-Type": "application/json", "Authorization": "Bearer test-admin"},
+            method="POST",
+        )
+        with urllib.request.urlopen(request) as response:
+            result = json.loads(response.read())
+        heartbeat = _post_json(
+            f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
+    finally:
+        tracker.stop()
+
+    assert result["status"] == "queued"
+    assert result["assignment"]["node_id"] == node["node_id"]
+    assert heartbeat["directives"][0]["action"] == "ADD_SHARD"
+    assert heartbeat["directives"][0]["model"] == "Qwen/Qwen2.5-0.5B-Instruct"
+
+
 def test_endpoint_key_distinguishes_same_port_different_hosts():
     from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key