routing tests, launch.configs, redirect, stats and route statistics

tests/test_dashboard.py

@@ -1,5 +1,6 @@
 """US-035: tracker web dashboard — served from any tracker, embedded asset."""
 
+import http.client
 import json
 import time
 import urllib.request
@@ -40,6 +41,23 @@ def test_dashboard_served_with_all_panels():
         tracker.stop()
 
 
+def test_tracker_root_redirects_to_dashboard():
+    tracker = TrackerServer()
+    port = tracker.start()
+    try:
+        connection = http.client.HTTPConnection("127.0.0.1", port)
+        connection.request("GET", "/")
+        response = connection.getresponse()
+        location = response.getheader("Location")
+        response.read()
+        connection.close()
+    finally:
+        tracker.stop()
+
+    assert response.status == 302
+    assert location == "/dashboard"
+
+
 def test_dashboard_chat_uses_streaming_fetch():
     tracker = TrackerServer(billing=BillingLedger())
     port = tracker.start()

tests/test_dynamic_routing.py

@@ -99,6 +99,21 @@ def test_route_stats_ewma_averages_samples():
 # ---- choose_route --------------------------------------------------------
 
 
+def test_route_stats_persist_historical_hop_latency_across_restart(tmp_path):
+    db_path = str(tmp_path / "routing-stats.sqlite")
+    signature = "m|gpu[0-11]->cpu[12-23]"
+    store = RouteStatsStore(RoutingConfig(stats_half_life_seconds=1e9), db_path=db_path)
+    store.record_sample("m", signature, tokens=100, elapsed_seconds=2.0, now=100.0)
+    store.save_to_db()
+
+    restored = RouteStatsStore(RoutingConfig(stats_half_life_seconds=1e9), db_path=db_path)
+    row = restored.model_rows("m", now=101.0)[0]
+    assert row["hop_count"] == 2
+    assert row["samples"] == 1
+    assert row["tps"] == 50.0
+    assert row["latency_ms"] == 2000.0
+
+
 def _candidates_two_routes():
     fast = RouteCandidate(nodes=[], signature="m|fast", prior_tps=100.0)
     slow = RouteCandidate(nodes=[], signature="m|slow", prior_tps=50.0)

tests/test_model_speed_latency.py

@@ -0,0 +1,153 @@
+"""Tracker-backed latency experiments and model-speed drill-down."""
+
+import http.server
+import json
+import threading
+import time
+import urllib.request
+
+import pytest
+
+from meshnet_tracker.server import TrackerServer
+
+
+MODELS = {
+    "qwen2.5-0.5b-instruct": (24, "Qwen/Qwen2.5-0.5B-Instruct"),
+    "qwen3.6-35b-a3b": (40, "unsloth/Qwen3.6-35B-A3B"),
+}
+
+
+def _post_json(url: str, payload: dict) -> dict:
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=10.0) as response:
+        return json.loads(response.read())
+
+
+def _get_json(url: str) -> dict:
+    with urllib.request.urlopen(url, timeout=10.0) as response:
+        return json.loads(response.read())
+
+
+class _LatencyNode(http.server.BaseHTTPRequestHandler):
+    """Synthetic node: every downstream Activation Seam adds deterministic delay."""
+
+    base_delay_seconds = 0.004
+    seam_delay_seconds = 0.006
+
+    def log_message(self, *_args):
+        pass
+
+    def do_POST(self):
+        self.rfile.read(int(self.headers.get("Content-Length", 0)))
+        downstream = json.loads(self.headers.get("X-Meshnet-Route", "[]"))
+        time.sleep(self.base_delay_seconds + self.seam_delay_seconds * len(downstream))
+        body = json.dumps({
+            "choices": [{"message": {"role": "assistant", "content": "ok " * 40}}],
+            "usage": {"prompt_tokens": 10, "completion_tokens": 40},
+        }).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+
+def _start_latency_nodes(count: int):
+    nodes = []
+    threads = []
+    for _ in range(count):
+        node = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _LatencyNode)
+        thread = threading.Thread(target=node.serve_forever, daemon=True)
+        thread.start()
+        nodes.append(node)
+        threads.append(thread)
+    return nodes, threads
+
+
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("hardware", ["gpu", "gpu-cpu", "cpu"])
+def test_tracker_records_increasing_hop_latency_for_model_and_hardware(model, hardware):
+    """One through five hops must preserve a measurable seam penalty in tracker stats."""
+    layer_count, hf_repo = MODELS[model]
+    nodes, threads = _start_latency_nodes(5)
+    tracker = TrackerServer(model_presets={
+        model: {
+            "layers_start": 0,
+            "layers_end": layer_count - 1,
+            "hf_repo": hf_repo,
+            "aliases": [model],
+        }
+    })
+    tracker_port = tracker.start()
+    try:
+        registered_ids = []
+        for index, node in enumerate(nodes):
+            start = (layer_count * index) // 5
+            end = (layer_count * (index + 1)) // 5 - 1
+            device = "cuda" if hardware == "gpu" or (hardware == "gpu-cpu" and index == 0) else "cpu"
+            data = _post_json(
+                f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
+                {
+                    "endpoint": f"http://127.0.0.1:{node.server_address[1]}",
+                    "model": model,
+                    "hf_repo": hf_repo,
+                    "num_layers": layer_count,
+                    "shard_start": start,
+                    "shard_end": end,
+                    "tracker_mode": index == 0,
+                    "hardware_profile": {"device": device},
+                    "vram_bytes": 8_000_000_000 if device == "cuda" else 0,
+                    "ram_bytes": 32_000_000_000,
+                    "benchmark_tokens_per_sec": 100.0,
+                },
+            )
+            registered_ids.append(data["node_id"])
+
+        for hops in range(1, 6):
+            route = registered_ids[:hops]
+            # Widen the final shard to make each pinned prefix a complete route.
+            with tracker._server.lock:
+                tracker._server.registry[route[-1]].shard_end = layer_count - 1
+            _post_json(
+                f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
+                {
+                    "model": model,
+                    "messages": [{"role": "user", "content": "measure"}],
+                    "route": route,
+                },
+            )
+
+        report = _get_json(f"http://127.0.0.1:{tracker_port}/v1/model-speed?model={model}")
+    finally:
+        tracker.stop()
+        for node, thread in zip(nodes, threads):
+            node.shutdown()
+            node.server_close()
+            thread.join(timeout=1.0)
+
+    routes = {entry["hop_count"]: entry for entry in report["routes"]}
+    assert set(routes) == {1, 2, 3, 4, 5}
+    assert routes[5]["latency_ms"] > routes[1]["latency_ms"]
+    assert routes[5]["latency_penalty_ms"] > 0
+    assert routes[5]["device_mix"] == hardware
+    assert report["model"] == model
+    assert report["nodes"]
+
+
+def test_model_speed_dashboard_includes_visualization_and_route_drilldown():
+    tracker = TrackerServer()
+    port = tracker.start()
+    try:
+        html = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard").read().decode()
+    finally:
+        tracker.stop()
+
+    assert "Model inference speed" in html
+    assert "model-speed-chart" in html
+    assert "renderModelSpeed" in html
+    assert "/v1/model-speed" in html

tests/test_real_distributed_inference.py

@@ -0,0 +1,62 @@
+"""Environment-gated acceptance test against real registered model shards.
+
+Run only after starting actual nodes (never synthetic HTTP handlers):
+
+MESHNET_REAL_INFERENCE_URL=http://localhost:8080 \
+MESHNET_REAL_INFERENCE_API_KEY=... \
+MESHNET_REAL_INFERENCE_MODEL=Qwen/Qwen2.5-0.5B-Instruct \
+MESHNET_REAL_INFERENCE_ROUTE=node-id-head,node-id-tail \
+.venv-rocm/bin/python -m pytest tests/test_real_distributed_inference.py -m integration -v
+"""
+
+import json
+import os
+import time
+import urllib.request
+
+import pytest
+
+
+BASE_URL = os.environ.get("MESHNET_REAL_INFERENCE_URL")
+API_KEY = os.environ.get("MESHNET_REAL_INFERENCE_API_KEY")
+MODEL = os.environ.get("MESHNET_REAL_INFERENCE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct")
+ROUTE = [node_id for node_id in os.environ.get("MESHNET_REAL_INFERENCE_ROUTE", "").split(",") if node_id]
+
+pytestmark = pytest.mark.integration
+
+
+def _required_real_stack() -> None:
+    if not BASE_URL or not API_KEY or len(ROUTE) < 2:
+        pytest.skip(
+            "set MESHNET_REAL_INFERENCE_URL, MESHNET_REAL_INFERENCE_API_KEY, and "
+            "a comma-separated MESHNET_REAL_INFERENCE_ROUTE with at least two real nodes"
+        )
+
+
+def test_real_registered_shards_complete_a_pinned_request():
+    """Acceptance test: tracker proxies an actual prompt through real registered shards."""
+    _required_real_stack()
+    assert BASE_URL is not None
+    assert API_KEY is not None
+    request = urllib.request.Request(
+        f"{BASE_URL.rstrip('/')}/v1/chat/completions",
+        data=json.dumps({
+            "model": MODEL,
+            "messages": [{"role": "user", "content": "What is 2 plus 2? Reply in one word."}],
+            "max_tokens": 8,
+            "temperature": 1.0,
+            "route": ROUTE,
+        }).encode(),
+        headers={"Content-Type": "application/json", "Authorization": f"Bearer {API_KEY}"},
+        method="POST",
+    )
+    started = time.monotonic()
+    with urllib.request.urlopen(request, timeout=300) as response:
+        payload = json.loads(response.read())
+    elapsed = time.monotonic() - started
+
+    content = payload["choices"][0]["message"]["content"].strip().lower()
+    assert content
+    assert "two" in content or "four" in content
+    assert elapsed > 0
+    assert payload["usage"]["total_tokens"] > 0