[verified] fix: preserve tracker precision eligibility

2026-07-13 10:27:45 +03:00
parent 377346c301
commit b5fa7245df
12 changed files with 557 additions and 48 deletions
--- a/tests/test_tracker_routing.py
+++ b/tests/test_tracker_routing.py
@@ -14,11 +14,13 @@ from meshnet_gateway.server import GatewayServer, _banned_route_wallet
 from meshnet_node.server import StubNodeServer
 from meshnet_contracts import LocalSolanaContracts
 from meshnet_tracker.auth import sign_hive_request
+from meshnet_tracker.capability import POLICY_COMPAT, POLICY_ENFORCE
 from meshnet_tracker.server import (
    TrackerServer,
    _NodeEntry,
    _available_quantizations,
    _memory_pool_map,
+    _rebalance_all_locked,
    _registration_ban_error,
    _scale_demanded_models_locked,
 )
@@ -2906,3 +2908,252 @@ def test_scale_demanded_models_queues_add_shard_on_spare_host():
        assert assignment["model"] == "org/ModelA"
    finally:
        tracker.stop()
+
+
+# ------------------------------------------------- RCR-001: legacy precision fallback
+
+
+class _EchoChatHandler(http.server.BaseHTTPRequestHandler):
+    """A tracker-mode node that answers the proxied chat request."""
+
+    def log_message(self, fmt, *args):
+        pass
+
+    def do_POST(self):
+        if self.path != "/v1/chat/completions":
+            self.send_response(404)
+            self.end_headers()
+            return
+        length = int(self.headers.get("Content-Length", 0))
+        self.rfile.read(length)
+        body = json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode()
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+
+def _legacy_registration(endpoint: str, quantization: str | None = None) -> dict:
+    """A registration as a pre-quantization-field node sends it: no `quantization`."""
+    payload = {
+        "endpoint": endpoint,
+        "model": "Qwen2.5-0.5B-Instruct",
+        "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
+        "num_layers": 24,
+        "shard_start": 0,
+        "shard_end": 23,
+        "tracker_mode": True,
+        "hardware_profile": {},
+        "score": 1.0,
+    }
+    if quantization is not None:
+        payload["quantization"] = quantization
+    return payload
+
+
+def _proxy_chat_status(policy: str, quantization: str | None) -> tuple[int, dict | None]:
+    """Register one legacy node under `policy` and proxy a default-precision chat."""
+    node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)
+    node_thread = threading.Thread(target=node.serve_forever, daemon=True)
+    node_thread.start()
+    tracker = TrackerServer(capability_policy=policy)
+    tracker_port = tracker.start()
+    try:
+        endpoint = f"http://127.0.0.1:{node.server_address[1]}"
+        _post_json(
+            f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
+            _legacy_registration(endpoint, quantization),
+        )
+        try:
+            response = _post_json(
+                f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
+                {"model": "qwen2.5-0.5b",
+                 "messages": [{"role": "user", "content": "hi"}]},
+            )
+        except urllib.error.HTTPError as exc:
+            return exc.code, None
+        return 200, response
+    finally:
+        tracker.stop()
+        node.shutdown()
+        node.server_close()
+        node_thread.join(timeout=1.0)
+
+
+def test_a_legacy_registration_without_quantization_serves_a_default_precision_request():
+    "An omitted field predates the protocol: it must not make a healthy node dark.\n\nTags: http, routing, tracker"
+    status, response = _proxy_chat_status(POLICY_COMPAT, quantization=None)
+    assert status == 200
+    assert response["choices"][0]["message"]["content"] == "ok"
+
+
+def test_a_legacy_registration_without_quantization_stays_dark_under_enforce():
+    "The precision fallback must not smuggle an absent capability report past `enforce`.\n\nTags: http, routing, tracker"
+    status, _ = _proxy_chat_status(POLICY_ENFORCE, quantization=None)
+    assert status == 503
+
+
+def test_a_node_declaring_an_unsupported_quantization_is_never_routed():
+    "Silence is grandfathered; an explicit unsupported precision is not.\n\nTags: http, routing, tracker"
+    status, _ = _proxy_chat_status(POLICY_COMPAT, quantization="fp4-experimental")
+    assert status == 503
+
+
+def test_a_node_declaring_a_null_quantization_is_never_routed():
+    "An explicit null states 'no usable precision' -- only an absent field is legacy.\n\nTags: http, routing, tracker"
+    node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)
+    node_thread = threading.Thread(target=node.serve_forever, daemon=True)
+    node_thread.start()
+    tracker = TrackerServer(capability_policy=POLICY_COMPAT)
+    tracker_port = tracker.start()
+    try:
+        endpoint = f"http://127.0.0.1:{node.server_address[1]}"
+        registration = _legacy_registration(endpoint)
+        registration["quantization"] = None  # declared, not omitted
+        _post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/register", registration)
+        with pytest.raises(urllib.error.HTTPError) as excinfo:
+            _post_json(
+                f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
+                {"model": "qwen2.5-0.5b",
+                 "messages": [{"role": "user", "content": "hi"}]},
+            )
+        assert excinfo.value.code == 503
+    finally:
+        tracker.stop()
+        node.shutdown()
+        node.server_close()
+        node_thread.join(timeout=1.0)
+
+
+def test_raft_apply_preserves_a_declared_precision_for_follower_routing():
+    "A follower that drops the declared precision would route unlike the leader.\n\nTags: raft, routing, tracker"
+    tracker = TrackerServer()
+    tracker.start()
+    try:
+        tracker._raft_apply("register", {
+            "node_id": "declared-node",
+            "endpoint": "http://127.0.0.1:1/",
+            "model": "Qwen2.5-0.5B-Instruct",
+            "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
+            "shard_start": 0,
+            "shard_end": 23,
+            "num_layers": 24,
+            "quantization": "nf4",
+            "quantizations": ["nf4", "int8"],
+        })
+        entry = tracker._registry["declared-node"]
+        assert entry.quantization == "nf4"
+        assert entry.quantizations == ["nf4", "int8"]
+    finally:
+        tracker.stop()
+
+
+def test_raft_apply_grandfathers_a_legacy_registration_that_omits_quantization():
+    "The follower must reach the same legacy fallback the leader did, not a bare None.\n\nTags: raft, routing, tracker"
+    tracker = TrackerServer()
+    tracker.start()
+    try:
+        tracker._raft_apply("register", {
+            "node_id": "legacy-node",
+            "endpoint": "http://127.0.0.1:1/",
+            "model": "Qwen2.5-0.5B-Instruct",
+            "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
+            "shard_start": 0,
+            "shard_end": 23,
+            "num_layers": 24,
+        })
+        entry = tracker._registry["legacy-node"]
+        assert entry.quantization == "bfloat16"
+    finally:
+        tracker.stop()
+
+
+def test_rebalancing_does_not_resurrect_a_node_that_declared_an_unusable_precision():
+    "Managed assignment picks a node's precision, so it must not overwrite an explicit unusable one.\n\nTags: http, routing, tracker"
+    tracker = TrackerServer(heartbeat_timeout=0.15, rebalance_interval=10.0)
+    tracker_port = tracker.start()
+    try:
+        managed = _post_json(
+            f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
+            {"endpoint": "http://127.0.0.1:9111", "model": "Qwen2.5-0.5B-Instruct",
+             "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", "num_layers": 24,
+             "shard_start": 0, "shard_end": 21, "managed_assignment": True,
+             "quantization": "fp4-experimental",
+             "vram_bytes": 1_000_000_000, "hardware_profile": {}, "score": 1.0},
+        )
+        # Letting this peer expire is what drives the rebalance that used to hand
+        # the managed node a fresh shard range -- and a default precision with it.
+        _post_json(
+            f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
+            {"endpoint": "http://127.0.0.1:9112", "model": "Qwen2.5-0.5B-Instruct",
+             "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", "num_layers": 24,
+             "shard_start": 22, "shard_end": 23,
+             "vram_bytes": 1_000_000_000, "hardware_profile": {}, "score": 1.0},
+        )
+
+        time.sleep(0.10)
+        _post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/{managed['node_id']}/heartbeat", {})
+        time.sleep(0.10)
+        hb = _post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/{managed['node_id']}/heartbeat", {})
+
+        entry = tracker._registry[managed["node_id"]]
+        assert entry.quantization is None
+        assert not [d for d in hb.get("directives", []) if d["action"] == "LOAD_SHARD"]
+    finally:
+        tracker.stop()
+
+
+def _register_shard(tracker_port: int, port: int, **overrides) -> dict:
+    payload = {
+        "endpoint": f"http://127.0.0.1:{port}",
+        "model": "Qwen2.5-0.5B-Instruct",
+        "hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
+        "num_layers": 24,
+        "vram_bytes": 40_000_000_000,
+        "hardware_profile": {},
+        "score": 1.0,
+    }
+    payload.update(overrides)
+    return _post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/register", payload)
+
+
+def test_a_node_with_an_unusable_precision_covers_no_layers():
+    "A node no request can reach must not report its shards as served coverage.\n\nTags: http, routing, tracker"
+    tracker = TrackerServer(rebalance_interval=10.0)
+    tracker_port = tracker.start()
+    try:
+        _register_shard(
+            tracker_port, 9121,
+            shard_start=0, shard_end=23, quantization="fp4-experimental",
+        )
+        models = _get_json(f"http://127.0.0.1:{tracker_port}/v1/models")
+        served = [m for m in models["data"] if m["id"] == "Qwen/Qwen2.5-0.5B-Instruct"]
+        assert [m["shard_coverage_percentage"] for m in served] == [0.0]
+    finally:
+        tracker.stop()
+
+
+def test_rebalancing_fills_the_layers_an_unusable_node_only_appears_to_cover():
+    "Counting a dark node as coverage hides the gap, so the rebalancer never fills it.\n\nTags: http, routing, tracker"
+    tracker = TrackerServer(rebalance_interval=10.0)
+    tracker_port = tracker.start()
+    try:
+        # The dark node claims the model's tail; the managed node holds the head.
+        # Both ends look covered, so the rebalancer used to see nothing to do.
+        dark = _register_shard(
+            tracker_port, 9131,
+            shard_start=12, shard_end=23, quantization="fp4-experimental",
+        )
+        managed = _register_shard(
+            tracker_port, 9132,
+            shard_start=0, shard_end=11, managed_assignment=True,
+        )
+        with tracker._lock:
+            _rebalance_all_locked(tracker._server)  # type: ignore[arg-type]
+
+        assert tracker._registry[dark["node_id"]].quantization is None
+        entry = tracker._registry[managed["node_id"]]
+        assert (entry.shard_start, entry.shard_end) == (0, 23)
+    finally:
+        tracker.stop()