popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[verified] fix: preserve tracker precision eligibility

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 10:27:45 +03:00
parent 377346c301
commit b5fa7245df
12 changed files with 557 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_hf_pricing_dispatch.py
View File

@@ -95,7 +95,7 @@ def test_refresh_loop_repriced_model_with_curated_alias(pricing_tracker):
tracker_url, ledger, tracker = pricing_tracker
new_price = _wait_for_price_change(ledger, PRICED_MODEL)
expected = round((0.93 + 3.00) / 2 / 1000 * 0.80, 6)
assert new_price == expected
assert new_price == pytest.approx(expected)
preset = tracker._model_presets[PRICED_MODEL]
assert preset["hf_last_price_per_1k"] == expected

8
tests/test_meshnet_sdk.py
View File

@@ -14,9 +14,13 @@ def test_sdk_support_helpers_are_deterministic_without_network():
"Non-socket coverage for SDK-facing gateway/tracker helper behavior.\n\nTags: gateway, http, sdk"
address = _payment_address_for_api_key("sdk-key")
# Registration always resolves a precision, so a registry node carries one;
# coverage counts only nodes that can actually serve the layers they claim.
nodes = [
_NodeEntry("a", "http://node-a", 0, 15, "stub-model", None, {}, None, 1.0),
_NodeEntry("b", "http://node-b", 16, 31, "stub-model", None, {}, None, 1.0),
_NodeEntry("a", "http://node-a", 0, 15, "stub-model", None, {}, None, 1.0,
quantization="bfloat16"),
_NodeEntry("b", "http://node-b", 16, 31, "stub-model", None, {}, None, 1.0,
quantization="bfloat16"),
]
assert 32 <= len(address) <= 44

52
tests/test_mining_cli.py
View File

@@ -553,9 +553,23 @@ def test_legacy_start_falls_back_to_env_tracker_and_model(monkeypatch):
assert captured["model_id"] == "Qwen/Qwen2.5-0.5B-Instruct"
def test_legacy_start_without_port_uses_next_available_port(monkeypatch):
"Omitting --port skips an occupied default port before startup loads the model.\n\nTags: general"
from meshnet_node.cli import main
def test_first_available_port_skips_an_occupied_custom_port():
"Port search skips an occupied custom base port.\n\nTags: general"
from meshnet_node.cli import _first_available_port
occupied = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
occupied.bind(("127.0.0.1", 0))
start = occupied.getsockname()[1]
occupied.listen(1)
try:
assert _first_available_port("127.0.0.1", start=start, attempts=100) > start
finally:
occupied.close()
def test_legacy_start_without_port_uses_port_search(monkeypatch):
"Omitting --port delegates legacy startup to the port-search helper.\n\nTags: general"
from meshnet_node import cli as cli_mod
captured = {}
@@ -566,26 +580,20 @@ def test_legacy_start_without_port_uses_next_available_port(monkeypatch):
def stop(self): pass
return _FakeNode()
occupied = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
occupied.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
occupied.bind(("127.0.0.1", 7000))
occupied.listen(1)
try:
monkeypatch.setattr(sys, "argv", [
"meshnet-node", "start",
"--tracker", "http://192.168.0.179:8081",
"--model", "Qwen/Qwen2.5-0.5B-Instruct",
"--host", "127.0.0.1",
])
monkeypatch.setattr(cli_mod, "_first_available_port", lambda *_args, **_kwargs: 7001)
monkeypatch.setattr(sys, "argv", [
"meshnet-node", "start",
"--tracker", "http://192.168.0.179:8081",
"--model", "Qwen/Qwen2.5-0.5B-Instruct",
"--host", "127.0.0.1",
])
with patch("meshnet_node.startup.run_startup", side_effect=fake_run_startup):
with patch("time.sleep", side_effect=KeyboardInterrupt):
try:
main()
except SystemExit as exc:
assert exc.code == 0
finally:
occupied.close()
with patch("meshnet_node.startup.run_startup", side_effect=fake_run_startup):
with patch("time.sleep", side_effect=KeyboardInterrupt):
try:
cli_mod.main()
except SystemExit as exc:
assert exc.code == 0
assert captured["port"] == 7001

2
tests/test_node_startup.py
View File

@@ -2454,7 +2454,9 @@ def test_cli_loads_local_env_before_config_defaults(tmp_path, monkeypatch):
from meshnet_node import config as config_mod
monkeypatch.delenv("MESHNET_DOWNLOAD_DIR", raising=False)
monkeypatch.delenv("HF_TOKEN", raising=False)
monkeypatch.chdir(tmp_path)
cli_mod = importlib.reload(cli_mod)
(tmp_path / ".env").write_text(
"MESHNET_DOWNLOAD_DIR=/run/media/popov/DATA/llm/safetensor/models\n"
"HF_TOKEN=hf_test_token\n"

31
tests/test_real_model_backend.py
View File

@@ -32,6 +32,15 @@ from meshnet_node.model_backend import (
from meshnet_node.torch_server import TorchNodeServer
def _require_functional_torch():
"""Skip tensor-behaviour tests when the installed torch namespace is incomplete."""
torch = pytest.importorskip("torch")
required = ("tensor", "zeros", "ones", "arange", "bfloat16", "long")
if not all(hasattr(torch, name) for name in required):
pytest.skip("requires a functional PyTorch tensor runtime")
return torch
class _FakeBackend:
model_id = "fake-model"
total_layers = 12
@@ -526,13 +535,14 @@ def test_distributed_generating_log_includes_tps(capsys):
out = capsys.readouterr().out
assert "generating step=1/1" in out
assert " tps=" in out
assert "generation complete tokens=1" in out
assert "generation complete session=" in out
assert "tokens=1" in out
assert out.count("generating step=1/1") == 1
def test_int_tensor_header_serializes_torch_tensors():
"Int tensor header serializes torch tensors\n\nTags: model, node, real-inference"
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
header = _int_tensor_header(torch.tensor([[1, 2, 3]], dtype=torch.long))
@@ -544,7 +554,7 @@ def test_bfloat16_wire_decode_views_owned_bytes_without_float32_round_trip():
Tags: model, performance, wire
"""
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
body = torch.tensor([[1, 2]], dtype=torch.bfloat16).view(torch.uint8).numpy().tobytes()
decoded = _tensor_from_bfloat16_bytes(body, [1, 2], torch)
@@ -555,7 +565,7 @@ def test_bfloat16_wire_decode_views_owned_bytes_without_float32_round_trip():
def test_decoder_attention_mask_is_causal_float_mask():
"Decoder attention mask is causal float mask\n\nTags: model, node, real-inference"
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
hidden_states = torch.zeros((1, 3, 8), dtype=torch.bfloat16)
mask = _decoder_attention_mask(torch.ones((1, 3), dtype=torch.long), hidden_states, torch)
@@ -573,7 +583,7 @@ def test_call_layer_passes_rotary_position_embeddings():
assert kwargs["position_embeddings"] == "rotary"
return hidden_states
hidden, cache_state = _call_layer(
hidden = _call_layer(
NeedsPositionEmbeddings(),
"hidden",
attention_mask=None,
@@ -582,7 +592,6 @@ def test_call_layer_passes_rotary_position_embeddings():
)
assert hidden == "hidden"
assert cache_state is None
def _fake_cache_shard(torch, *, max_sessions=16, ttl=600.0):
@@ -618,7 +627,7 @@ def _fake_cache_shard(torch, *, max_sessions=16, ttl=600.0):
def test_shard_cache_prefill_then_decode_reuses_opaque_layer_state():
"Shard cache prefill then decode reuses opaque layer state\n\nTags: cache, model, node, real-inference"
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
shard = _fake_cache_shard(torch)
prefill_hidden = torch.zeros((1, 4, 2), dtype=torch.bfloat16)
@@ -658,7 +667,7 @@ def test_shard_cache_prefill_then_decode_reuses_opaque_layer_state():
def test_shard_cache_decode_miss_is_explicit():
"Shard cache decode miss is explicit\n\nTags: cache, model, node, real-inference"
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
shard = _fake_cache_shard(torch)
with pytest.raises(KVCacheMiss):
@@ -674,7 +683,7 @@ def test_shard_cache_decode_miss_is_explicit():
def test_shard_cache_lru_bounds_sessions():
"Shard cache lru bounds sessions\n\nTags: cache, model, node, real-inference"
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
shard = _fake_cache_shard(torch, max_sessions=1)
for session in ("old", "new"):
@@ -1154,7 +1163,7 @@ def test_torch_model_shard_prefers_partial_loader_for_local_snapshot(tmp_path, m
"torch",
types.SimpleNamespace(
cuda=types.SimpleNamespace(is_available=lambda: False),
device=lambda value: value,
device=lambda value: types.SimpleNamespace(type=value),
bfloat16="bf16",
),
)
@@ -1194,7 +1203,7 @@ def test_two_node_gpt2_completion_is_deterministic():
"Two node gpt2 completion is deterministic\n\nTags: model, node, real-inference"
if os.environ.get("CI"):
pytest.skip("GPT-2 integration test is skipped in CI")
torch = pytest.importorskip("torch")
torch = _require_functional_torch()
pytest.importorskip("transformers")
pytest.importorskip("safetensors")
pytest.importorskip("accelerate")

251
tests/test_tracker_routing.py
View File

@@ -14,11 +14,13 @@ from meshnet_gateway.server import GatewayServer, _banned_route_wallet
from meshnet_node.server import StubNodeServer
from meshnet_contracts import LocalSolanaContracts
from meshnet_tracker.auth import sign_hive_request
from meshnet_tracker.capability import POLICY_COMPAT, POLICY_ENFORCE
from meshnet_tracker.server import (
TrackerServer,
_NodeEntry,
_available_quantizations,
_memory_pool_map,
_rebalance_all_locked,
_registration_ban_error,
_scale_demanded_models_locked,
)
@@ -2906,3 +2908,252 @@ def test_scale_demanded_models_queues_add_shard_on_spare_host():
assert assignment["model"] == "org/ModelA"
finally:
tracker.stop()
# ------------------------------------------------- RCR-001: legacy precision fallback
class _EchoChatHandler(http.server.BaseHTTPRequestHandler):
"""A tracker-mode node that answers the proxied chat request."""
def log_message(self, fmt, *args):
pass
def do_POST(self):
if self.path != "/v1/chat/completions":
self.send_response(404)
self.end_headers()
return
length = int(self.headers.get("Content-Length", 0))
self.rfile.read(length)
body = json.dumps({"choices": [{"message": {"content": "ok"}}]}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _legacy_registration(endpoint: str, quantization: str | None = None) -> dict:
"""A registration as a pre-quantization-field node sends it: no `quantization`."""
payload = {
"endpoint": endpoint,
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"num_layers": 24,
"shard_start": 0,
"shard_end": 23,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0,
}
if quantization is not None:
payload["quantization"] = quantization
return payload
def _proxy_chat_status(policy: str, quantization: str | None) -> tuple[int, dict | None]:
"""Register one legacy node under `policy` and proxy a default-precision chat."""
node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(capability_policy=policy)
tracker_port = tracker.start()
try:
endpoint = f"http://127.0.0.1:{node.server_address[1]}"
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
_legacy_registration(endpoint, quantization),
)
try:
response = _post_json(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
{"model": "qwen2.5-0.5b",
"messages": [{"role": "user", "content": "hi"}]},
)
except urllib.error.HTTPError as exc:
return exc.code, None
return 200, response
finally:
tracker.stop()
node.shutdown()
node.server_close()
node_thread.join(timeout=1.0)
def test_a_legacy_registration_without_quantization_serves_a_default_precision_request():
"An omitted field predates the protocol: it must not make a healthy node dark.\n\nTags: http, routing, tracker"
status, response = _proxy_chat_status(POLICY_COMPAT, quantization=None)
assert status == 200
assert response["choices"][0]["message"]["content"] == "ok"
def test_a_legacy_registration_without_quantization_stays_dark_under_enforce():
"The precision fallback must not smuggle an absent capability report past `enforce`.\n\nTags: http, routing, tracker"
status, _ = _proxy_chat_status(POLICY_ENFORCE, quantization=None)
assert status == 503
def test_a_node_declaring_an_unsupported_quantization_is_never_routed():
"Silence is grandfathered; an explicit unsupported precision is not.\n\nTags: http, routing, tracker"
status, _ = _proxy_chat_status(POLICY_COMPAT, quantization="fp4-experimental")
assert status == 503
def test_a_node_declaring_a_null_quantization_is_never_routed():
"An explicit null states 'no usable precision' -- only an absent field is legacy.\n\nTags: http, routing, tracker"
node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(capability_policy=POLICY_COMPAT)
tracker_port = tracker.start()
try:
endpoint = f"http://127.0.0.1:{node.server_address[1]}"
registration = _legacy_registration(endpoint)
registration["quantization"] = None # declared, not omitted
_post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/register", registration)
with pytest.raises(urllib.error.HTTPError) as excinfo:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
{"model": "qwen2.5-0.5b",
"messages": [{"role": "user", "content": "hi"}]},
)
assert excinfo.value.code == 503
finally:
tracker.stop()
node.shutdown()
node.server_close()
node_thread.join(timeout=1.0)
def test_raft_apply_preserves_a_declared_precision_for_follower_routing():
"A follower that drops the declared precision would route unlike the leader.\n\nTags: raft, routing, tracker"
tracker = TrackerServer()
tracker.start()
try:
tracker._raft_apply("register", {
"node_id": "declared-node",
"endpoint": "http://127.0.0.1:1/",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"shard_start": 0,
"shard_end": 23,
"num_layers": 24,
"quantization": "nf4",
"quantizations": ["nf4", "int8"],
})
entry = tracker._registry["declared-node"]
assert entry.quantization == "nf4"
assert entry.quantizations == ["nf4", "int8"]
finally:
tracker.stop()
def test_raft_apply_grandfathers_a_legacy_registration_that_omits_quantization():
"The follower must reach the same legacy fallback the leader did, not a bare None.\n\nTags: raft, routing, tracker"
tracker = TrackerServer()
tracker.start()
try:
tracker._raft_apply("register", {
"node_id": "legacy-node",
"endpoint": "http://127.0.0.1:1/",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"shard_start": 0,
"shard_end": 23,
"num_layers": 24,
})
entry = tracker._registry["legacy-node"]
assert entry.quantization == "bfloat16"
finally:
tracker.stop()
def test_rebalancing_does_not_resurrect_a_node_that_declared_an_unusable_precision():
"Managed assignment picks a node's precision, so it must not overwrite an explicit unusable one.\n\nTags: http, routing, tracker"
tracker = TrackerServer(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker_port = tracker.start()
try:
managed = _post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://127.0.0.1:9111", "model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", "num_layers": 24,
"shard_start": 0, "shard_end": 21, "managed_assignment": True,
"quantization": "fp4-experimental",
"vram_bytes": 1_000_000_000, "hardware_profile": {}, "score": 1.0},
)
# Letting this peer expire is what drives the rebalance that used to hand
# the managed node a fresh shard range -- and a default precision with it.
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://127.0.0.1:9112", "model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct", "num_layers": 24,
"shard_start": 22, "shard_end": 23,
"vram_bytes": 1_000_000_000, "hardware_profile": {}, "score": 1.0},
)
time.sleep(0.10)
_post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/{managed['node_id']}/heartbeat", {})
time.sleep(0.10)
hb = _post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/{managed['node_id']}/heartbeat", {})
entry = tracker._registry[managed["node_id"]]
assert entry.quantization is None
assert not [d for d in hb.get("directives", []) if d["action"] == "LOAD_SHARD"]
finally:
tracker.stop()
def _register_shard(tracker_port: int, port: int, **overrides) -> dict:
payload = {
"endpoint": f"http://127.0.0.1:{port}",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"num_layers": 24,
"vram_bytes": 40_000_000_000,
"hardware_profile": {},
"score": 1.0,
}
payload.update(overrides)
return _post_json(f"http://127.0.0.1:{tracker_port}/v1/nodes/register", payload)
def test_a_node_with_an_unusable_precision_covers_no_layers():
"A node no request can reach must not report its shards as served coverage.\n\nTags: http, routing, tracker"
tracker = TrackerServer(rebalance_interval=10.0)
tracker_port = tracker.start()
try:
_register_shard(
tracker_port, 9121,
shard_start=0, shard_end=23, quantization="fp4-experimental",
)
models = _get_json(f"http://127.0.0.1:{tracker_port}/v1/models")
served = [m for m in models["data"] if m["id"] == "Qwen/Qwen2.5-0.5B-Instruct"]
assert [m["shard_coverage_percentage"] for m in served] == [0.0]
finally:
tracker.stop()
def test_rebalancing_fills_the_layers_an_unusable_node_only_appears_to_cover():
"Counting a dark node as coverage hides the gap, so the rebalancer never fills it.\n\nTags: http, routing, tracker"
tracker = TrackerServer(rebalance_interval=10.0)
tracker_port = tracker.start()
try:
# The dark node claims the model's tail; the managed node holds the head.
# Both ends look covered, so the rebalancer used to see nothing to do.
dark = _register_shard(
tracker_port, 9131,
shard_start=12, shard_end=23, quantization="fp4-experimental",
)
managed = _register_shard(
tracker_port, 9132,
shard_start=0, shard_end=11, managed_assignment=True,
)
with tracker._lock:
_rebalance_all_locked(tracker._server) # type: ignore[arg-type]
assert tracker._registry[dark["node_id"]].quantization is None
entry = tracker._registry[managed["node_id"]]
assert (entry.shard_start, entry.shard_end) == (0, 23)
finally:
tracker.stop()