popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

qol improvements

Browse Source
This commit is contained in:
Dobromir Popov
2026-06-30 19:27:46 +02:00
parent d8a723a4c7
commit f1e4870124
4 changed files with 317 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
QUICKSTART.md
View File

@@ -129,7 +129,6 @@ $env:HF_HOME = "D:\DEV\models"
One-line variants:
```powershell
// only this works - when not behind NAT(wsl), and via IP. Revisit when relaying/RPC is implemented
.\.venv\Scripts\meshnet-node.exe start --tracker http://192.168.0.179:8081 --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20
.\.venv\Scripts\meshnet-node.exe start --tracker http://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --advertise-host 192.168.0.20
```
@@ -186,6 +185,22 @@ Then a node only needs the public tracker address:
--model Qwen/Qwen2.5-0.5B-Instruct
```
No authentication is required in the prototype. The first public node for a model
must still choose that model with `--model` or a saved wizard config. After at
least one HF model node is registered, later nodes can join the public network
with only the tracker URL; the tracker assigns an uncovered shard if one exists:
```bash
.venv/bin/meshnet-node start --tracker https://ai.neuron.d-popov.com
```
Use the public tracker to verify registration and routing:
```bash
curl -s "https://ai.neuron.d-popov.com/v1/network/map" | python3 -m json.tool
curl -s "https://ai.neuron.d-popov.com/v1/route?model=qwen2.5-0.5b" | python3 -m json.tool
```
---
## Step 1 — Start the tracker (Terminal 1)
@@ -258,6 +273,9 @@ For gated models (Llama), run `huggingface-cli login` first.
## Step 3 — Send an inference request (Terminal 3)
If you started the node with `--port 8001`, send the request directly to that
head node:
```bash Qwen2.5-0.5B-Instruct
curl -s http://localhost:8001/v1/chat/completions \
-H "Content-Type: application/json" \
@@ -268,6 +286,22 @@ curl -s http://localhost:8001/v1/chat/completions \
}' | python3 -m json.tool
```
If you did not pass `--port`, `meshnet-node start` uses the first free port at
or above `7000`. Use the `Endpoint:` printed by the node instead of `8001`.
To test tracker routing/proxying, send the same OpenAI-compatible request to the
tracker, using either the full HuggingFace repo or the quick alias:
```bash
curl -s http://localhost:8080/v1/chat/completions \
-H "Content-Type: application/json" \
-d '{
"model": "qwen2.5-0.5b",
"messages": [{"role": "user", "content": "What is 7 times 8? Answer in one word."}],
"stream": false
}' | python3 -m json.tool
```
Or use the test script:
```bash
@@ -313,6 +347,12 @@ activations to Node B, and streams the final response back.
See `docs/TWO_MACHINE_TEST.md` (created by US-018).
For WSL2 nodes, registration only proves the node can reach the tracker
outbound. Tracker-routed inference also requires the tracker to reach the node's
advertised endpoint inbound. Either run the node in native Windows PowerShell,
configure Windows port forwarding into WSL for the node port, or start the
tracker with a relay URL so the node registers a `relay_addr`.
---
## Browse available models

31
packages/tracker/meshnet_tracker/server.py
View File

@@ -56,6 +56,27 @@ DEFAULT_QUANTIZATIONS = ["bfloat16"]
DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
def _model_aliases(model: str | None) -> set[str]:
"""Return stable lookup aliases for a model repo or display name."""
if not model:
return set()
aliases = {model}
short = model.rsplit("/", 1)[-1]
aliases.add(short)
lowered = short.lower()
aliases.add(lowered)
if lowered.endswith("-instruct"):
aliases.add(lowered.removesuffix("-instruct"))
return aliases
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
requested = _model_aliases(model)
if not requested:
return False
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
class _RollingCounter:
"""Circular-bucket request counter.
@@ -549,7 +570,7 @@ def _nodes_and_bounds_for_model(
nodes = [
node for node in server.registry.values()
if (node.hf_repo == model or node.model == model)
if _node_matches_model(node, model)
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None
@@ -1028,7 +1049,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._purge_expired_nodes()
candidates = [
n for n in server.registry.values()
if n.tracker_mode and (n.model == model or n.hf_repo == model)
if n.tracker_mode and _node_matches_model(n, model)
]
if not candidates:
@@ -1036,7 +1057,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
candidates = [
n for n in server.registry.values()
if n.shard_start == 0 and (n.model == model or n.hf_repo == model)
if n.shard_start == 0 and _node_matches_model(n, model)
]
if not candidates:
@@ -1064,7 +1085,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else:
all_nodes = [
n for n in server.registry.values()
if (n.hf_repo == route_model or n.model == route_model)
if _node_matches_model(n, route_model)
and n.shard_start is not None and n.num_layers is not None
]
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
@@ -1745,7 +1766,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# HF model routing: match by hf_repo (full) or model short name.
alive = [
node for node in server.registry.values()
if (node.hf_repo == model or node.model == model)
if _node_matches_model(node, model)
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None

165
tests/test_node_startup.py
View File

@@ -391,6 +391,171 @@ def test_real_model_startup_summary_shows_total_layers(tmp_path, monkeypatch, ca
assert "Node ID: node-test-123" in output
def test_public_tracker_model_node_registers_relay_metadata_from_tracker_url_only(
tmp_path,
monkeypatch,
):
"""A node only needs the public tracker URL to discover relay metadata and register."""
import meshnet_node.startup as startup_mod
class FakeBackend:
total_layers = 24
class FakeTorchNodeServer:
def __init__(self, **kwargs):
self.kwargs = kwargs
self.backend = FakeBackend()
self.port = None
self.chat_completion_count = 0
self.total_requests = 0
self.failed_requests = 0
self.queue_depth = 0
def start(self):
self.port = 8001
return self.port
def stop(self):
pass
class FakeRelayHttpBridge:
def __init__(self, relay_url, peer_id, local_base_url, advertised_addr):
self.relay_url = relay_url
self.peer_id = peer_id
self.local_base_url = local_base_url
self.advertised_addr = advertised_addr
@property
def relay_addr(self):
return f"ws://public-relay.example/rpc/{self.peer_id}"
def start(self):
return types.SimpleNamespace(
peer_id=self.peer_id,
relay_addr=self.relay_addr,
)
def wait_connected(self, timeout=5.0):
return True
def stop(self):
pass
monkeypatch.setattr(
startup_mod,
"detect_hardware",
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
)
monkeypatch.setattr(startup_mod, "_detect_num_layers", lambda _model_id: 24)
monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer)
monkeypatch.setattr(startup_mod, "RelayHttpBridge", FakeRelayHttpBridge)
tracker = TrackerServer(relay_url="ws://public-relay.example/ws")
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
try:
node = run_startup(
tracker_url=tracker_url,
model_id="Qwen/Qwen2.5-0.5B-Instruct",
advertise_host="203.0.113.10",
wallet_path=tmp_path / "wallet.json",
)
try:
network_map = _get_json(f"{tracker_url}/v1/network/map")
finally:
node.stop()
finally:
tracker.stop()
assert network_map["relay_url"] == "ws://public-relay.example/ws"
assert len(network_map["nodes"]) == 1
registered = network_map["nodes"][0]
assert registered["hf_repo"] == "Qwen/Qwen2.5-0.5B-Instruct"
assert registered["endpoint"] == "http://203.0.113.10:8001"
assert registered["relay_addr"].startswith("ws://public-relay.example/rpc/")
assert registered["peer_id"]
def test_later_node_auto_joins_existing_public_hf_model_with_only_tracker_url(
tmp_path,
monkeypatch,
):
"""After a model exists, a node can join by knowing only the public tracker URL."""
import meshnet_node.startup as startup_mod
captured = {}
class FakeBackend:
total_layers = 24
class FakeTorchNodeServer:
def __init__(self, **kwargs):
captured.update(kwargs)
self.backend = FakeBackend()
self.port = None
self.chat_completion_count = 0
self.total_requests = 0
self.failed_requests = 0
self.queue_depth = 0
def start(self):
self.port = 8002
return self.port
def stop(self):
pass
monkeypatch.setattr(
startup_mod,
"detect_hardware",
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
)
monkeypatch.setattr(startup_mod, "TorchNodeServer", FakeTorchNodeServer)
tracker = TrackerServer()
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
try:
data = json.dumps({
"endpoint": "http://203.0.113.20:8001",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"num_layers": 24,
"shard_start": 0,
"shard_end": 11,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0,
}).encode()
req = urllib.request.Request(
f"{tracker_url}/v1/nodes/register",
data=data,
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req) as resp:
resp.read()
node = run_startup(
tracker_url=tracker_url,
advertise_host="203.0.113.21",
wallet_path=tmp_path / "wallet.json",
)
try:
route_resp = _get_json(
f"{tracker_url}/v1/route?model=Qwen/Qwen2.5-0.5B-Instruct"
)
finally:
node.stop()
finally:
tracker.stop()
assert captured["model_id"] == "Qwen/Qwen2.5-0.5B-Instruct"
assert captured["shard_start"] == 12
assert captured["shard_end"] == 23
assert route_resp["route"] == ["http://203.0.113.20:8001", "http://203.0.113.21:8002"]
# ---------------------------------------------------------------------------
# Full startup integration test
# ---------------------------------------------------------------------------

84
tests/test_tracker_routing.py
View File

@@ -113,6 +113,90 @@ def test_tracker_serves_health_while_proxy_request_is_in_flight():
slow_thread.join(timeout=1.0)
def test_tracker_routes_hf_model_alias_from_quickstart():
"""The documented qwen2.5-0.5b alias resolves a full HF repo registration."""
tracker = TrackerServer()
tracker_port = tracker.start()
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://127.0.0.1:9100",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"num_layers": 24,
"shard_start": 0,
"shard_end": 23,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
route_resp = _get_json(
f"http://127.0.0.1:{tracker_port}/v1/route?model=qwen2.5-0.5b"
)
finally:
tracker.stop()
assert route_resp["route"] == ["http://127.0.0.1:9100"]
def test_tracker_proxy_accepts_hf_model_alias_from_quickstart():
"""The tracker proxy accepts the same model alias used by the quickstart curl."""
class ChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
pass
def do_POST(self):
if self.path != "/v1/chat/completions":
self.send_response(404)
self.end_headers()
return
length = int(self.headers.get("Content-Length", 0))
request_body = json.loads(self.rfile.read(length) or b"{}")
body = json.dumps({
"model": request_body["model"],
"choices": [{"message": {"content": "56"}}],
}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
node = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer()
tracker_port = tracker.start()
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
"model": "Qwen2.5-0.5B-Instruct",
"hf_repo": "Qwen/Qwen2.5-0.5B-Instruct",
"num_layers": 24,
"shard_start": 0,
"shard_end": 23,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
response = _post_json(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
{"model": "qwen2.5-0.5b",
"messages": [{"role": "user", "content": "What is 7 times 8?"}]},
)
finally:
tracker.stop()
node.shutdown()
node.server_close()
node_thread.join(timeout=1.0)
assert response["choices"][0]["message"]["content"] == "56"
def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix():
tracker = TrackerServer()
tracker_port = tracker.start()