routing
This commit is contained in:
@@ -829,6 +829,7 @@ def run_startup(
|
|||||||
shard_label = _format_shard_label(shard_start, shard_end, total_layers)
|
shard_label = _format_shard_label(shard_start, shard_end, total_layers)
|
||||||
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
||||||
endpoint = f"http://{public_host}:{actual_port}"
|
endpoint = f"http://{public_host}:{actual_port}"
|
||||||
|
node.set_advertised_endpoint(endpoint)
|
||||||
local_base_url = f"http://127.0.0.1:{actual_port}"
|
local_base_url = f"http://127.0.0.1:{actual_port}"
|
||||||
relay_bridge, relay_fields = _start_relay_bridge_if_available(
|
relay_bridge, relay_fields = _start_relay_bridge_if_available(
|
||||||
tracker_url,
|
tracker_url,
|
||||||
@@ -977,6 +978,7 @@ def run_startup(
|
|||||||
actual_port = node.start()
|
actual_port = node.start()
|
||||||
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
||||||
endpoint = f"http://{public_host}:{actual_port}"
|
endpoint = f"http://{public_host}:{actual_port}"
|
||||||
|
node.set_advertised_endpoint(endpoint)
|
||||||
local_base_url = f"http://127.0.0.1:{actual_port}"
|
local_base_url = f"http://127.0.0.1:{actual_port}"
|
||||||
relay_bridge, relay_fields = _start_relay_bridge_if_available(
|
relay_bridge, relay_fields = _start_relay_bridge_if_available(
|
||||||
tracker_url,
|
tracker_url,
|
||||||
@@ -1154,6 +1156,7 @@ def run_startup(
|
|||||||
shard_label = f"{shard_label} (pinned)"
|
shard_label = f"{shard_label} (pinned)"
|
||||||
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
||||||
endpoint = f"http://{public_host}:{actual_port}"
|
endpoint = f"http://{public_host}:{actual_port}"
|
||||||
|
node.set_advertised_endpoint(endpoint)
|
||||||
local_base_url = f"http://127.0.0.1:{actual_port}"
|
local_base_url = f"http://127.0.0.1:{actual_port}"
|
||||||
relay_bridge, relay_fields = _start_relay_bridge_if_available(
|
relay_bridge, relay_fields = _start_relay_bridge_if_available(
|
||||||
tracker_url,
|
tracker_url,
|
||||||
|
|||||||
@@ -31,6 +31,52 @@ from .server import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoint_key(url: str) -> str:
|
||||||
|
"""Normalize http(s) endpoints for host:port comparison."""
|
||||||
|
parsed = urllib.parse.urlparse(url.rstrip("/"))
|
||||||
|
host = (parsed.hostname or "").lower()
|
||||||
|
if not host:
|
||||||
|
return url.rstrip("/").lower()
|
||||||
|
port = parsed.port
|
||||||
|
if port is None:
|
||||||
|
port = 443 if parsed.scheme == "https" else 80
|
||||||
|
return f"{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
|
def _own_endpoint_key(server: _TorchHTTPServer) -> str:
|
||||||
|
advertised = getattr(server, "advertised_endpoint", None)
|
||||||
|
if advertised:
|
||||||
|
return _endpoint_key(advertised)
|
||||||
|
host, port = server.server_address
|
||||||
|
return _endpoint_key(f"http://{host}:{port}")
|
||||||
|
|
||||||
|
|
||||||
|
def _clamp_downstream_hops(
|
||||||
|
hops: list[dict],
|
||||||
|
backend: TorchModelShard | None,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Ensure downstream start_layer continues after this shard's layers."""
|
||||||
|
if not hops or backend is None:
|
||||||
|
return hops
|
||||||
|
shard_end = getattr(backend, "shard_end", None)
|
||||||
|
if shard_end is None:
|
||||||
|
return hops
|
||||||
|
min_start = int(shard_end) + 1
|
||||||
|
clamped: list[dict] = []
|
||||||
|
for hop in hops:
|
||||||
|
adjusted = dict(hop)
|
||||||
|
if int(adjusted.get("start_layer", 0)) < min_start:
|
||||||
|
adjusted["start_layer"] = min_start
|
||||||
|
clamped.append(adjusted)
|
||||||
|
return clamped
|
||||||
|
|
||||||
|
|
||||||
|
def _format_downstream_route(hops: list[dict]) -> str:
|
||||||
|
return ", ".join(
|
||||||
|
f"{h['endpoint']}@{h.get('start_layer', 0)}" for h in hops
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _write_progress_line(state: list[bool], message: str, *, final: bool = False) -> None:
|
def _write_progress_line(state: list[bool], message: str, *, final: bool = False) -> None:
|
||||||
"""Rewrite one in-place progress line (\\r) or finish with a newline."""
|
"""Rewrite one in-place progress line (\\r) or finish with a newline."""
|
||||||
if final:
|
if final:
|
||||||
@@ -104,6 +150,7 @@ class _TorchHTTPServer(http.server.HTTPServer):
|
|||||||
self.route_timeout = route_timeout
|
self.route_timeout = route_timeout
|
||||||
self.debug = debug
|
self.debug = debug
|
||||||
self.max_loaded_shards = max(1, max_loaded_shards)
|
self.max_loaded_shards = max(1, max_loaded_shards)
|
||||||
|
self.advertised_endpoint: str | None = None
|
||||||
self.total_requests: int = 0
|
self.total_requests: int = 0
|
||||||
self.failed_requests: int = 0
|
self.failed_requests: int = 0
|
||||||
self.queue_depth: int = 0
|
self.queue_depth: int = 0
|
||||||
@@ -572,6 +619,9 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
start_layer tells each downstream node which layer to begin from,
|
start_layer tells each downstream node which layer to begin from,
|
||||||
enabling correct execution when shard ranges overlap.
|
enabling correct execution when shard ranges overlap.
|
||||||
"""
|
"""
|
||||||
|
server: _TorchHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
active_backend = backend or server.backend
|
||||||
|
|
||||||
# Fast path: tracker pre-resolved the downstream route and injected it as a header.
|
# Fast path: tracker pre-resolved the downstream route and injected it as a header.
|
||||||
injected = self.headers.get("X-Meshnet-Route")
|
injected = self.headers.get("X-Meshnet-Route")
|
||||||
if injected:
|
if injected:
|
||||||
@@ -590,14 +640,16 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
hops.append(hop)
|
hops.append(hop)
|
||||||
elif isinstance(item, str):
|
elif isinstance(item, str):
|
||||||
hops.append({"endpoint": item, "start_layer": 0})
|
hops.append({"endpoint": item, "start_layer": 0})
|
||||||
print(f" [node] using injected downstream route: {[h['endpoint'] for h in hops]}", flush=True)
|
hops = _clamp_downstream_hops(hops, active_backend)
|
||||||
|
print(
|
||||||
|
f" [node] using injected downstream route: {_format_downstream_route(hops)}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
return hops
|
return hops
|
||||||
except (json.JSONDecodeError, TypeError, KeyError):
|
except (json.JSONDecodeError, TypeError, KeyError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Slow path: query the tracker (direct node-to-node calls, or tracker didn't inject).
|
# Slow path: query the tracker (direct node-to-node calls, or tracker didn't inject).
|
||||||
server: _TorchHTTPServer = self.server # type: ignore[assignment]
|
|
||||||
active_backend = backend or server.backend
|
|
||||||
if server.tracker_url is None:
|
if server.tracker_url is None:
|
||||||
return []
|
return []
|
||||||
route_model = getattr(active_backend, "model_id", None) or model
|
route_model = getattr(active_backend, "model_id", None) or model
|
||||||
@@ -605,23 +657,31 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
url = f"{server.tracker_url}/v1/route?model={urllib.parse.quote(route_model)}"
|
url = f"{server.tracker_url}/v1/route?model={urllib.parse.quote(route_model)}"
|
||||||
with urllib.request.urlopen(url, timeout=server.route_timeout) as r:
|
with urllib.request.urlopen(url, timeout=server.route_timeout) as r:
|
||||||
route_resp = json.loads(r.read())
|
route_resp = json.loads(r.read())
|
||||||
own_port = server.server_address[1]
|
own_key = _own_endpoint_key(server)
|
||||||
nodes_info = route_resp.get("nodes", [])
|
nodes_info = route_resp.get("nodes", [])
|
||||||
hops = []
|
hops: list[dict] = []
|
||||||
covered_up_to: int | None = None
|
passed_self = False
|
||||||
for node_info in nodes_info:
|
for node_info in nodes_info:
|
||||||
ep = node_info.get("endpoint", "")
|
ep = node_info.get("endpoint", "")
|
||||||
if ep.rstrip("/").endswith(f":{own_port}"):
|
if not ep:
|
||||||
covered_up_to = node_info.get("shard_end")
|
|
||||||
continue
|
continue
|
||||||
if covered_up_to is None:
|
if _endpoint_key(ep) == own_key:
|
||||||
covered_up_to = (node_info.get("shard_start") or 1) - 1
|
passed_self = True
|
||||||
hop = {"endpoint": ep, "start_layer": covered_up_to + 1}
|
continue
|
||||||
|
if not passed_self:
|
||||||
|
continue
|
||||||
|
hop = {
|
||||||
|
"endpoint": ep,
|
||||||
|
"start_layer": int(node_info.get("start_layer", 0)),
|
||||||
|
}
|
||||||
if node_info.get("relay_addr"):
|
if node_info.get("relay_addr"):
|
||||||
hop["relay_addr"] = str(node_info["relay_addr"])
|
hop["relay_addr"] = str(node_info["relay_addr"])
|
||||||
hops.append(hop)
|
hops.append(hop)
|
||||||
covered_up_to = node_info.get("shard_end", covered_up_to)
|
hops = _clamp_downstream_hops(hops, active_backend)
|
||||||
print(f" [node] tracker downstream route: {[h['endpoint'] for h in hops]}", flush=True)
|
print(
|
||||||
|
f" [node] tracker downstream route: {_format_downstream_route(hops)}",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
return hops
|
return hops
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
print(f" [node] WARNING: route lookup failed for {route_model!r}: {exc}", flush=True)
|
print(f" [node] WARNING: route lookup failed for {route_model!r}: {exc}", flush=True)
|
||||||
@@ -1039,6 +1099,11 @@ class TorchNodeServer:
|
|||||||
self._thread.start()
|
self._thread.start()
|
||||||
return self.port
|
return self.port
|
||||||
|
|
||||||
|
def set_advertised_endpoint(self, endpoint: str) -> None:
|
||||||
|
"""Set the LAN-facing endpoint used for route self-detection."""
|
||||||
|
if self._server is not None:
|
||||||
|
self._server.advertised_endpoint = endpoint
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
if self._server is None:
|
if self._server is None:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -293,14 +293,14 @@ function buildModelAliasMap(map) {
|
|||||||
if (!display) return;
|
if (!display) return;
|
||||||
for (const name of names) {
|
for (const name of names) {
|
||||||
const key = modelAliasKey(name);
|
const key = modelAliasKey(name);
|
||||||
if (key) byAlias.set(key, display);
|
if (key && !byAlias.has(key)) byAlias.set(key, display);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
for (const entry of (map && map.recommended_models) || []) {
|
for (const entry of (map && map.recommended_models) || []) {
|
||||||
register(entry.id, entry.id, entry.hf_repo, ...(entry.aliases || []));
|
register(entry.id, entry.id, entry.hf_repo, ...(entry.aliases || []));
|
||||||
}
|
}
|
||||||
for (const entry of availableModels || []) {
|
for (const entry of availableModels || []) {
|
||||||
register(entry.name || entry.id, entry.id, entry.name, ...(entry.aliases || []));
|
register(entry.id, entry.id, entry.name, entry.hf_repo, ...(entry.aliases || []));
|
||||||
}
|
}
|
||||||
return byAlias;
|
return byAlias;
|
||||||
}
|
}
|
||||||
@@ -311,7 +311,9 @@ function resolveModelGroup(node, aliasMap) {
|
|||||||
const hit = aliasMap.get(modelAliasKey(candidate));
|
const hit = aliasMap.get(modelAliasKey(candidate));
|
||||||
if (hit) return hit;
|
if (hit) return hit;
|
||||||
}
|
}
|
||||||
return node.hf_repo || node.model || "?";
|
const raw = node.hf_repo || node.model || "?";
|
||||||
|
const key = modelAliasKey(raw);
|
||||||
|
return aliasMap.get(key) || key || "?";
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchJson(path) {
|
async function fetchJson(path) {
|
||||||
@@ -1575,6 +1577,7 @@ async function refresh() {
|
|||||||
availableModels = ((models && models.data) || []).map(model => ({
|
availableModels = ((models && models.data) || []).map(model => ({
|
||||||
id: model.id,
|
id: model.id,
|
||||||
name: model.name || model.id,
|
name: model.name || model.id,
|
||||||
|
hf_repo: model.hf_repo,
|
||||||
recommended: Boolean(model.recommended),
|
recommended: Boolean(model.recommended),
|
||||||
aliases: model.aliases || [],
|
aliases: model.aliases || [],
|
||||||
})).filter(model => model.id);
|
})).filter(model => model.id);
|
||||||
|
|||||||
@@ -159,6 +159,18 @@ DEFAULT_CALLER_CREDIT_USDT = 1.0
|
|||||||
DEFAULT_DEVNET_TOPUP_USDT = 1.0
|
DEFAULT_DEVNET_TOPUP_USDT = 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoint_key(url: str) -> str:
|
||||||
|
"""Normalize http(s) endpoints for host:port comparison."""
|
||||||
|
parsed = urllib.parse.urlparse(url.rstrip("/"))
|
||||||
|
host = (parsed.hostname or "").lower()
|
||||||
|
if not host:
|
||||||
|
return url.rstrip("/").lower()
|
||||||
|
port = parsed.port
|
||||||
|
if port is None:
|
||||||
|
port = 443 if parsed.scheme == "https" else 80
|
||||||
|
return f"{host}:{port}"
|
||||||
|
|
||||||
|
|
||||||
def _model_aliases(model: str | None) -> set[str]:
|
def _model_aliases(model: str | None) -> set[str]:
|
||||||
"""Return stable lookup aliases for a model repo or display name."""
|
"""Return stable lookup aliases for a model repo or display name."""
|
||||||
if not model:
|
if not model:
|
||||||
@@ -3168,7 +3180,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
node_work.append((rn.wallet_address, max(0, effective_end - covered_up_to)))
|
node_work.append((rn.wallet_address, max(0, effective_end - covered_up_to)))
|
||||||
covered_up_to = effective_end
|
covered_up_to = effective_end
|
||||||
# Strip the first-shard node we're about to proxy to — it's already handling the request.
|
# Strip the first-shard node we're about to proxy to — it's already handling the request.
|
||||||
downstream_hops = [h for h in route_hops if h["endpoint"].rstrip("/") != node.endpoint.rstrip("/")]
|
downstream_hops = [
|
||||||
|
h for h in route_hops
|
||||||
|
if _endpoint_key(h["endpoint"]) != _endpoint_key(node.endpoint)
|
||||||
|
]
|
||||||
downstream_urls = json.dumps(downstream_hops)
|
downstream_urls = json.dumps(downstream_hops)
|
||||||
route_debug = " -> ".join(
|
route_debug = " -> ".join(
|
||||||
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
|
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
|
||||||
@@ -5447,7 +5462,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||||||
]
|
]
|
||||||
|
|
||||||
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
|
route_model = resolved_name or model
|
||||||
|
candidates = _enumerate_routes(
|
||||||
|
alive,
|
||||||
|
required_start,
|
||||||
|
required_end,
|
||||||
|
model=route_model,
|
||||||
|
contracts=server.contracts,
|
||||||
|
max_candidates=server.route_stats.config.max_candidate_routes,
|
||||||
|
)
|
||||||
|
if candidates:
|
||||||
|
# Prefer a distributed multi-hop route when available. Greedy
|
||||||
|
# _select_route alone would pick a single full-shard node and omit
|
||||||
|
# partial head shards that share the same port on another host.
|
||||||
|
route = max(candidates, key=lambda cand: len(cand.nodes)).nodes
|
||||||
|
error = ""
|
||||||
|
else:
|
||||||
|
route, error = _select_route(
|
||||||
|
alive, required_start, required_end, contracts=server.contracts,
|
||||||
|
)
|
||||||
if error:
|
if error:
|
||||||
_tracker_log(
|
_tracker_log(
|
||||||
server,
|
server,
|
||||||
|
|||||||
@@ -28,9 +28,10 @@ def test_dashboard_served_with_all_panels():
|
|||||||
).read().decode()
|
).read().decode()
|
||||||
for panel in PANELS:
|
for panel in PANELS:
|
||||||
assert panel in html
|
assert panel in html
|
||||||
assert "<script>" in html # polling client embedded, no build step
|
assert "<script>" in html # polling client embedded, no build step
|
||||||
assert "resolveModelGroup" in html
|
assert "resolveModelGroup" in html
|
||||||
assert "buildModelAliasMap" in html
|
assert "buildModelAliasMap" in html
|
||||||
|
assert "modelAliasKey(raw)" in html
|
||||||
finally:
|
finally:
|
||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|||||||
@@ -288,3 +288,19 @@ def test_proxy_head_is_route_head_and_routing_endpoint_lists_routes():
|
|||||||
assert len(table["routes"]) == 2
|
assert len(table["routes"]) == 2
|
||||||
sampled = [r for r in table["routes"] if r["samples"] > 0]
|
sampled = [r for r in table["routes"] if r["samples"] > 0]
|
||||||
assert sampled, "completed requests must produce route samples"
|
assert sampled, "completed requests must produce route samples"
|
||||||
|
|
||||||
|
|
||||||
|
def test_endpoint_key_distinguishes_same_port_different_hosts():
|
||||||
|
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
|
||||||
|
|
||||||
|
assert _endpoint_key("http://192.168.0.20:7000") == "192.168.0.20:7000"
|
||||||
|
assert _endpoint_key("http://192.168.0.179:7000") == "192.168.0.179:7000"
|
||||||
|
assert _endpoint_key("http://192.168.0.20:7000") != _endpoint_key("http://192.168.0.179:7000")
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
shard_end = 21
|
||||||
|
|
||||||
|
hops = [{"endpoint": "http://192.168.0.179:7000", "start_layer": 0}]
|
||||||
|
assert _clamp_downstream_hops(hops, Backend()) == [
|
||||||
|
{"endpoint": "http://192.168.0.179:7000", "start_layer": 22},
|
||||||
|
]
|
||||||
|
|||||||
@@ -910,6 +910,57 @@ def test_tracker_route_endpoint_ignores_model_case_and_outer_whitespace():
|
|||||||
assert response["route"] == ["http://127.0.0.1:9101"]
|
assert response["route"] == ["http://127.0.0.1:9101"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_route_prefers_distributed_over_single_full_shard():
|
||||||
|
"""When a full 0-39 node and a partial 0-21 head coexist, /v1/route
|
||||||
|
should return both hops — not the full shard alone."""
|
||||||
|
tracker = TrackerServer(model_presets={
|
||||||
|
"qwen3.6-35b-a3b": {
|
||||||
|
"layers_start": 0,
|
||||||
|
"layers_end": 39,
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"aliases": ["Qwen3.6-35B-A3B"],
|
||||||
|
}
|
||||||
|
})
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": "http://192.168.0.179:7000",
|
||||||
|
"model": "qwen3.6-35b-a3b",
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"num_layers": 40,
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 39,
|
||||||
|
"tracker_mode": True,
|
||||||
|
"hardware_profile": {},
|
||||||
|
"score": 1.0},
|
||||||
|
)
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": "http://192.168.0.20:7000",
|
||||||
|
"model": "Qwen3.6-35B-A3B",
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"num_layers": 40,
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 21,
|
||||||
|
"tracker_mode": True,
|
||||||
|
"hardware_profile": {},
|
||||||
|
"score": 1.0},
|
||||||
|
)
|
||||||
|
|
||||||
|
response = _get_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/route?model=qwen3.6-35b-a3b"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert response["route"] == [
|
||||||
|
"http://192.168.0.20:7000",
|
||||||
|
"http://192.168.0.179:7000",
|
||||||
|
]
|
||||||
|
assert [node["start_layer"] for node in response["nodes"]] == [0, 22]
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_proxy_ignores_model_case_and_outer_whitespace():
|
def test_tracker_proxy_ignores_model_case_and_outer_whitespace():
|
||||||
class ChatHandler(http.server.BaseHTTPRequestHandler):
|
class ChatHandler(http.server.BaseHTTPRequestHandler):
|
||||||
def log_message(self, fmt, *args):
|
def log_message(self, fmt, *args):
|
||||||
|
|||||||
Reference in New Issue
Block a user