2 Commits

Author SHA1 Message Date
Dobromir Popov
5cdce1a5b0 Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai 2026-07-07 14:56:08 +02:00
Dobromir Popov
80bd83ae0a routing and dashboard fixes 2026-07-07 14:53:37 +02:00
5 changed files with 387 additions and 96 deletions

View File

@@ -42,3 +42,4 @@ Historical handoff note: `/mnt/c/Users/popov/Downloads/neuron-tai-alpha-handoff-
- Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures. - Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures.
- Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`. - Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`.
- Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events. - Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events.
- Qwen3.6-35B-A3B reserve-based split is expected: an 79 GB CPU node may be assigned layers 0-36, and a second node fills 37-39. Do not "fix" this by bypassing the 20% assignment reserve unless the shard-planning policy changes.

View File

@@ -193,22 +193,33 @@ def _configure_torch_threads(
return active return active
def _max_assignable_layers(memory_mb: int, total_layers: int | None) -> int: def _max_assignable_layers(
memory_mb: int,
total_layers: int | None,
bytes_per_layer: int | None = None,
) -> int:
if total_layers is None or total_layers <= 0 or memory_mb <= 0: if total_layers is None or total_layers <= 0 or memory_mb <= 0:
return 0 return 0
budget_bytes = memory_mb * 1024 * 1024 budget_bytes = memory_mb * 1024 * 1024
return min(total_layers, int((budget_bytes * 0.8) // _DEFAULT_BYTES_PER_LAYER)) layer_bytes = bytes_per_layer or _DEFAULT_BYTES_PER_LAYER
return min(total_layers, int((budget_bytes * 0.8) // layer_bytes))
def _shard_budget_line(memory_mb: int, memory_source: str, total_layers: int | None, quantization: str) -> str: def _shard_budget_line(
memory_mb: int,
memory_source: str,
total_layers: int | None,
quantization: str,
bytes_per_layer: int | None = None,
) -> str:
memory_gb = memory_mb / 1024 memory_gb = memory_mb / 1024
gb_str = f"{memory_gb:.1f} GB" gb_str = f"{memory_gb:.1f} GB"
budget_quantization = "bfloat16" if quantization == "auto" else quantization budget_quantization = "bfloat16" if quantization == "auto" else quantization
if total_layers is None or total_layers <= 0: if total_layers is None or total_layers <= 0:
return f"Memory budget: {gb_str} {memory_source}; shard budget: unknown model layer count" return f"Memory budget: {gb_str} {memory_source}; shard budget: unknown model layer count"
max_layers = _max_assignable_layers(memory_mb, total_layers) max_layers = _max_assignable_layers(memory_mb, total_layers, bytes_per_layer=bytes_per_layer)
# Remaining capacity after one full model load (rough estimate) # Remaining capacity after one full model load (rough estimate)
shard_bytes = max_layers * _DEFAULT_BYTES_PER_LAYER shard_bytes = max_layers * (bytes_per_layer or _DEFAULT_BYTES_PER_LAYER)
remaining_gb = (memory_mb * 1024 * 1024 - shard_bytes) / (1024 ** 3) remaining_gb = (memory_mb * 1024 * 1024 - shard_bytes) / (1024 ** 3)
remaining_str = f"; {remaining_gb:.1f} GB remaining after full load" if remaining_gb > 1 else "" remaining_str = f"; {remaining_gb:.1f} GB remaining after full load" if remaining_gb > 1 else ""
return ( return (
@@ -218,6 +229,23 @@ def _shard_budget_line(memory_mb: int, memory_source: str, total_layers: int | N
) )
def _assignment_bytes_per_layer(assignment: dict, quantization: str) -> int | None:
bytes_per_layer = assignment.get("bytes_per_layer")
if isinstance(bytes_per_layer, int) and bytes_per_layer > 0:
return bytes_per_layer
if not isinstance(bytes_per_layer, dict):
return None
keys = [quantization, "bfloat16", "bf16", "int8", "nf4"]
for key in keys:
value = bytes_per_layer.get(key)
if isinstance(value, int) and value > 0:
return value
for value in bytes_per_layer.values():
if isinstance(value, int) and value > 0:
return value
return None
def _post_json(url: str, payload: dict, timeout: float = 10.0) -> dict: def _post_json(url: str, payload: dict, timeout: float = 10.0) -> dict:
data = json.dumps(payload).encode() data = json.dumps(payload).encode()
req = urllib.request.Request( req = urllib.request.Request(
@@ -869,6 +897,7 @@ def run_startup(
hf_repo: str | None = assignment.get("hf_repo") hf_repo: str | None = assignment.get("hf_repo")
peers: list[dict] = assignment.get("peers", []) peers: list[dict] = assignment.get("peers", [])
model_sources: list[dict] = [] if tracker_source_disabled else assignment.get("model_sources", []) model_sources: list[dict] = [] if tracker_source_disabled else assignment.get("model_sources", [])
assignment_bytes_per_layer = _assignment_bytes_per_layer(assignment, quantization)
print(f" Shard: layers {shard_start}-{shard_end} of {assigned_model}", flush=True) print(f" Shard: layers {shard_start}-{shard_end} of {assigned_model}", flush=True)
# 4. Download shard # 4. Download shard
@@ -955,7 +984,7 @@ def run_startup(
f"meshnet-node ready\n" f"meshnet-node ready\n"
f" Wallet: {address}\n" f" Wallet: {address}\n"
f" Shard: layers {shard_start}-{shard_end} ({assigned_model})\n" f" Shard: layers {shard_start}-{shard_end} ({assigned_model})\n"
f" {_shard_budget_line(memory_budget_mb, memory_budget_source, assignment.get('model_layers_end', shard_end) + 1, quantization)}\n" f" {_shard_budget_line(memory_budget_mb, memory_budget_source, assignment.get('model_layers_end', shard_end) + 1, quantization, bytes_per_layer=assignment_bytes_per_layer)}\n"
f" Endpoint: {endpoint}\n" f" Endpoint: {endpoint}\n"
f" Node ID: {node_id}\n" f" Node ID: {node_id}\n"
f" Hardware: {hw_str}\n" f" Hardware: {hw_str}\n"

View File

@@ -75,6 +75,7 @@
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section> <section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section> <section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section> <section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
<section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section>
</main> </main>
<script> <script>
"use strict"; "use strict";
@@ -209,9 +210,9 @@ function renderStats(stats) {
} }
function renderThroughput(stats) { function renderThroughput(stats) {
const nodes = (stats && stats.nodes) || {}; const nodes = (stats && stats.nodes) || {};
const rows = []; const rows = [];
for (const [nodeId, nodeStats] of Object.entries(nodes)) { for (const [nodeId, nodeStats] of Object.entries(nodes)) {
for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) { for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) {
rows.push([ rows.push([
esc(short(nodeId)), esc(short(nodeId)),
@@ -220,10 +221,52 @@ function renderThroughput(stats) {
`<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`, `<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`,
]); ]);
} }
} }
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows); $("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
} }
function renderInferenceHistory(data) {
const events = (data && data.events) || [];
const started = new Map();
const completed = [];
for (const e of events) {
const f = e.fields || {};
const id = f.request_id;
if (!id) continue;
if (e.message === "proxy route selected") {
started.set(id, e);
} else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") {
completed.push(e);
started.delete(id);
}
}
const activeByModel = {};
for (const e of started.values()) {
const f = e.fields || {};
const model = f.model || f.route_model || "?";
activeByModel[model] = (activeByModel[model] || 0) + 1;
}
const active = Object.entries(activeByModel)
.map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`)
.join(" &middot; ");
const rows = completed.slice(-20).reverse().map(e => {
const f = e.fields || {};
return [
new Date((e.ts || 0) * 1000).toLocaleTimeString(),
esc(short(f.model || f.route_model || "?", 28)),
esc(short(f.request_id || "?", 18)),
`<span class="num">${esc(tps(f.tokens_per_sec))}</span>`,
`<span class="num">${esc(String(f.tokens ?? "?"))}</span>`,
`<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`,
f.stream ? "stream" : "json",
];
});
$("inference-history").innerHTML =
`<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` +
(rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows)
: '<div class="empty">no completed inference requests</div>');
}
function renderConsole(data) { function renderConsole(data) {
const events = (data && data.events) || []; const events = (data && data.events) || [];
if (!events.length) { if (!events.length) {
@@ -406,9 +449,10 @@ async function refresh() {
renderNodes(map); renderNodes(map);
renderBilling(summary); renderBilling(summary);
renderSettlements(settlements); renderSettlements(settlements);
renderFraud(wallets, summary); renderFraud(wallets, summary);
renderStats(stats); renderStats(stats);
renderThroughput(stats); renderThroughput(stats);
renderInferenceHistory(consoleData);
renderConsole(consoleData); renderConsole(consoleData);
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString(); $("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
} }

View File

@@ -636,7 +636,7 @@ def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
return 0.5 + 0.5 * reputation return 0.5 + 0.5 * reputation
def _select_route( def _select_route(
nodes: list[_NodeEntry], nodes: list[_NodeEntry],
required_start: int, required_start: int,
required_end: int, required_end: int,
@@ -678,10 +678,25 @@ def _select_route(
covered_up_to = best.shard_end covered_up_to = best.shard_end
candidates = [n for n in candidates if n is not best] candidates = [n for n in candidates if n is not best]
return route, "" return route, ""
def _coverage_percentage( def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
return [
{
"node_id": node.node_id,
"model": node.model,
"hf_repo": node.hf_repo,
"endpoint": node.endpoint,
"shard": f"{node.shard_start}-{node.shard_end}",
"num_layers": node.num_layers,
"queue_depth": node.queue_depth,
}
for node in nodes
]
def _coverage_percentage(
nodes: list[_NodeEntry], nodes: list[_NodeEntry],
required_start: int, required_start: int,
required_end: int, required_end: int,
@@ -746,14 +761,14 @@ def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
return DEFAULT_RAM_BYTES, "ram-default" return DEFAULT_RAM_BYTES, "ram-default"
def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int: def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
bytes_per_layer = _preset_bytes_per_layer(preset) bytes_per_layer = _preset_bytes_per_layer(preset)
quantization = _node_quantization(node, preset) quantization = _node_quantization(node, preset)
layer_bytes = bytes_per_layer[quantization] layer_bytes = bytes_per_layer[quantization]
if layer_bytes <= 0: if layer_bytes <= 0:
return 0 return 0
memory_budget_bytes, _ = _node_memory_budget_bytes(node) memory_budget_bytes, _ = _node_memory_budget_bytes(node)
return int((memory_budget_bytes * 0.8) // layer_bytes) return int((memory_budget_bytes * 0.8) // layer_bytes)
def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict: def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
@@ -817,16 +832,17 @@ def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
} }
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int: def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
if total_layers <= 0: if total_layers <= 0:
return 0 return 0
if memory_mb <= 0: if memory_mb <= 0:
return max(1, total_layers // 2) return max(1, total_layers // 2)
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024 memory_bytes = memory_mb * 1024 * 1024
return min( bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
total_layers, return min(
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)), total_layers,
) max(1, int((memory_bytes * 0.8) // bytes_per_layer)),
)
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict: def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
@@ -2330,17 +2346,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Pre-resolve the downstream route so the first-shard node skips its own # Pre-resolve the downstream route so the first-shard node skips its own
# tracker query. We already hold the full registry picture — no need for # tracker query. We already hold the full registry picture — no need for
# a second round-trip. # a second round-trip.
route_model = node.hf_repo or node.model or model route_model = node.hf_repo or node.model or model
with server.lock: with server.lock:
if route_model in server.model_presets: resolved_route_model, route_preset = _resolve_model_preset(server.model_presets, route_model)
preset = server.model_presets[route_model] if route_preset is not None:
rs, re = _preset_layer_bounds(preset) route_model = resolved_route_model or route_model
all_nodes: list = [n for n in server.registry.values() if n.model == route_model] preset = route_preset
else: rs, re = _preset_layer_bounds(preset)
all_nodes = [ all_nodes: list = [
n for n in server.registry.values() n for n in server.registry.values()
if _node_matches_model(n, route_model) if _node_matches_preset(n, route_model, preset)
and n.shard_start is not None and n.num_layers is not None and n.shard_start is not None
and n.shard_end is not None
]
else:
all_nodes = [
n for n in server.registry.values()
if _node_matches_model(n, route_model)
and n.shard_start is not None and n.num_layers is not None
] ]
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1) rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
if pinned_nodes is not None: if pinned_nodes is not None:
@@ -2356,6 +2379,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
route_model=route_model, route_model=route_model,
error=route_error, error=route_error,
candidate_count=len(all_nodes), candidate_count=len(all_nodes),
candidates=_node_route_summary(all_nodes),
) )
self._send_json(503, {"error": { self._send_json(503, {"error": {
"message": route_error, "message": route_error,
@@ -2393,6 +2417,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
head_endpoint=node.endpoint, head_endpoint=node.endpoint,
downstream=downstream_urls, downstream=downstream_urls,
route=route_debug or "<empty>", route=route_debug or "<empty>",
nodes=_node_route_summary(route_nodes),
) )
req = urllib.request.Request( req = urllib.request.Request(
@@ -2450,12 +2475,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body) in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body)
except (json.JSONDecodeError, TypeError): except (json.JSONDecodeError, TypeError):
in_tokens, out_tokens = 0, 0 in_tokens, out_tokens = 0, 0
tokens = in_tokens + out_tokens tokens = in_tokens + out_tokens
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes) self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
self._bill_completed( _tracker_log(
api_key, model, tokens, node_work, server,
input_tokens=in_tokens, output_tokens=out_tokens, "info",
) "proxy complete",
request_id=request_id,
model=model,
route_model=route_model,
status=int(first.get("status", 503)),
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
return return
_tracker_log( _tracker_log(
server, server,
@@ -2539,35 +2577,57 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
in_tokens, out_tokens = _stream_billable_split( in_tokens, out_tokens = _stream_billable_split(
observed_stream_tokens, stream_usage, body observed_stream_tokens, stream_usage, body
) )
self._record_observed_throughput( self._record_observed_throughput(
model, route_model, in_tokens + out_tokens, elapsed, route_nodes model, route_model, in_tokens + out_tokens, elapsed, route_nodes
) )
self._bill_completed( tokens = in_tokens + out_tokens
api_key, model, in_tokens + out_tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
else:
# Non-streaming: buffer and relay
resp_body = upstream.read()
elapsed = time.monotonic() - started
_tracker_log( _tracker_log(
server, server,
"info", "info",
"proxy complete", "proxy complete",
request_id=request_id, request_id=request_id,
target_url=target_url, model=model,
bytes=len(resp_body), route_model=route_model,
status=200,
stream=True,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
) )
observed_output = "" self._bill_completed(
try: api_key, model, tokens, node_work,
response_payload = json.loads(resp_body) input_tokens=in_tokens, output_tokens=out_tokens,
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body) )
observed_output = _observed_output_from_non_stream_payload(response_payload) else:
except json.JSONDecodeError: # Non-streaming: buffer and relay
in_tokens, out_tokens = 0, 0 resp_body = upstream.read()
tokens = in_tokens + out_tokens elapsed = time.monotonic() - started
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes) observed_output = ""
self._record_validation_event(request_id, model, body, observed_output, route_nodes) try:
response_payload = json.loads(resp_body)
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
observed_output = _observed_output_from_non_stream_payload(response_payload)
except json.JSONDecodeError:
in_tokens, out_tokens = 0, 0
tokens = in_tokens + out_tokens
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
_tracker_log(
server,
"info",
"proxy complete",
request_id=request_id,
model=model,
route_model=route_model,
target_url=target_url,
status=200,
bytes=len(resp_body),
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
self._bill_completed( self._bill_completed(
api_key, model, tokens, node_work, api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens, input_tokens=in_tokens, output_tokens=out_tokens,
@@ -2740,13 +2800,30 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
in_tokens, out_tokens = _stream_billable_split( in_tokens, out_tokens = _stream_billable_split(
observed_stream_tokens, stream_usage, request_body observed_stream_tokens, stream_usage, request_body
) )
self._record_observed_throughput( self._record_observed_throughput(
model, route_model, in_tokens + out_tokens, elapsed, route_nodes model, route_model, in_tokens + out_tokens, elapsed, route_nodes
) )
self._bill_completed( tokens = in_tokens + out_tokens
api_key, model, in_tokens + out_tokens, node_work, server: _TrackerHTTPServer = self.server # type: ignore[assignment]
input_tokens=in_tokens, output_tokens=out_tokens, _tracker_log(
) server,
"info",
"proxy complete",
request_id=session_id,
model=model,
route_model=route_model,
status=200,
stream=True,
relay=True,
tokens=tokens,
elapsed_seconds=round(elapsed, 4),
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
route=_node_route_summary(route_nodes),
)
self._bill_completed(
api_key, model, tokens, node_work,
input_tokens=in_tokens, output_tokens=out_tokens,
)
def _send_relayed_response(self, response: dict) -> None: def _send_relayed_response(self, response: dict) -> None:
status = int(response.get("status", 503)) status = int(response.get("status", 503))
@@ -4140,11 +4217,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"shard_end": shard_end, "shard_end": shard_end,
"num_layers": total_l, "num_layers": total_l,
"gap_found": True, "gap_found": True,
"price_per_token": 0.0, "price_per_token": 0.0,
"deployment": _deployment_summary(pool_with_caller, preset), "bytes_per_layer": _preset_bytes_per_layer(preset),
"model_sources": self._model_sources( "deployment": _deployment_summary(pool_with_caller, preset),
str(resolved_name), "model_sources": self._model_sources(
preset, str(resolved_name),
preset,
shard_start, shard_start,
shard_end, shard_end,
), ),
@@ -4236,12 +4314,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"shard_start": shard_start, "shard_start": shard_start,
"shard_end": shard_end, "shard_end": shard_end,
"num_layers": total_l, "num_layers": total_l,
"gap_found": gap_found, "gap_found": gap_found,
"price_per_token": 0.0, "price_per_token": 0.0,
"model_sources": self._model_sources( "bytes_per_layer": _preset_bytes_per_layer(best_preset) if best_preset is not None else None,
str(resolved_name or best_repo), "model_sources": self._model_sources(
best_preset or {"hf_repo": best_repo}, str(resolved_name or best_repo),
shard_start, best_preset or {"hf_repo": best_repo},
shard_start,
shard_end, shard_end,
), ),
}) })
@@ -4287,10 +4366,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
] ]
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts) route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
if error: if error:
self._send_json(503, {"error": error}) _tracker_log(
return server,
"warn",
"route unavailable",
model=model,
route_model=resolved_name or model,
error=error,
candidate_count=len(alive),
candidates=_node_route_summary(alive),
)
self._send_json(503, {"error": error})
return
covered_up_to = required_start - 1 covered_up_to = required_start - 1
route_with_start: list[tuple] = [] route_with_start: list[tuple] = []

View File

@@ -355,6 +355,134 @@ def test_tracker_proxy_accepts_hf_model_alias_from_quickstart():
assert response["choices"][0]["message"]["content"] == "56" assert response["choices"][0]["message"]["content"] == "56"
def test_tracker_proxy_routes_split_preset_nodes_by_alias():
"""The chat proxy must build routes from alias-matched preset nodes."""
class ChatHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args):
pass
def do_POST(self):
if self.path != "/v1/chat/completions":
self.send_response(404)
self.end_headers()
return
length = int(self.headers.get("Content-Length", 0))
self.rfile.read(length)
route_header = self.headers.get("X-Meshnet-Route", "[]")
body = json.dumps({
"choices": [{"message": {"content": route_header}}],
"usage": {"prompt_tokens": 1, "completion_tokens": 1},
}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
head = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
head_thread = threading.Thread(target=head.serve_forever, daemon=True)
head_thread.start()
tracker = TrackerServer(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"aliases": ["Qwen3.6-35B-A3B"],
}
})
tracker_port = tracker.start()
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": f"http://127.0.0.1:{head.server_address[1]}",
"model": "qwen3.6-35b-a3b",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 0,
"shard_end": 21,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://127.0.0.1:9199",
"model": "Qwen3.6-35B-A3B",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 22,
"shard_end": 39,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
response = _post_json(
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
{"model": "Qwen3.6-35B-A3B",
"messages": [{"role": "user", "content": "hi"}]},
)
console = _get_json(f"http://127.0.0.1:{tracker_port}/v1/console")
finally:
tracker.stop()
head.shutdown()
head.server_close()
head_thread.join(timeout=1.0)
route = json.loads(response["choices"][0]["message"]["content"])
assert route == [{"endpoint": "http://127.0.0.1:9199", "start_layer": 22}]
complete = [event for event in console["events"] if event["message"] == "proxy complete"][-1]
assert complete["fields"]["tokens"] == 2
assert complete["fields"]["tokens_per_sec"] >= 0
def test_tracker_route_endpoint_routes_split_preset_nodes_by_alias():
tracker = TrackerServer(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"aliases": ["Qwen3.6-35B-A3B"],
}
})
tracker_port = tracker.start()
try:
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://127.0.0.1:9101",
"model": "qwen3.6-35b-a3b",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 0,
"shard_end": 21,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
_post_json(
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
{"endpoint": "http://127.0.0.1:9102",
"model": "Qwen3.6-35B-A3B",
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
"num_layers": 40,
"shard_start": 22,
"shard_end": 39,
"tracker_mode": True,
"hardware_profile": {},
"score": 1.0},
)
response = _get_json(
f"http://127.0.0.1:{tracker_port}/v1/route?model=Qwen3.6-35B-A3B"
)
finally:
tracker.stop()
assert response["route"] == ["http://127.0.0.1:9101", "http://127.0.0.1:9102"]
assert [node["start_layer"] for node in response["nodes"]] == [0, 22]
def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix(): def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix():
tracker = TrackerServer() tracker = TrackerServer()
tracker_port = tracker.start() tracker_port = tracker.start()