Merge branch 'master' of https://git.d-popov.com/popov/neuron-tai
This commit is contained in:
@@ -42,3 +42,4 @@ Historical handoff note: `/mnt/c/Users/popov/Downloads/neuron-tai-alpha-handoff-
|
|||||||
- Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures.
|
- Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures.
|
||||||
- Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`.
|
- Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`.
|
||||||
- Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events.
|
- Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events.
|
||||||
|
- Qwen3.6-35B-A3B reserve-based split is expected: an 79 GB CPU node may be assigned layers 0-36, and a second node fills 37-39. Do not "fix" this by bypassing the 20% assignment reserve unless the shard-planning policy changes.
|
||||||
|
|||||||
@@ -193,22 +193,33 @@ def _configure_torch_threads(
|
|||||||
return active
|
return active
|
||||||
|
|
||||||
|
|
||||||
def _max_assignable_layers(memory_mb: int, total_layers: int | None) -> int:
|
def _max_assignable_layers(
|
||||||
|
memory_mb: int,
|
||||||
|
total_layers: int | None,
|
||||||
|
bytes_per_layer: int | None = None,
|
||||||
|
) -> int:
|
||||||
if total_layers is None or total_layers <= 0 or memory_mb <= 0:
|
if total_layers is None or total_layers <= 0 or memory_mb <= 0:
|
||||||
return 0
|
return 0
|
||||||
budget_bytes = memory_mb * 1024 * 1024
|
budget_bytes = memory_mb * 1024 * 1024
|
||||||
return min(total_layers, int((budget_bytes * 0.8) // _DEFAULT_BYTES_PER_LAYER))
|
layer_bytes = bytes_per_layer or _DEFAULT_BYTES_PER_LAYER
|
||||||
|
return min(total_layers, int((budget_bytes * 0.8) // layer_bytes))
|
||||||
|
|
||||||
|
|
||||||
def _shard_budget_line(memory_mb: int, memory_source: str, total_layers: int | None, quantization: str) -> str:
|
def _shard_budget_line(
|
||||||
|
memory_mb: int,
|
||||||
|
memory_source: str,
|
||||||
|
total_layers: int | None,
|
||||||
|
quantization: str,
|
||||||
|
bytes_per_layer: int | None = None,
|
||||||
|
) -> str:
|
||||||
memory_gb = memory_mb / 1024
|
memory_gb = memory_mb / 1024
|
||||||
gb_str = f"{memory_gb:.1f} GB"
|
gb_str = f"{memory_gb:.1f} GB"
|
||||||
budget_quantization = "bfloat16" if quantization == "auto" else quantization
|
budget_quantization = "bfloat16" if quantization == "auto" else quantization
|
||||||
if total_layers is None or total_layers <= 0:
|
if total_layers is None or total_layers <= 0:
|
||||||
return f"Memory budget: {gb_str} {memory_source}; shard budget: unknown model layer count"
|
return f"Memory budget: {gb_str} {memory_source}; shard budget: unknown model layer count"
|
||||||
max_layers = _max_assignable_layers(memory_mb, total_layers)
|
max_layers = _max_assignable_layers(memory_mb, total_layers, bytes_per_layer=bytes_per_layer)
|
||||||
# Remaining capacity after one full model load (rough estimate)
|
# Remaining capacity after one full model load (rough estimate)
|
||||||
shard_bytes = max_layers * _DEFAULT_BYTES_PER_LAYER
|
shard_bytes = max_layers * (bytes_per_layer or _DEFAULT_BYTES_PER_LAYER)
|
||||||
remaining_gb = (memory_mb * 1024 * 1024 - shard_bytes) / (1024 ** 3)
|
remaining_gb = (memory_mb * 1024 * 1024 - shard_bytes) / (1024 ** 3)
|
||||||
remaining_str = f"; {remaining_gb:.1f} GB remaining after full load" if remaining_gb > 1 else ""
|
remaining_str = f"; {remaining_gb:.1f} GB remaining after full load" if remaining_gb > 1 else ""
|
||||||
return (
|
return (
|
||||||
@@ -218,6 +229,23 @@ def _shard_budget_line(memory_mb: int, memory_source: str, total_layers: int | N
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _assignment_bytes_per_layer(assignment: dict, quantization: str) -> int | None:
|
||||||
|
bytes_per_layer = assignment.get("bytes_per_layer")
|
||||||
|
if isinstance(bytes_per_layer, int) and bytes_per_layer > 0:
|
||||||
|
return bytes_per_layer
|
||||||
|
if not isinstance(bytes_per_layer, dict):
|
||||||
|
return None
|
||||||
|
keys = [quantization, "bfloat16", "bf16", "int8", "nf4"]
|
||||||
|
for key in keys:
|
||||||
|
value = bytes_per_layer.get(key)
|
||||||
|
if isinstance(value, int) and value > 0:
|
||||||
|
return value
|
||||||
|
for value in bytes_per_layer.values():
|
||||||
|
if isinstance(value, int) and value > 0:
|
||||||
|
return value
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _post_json(url: str, payload: dict, timeout: float = 10.0) -> dict:
|
def _post_json(url: str, payload: dict, timeout: float = 10.0) -> dict:
|
||||||
data = json.dumps(payload).encode()
|
data = json.dumps(payload).encode()
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
@@ -869,6 +897,7 @@ def run_startup(
|
|||||||
hf_repo: str | None = assignment.get("hf_repo")
|
hf_repo: str | None = assignment.get("hf_repo")
|
||||||
peers: list[dict] = assignment.get("peers", [])
|
peers: list[dict] = assignment.get("peers", [])
|
||||||
model_sources: list[dict] = [] if tracker_source_disabled else assignment.get("model_sources", [])
|
model_sources: list[dict] = [] if tracker_source_disabled else assignment.get("model_sources", [])
|
||||||
|
assignment_bytes_per_layer = _assignment_bytes_per_layer(assignment, quantization)
|
||||||
print(f" Shard: layers {shard_start}-{shard_end} of {assigned_model}", flush=True)
|
print(f" Shard: layers {shard_start}-{shard_end} of {assigned_model}", flush=True)
|
||||||
|
|
||||||
# 4. Download shard
|
# 4. Download shard
|
||||||
@@ -955,7 +984,7 @@ def run_startup(
|
|||||||
f"meshnet-node ready\n"
|
f"meshnet-node ready\n"
|
||||||
f" Wallet: {address}\n"
|
f" Wallet: {address}\n"
|
||||||
f" Shard: layers {shard_start}-{shard_end} ({assigned_model})\n"
|
f" Shard: layers {shard_start}-{shard_end} ({assigned_model})\n"
|
||||||
f" {_shard_budget_line(memory_budget_mb, memory_budget_source, assignment.get('model_layers_end', shard_end) + 1, quantization)}\n"
|
f" {_shard_budget_line(memory_budget_mb, memory_budget_source, assignment.get('model_layers_end', shard_end) + 1, quantization, bytes_per_layer=assignment_bytes_per_layer)}\n"
|
||||||
f" Endpoint: {endpoint}\n"
|
f" Endpoint: {endpoint}\n"
|
||||||
f" Node ID: {node_id}\n"
|
f" Node ID: {node_id}\n"
|
||||||
f" Hardware: {hw_str}\n"
|
f" Hardware: {hw_str}\n"
|
||||||
|
|||||||
@@ -75,6 +75,7 @@
|
|||||||
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
||||||
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
|
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
|
||||||
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
|
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
|
||||||
|
<section class="wide"><h2>Inference history</h2><div id="inference-history" class="empty">loading...</div></section>
|
||||||
</main>
|
</main>
|
||||||
<script>
|
<script>
|
||||||
"use strict";
|
"use strict";
|
||||||
@@ -209,9 +210,9 @@ function renderStats(stats) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function renderThroughput(stats) {
|
function renderThroughput(stats) {
|
||||||
const nodes = (stats && stats.nodes) || {};
|
const nodes = (stats && stats.nodes) || {};
|
||||||
const rows = [];
|
const rows = [];
|
||||||
for (const [nodeId, nodeStats] of Object.entries(nodes)) {
|
for (const [nodeId, nodeStats] of Object.entries(nodes)) {
|
||||||
for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) {
|
for (const [model, s] of Object.entries((nodeStats && nodeStats.models) || {})) {
|
||||||
rows.push([
|
rows.push([
|
||||||
esc(short(nodeId)),
|
esc(short(nodeId)),
|
||||||
@@ -220,10 +221,52 @@ function renderThroughput(stats) {
|
|||||||
`<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`,
|
`<span class="num">${esc(String(s.sample_count_last_hour ?? 0))}</span>`,
|
||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
|
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderInferenceHistory(data) {
|
||||||
|
const events = (data && data.events) || [];
|
||||||
|
const started = new Map();
|
||||||
|
const completed = [];
|
||||||
|
for (const e of events) {
|
||||||
|
const f = e.fields || {};
|
||||||
|
const id = f.request_id;
|
||||||
|
if (!id) continue;
|
||||||
|
if (e.message === "proxy route selected") {
|
||||||
|
started.set(id, e);
|
||||||
|
} else if (e.message === "proxy complete" || e.message === "proxy failed" || e.message === "direct proxy failed after relay") {
|
||||||
|
completed.push(e);
|
||||||
|
started.delete(id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const activeByModel = {};
|
||||||
|
for (const e of started.values()) {
|
||||||
|
const f = e.fields || {};
|
||||||
|
const model = f.model || f.route_model || "?";
|
||||||
|
activeByModel[model] = (activeByModel[model] || 0) + 1;
|
||||||
|
}
|
||||||
|
const active = Object.entries(activeByModel)
|
||||||
|
.map(([model, count]) => `${esc(model)}: <span class="num">${count}</span> active`)
|
||||||
|
.join(" · ");
|
||||||
|
const rows = completed.slice(-20).reverse().map(e => {
|
||||||
|
const f = e.fields || {};
|
||||||
|
return [
|
||||||
|
new Date((e.ts || 0) * 1000).toLocaleTimeString(),
|
||||||
|
esc(short(f.model || f.route_model || "?", 28)),
|
||||||
|
esc(short(f.request_id || "?", 18)),
|
||||||
|
`<span class="num">${esc(tps(f.tokens_per_sec))}</span>`,
|
||||||
|
`<span class="num">${esc(String(f.tokens ?? "?"))}</span>`,
|
||||||
|
`<span class="num">${esc(String(f.elapsed_seconds ?? "?"))}</span>`,
|
||||||
|
f.stream ? "stream" : "json",
|
||||||
|
];
|
||||||
|
});
|
||||||
|
$("inference-history").innerHTML =
|
||||||
|
`<div class="dim" style="margin-bottom:6px">${active || "no active requests"}</div>` +
|
||||||
|
(rows.length ? table(["time", "model", "request", "tps", "tokens", "sec", "mode"], rows)
|
||||||
|
: '<div class="empty">no completed inference requests</div>');
|
||||||
|
}
|
||||||
|
|
||||||
function renderConsole(data) {
|
function renderConsole(data) {
|
||||||
const events = (data && data.events) || [];
|
const events = (data && data.events) || [];
|
||||||
if (!events.length) {
|
if (!events.length) {
|
||||||
@@ -406,9 +449,10 @@ async function refresh() {
|
|||||||
renderNodes(map);
|
renderNodes(map);
|
||||||
renderBilling(summary);
|
renderBilling(summary);
|
||||||
renderSettlements(settlements);
|
renderSettlements(settlements);
|
||||||
renderFraud(wallets, summary);
|
renderFraud(wallets, summary);
|
||||||
renderStats(stats);
|
renderStats(stats);
|
||||||
renderThroughput(stats);
|
renderThroughput(stats);
|
||||||
|
renderInferenceHistory(consoleData);
|
||||||
renderConsole(consoleData);
|
renderConsole(consoleData);
|
||||||
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
|
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -636,7 +636,7 @@ def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
|
|||||||
return 0.5 + 0.5 * reputation
|
return 0.5 + 0.5 * reputation
|
||||||
|
|
||||||
|
|
||||||
def _select_route(
|
def _select_route(
|
||||||
nodes: list[_NodeEntry],
|
nodes: list[_NodeEntry],
|
||||||
required_start: int,
|
required_start: int,
|
||||||
required_end: int,
|
required_end: int,
|
||||||
@@ -678,10 +678,25 @@ def _select_route(
|
|||||||
covered_up_to = best.shard_end
|
covered_up_to = best.shard_end
|
||||||
candidates = [n for n in candidates if n is not best]
|
candidates = [n for n in candidates if n is not best]
|
||||||
|
|
||||||
return route, ""
|
return route, ""
|
||||||
|
|
||||||
|
|
||||||
def _coverage_percentage(
|
def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"node_id": node.node_id,
|
||||||
|
"model": node.model,
|
||||||
|
"hf_repo": node.hf_repo,
|
||||||
|
"endpoint": node.endpoint,
|
||||||
|
"shard": f"{node.shard_start}-{node.shard_end}",
|
||||||
|
"num_layers": node.num_layers,
|
||||||
|
"queue_depth": node.queue_depth,
|
||||||
|
}
|
||||||
|
for node in nodes
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _coverage_percentage(
|
||||||
nodes: list[_NodeEntry],
|
nodes: list[_NodeEntry],
|
||||||
required_start: int,
|
required_start: int,
|
||||||
required_end: int,
|
required_end: int,
|
||||||
@@ -746,14 +761,14 @@ def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
|
|||||||
return DEFAULT_RAM_BYTES, "ram-default"
|
return DEFAULT_RAM_BYTES, "ram-default"
|
||||||
|
|
||||||
|
|
||||||
def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
|
def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
|
||||||
bytes_per_layer = _preset_bytes_per_layer(preset)
|
bytes_per_layer = _preset_bytes_per_layer(preset)
|
||||||
quantization = _node_quantization(node, preset)
|
quantization = _node_quantization(node, preset)
|
||||||
layer_bytes = bytes_per_layer[quantization]
|
layer_bytes = bytes_per_layer[quantization]
|
||||||
if layer_bytes <= 0:
|
if layer_bytes <= 0:
|
||||||
return 0
|
return 0
|
||||||
memory_budget_bytes, _ = _node_memory_budget_bytes(node)
|
memory_budget_bytes, _ = _node_memory_budget_bytes(node)
|
||||||
return int((memory_budget_bytes * 0.8) // layer_bytes)
|
return int((memory_budget_bytes * 0.8) // layer_bytes)
|
||||||
|
|
||||||
|
|
||||||
def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
|
def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
|
||||||
@@ -817,16 +832,17 @@ def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
|
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
|
||||||
if total_layers <= 0:
|
if total_layers <= 0:
|
||||||
return 0
|
return 0
|
||||||
if memory_mb <= 0:
|
if memory_mb <= 0:
|
||||||
return max(1, total_layers // 2)
|
return max(1, total_layers // 2)
|
||||||
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
|
memory_bytes = memory_mb * 1024 * 1024
|
||||||
return min(
|
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
|
||||||
total_layers,
|
return min(
|
||||||
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
|
total_layers,
|
||||||
)
|
max(1, int((memory_bytes * 0.8) // bytes_per_layer)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
|
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
|
||||||
@@ -2330,17 +2346,24 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
# Pre-resolve the downstream route so the first-shard node skips its own
|
# Pre-resolve the downstream route so the first-shard node skips its own
|
||||||
# tracker query. We already hold the full registry picture — no need for
|
# tracker query. We already hold the full registry picture — no need for
|
||||||
# a second round-trip.
|
# a second round-trip.
|
||||||
route_model = node.hf_repo or node.model or model
|
route_model = node.hf_repo or node.model or model
|
||||||
with server.lock:
|
with server.lock:
|
||||||
if route_model in server.model_presets:
|
resolved_route_model, route_preset = _resolve_model_preset(server.model_presets, route_model)
|
||||||
preset = server.model_presets[route_model]
|
if route_preset is not None:
|
||||||
rs, re = _preset_layer_bounds(preset)
|
route_model = resolved_route_model or route_model
|
||||||
all_nodes: list = [n for n in server.registry.values() if n.model == route_model]
|
preset = route_preset
|
||||||
else:
|
rs, re = _preset_layer_bounds(preset)
|
||||||
all_nodes = [
|
all_nodes: list = [
|
||||||
n for n in server.registry.values()
|
n for n in server.registry.values()
|
||||||
if _node_matches_model(n, route_model)
|
if _node_matches_preset(n, route_model, preset)
|
||||||
and n.shard_start is not None and n.num_layers is not None
|
and n.shard_start is not None
|
||||||
|
and n.shard_end is not None
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
all_nodes = [
|
||||||
|
n for n in server.registry.values()
|
||||||
|
if _node_matches_model(n, route_model)
|
||||||
|
and n.shard_start is not None and n.num_layers is not None
|
||||||
]
|
]
|
||||||
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
|
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
|
||||||
if pinned_nodes is not None:
|
if pinned_nodes is not None:
|
||||||
@@ -2356,6 +2379,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
route_model=route_model,
|
route_model=route_model,
|
||||||
error=route_error,
|
error=route_error,
|
||||||
candidate_count=len(all_nodes),
|
candidate_count=len(all_nodes),
|
||||||
|
candidates=_node_route_summary(all_nodes),
|
||||||
)
|
)
|
||||||
self._send_json(503, {"error": {
|
self._send_json(503, {"error": {
|
||||||
"message": route_error,
|
"message": route_error,
|
||||||
@@ -2393,6 +2417,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
head_endpoint=node.endpoint,
|
head_endpoint=node.endpoint,
|
||||||
downstream=downstream_urls,
|
downstream=downstream_urls,
|
||||||
route=route_debug or "<empty>",
|
route=route_debug or "<empty>",
|
||||||
|
nodes=_node_route_summary(route_nodes),
|
||||||
)
|
)
|
||||||
|
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
@@ -2450,12 +2475,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body)
|
in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body)
|
||||||
except (json.JSONDecodeError, TypeError):
|
except (json.JSONDecodeError, TypeError):
|
||||||
in_tokens, out_tokens = 0, 0
|
in_tokens, out_tokens = 0, 0
|
||||||
tokens = in_tokens + out_tokens
|
tokens = in_tokens + out_tokens
|
||||||
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
||||||
self._bill_completed(
|
_tracker_log(
|
||||||
api_key, model, tokens, node_work,
|
server,
|
||||||
input_tokens=in_tokens, output_tokens=out_tokens,
|
"info",
|
||||||
)
|
"proxy complete",
|
||||||
|
request_id=request_id,
|
||||||
|
model=model,
|
||||||
|
route_model=route_model,
|
||||||
|
status=int(first.get("status", 503)),
|
||||||
|
tokens=tokens,
|
||||||
|
elapsed_seconds=round(elapsed, 4),
|
||||||
|
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||||||
|
route=_node_route_summary(route_nodes),
|
||||||
|
)
|
||||||
|
self._bill_completed(
|
||||||
|
api_key, model, tokens, node_work,
|
||||||
|
input_tokens=in_tokens, output_tokens=out_tokens,
|
||||||
|
)
|
||||||
return
|
return
|
||||||
_tracker_log(
|
_tracker_log(
|
||||||
server,
|
server,
|
||||||
@@ -2539,35 +2577,57 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
in_tokens, out_tokens = _stream_billable_split(
|
in_tokens, out_tokens = _stream_billable_split(
|
||||||
observed_stream_tokens, stream_usage, body
|
observed_stream_tokens, stream_usage, body
|
||||||
)
|
)
|
||||||
self._record_observed_throughput(
|
self._record_observed_throughput(
|
||||||
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
|
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
|
||||||
)
|
)
|
||||||
self._bill_completed(
|
tokens = in_tokens + out_tokens
|
||||||
api_key, model, in_tokens + out_tokens, node_work,
|
|
||||||
input_tokens=in_tokens, output_tokens=out_tokens,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Non-streaming: buffer and relay
|
|
||||||
resp_body = upstream.read()
|
|
||||||
elapsed = time.monotonic() - started
|
|
||||||
_tracker_log(
|
_tracker_log(
|
||||||
server,
|
server,
|
||||||
"info",
|
"info",
|
||||||
"proxy complete",
|
"proxy complete",
|
||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
target_url=target_url,
|
model=model,
|
||||||
bytes=len(resp_body),
|
route_model=route_model,
|
||||||
|
status=200,
|
||||||
|
stream=True,
|
||||||
|
tokens=tokens,
|
||||||
|
elapsed_seconds=round(elapsed, 4),
|
||||||
|
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||||||
|
route=_node_route_summary(route_nodes),
|
||||||
)
|
)
|
||||||
observed_output = ""
|
self._bill_completed(
|
||||||
try:
|
api_key, model, tokens, node_work,
|
||||||
response_payload = json.loads(resp_body)
|
input_tokens=in_tokens, output_tokens=out_tokens,
|
||||||
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
|
)
|
||||||
observed_output = _observed_output_from_non_stream_payload(response_payload)
|
else:
|
||||||
except json.JSONDecodeError:
|
# Non-streaming: buffer and relay
|
||||||
in_tokens, out_tokens = 0, 0
|
resp_body = upstream.read()
|
||||||
tokens = in_tokens + out_tokens
|
elapsed = time.monotonic() - started
|
||||||
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
observed_output = ""
|
||||||
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
|
try:
|
||||||
|
response_payload = json.loads(resp_body)
|
||||||
|
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
|
||||||
|
observed_output = _observed_output_from_non_stream_payload(response_payload)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
in_tokens, out_tokens = 0, 0
|
||||||
|
tokens = in_tokens + out_tokens
|
||||||
|
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
||||||
|
_tracker_log(
|
||||||
|
server,
|
||||||
|
"info",
|
||||||
|
"proxy complete",
|
||||||
|
request_id=request_id,
|
||||||
|
model=model,
|
||||||
|
route_model=route_model,
|
||||||
|
target_url=target_url,
|
||||||
|
status=200,
|
||||||
|
bytes=len(resp_body),
|
||||||
|
tokens=tokens,
|
||||||
|
elapsed_seconds=round(elapsed, 4),
|
||||||
|
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||||||
|
route=_node_route_summary(route_nodes),
|
||||||
|
)
|
||||||
|
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
|
||||||
self._bill_completed(
|
self._bill_completed(
|
||||||
api_key, model, tokens, node_work,
|
api_key, model, tokens, node_work,
|
||||||
input_tokens=in_tokens, output_tokens=out_tokens,
|
input_tokens=in_tokens, output_tokens=out_tokens,
|
||||||
@@ -2740,13 +2800,30 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
in_tokens, out_tokens = _stream_billable_split(
|
in_tokens, out_tokens = _stream_billable_split(
|
||||||
observed_stream_tokens, stream_usage, request_body
|
observed_stream_tokens, stream_usage, request_body
|
||||||
)
|
)
|
||||||
self._record_observed_throughput(
|
self._record_observed_throughput(
|
||||||
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
|
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
|
||||||
)
|
)
|
||||||
self._bill_completed(
|
tokens = in_tokens + out_tokens
|
||||||
api_key, model, in_tokens + out_tokens, node_work,
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
input_tokens=in_tokens, output_tokens=out_tokens,
|
_tracker_log(
|
||||||
)
|
server,
|
||||||
|
"info",
|
||||||
|
"proxy complete",
|
||||||
|
request_id=session_id,
|
||||||
|
model=model,
|
||||||
|
route_model=route_model,
|
||||||
|
status=200,
|
||||||
|
stream=True,
|
||||||
|
relay=True,
|
||||||
|
tokens=tokens,
|
||||||
|
elapsed_seconds=round(elapsed, 4),
|
||||||
|
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||||||
|
route=_node_route_summary(route_nodes),
|
||||||
|
)
|
||||||
|
self._bill_completed(
|
||||||
|
api_key, model, tokens, node_work,
|
||||||
|
input_tokens=in_tokens, output_tokens=out_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
def _send_relayed_response(self, response: dict) -> None:
|
def _send_relayed_response(self, response: dict) -> None:
|
||||||
status = int(response.get("status", 503))
|
status = int(response.get("status", 503))
|
||||||
@@ -4140,11 +4217,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"shard_end": shard_end,
|
"shard_end": shard_end,
|
||||||
"num_layers": total_l,
|
"num_layers": total_l,
|
||||||
"gap_found": True,
|
"gap_found": True,
|
||||||
"price_per_token": 0.0,
|
"price_per_token": 0.0,
|
||||||
"deployment": _deployment_summary(pool_with_caller, preset),
|
"bytes_per_layer": _preset_bytes_per_layer(preset),
|
||||||
"model_sources": self._model_sources(
|
"deployment": _deployment_summary(pool_with_caller, preset),
|
||||||
str(resolved_name),
|
"model_sources": self._model_sources(
|
||||||
preset,
|
str(resolved_name),
|
||||||
|
preset,
|
||||||
shard_start,
|
shard_start,
|
||||||
shard_end,
|
shard_end,
|
||||||
),
|
),
|
||||||
@@ -4236,12 +4314,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"shard_start": shard_start,
|
"shard_start": shard_start,
|
||||||
"shard_end": shard_end,
|
"shard_end": shard_end,
|
||||||
"num_layers": total_l,
|
"num_layers": total_l,
|
||||||
"gap_found": gap_found,
|
"gap_found": gap_found,
|
||||||
"price_per_token": 0.0,
|
"price_per_token": 0.0,
|
||||||
"model_sources": self._model_sources(
|
"bytes_per_layer": _preset_bytes_per_layer(best_preset) if best_preset is not None else None,
|
||||||
str(resolved_name or best_repo),
|
"model_sources": self._model_sources(
|
||||||
best_preset or {"hf_repo": best_repo},
|
str(resolved_name or best_repo),
|
||||||
shard_start,
|
best_preset or {"hf_repo": best_repo},
|
||||||
|
shard_start,
|
||||||
shard_end,
|
shard_end,
|
||||||
),
|
),
|
||||||
})
|
})
|
||||||
@@ -4287,10 +4366,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||||||
]
|
]
|
||||||
|
|
||||||
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
|
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
|
||||||
if error:
|
if error:
|
||||||
self._send_json(503, {"error": error})
|
_tracker_log(
|
||||||
return
|
server,
|
||||||
|
"warn",
|
||||||
|
"route unavailable",
|
||||||
|
model=model,
|
||||||
|
route_model=resolved_name or model,
|
||||||
|
error=error,
|
||||||
|
candidate_count=len(alive),
|
||||||
|
candidates=_node_route_summary(alive),
|
||||||
|
)
|
||||||
|
self._send_json(503, {"error": error})
|
||||||
|
return
|
||||||
|
|
||||||
covered_up_to = required_start - 1
|
covered_up_to = required_start - 1
|
||||||
route_with_start: list[tuple] = []
|
route_with_start: list[tuple] = []
|
||||||
|
|||||||
@@ -355,6 +355,134 @@ def test_tracker_proxy_accepts_hf_model_alias_from_quickstart():
|
|||||||
assert response["choices"][0]["message"]["content"] == "56"
|
assert response["choices"][0]["message"]["content"] == "56"
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_proxy_routes_split_preset_nodes_by_alias():
|
||||||
|
"""The chat proxy must build routes from alias-matched preset nodes."""
|
||||||
|
|
||||||
|
class ChatHandler(http.server.BaseHTTPRequestHandler):
|
||||||
|
def log_message(self, fmt, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
if self.path != "/v1/chat/completions":
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
return
|
||||||
|
length = int(self.headers.get("Content-Length", 0))
|
||||||
|
self.rfile.read(length)
|
||||||
|
route_header = self.headers.get("X-Meshnet-Route", "[]")
|
||||||
|
body = json.dumps({
|
||||||
|
"choices": [{"message": {"content": route_header}}],
|
||||||
|
"usage": {"prompt_tokens": 1, "completion_tokens": 1},
|
||||||
|
}).encode()
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
head = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
|
||||||
|
head_thread = threading.Thread(target=head.serve_forever, daemon=True)
|
||||||
|
head_thread.start()
|
||||||
|
tracker = TrackerServer(model_presets={
|
||||||
|
"qwen3.6-35b-a3b": {
|
||||||
|
"layers_start": 0,
|
||||||
|
"layers_end": 39,
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"aliases": ["Qwen3.6-35B-A3B"],
|
||||||
|
}
|
||||||
|
})
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": f"http://127.0.0.1:{head.server_address[1]}",
|
||||||
|
"model": "qwen3.6-35b-a3b",
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"num_layers": 40,
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 21,
|
||||||
|
"tracker_mode": True,
|
||||||
|
"hardware_profile": {},
|
||||||
|
"score": 1.0},
|
||||||
|
)
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": "http://127.0.0.1:9199",
|
||||||
|
"model": "Qwen3.6-35B-A3B",
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"num_layers": 40,
|
||||||
|
"shard_start": 22,
|
||||||
|
"shard_end": 39,
|
||||||
|
"tracker_mode": True,
|
||||||
|
"hardware_profile": {},
|
||||||
|
"score": 1.0},
|
||||||
|
)
|
||||||
|
|
||||||
|
response = _post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
|
||||||
|
{"model": "Qwen3.6-35B-A3B",
|
||||||
|
"messages": [{"role": "user", "content": "hi"}]},
|
||||||
|
)
|
||||||
|
console = _get_json(f"http://127.0.0.1:{tracker_port}/v1/console")
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
head.shutdown()
|
||||||
|
head.server_close()
|
||||||
|
head_thread.join(timeout=1.0)
|
||||||
|
|
||||||
|
route = json.loads(response["choices"][0]["message"]["content"])
|
||||||
|
assert route == [{"endpoint": "http://127.0.0.1:9199", "start_layer": 22}]
|
||||||
|
complete = [event for event in console["events"] if event["message"] == "proxy complete"][-1]
|
||||||
|
assert complete["fields"]["tokens"] == 2
|
||||||
|
assert complete["fields"]["tokens_per_sec"] >= 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_route_endpoint_routes_split_preset_nodes_by_alias():
|
||||||
|
tracker = TrackerServer(model_presets={
|
||||||
|
"qwen3.6-35b-a3b": {
|
||||||
|
"layers_start": 0,
|
||||||
|
"layers_end": 39,
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"aliases": ["Qwen3.6-35B-A3B"],
|
||||||
|
}
|
||||||
|
})
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": "http://127.0.0.1:9101",
|
||||||
|
"model": "qwen3.6-35b-a3b",
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"num_layers": 40,
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 21,
|
||||||
|
"tracker_mode": True,
|
||||||
|
"hardware_profile": {},
|
||||||
|
"score": 1.0},
|
||||||
|
)
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{"endpoint": "http://127.0.0.1:9102",
|
||||||
|
"model": "Qwen3.6-35B-A3B",
|
||||||
|
"hf_repo": "unsloth/Qwen3.6-35B-A3B",
|
||||||
|
"num_layers": 40,
|
||||||
|
"shard_start": 22,
|
||||||
|
"shard_end": 39,
|
||||||
|
"tracker_mode": True,
|
||||||
|
"hardware_profile": {},
|
||||||
|
"score": 1.0},
|
||||||
|
)
|
||||||
|
|
||||||
|
response = _get_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/route?model=Qwen3.6-35B-A3B"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert response["route"] == ["http://127.0.0.1:9101", "http://127.0.0.1:9102"]
|
||||||
|
assert [node["start_layer"] for node in response["nodes"]] == [0, 22]
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix():
|
def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix():
|
||||||
tracker = TrackerServer()
|
tracker = TrackerServer()
|
||||||
tracker_port = tracker.start()
|
tracker_port = tracker.start()
|
||||||
|
|||||||
Reference in New Issue
Block a user