popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge commit '47b243cd98fd94da7918cacf5725373b099208e5' into ralph/distributed-gguf-runtime

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-15 23:04:52 +02:00
parent 1fe31ef38d 47b243cd98
commit 254627629b
12 changed files with 620 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

132
packages/tracker/meshnet_tracker/server.py
View File

@@ -87,7 +87,7 @@ from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode
_CONSOLE_LIMIT = 300
_CONSOLE_LIMIT = 1000
_PROXY_PROGRESS_LOG_INTERVAL = 5.0
_SESSION_COOKIE_NAME = "meshnet_session"
@@ -1121,12 +1121,15 @@ def _registration_quantization(body: dict, quantizations: list[str]) -> str | No
An absent field predates the protocol adding it: it means "unknown", not
"unsupported", so the node keeps the best precision it advertises and stays
routable. Anything the node states explicitly is taken at its word -- a null,
a non-string, or an unsupported name leaves it with no usable precision and
routing excludes it.
routable. An explicit "auto" means the same thing — the node's CLI default
delegates the choice, it does not refuse one. Anything else the node states
explicitly is taken at its word -- a null, a non-string, or an unsupported
name leaves it with no usable precision and routing excludes it.
"""
if "quantization" in body:
return _normalize_quantization(body["quantization"])
declared = body.get("quantization")
declared_auto = isinstance(declared, str) and declared.strip().lower() == "auto"
if "quantization" in body and not declared_auto:
return _normalize_quantization(declared)
supported = [
normalized for value in quantizations
if (normalized := _normalize_quantization(value)) is not None
@@ -1245,6 +1248,7 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
"quantization": node.quantization,
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
"effective_throughput": round(_effective_throughput(node), 4),
"loaded_model_bytes": _assignment_memory_bytes(node, preset),
}
if preset is not None:
summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
@@ -1514,7 +1518,9 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
break
def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
def _request_model_load_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> dict | None:
"""Queue an explicitly requested model on the best available joined node."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None or not preset.get("hf_repo"):
@@ -1530,6 +1536,8 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
continue
host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
placeable = [node for node in host_nodes if _has_usable_quantization(node)]
if node_id is not None:
placeable = [node for node in placeable if node.node_id == node_id]
if not placeable:
continue
anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec)
@@ -1548,21 +1556,26 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
return None
def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
def _force_model_load_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> dict | None:
"""Replace the fastest ready assignment after an explicit admin eviction."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None or not preset.get("hf_repo"):
return None
start, end = _preset_layer_bounds(preset)
candidates = [node for node in server.registry.values()
if node.status == "ready" and node.pending_new_assignment is None
and _has_usable_quantization(node)]
# An explicit admin eviction is permitted to recover a stuck/loading node
# and to use the preset default precision. It must only avoid a node that
# already has another assignment in flight.
candidates = [
node for node in server.registry.values()
if node.pending_new_assignment is None
and (node_id is None or node.node_id == node_id)
]
if not candidates:
return None
node = max(candidates, key=lambda item: item.benchmark_tokens_per_sec)
shard_end = min(end, start + min(_node_layer_capacity(node, preset), end - start + 1) - 1)
if shard_end < start:
return None
shard_end = min(end, start + max(1, min(_node_layer_capacity(node, preset), end - start + 1)) - 1)
quantization = _node_quantization(node, preset)
directive = _load_directive(node, str(preset["hf_repo"]), start, shard_end, quantization)
replaced = node.hf_repo or node.model
@@ -1576,13 +1589,17 @@ def _force_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> di
"shard_start": start, "shard_end": shard_end, "replaced_model": replaced}
def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
def _release_model_locked(
server: "_TrackerHTTPServer", model_key: str, node_id: str | None = None,
) -> int:
"""Queue DROP_SHARD for every served shard and remove it from routing immediately."""
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
if preset is None:
return 0
released = 0
for node in server.registry.values():
if node_id is not None and node.node_id != node_id:
continue
if not _node_matches_preset(node, resolved_name, preset) or node.shard_start is None or node.shard_end is None:
continue
node.pending_directives.append(_drop_directive(node, str(preset.get("hf_repo") or resolved_name), node.shard_start, node.shard_end, node.quantization or "bfloat16"))
@@ -1591,6 +1608,16 @@ def _release_model_locked(server: "_TrackerHTTPServer", model_key: str) -> int:
return released
def _release_all_node_models_locked(server: "_TrackerHTTPServer", node_id: str) -> int:
"""Queue removal of every loaded assignment on one node."""
node = server.registry.get(node_id)
if node is None or node.shard_start is None or node.shard_end is None:
return 0
node.pending_directives.append({"action": "DROP_ALL_SHARDS"})
node.status = "loading"
return 1
def _preferred_node_quantization(
node: _NodeEntry,
preset: dict,
@@ -3109,6 +3136,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if self.path == "/v1/models/release":
self._handle_model_release_request()
return
if self.path == "/v1/nodes/release-all":
self._handle_node_release_all_request()
return
if self.path == "/v1/models/vote":
self._handle_model_coverage_vote()
return
@@ -3285,7 +3315,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
node.hf_repo or node.model
for node in alive
if node.model is not None
and node.model not in server.model_presets
# The same model can be registered under its HF repository while
# the catalogue exposes its short preset id. Do not emit a second
# repo-keyed entry when either node identifier resolves to a preset.
and _resolve_model_preset(
server.model_presets, node.hf_repo or node.model,
)[1] is None
and node.shard_start is not None
and node.shard_end is not None
and node.num_layers is not None
@@ -3386,6 +3421,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"endpoint": node.endpoint,
"relay_addr": node.relay_addr,
"peer_id": node.peer_id,
"wallet_address": node.wallet_address,
"hardware_profile": dict(node.hardware_profile),
"ram_bytes": node.ram_bytes,
"vram_bytes": node.vram_bytes,
"max_loaded_shards": node.max_loaded_shards,
}
for node in tracker_nodes
],
@@ -3409,12 +3449,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
memory_pool = _memory_pool_map(server)
def capacity_for(node: _NodeEntry) -> dict:
preset = None
if node.model:
preset = server.model_presets.get(node.model)
if preset is None and node.hf_repo and node.num_layers:
preset = _hf_rebalance_preset([node])
return _node_capacity_summary(node, preset)
return _node_capacity_summary(node, _preset_for_node(server, node))
def throughput_for(node: _NodeEntry) -> dict:
if server.stats is None:
@@ -4832,6 +4867,20 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
entry.uptime_seconds = float(body["uptime_seconds"])
if "status" in body and body["status"] in ("ready", "loading"):
entry.status = body["status"]
completed_directives = body.get("completed_directives", [])
if isinstance(completed_directives, list):
for directive in completed_directives:
if not isinstance(directive, dict) or directive.get("action") not in {"DROP_SHARD", "DROP_ALL_SHARDS"}:
continue
# A node has confirmed the release. Stop advertising its
# old route immediately so the dashboard and routing state
# agree with the runtime.
entry.model = "stub-model"
entry.hf_repo = None
entry.shard_start = None
entry.shard_end = None
entry.tracker_mode = False
entry.status = "ready"
if "friendly_name" in body:
try:
entry.friendly_name = _normalize_friendly_name(body.get("friendly_name"))
@@ -4903,11 +4952,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model, str) or not model.strip():
self._send_json(400, {"error": "model is required"})
return
node_id = body.get("node_id")
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
_resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None or str(preset.get("hf_repo") or "").strip().lower() == "stub-model":
self._send_json(400, {"error": "stub-model is a local test backend and cannot be loaded onto a node"})
return
with server.lock:
self._purge_expired_nodes()
assignment = _request_model_load_locked(server, model)
assignment = _request_model_load_locked(server, model, node_id)
if assignment is None and body.get("force") is True:
assignment = _force_model_load_locked(server, model)
assignment = _force_model_load_locked(server, model, node_id)
if assignment is None:
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
return
@@ -4924,14 +4981,39 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not isinstance(model, str) or not model.strip():
self._send_json(400, {"error": "model is required"})
return
node_id = body.get("node_id")
if node_id is not None and (not isinstance(node_id, str) or not node_id):
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock:
self._purge_expired_nodes()
released = _release_model_locked(server, model)
released = _release_model_locked(server, model, node_id)
if not released:
self._send_json(404, {"error": "no served shards found for model"})
return
self._send_json(202, {"status": "release_queued", "released": released})
def _handle_node_release_all_request(self):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if not self._require_role("admin", "validator"):
return
body = self._read_json_body()
if body is None:
return
node_id = body.get("node_id")
if not isinstance(node_id, str) or not node_id:
self._send_json(400, {"error": "node_id must be a non-empty string"})
return
with server.lock:
self._purge_expired_nodes()
released = _release_all_node_models_locked(server, node_id)
if not released:
self._send_json(404, {"error": "no loaded models found for node"})
return
self._send_json(202, {
"status": "release_queued", "released": released, "node_id": node_id,
})
def _handle_model_coverage_vote(self):
"""Record a rolling wish-list signal for an unavailable precision."""
server: _TrackerHTTPServer = self.server # type: ignore[assignment]