popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[verified] fix: preserve tracker precision eligibility

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-13 10:27:45 +03:00
parent 377346c301
commit b5fa7245df
12 changed files with 557 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
packages/tracker/meshnet_tracker/server.py
View File

@@ -972,7 +972,7 @@ def _coverage_percentage(
intervals = sorted(
(
(max(required_start, node.shard_start), min(required_end, node.shard_end))
for node in nodes
for node in _serviceable_nodes(nodes)
if node.shard_start is not None
and node.shard_end is not None
if node.shard_end >= required_start and node.shard_start <= required_end
@@ -1004,7 +1004,7 @@ def _served_model_copies(
layer_counts = []
for layer in range(required_start, required_end + 1):
count = 0
for node in nodes:
for node in _serviceable_nodes(nodes):
if node.shard_start is None or node.shard_end is None:
continue
if node.shard_start <= layer <= node.shard_end:
@@ -1092,6 +1092,57 @@ def _normalize_quantization(value: object) -> str | None:
return normalized if normalized in _QUANTIZATION_QUALITY else None
def _registration_quantization(body: dict, quantizations: list[str]) -> str | None:
"""Resolve the active precision a registration is routable at.
Only the raw body can tell an *absent* `quantization` from an explicit null,
and the two mean opposite things, so this must run at the registration seam
rather than in `_NodeEntry` (both reach the constructor as None).
An absent field predates the protocol adding it: it means "unknown", not
"unsupported", so the node keeps the best precision it advertises and stays
routable. Anything the node states explicitly is taken at its word -- a null,
a non-string, or an unsupported name leaves it with no usable precision and
routing excludes it.
"""
if "quantization" in body:
return _normalize_quantization(body["quantization"])
supported = [
normalized for value in quantizations
if (normalized := _normalize_quantization(value)) is not None
]
if not supported:
return None
return max(supported, key=lambda value: _QUANTIZATION_QUALITY[value])
def _has_usable_quantization(node: _NodeEntry) -> bool:
"""Whether a node declared a precision it can actually serve.
`quantization` is None only for a registration that explicitly declared a
null, malformed, or unsupported precision -- a legacy registration that omits
the field is resolved to a concrete precision at the registration seam. Such a
node is already dark to routing, and shard placement has to keep it that way:
the assignment paths pick a precision via `_node_quantization`, which falls
back to the advertised list and then to a preset default, so an unguarded
assignment would overwrite the declared-unusable precision and silently make
the node routable again.
"""
return node.quantization is not None
def _serviceable_nodes(nodes: list[_NodeEntry]) -> list[_NodeEntry]:
"""The nodes a coverage calculation is allowed to count.
A node with no usable precision is excluded from every routing path, so it
serves none of the layers it claims. Counting its shard range as covered
would report a complete model that no request can reach, and would hide the
hole from the rebalancer whose whole job is to fill it -- the managed nodes
that could cover those layers stay idle because the gap never appears.
"""
return [node for node in nodes if _has_usable_quantization(node)]
def _quantization_satisfies(available: object, requested: object) -> bool:
"""Return whether a shard meets a request's minimum precision."""
available_normalized = _normalize_quantization(available)
@@ -1406,7 +1457,10 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
continue
if any((n.hf_repo or n.model) == hf_repo for n in host_nodes):
continue
anchor = max(host_nodes, key=lambda n: n.benchmark_tokens_per_sec)
placeable = [n for n in host_nodes if _has_usable_quantization(n)]
if not placeable:
continue
anchor = max(placeable, key=lambda n: n.benchmark_tokens_per_sec)
if anchor.status != "ready" or anchor.pending_new_assignment is not None:
continue
capacity = min(_node_layer_capacity(anchor, preset), total_layers)
@@ -1455,9 +1509,10 @@ def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) ->
if host["spare_slots"] <= 0 and not has_spare_memory:
continue
host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
if not host_nodes:
placeable = [node for node in host_nodes if _has_usable_quantization(node)]
if not placeable:
continue
anchor = max(host_nodes, key=lambda node: node.benchmark_tokens_per_sec)
anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec)
if anchor.status != "ready" or anchor.pending_new_assignment is not None:
continue
capacity = min(_node_layer_capacity(anchor, preset), total_layers)
@@ -1479,6 +1534,8 @@ def _preferred_node_quantization(
requested: str,
) -> str | None:
"""Choose the highest supported precision that satisfies a request."""
if not _has_usable_quantization(node):
return None
supported = {
normalized
for value in ([node.quantization] if node.quantization else []) + list(node.quantizations)
@@ -1631,7 +1688,7 @@ def _coverage_map(
layer_counts = []
for layer in range(required_start, required_end + 1):
count = 0
for node in nodes:
for node in _serviceable_nodes(nodes):
if node.shard_start is None or node.shard_end is None:
continue
if node.shard_start <= layer <= node.shard_end:
@@ -1680,8 +1737,10 @@ def _coverage_map_detailed(
) -> list[dict]:
"""Like _coverage_map but with per-node identity and health in each band.
Includes all nodes (alive and stale) so operators can see both coverage
holes and which dead nodes used to fill them.
Includes all nodes (alive and stale, serviceable or not) so operators can see
both coverage holes and which nodes used to fill them. This is why it does not
share `_coverage_map`'s serviceability filter: a node dark for a declared bad
precision is exactly what an operator staring at a hole needs to see.
"""
now = time.monotonic()
@@ -1793,6 +1852,8 @@ def _assign_redundant_managed_nodes(
):
if node.shard_start is not None and node.shard_end is not None:
continue
if not _has_usable_quantization(node):
continue
capacity = min(_node_layer_capacity(node, preset), total_layers)
if capacity <= 0:
continue
@@ -2225,7 +2286,8 @@ def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
eligible_nodes = [
node for node in managed_nodes
if _node_layer_capacity(node, preset) > 0
if _has_usable_quantization(node)
and _node_layer_capacity(node, preset) > 0
]
node_index = 0
for gap_start, gap_end in gaps:
@@ -2315,7 +2377,8 @@ def _rebalance_hf_model_locked(server: "_TrackerHTTPServer", hf_repo: str) -> No
eligible_nodes = [
node for node in managed_nodes
if _node_layer_capacity(node, preset) > 0
if _has_usable_quantization(node)
and _node_layer_capacity(node, preset) > 0
]
node_index = 0
for gap_start, gap_end in gaps:
@@ -4424,10 +4487,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(400, {"error": "quantizations must be a non-empty string array"})
return
quantizations = list(quantizations_body)
quantization = body.get("quantization")
if quantization is not None and not isinstance(quantization, str):
if body.get("quantization") is not None and not isinstance(body["quantization"], str):
self._send_json(400, {"error": "quantization must be a string"})
return
quantization = _registration_quantization(body, quantizations)
wallet_address = body.get("wallet_address")
if wallet_address is not None and not isinstance(wallet_address, str):
self._send_json(400, {"error": "wallet_address must be a string"})
@@ -6875,6 +6938,21 @@ class TrackerServer:
friendly_name = _normalize_friendly_name(payload.get("friendly_name"))
except ValueError:
friendly_name = None
# The replicated payload is the raw registration body, so the follower can
# resolve precision exactly as the leader did -- including telling a legacy
# absent `quantization` from a declared one. Dropping these fields here
# would rebuild every node at the default precision and diverge follower
# routing from leader routing.
quantizations_payload = payload.get("quantizations")
quantizations = (
list(quantizations_payload)
if (
isinstance(quantizations_payload, list)
and quantizations_payload
and all(isinstance(item, str) and item for item in quantizations_payload)
)
else list(DEFAULT_QUANTIZATIONS)
)
entry = _NodeEntry(
node_id=node_id,
endpoint=endpoint.rstrip("/"),
@@ -6885,6 +6963,8 @@ class TrackerServer:
hardware_profile=payload.get("hardware_profile", {}),
wallet_address=payload.get("wallet_address"),
score=float(payload.get("score", 1.0)),
quantizations=quantizations,
quantization=_registration_quantization(payload, quantizations),
tracker_mode=bool(payload.get("tracker_mode", False)),
hf_repo=payload.get("hf_repo"),
num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,