popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[verified] feat: complete Ralph task workstreams

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-12 11:17:03 +03:00
parent 9a1b15c020
commit 377346c301
37 changed files with 5862 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

212
packages/tracker/meshnet_tracker/server.py
View File

@@ -6,10 +6,14 @@ HTTP API contract:
"endpoint": "http://host:port", "shard_start": int, "shard_end": int,
"model": str optional, "shard_checksum": str optional,
"hardware_profile": object, "wallet_address": str optional,
"score": number optional
"score": number optional,
"capability_report": object optional, # ADR-0023 proof of this exact shard
"recipe_id": str optional, "recipe_version": str optional
}
Response 200: {"node_id": str}
Response 200: {"node_id": str, "capability": {"state": str, "routable": bool, ...}}
Response 400: {"error": str}
A node whose proof does not admit what it advertises still registers (so the
operator can see why it is dark) but is not routable -- see `capability.py`.
- POST /v1/nodes/{node_id}/heartbeat
Response 200: {}
Response 404: {"error": "node not found"}
@@ -48,6 +52,20 @@ from typing import Any
from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
from .auth import is_validator_token, sign_hive_request, verify_hive_request
from .capability import (
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
POLICY_COMPAT,
POLICY_ENFORCE,
STATE_ABSENT,
STATE_ADMITTED,
STATE_MODEL_MISMATCH,
STATE_SHARD_MISMATCH,
CapabilityState,
absent_state,
evaluate_report,
normalize_policy,
policy_from_env,
)
from .wallet_proof import binding_message, verify_wallet_signature
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
@@ -587,6 +605,8 @@ class _NodeEntry:
"heartbeats_expected", "heartbeats_received",
# dynamic reassignment queued by the tracker
"pending_new_assignment",
# the tracker's verdict on the capability proof this node presented
"capability",
)
def __init__(
@@ -616,6 +636,7 @@ class _NodeEntry:
cert_fingerprint: str | None = None,
peer_id: str | None = None,
friendly_name: str | None = None,
capability: "CapabilityState | None" = None,
) -> None:
self.node_id = node_id
self.endpoint = endpoint
@@ -643,6 +664,9 @@ class _NodeEntry:
self.cert_fingerprint = cert_fingerprint
self.peer_id = peer_id
self.friendly_name = friendly_name
# No proof presented is `absent`, never `admitted` — a node can only earn
# `admitted` by presenting a report that covers what it advertises.
self.capability: CapabilityState = capability or absent_state()
self.pending_directives: list[dict] = []
self.last_heartbeat: float = time.monotonic()
self.total_requests: int = 0
@@ -731,12 +755,88 @@ def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
return 0.5 + 0.5 * reputation
def _node_admission(node: "_NodeEntry") -> CapabilityState:
"""The node's recorded verdict, re-checked against what it advertises *now*.
A proof covers one model/shard combination. The tracker can move a node to a
different range after it registered (rebalance, `pending_new_assignment`), and
the proof does not travel with it: until the node re-registers with a proof
for the range it now advertises, it is shard-mismatched and not routable.
"""
state = node.capability
if not state.proven:
return state
if state.model_id and not _node_matches_model(node, state.model_id):
return state.with_state(
STATE_MODEL_MISMATCH,
f"proof is for model {state.model_id!r}, but the node now serves "
f"{node.hf_repo or node.model!r}",
)
if (
node.shard_start is not None
and node.shard_end is not None
and (state.shard_start, state.shard_end) != (node.shard_start, node.shard_end)
):
return state.with_state(
STATE_SHARD_MISMATCH,
f"proof is for layers {state.shard_start}–{state.shard_end}, but the "
f"node now serves layers {node.shard_start}–{node.shard_end}",
)
return state
def _capability_from_registration(
payload: dict,
*,
model: str | None,
hf_repo: str | None,
shard_start: int | None,
shard_end: int | None,
) -> CapabilityState:
"""The tracker's verdict on the proof carried by one registration payload.
Used by the register handler and by the Raft follower path, so a replicated
registration lands with the same verdict as the one the leader recorded.
"""
aliases = _model_aliases(model) | _model_aliases(hf_repo)
recipe_id = payload.get("recipe_id")
recipe_version = payload.get("recipe_version")
return evaluate_report(
payload.get("capability_report"),
model_matches=lambda reported: bool(_model_aliases(reported) & aliases),
advertised_model=hf_repo or model,
shard_start=shard_start,
shard_end=shard_end,
declared_recipe_id=recipe_id if isinstance(recipe_id, str) else None,
declared_recipe_version=(
recipe_version if isinstance(recipe_version, str) else None
),
)
def _capability_routable(node: "_NodeEntry", policy: str) -> bool:
"""May this node carry traffic under the tracker's capability policy?"""
return _node_admission(node).routable_under(policy)
def _admitted_nodes(nodes: list["_NodeEntry"], policy: str | None) -> list["_NodeEntry"]:
"""Drop every candidate whose capability proof does not admit it (ADR-0023).
This is the single gate every route path goes through. It removes candidates;
it never reorders or reweights them, so coverage-first selection and
throughput-weighted preference among the survivors are unchanged.
"""
effective = normalize_policy(policy)
return [node for node in nodes if _capability_routable(node, effective)]
def _select_route(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
model: str | None = None,
contracts: Any | None = None,
policy: str | None = None,
) -> tuple[list[_NodeEntry], str]:
"""Greedy interval-cover biased toward fast, lightly-loaded, reputable nodes.
@@ -747,7 +847,10 @@ def _select_route(
Tiebreak: higher shard_end (fewer hops).
"""
candidates = sorted(
[node for node in nodes if node.shard_start is not None and node.shard_end is not None],
[
node for node in _admitted_nodes(nodes, policy)
if node.shard_start is not None and node.shard_end is not None
],
key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
)
route: list[_NodeEntry] = []
@@ -783,6 +886,7 @@ def _enumerate_routes(
model: str | None = None,
contracts: Any | None = None,
max_candidates: int = 8,
policy: str | None = None,
) -> list["RouteCandidate"]:
"""Enumerate viable route candidates for bandit selection (ADR-0021).
@@ -791,9 +895,13 @@ def _enumerate_routes(
with the longest-advancing hops. The route's prior throughput estimate is
its bottleneck hop's queue-adjusted effective throughput — used only until
observed route samples exist.
Candidates that are not admitted by their capability proof are dropped before
enumeration, so an unproven node cannot appear in any route candidate — not
even as a scouted one.
"""
sharded = [
n for n in nodes
n for n in _admitted_nodes(nodes, policy)
if n.shard_start is not None and n.shard_end is not None
]
# Heads must start the pipeline at the first required layer (they tokenize
@@ -2675,9 +2783,13 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
route_stats: "RouteStatsStore | None" = None,
relay_status: dict | None = None,
test_runner: "TestRunManager | None" = None,
capability_policy: str | None = None,
) -> None:
super().__init__(addr, handler)
self.registry = registry
self.capability_policy = normalize_policy(
capability_policy if capability_policy is not None else policy_from_env()
)
self.lock = lock
self.heartbeat_timeout = heartbeat_timeout
self.model_presets = model_presets
@@ -3177,9 +3289,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def model_supply_for(node: _NodeEntry) -> dict:
return _model_health_summary(server, node.model, node.hf_repo)
def capability_for(node: _NodeEntry) -> dict:
# Re-verdicted against what the node advertises now, so the operator
# view and the routing gate can never disagree.
state = _node_admission(node)
return {
**state.to_dict(),
"routable": state.routable_under(server.capability_policy),
}
self._send_json(200, {
"relay_url": server.relay_url,
"relay": dict(server.relay_status),
"capability_policy": server.capability_policy,
"pool": _pool_summary(nodes),
"memory_pool": memory_pool,
"recommended_models": [
@@ -3213,6 +3335,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"capacity": capacity_for(node),
"model_supply": model_supply_for(node),
"throughput": throughput_for(node),
"capability": capability_for(node),
"stats": _node_health(node, server.heartbeat_timeout),
}
for node in nodes
@@ -3365,6 +3488,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if n.tracker_mode
and _node_matches_model(n, model)
and _quantization_satisfies(n.quantization, requested_quantization)
and _capability_routable(n, server.capability_policy)
]
if not candidates:
@@ -3375,6 +3499,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if n.shard_start == 0
and _node_matches_model(n, model)
and _quantization_satisfies(n.quantization, requested_quantization)
and _capability_routable(n, server.capability_policy)
]
if not candidates:
@@ -3480,6 +3605,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model=route_model,
contracts=server.contracts,
max_candidates=server.route_stats.config.max_candidate_routes,
policy=server.capability_policy,
)
picked, routing_decision = choose_route(
route_candidates, server.route_stats, route_model, rng=server.route_rng,
@@ -3489,7 +3615,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else:
# No head-anchored candidate — legacy greedy cover as fallback
# (also produces the layer-gap error message).
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
route_nodes, route_error = _select_route(
all_nodes, rs, re,
model=route_model,
contracts=server.contracts,
policy=server.capability_policy,
)
routing_decision = {"mode": "greedy-fallback"}
if route_error:
_tracker_log(
@@ -4344,6 +4475,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(400, {"error": str(exc)})
return
recipe_id = body.get("recipe_id")
recipe_version = body.get("recipe_version")
if (recipe_id is not None and not isinstance(recipe_id, str)) or (
recipe_version is not None and not isinstance(recipe_version, str)
):
self._send_json(400, {"error": "recipe_id and recipe_version must be strings"})
return
# The capability proof (ADR-0023). A bad proof does not fail registration --
# the node is recorded with its verdict so the operator can see *why* it is
# not routing -- but only an `admitted` verdict makes it routable.
capability = _capability_from_registration(
body,
model=model,
hf_repo=hf_repo,
shard_start=shard_start,
shard_end=shard_end,
)
node_id = _node_id_for_registration(
endpoint,
model,
@@ -4378,6 +4528,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
cert_fingerprint=cert_fingerprint,
peer_id=peer_id,
friendly_name=friendly_name,
capability=capability,
)
with server.lock:
self._purge_expired_nodes()
@@ -4433,6 +4584,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
old_model_health=_model_health_summary(server, old.model, old.hf_repo),
model_health=model_health,
)
routable = _capability_routable(entry, server.capability_policy)
_tracker_log(
server,
"info",
@@ -4444,7 +4596,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
shard=f"{entry.shard_start}-{entry.shard_end}",
tracker_mode=entry.tracker_mode,
model_health=model_health,
capability=entry.capability.state,
routable=routable,
)
if not routable:
_tracker_log(
server,
"warn",
"node registered but is not routable",
node_id=node_id,
endpoint=entry.endpoint,
capability=entry.capability.state,
detail=entry.capability.detail,
)
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
repo_info = f" [{hf_repo}]" if hf_repo else ""
@@ -4456,7 +4620,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
flush=True,
)
payload = {"node_id": node_id}
payload = {
"node_id": node_id,
# Tell the node what the tracker made of its proof: a node that is
# registered but not routable must be able to see that it is dark.
"capability": {
"state": entry.capability.state,
"detail": entry.capability.detail,
"routable": routable,
"policy": server.capability_policy,
},
}
if assignment_directive is not None:
payload["directive"] = assignment_directive
self._send_json(200, payload)
@@ -5989,6 +6163,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model=route_model,
contracts=server.contracts,
max_candidates=server.route_stats.config.max_candidate_routes,
policy=server.capability_policy,
)
if candidates:
# Prefer a distributed multi-hop route when available. Greedy
@@ -5998,7 +6173,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
error = ""
else:
route, error = _select_route(
alive, required_start, required_end, contracts=server.contracts,
alive, required_start, required_end,
contracts=server.contracts,
policy=server.capability_policy,
)
if error:
_tracker_log(
@@ -6084,6 +6261,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model=stats_key,
contracts=server.contracts,
max_candidates=cfg.max_candidate_routes,
policy=server.capability_policy,
)
out[stats_key] = {
"epoch": server.route_stats.epoch(stats_key),
@@ -6177,7 +6355,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
routes = []
remaining = list(candidates)
for _ in range(redundancy):
route, error = _select_route(remaining, required_start, required_end, contracts=server.contracts)
route, error = _select_route(
remaining, required_start, required_end,
contracts=server.contracts,
policy=server.capability_policy,
)
if error:
self._send_json(503, {"error": error})
return
@@ -6262,7 +6444,11 @@ class TrackerServer:
routing_config: RoutingConfig | None = None,
enable_test_runner: bool = False,
test_runner: TestRunManager | None = None,
capability_policy: str | None = None,
) -> None:
self._capability_policy = normalize_policy(
capability_policy if capability_policy is not None else policy_from_env()
)
self._host = host
self._requested_port = port
self._heartbeat_timeout = heartbeat_timeout
@@ -6475,6 +6661,7 @@ class TrackerServer:
route_stats=self._route_stats,
relay_status=http_relay_status,
test_runner=self._test_runner,
capability_policy=self._capability_policy,
)
self.port = self._server.server_address[1]
@@ -6708,6 +6895,15 @@ class TrackerServer:
else None
),
friendly_name=_normalize_friendly_name(payload.get("friendly_name")),
# A replicated registration carries its proof: without this, a proven
# node would be routable on the leader and dark on every follower.
capability=_capability_from_registration(
payload,
model=payload.get("model", "stub-model"),
hf_repo=payload.get("hf_repo"),
shard_start=shard_start,
shard_end=shard_end,
),
)
with self._lock:
self._registry[node_id] = entry