popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[verified] feat: complete Ralph task workstreams

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-12 11:17:03 +03:00
parent 9a1b15c020
commit 377346c301
37 changed files with 5862 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

415
packages/tracker/meshnet_tracker/capability.py Normal file
View File

@@ -0,0 +1,415 @@
"""Tracker-side validation of the capability report a Node presents at registration.
A Node proves locally that it can execute one exact combination — model artifact,
shard range, recipe, backend/device — and ships that proof with its registration
(ADR-0023, NCA-001/002/003). The tracker does not re-run the forward; it decides
whether the presented proof *covers what the node is advertising*, records the
verdict as a small sanitized enum, and routes only to nodes whose verdict is
`admitted`.
Two properties this module deliberately keeps:
* **No model knowledge.** Model ids, recipe ids, backend ids and device names are
opaque labels. They are compared, never interpreted; no vendor string is a
code path here.
* **Evidence, not assertion.** A report is treated as a claim about identity, and
the tracker only ever *narrows* what a node may serve with it. Nothing in a
report can widen a node's eligibility or its routing weight — throughput
routing stays measurement-driven (ADR-0013/0021).
Older nodes that predate the capability protocol present no report at all. They
are handled by an explicit policy (`POLICY_COMPAT` vs `POLICY_ENFORCE`), never by
silently treating "no proof" as "proven" — see `docs/adr/0023-…` for the rollout.
"""
from __future__ import annotations
import os
import re
import time
from dataclasses import dataclass, replace
from typing import Any, Callable, Mapping
# The capability report layout this tracker reads (meshnet_node.capability).
SUPPORTED_SCHEMA_VERSION = 1
# The oldest recipe catalogue whose recipe semantics this tracker still trusts.
# A node carrying an older catalogue may be running a recipe whose id has since
# been redefined, so its proof cannot be matched to a name reliably.
MIN_CATALOGUE_VERSION = "2026.07.1"
# How old a proof may be *at the moment it is presented*. Freshness after that is
# carried by liveness: a registration is re-asserted on tracker restart and the
# node is purged once heartbeats stop.
DEFAULT_MAX_REPORT_AGE_SECONDS = 900.0
# A proof timestamped further ahead than this is not fresh, it is wrong.
MAX_CLOCK_SKEW_SECONDS = 60.0
STATUS_PASSED = "passed"
# --- Admission verdicts. `admitted` is the only routable one under `enforce`. ---
STATE_ADMITTED = "admitted"
STATE_ABSENT = "absent"
STATE_INVALID = "invalid"
STATE_FAILED = "failed"
STATE_STALE = "stale"
STATE_MODEL_MISMATCH = "model-mismatch"
STATE_SHARD_MISMATCH = "shard-mismatch"
STATE_RECIPE_MISMATCH = "recipe-mismatch"
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
ALL_STATES = (
STATE_ADMITTED,
STATE_ABSENT,
STATE_INVALID,
STATE_FAILED,
STATE_STALE,
STATE_MODEL_MISMATCH,
STATE_SHARD_MISMATCH,
STATE_RECIPE_MISMATCH,
STATE_CATALOGUE_INCOMPATIBLE,
)
# --- Compatibility policy for nodes that predate the capability protocol. ---
# `compat` — a node presenting *no* proof still routes (legacy behaviour), but a
# node presenting a *bad* proof never does. Presenting a broken or
# mismatched proof is a stronger signal than presenting none.
# `enforce` — only `admitted` routes. Absent proof is not routable.
POLICY_COMPAT = "compat"
POLICY_ENFORCE = "enforce"
ALL_POLICIES = (POLICY_COMPAT, POLICY_ENFORCE)
DEFAULT_POLICY = POLICY_COMPAT
POLICY_ENV_VAR = "MESHNET_TRACKER_CAPABILITY_POLICY"
# Operator-facing detail strings are short and never carry a raw exception.
_MAX_DETAIL_CHARS = 240
_MAX_DIAGNOSTICS = 3
_CREDENTIAL_PATTERNS = (
re.compile(r"\b[A-Za-z0-9_]{2,6}_[A-Za-z0-9]{16,}\b"), # hf_…, ghp_…, sk_live_…
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
re.compile(r"(?i)\bbearer\s+\S+"),
re.compile(r"(?i)\b(?:token|api[_-]?key|password|secret)\s*[=:]\s*\S+"),
)
_REDACTED = "[redacted]"
def normalize_policy(value: Any) -> str:
"""Return a known policy name, falling back to the default for anything else."""
if isinstance(value, str) and value.strip().lower() in ALL_POLICIES:
return value.strip().lower()
return DEFAULT_POLICY
def policy_from_env(environ: Mapping[str, str] | None = None) -> str:
env = os.environ if environ is None else environ
return normalize_policy(env.get(POLICY_ENV_VAR))
def sanitize_detail(text: Any) -> str:
"""Collapse, redact and clip a string bound for an operator view."""
cleaned = " ".join(str(text).split())
for pattern in _CREDENTIAL_PATTERNS:
cleaned = pattern.sub(_REDACTED, cleaned)
if len(cleaned) > _MAX_DETAIL_CHARS:
cleaned = cleaned[: _MAX_DETAIL_CHARS - 1].rstrip() + "…"
return cleaned
def catalogue_is_compatible(version: Any) -> bool:
"""True when `version` is at least `MIN_CATALOGUE_VERSION`.
Versions are dotted integer sequences (`2026.07.1`). Anything that does not
parse is incompatible — an unparseable catalogue version cannot be shown to
be new enough.
"""
parsed = _parse_version(version)
if parsed is None:
return False
return parsed >= _parse_version(MIN_CATALOGUE_VERSION) # type: ignore[operator]
def _parse_version(value: Any) -> tuple[int, ...] | None:
if not isinstance(value, str) or not value.strip():
return None
parts = value.strip().split(".")
try:
return tuple(int(part) for part in parts)
except ValueError:
return None
@dataclass(frozen=True)
class CapabilityState:
"""The tracker's sanitized verdict on one node's presented proof.
This is what the network map exposes and what route selection consults. It
holds identity labels and a verdict — never a raw exception, a file path, or
a credential.
"""
state: str
detail: str = ""
model_id: str | None = None
shard_start: int | None = None
shard_end: int | None = None
recipe_id: str | None = None
recipe_version: str | None = None
catalogue_version: str | None = None
backend_id: str | None = None
device: str | None = None
quantization: str | None = None
validated_at: float | None = None
recorded_at: float = 0.0
schema_version: int | None = None
diagnostics: tuple[str, ...] = ()
@property
def proven(self) -> bool:
"""The presented proof covers exactly what the node advertised."""
return self.state == STATE_ADMITTED
def routable_under(self, policy: str) -> bool:
if self.proven:
return True
return self.state == STATE_ABSENT and normalize_policy(policy) == POLICY_COMPAT
def with_state(self, state: str, detail: str) -> CapabilityState:
"""Re-verdict a recorded proof against what the node advertises *now*."""
return replace(self, state=state, detail=sanitize_detail(detail))
def to_dict(self) -> dict:
return {
"state": self.state,
"detail": self.detail,
"model_id": self.model_id,
"shard_start": self.shard_start,
"shard_end": self.shard_end,
"recipe_id": self.recipe_id,
"recipe_version": self.recipe_version,
"catalogue_version": self.catalogue_version,
"backend_id": self.backend_id,
"device": self.device,
"quantization": self.quantization,
"validated_at": self.validated_at,
"recorded_at": self.recorded_at,
"schema_version": self.schema_version,
"diagnostics": list(self.diagnostics),
}
def absent_state(detail: str = "", *, now: float | None = None) -> CapabilityState:
"""The verdict for a node that presented no proof at all (legacy node)."""
return CapabilityState(
state=STATE_ABSENT,
detail=sanitize_detail(
detail
or "node registered without a capability report; it predates the "
"capability protocol or ran with admission disabled"
),
recorded_at=time.time() if now is None else now,
)
def evaluate_report(
report: Any,
*,
model_matches: Callable[[str], bool],
advertised_model: str | None,
shard_start: int | None,
shard_end: int | None,
declared_recipe_id: str | None = None,
declared_recipe_version: str | None = None,
now: float | None = None,
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
) -> CapabilityState:
"""Judge the proof a node presented against what that node is advertising.
`model_matches` is the tracker's own alias-aware comparison against the
node's registered model / hf_repo, so an opaque model id never has to be
parsed here.
Returns a verdict for *every* input, including malformed ones: a bad proof is
recorded and shown to the operator rather than dropped, so "why is my node not
routing" has an answer in the network map.
"""
now = time.time() if now is None else now
if report is None:
return absent_state(now=now)
if not isinstance(report, Mapping):
return CapabilityState(
state=STATE_INVALID,
detail=sanitize_detail(
f"capability_report must be a JSON object, got "
f"{type(report).__name__}"
),
recorded_at=now,
)
try:
parsed = _parse_report(report)
except _ReportError as exc:
return CapabilityState(
state=STATE_INVALID,
detail=sanitize_detail(str(exc)),
recorded_at=now,
schema_version=_maybe_int(report.get("schema_version")),
)
status = parsed.pop("_status")
base = CapabilityState(state=STATE_ADMITTED, recorded_at=now, **parsed)
if base.schema_version != SUPPORTED_SCHEMA_VERSION:
return base.with_state(
STATE_INVALID,
f"capability report declares schema version {base.schema_version}; "
f"this tracker reads version {SUPPORTED_SCHEMA_VERSION}",
)
if not catalogue_is_compatible(base.catalogue_version):
return base.with_state(
STATE_CATALOGUE_INCOMPATIBLE,
f"recipe catalogue {base.catalogue_version!r} is older than the "
f"minimum this tracker trusts ({MIN_CATALOGUE_VERSION}); upgrade the node",
)
if not model_matches(base.model_id or ""):
return base.with_state(
STATE_MODEL_MISMATCH,
f"proof is for model {base.model_id!r}, but the node registered "
f"{advertised_model!r}",
)
if shard_start is not None and shard_end is not None:
if (base.shard_start, base.shard_end) != (shard_start, shard_end):
return base.with_state(
STATE_SHARD_MISMATCH,
f"proof is for layers {base.shard_start}–{base.shard_end}, but the "
f"node registered layers {shard_start}–{shard_end}",
)
if declared_recipe_id is not None and base.recipe_id != declared_recipe_id:
return base.with_state(
STATE_RECIPE_MISMATCH,
f"proof is for recipe {base.recipe_id!r}, but the node declared it "
f"serves with {declared_recipe_id!r}",
)
if (
declared_recipe_version is not None
and base.recipe_version != declared_recipe_version
):
return base.with_state(
STATE_RECIPE_MISMATCH,
f"proof is for recipe {base.recipe_id!r} v{base.recipe_version}, but "
f"the node declared v{declared_recipe_version}",
)
if status != STATUS_PASSED:
return base.with_state(
STATE_FAILED,
f"capability validation {status} on the node"
+ (f" — {' '.join(base.diagnostics)}" if base.diagnostics else ""),
)
age = now - (base.validated_at or 0.0)
if age > max_age_seconds:
return base.with_state(
STATE_STALE,
f"proof is {age / 60:.0f} min old (limit {max_age_seconds / 60:.0f} min); "
"the node must re-validate before it can be routed",
)
if age < -MAX_CLOCK_SKEW_SECONDS:
return base.with_state(
STATE_STALE,
f"proof is timestamped {-age:.0f}s in the future; check the node's clock",
)
return base.with_state(
STATE_ADMITTED,
f"{base.model_id} layers {base.shard_start}–{base.shard_end} proven on "
f"{base.device} with recipe {base.recipe_id} (v{base.recipe_version})",
)
class _ReportError(ValueError):
"""Malformed report input. Messages name the field, never echo a payload."""
def _parse_report(doc: Mapping[str, Any]) -> dict:
model = _object(doc.get("model"), "model")
shard = _object(doc.get("shard"), "shard")
recipe = _object(doc.get("recipe"), "recipe")
backend = _object(doc.get("backend"), "backend")
validated_at = doc.get("validated_at")
if isinstance(validated_at, bool) or not isinstance(validated_at, (int, float)):
raise _ReportError("'validated_at' must be a Unix timestamp")
schema_version = doc.get("schema_version")
if isinstance(schema_version, bool) or not isinstance(schema_version, int):
raise _ReportError("'schema_version' must be an integer")
return {
"model_id": _text(model.get("model_id"), "model.model_id"),
"shard_start": _index(shard.get("start"), "shard.start"),
"shard_end": _index(shard.get("end"), "shard.end"),
"recipe_id": _text(recipe.get("recipe_id"), "recipe.recipe_id"),
"recipe_version": _text(recipe.get("recipe_version"), "recipe.recipe_version"),
"catalogue_version": _text(
recipe.get("catalogue_version"), "recipe.catalogue_version"
),
"backend_id": _text(backend.get("backend_id"), "backend.backend_id"),
"device": _text(backend.get("device"), "backend.device"),
"quantization": _optional_text(
backend.get("quantization"), "backend.quantization"
),
"validated_at": float(validated_at),
"schema_version": schema_version,
"diagnostics": _diagnostics(doc.get("diagnostics")),
"_status": _text(doc.get("status"), "status"),
}
def _object(value: Any, field_name: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise _ReportError(f"{field_name!r} must be a JSON object")
return value
def _text(value: Any, field_name: str) -> str:
if not isinstance(value, str) or not value.strip():
raise _ReportError(f"{field_name!r} must be a non-empty string")
return value
def _optional_text(value: Any, field_name: str) -> str | None:
if value is None:
return None
return _text(value, field_name)
def _index(value: Any, field_name: str) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
raise _ReportError(f"{field_name!r} must be a non-negative integer")
return value
def _maybe_int(value: Any) -> int | None:
if isinstance(value, bool) or not isinstance(value, int):
return None
return value
def _diagnostics(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()
out = [
sanitize_detail(item)
for item in value[:_MAX_DIAGNOSTICS]
if isinstance(item, str) and item.strip()
]
return tuple(out)

14
packages/tracker/meshnet_tracker/cli.py
View File

@@ -9,6 +9,7 @@ from pathlib import Path
from .accounts import DEFAULT_ACCOUNTS_DB_PATH
from .billing import DEFAULT_BILLING_DB_PATH
from .capability import ALL_POLICIES as ALL_CAPABILITY_POLICIES
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH
from .logging_setup import (
DEFAULT_LOG_BACKUP_COUNT,
@@ -105,6 +106,18 @@ def main() -> None:
metavar="PATH",
help="SQLite database path for persistent model usage statistics",
)
common.add_argument(
"--capability-policy",
choices=list(ALL_CAPABILITY_POLICIES),
default=None,
help=(
"How to treat nodes that present no capability proof (ADR-0023): "
"'compat' (default) still routes pre-capability nodes; 'enforce' routes "
"only nodes whose proof covers the model and shard they advertise. "
"A broken or mismatched proof is never routed under either policy. "
"Falls back to $MESHNET_TRACKER_CAPABILITY_POLICY when omitted."
),
)
common.add_argument(
"--relay-url",
default=None,
@@ -416,6 +429,7 @@ def main() -> None:
cluster_peers=cluster_peers or None,
cluster_self_url=args.self_url,
stats_db=getattr(args, "stats_db", None),
capability_policy=getattr(args, "capability_policy", None),
relay_url=relay_url,
embedded_relay=args.embedded_relay,
embedded_relay_host=args.relay_host,

212
packages/tracker/meshnet_tracker/server.py
View File

@@ -6,10 +6,14 @@ HTTP API contract:
"endpoint": "http://host:port", "shard_start": int, "shard_end": int,
"model": str optional, "shard_checksum": str optional,
"hardware_profile": object, "wallet_address": str optional,
"score": number optional
"score": number optional,
"capability_report": object optional, # ADR-0023 proof of this exact shard
"recipe_id": str optional, "recipe_version": str optional
}
Response 200: {"node_id": str}
Response 200: {"node_id": str, "capability": {"state": str, "routable": bool, ...}}
Response 400: {"error": str}
A node whose proof does not admit what it advertises still registers (so the
operator can see why it is dark) but is not routable -- see `capability.py`.
- POST /v1/nodes/{node_id}/heartbeat
Response 200: {}
Response 404: {"error": "node not found"}
@@ -48,6 +52,20 @@ from typing import Any
from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
from .auth import is_validator_token, sign_hive_request, verify_hive_request
from .capability import (
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
POLICY_COMPAT,
POLICY_ENFORCE,
STATE_ABSENT,
STATE_ADMITTED,
STATE_MODEL_MISMATCH,
STATE_SHARD_MISMATCH,
CapabilityState,
absent_state,
evaluate_report,
normalize_policy,
policy_from_env,
)
from .wallet_proof import binding_message, verify_wallet_signature
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
@@ -587,6 +605,8 @@ class _NodeEntry:
"heartbeats_expected", "heartbeats_received",
# dynamic reassignment queued by the tracker
"pending_new_assignment",
# the tracker's verdict on the capability proof this node presented
"capability",
)
def __init__(
@@ -616,6 +636,7 @@ class _NodeEntry:
cert_fingerprint: str | None = None,
peer_id: str | None = None,
friendly_name: str | None = None,
capability: "CapabilityState | None" = None,
) -> None:
self.node_id = node_id
self.endpoint = endpoint
@@ -643,6 +664,9 @@ class _NodeEntry:
self.cert_fingerprint = cert_fingerprint
self.peer_id = peer_id
self.friendly_name = friendly_name
# No proof presented is `absent`, never `admitted` — a node can only earn
# `admitted` by presenting a report that covers what it advertises.
self.capability: CapabilityState = capability or absent_state()
self.pending_directives: list[dict] = []
self.last_heartbeat: float = time.monotonic()
self.total_requests: int = 0
@@ -731,12 +755,88 @@ def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
return 0.5 + 0.5 * reputation
def _node_admission(node: "_NodeEntry") -> CapabilityState:
"""The node's recorded verdict, re-checked against what it advertises *now*.
A proof covers one model/shard combination. The tracker can move a node to a
different range after it registered (rebalance, `pending_new_assignment`), and
the proof does not travel with it: until the node re-registers with a proof
for the range it now advertises, it is shard-mismatched and not routable.
"""
state = node.capability
if not state.proven:
return state
if state.model_id and not _node_matches_model(node, state.model_id):
return state.with_state(
STATE_MODEL_MISMATCH,
f"proof is for model {state.model_id!r}, but the node now serves "
f"{node.hf_repo or node.model!r}",
)
if (
node.shard_start is not None
and node.shard_end is not None
and (state.shard_start, state.shard_end) != (node.shard_start, node.shard_end)
):
return state.with_state(
STATE_SHARD_MISMATCH,
f"proof is for layers {state.shard_start}–{state.shard_end}, but the "
f"node now serves layers {node.shard_start}–{node.shard_end}",
)
return state
def _capability_from_registration(
payload: dict,
*,
model: str | None,
hf_repo: str | None,
shard_start: int | None,
shard_end: int | None,
) -> CapabilityState:
"""The tracker's verdict on the proof carried by one registration payload.
Used by the register handler and by the Raft follower path, so a replicated
registration lands with the same verdict as the one the leader recorded.
"""
aliases = _model_aliases(model) | _model_aliases(hf_repo)
recipe_id = payload.get("recipe_id")
recipe_version = payload.get("recipe_version")
return evaluate_report(
payload.get("capability_report"),
model_matches=lambda reported: bool(_model_aliases(reported) & aliases),
advertised_model=hf_repo or model,
shard_start=shard_start,
shard_end=shard_end,
declared_recipe_id=recipe_id if isinstance(recipe_id, str) else None,
declared_recipe_version=(
recipe_version if isinstance(recipe_version, str) else None
),
)
def _capability_routable(node: "_NodeEntry", policy: str) -> bool:
"""May this node carry traffic under the tracker's capability policy?"""
return _node_admission(node).routable_under(policy)
def _admitted_nodes(nodes: list["_NodeEntry"], policy: str | None) -> list["_NodeEntry"]:
"""Drop every candidate whose capability proof does not admit it (ADR-0023).
This is the single gate every route path goes through. It removes candidates;
it never reorders or reweights them, so coverage-first selection and
throughput-weighted preference among the survivors are unchanged.
"""
effective = normalize_policy(policy)
return [node for node in nodes if _capability_routable(node, effective)]
def _select_route(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
model: str | None = None,
contracts: Any | None = None,
policy: str | None = None,
) -> tuple[list[_NodeEntry], str]:
"""Greedy interval-cover biased toward fast, lightly-loaded, reputable nodes.
@@ -747,7 +847,10 @@ def _select_route(
Tiebreak: higher shard_end (fewer hops).
"""
candidates = sorted(
[node for node in nodes if node.shard_start is not None and node.shard_end is not None],
[
node for node in _admitted_nodes(nodes, policy)
if node.shard_start is not None and node.shard_end is not None
],
key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
)
route: list[_NodeEntry] = []
@@ -783,6 +886,7 @@ def _enumerate_routes(
model: str | None = None,
contracts: Any | None = None,
max_candidates: int = 8,
policy: str | None = None,
) -> list["RouteCandidate"]:
"""Enumerate viable route candidates for bandit selection (ADR-0021).
@@ -791,9 +895,13 @@ def _enumerate_routes(
with the longest-advancing hops. The route's prior throughput estimate is
its bottleneck hop's queue-adjusted effective throughput — used only until
observed route samples exist.
Candidates that are not admitted by their capability proof are dropped before
enumeration, so an unproven node cannot appear in any route candidate — not
even as a scouted one.
"""
sharded = [
n for n in nodes
n for n in _admitted_nodes(nodes, policy)
if n.shard_start is not None and n.shard_end is not None
]
# Heads must start the pipeline at the first required layer (they tokenize
@@ -2675,9 +2783,13 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
route_stats: "RouteStatsStore | None" = None,
relay_status: dict | None = None,
test_runner: "TestRunManager | None" = None,
capability_policy: str | None = None,
) -> None:
super().__init__(addr, handler)
self.registry = registry
self.capability_policy = normalize_policy(
capability_policy if capability_policy is not None else policy_from_env()
)
self.lock = lock
self.heartbeat_timeout = heartbeat_timeout
self.model_presets = model_presets
@@ -3177,9 +3289,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def model_supply_for(node: _NodeEntry) -> dict:
return _model_health_summary(server, node.model, node.hf_repo)
def capability_for(node: _NodeEntry) -> dict:
# Re-verdicted against what the node advertises now, so the operator
# view and the routing gate can never disagree.
state = _node_admission(node)
return {
**state.to_dict(),
"routable": state.routable_under(server.capability_policy),
}
self._send_json(200, {
"relay_url": server.relay_url,
"relay": dict(server.relay_status),
"capability_policy": server.capability_policy,
"pool": _pool_summary(nodes),
"memory_pool": memory_pool,
"recommended_models": [
@@ -3213,6 +3335,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"capacity": capacity_for(node),
"model_supply": model_supply_for(node),
"throughput": throughput_for(node),
"capability": capability_for(node),
"stats": _node_health(node, server.heartbeat_timeout),
}
for node in nodes
@@ -3365,6 +3488,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if n.tracker_mode
and _node_matches_model(n, model)
and _quantization_satisfies(n.quantization, requested_quantization)
and _capability_routable(n, server.capability_policy)
]
if not candidates:
@@ -3375,6 +3499,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if n.shard_start == 0
and _node_matches_model(n, model)
and _quantization_satisfies(n.quantization, requested_quantization)
and _capability_routable(n, server.capability_policy)
]
if not candidates:
@@ -3480,6 +3605,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model=route_model,
contracts=server.contracts,
max_candidates=server.route_stats.config.max_candidate_routes,
policy=server.capability_policy,
)
picked, routing_decision = choose_route(
route_candidates, server.route_stats, route_model, rng=server.route_rng,
@@ -3489,7 +3615,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else:
# No head-anchored candidate — legacy greedy cover as fallback
# (also produces the layer-gap error message).
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
route_nodes, route_error = _select_route(
all_nodes, rs, re,
model=route_model,
contracts=server.contracts,
policy=server.capability_policy,
)
routing_decision = {"mode": "greedy-fallback"}
if route_error:
_tracker_log(
@@ -4344,6 +4475,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(400, {"error": str(exc)})
return
recipe_id = body.get("recipe_id")
recipe_version = body.get("recipe_version")
if (recipe_id is not None and not isinstance(recipe_id, str)) or (
recipe_version is not None and not isinstance(recipe_version, str)
):
self._send_json(400, {"error": "recipe_id and recipe_version must be strings"})
return
# The capability proof (ADR-0023). A bad proof does not fail registration --
# the node is recorded with its verdict so the operator can see *why* it is
# not routing -- but only an `admitted` verdict makes it routable.
capability = _capability_from_registration(
body,
model=model,
hf_repo=hf_repo,
shard_start=shard_start,
shard_end=shard_end,
)
node_id = _node_id_for_registration(
endpoint,
model,
@@ -4378,6 +4528,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
cert_fingerprint=cert_fingerprint,
peer_id=peer_id,
friendly_name=friendly_name,
capability=capability,
)
with server.lock:
self._purge_expired_nodes()
@@ -4433,6 +4584,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
old_model_health=_model_health_summary(server, old.model, old.hf_repo),
model_health=model_health,
)
routable = _capability_routable(entry, server.capability_policy)
_tracker_log(
server,
"info",
@@ -4444,7 +4596,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
shard=f"{entry.shard_start}-{entry.shard_end}",
tracker_mode=entry.tracker_mode,
model_health=model_health,
capability=entry.capability.state,
routable=routable,
)
if not routable:
_tracker_log(
server,
"warn",
"node registered but is not routable",
node_id=node_id,
endpoint=entry.endpoint,
capability=entry.capability.state,
detail=entry.capability.detail,
)
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
repo_info = f" [{hf_repo}]" if hf_repo else ""
@@ -4456,7 +4620,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
flush=True,
)
payload = {"node_id": node_id}
payload = {
"node_id": node_id,
# Tell the node what the tracker made of its proof: a node that is
# registered but not routable must be able to see that it is dark.
"capability": {
"state": entry.capability.state,
"detail": entry.capability.detail,
"routable": routable,
"policy": server.capability_policy,
},
}
if assignment_directive is not None:
payload["directive"] = assignment_directive
self._send_json(200, payload)
@@ -5989,6 +6163,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model=route_model,
contracts=server.contracts,
max_candidates=server.route_stats.config.max_candidate_routes,
policy=server.capability_policy,
)
if candidates:
# Prefer a distributed multi-hop route when available. Greedy
@@ -5998,7 +6173,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
error = ""
else:
route, error = _select_route(
alive, required_start, required_end, contracts=server.contracts,
alive, required_start, required_end,
contracts=server.contracts,
policy=server.capability_policy,
)
if error:
_tracker_log(
@@ -6084,6 +6261,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
model=stats_key,
contracts=server.contracts,
max_candidates=cfg.max_candidate_routes,
policy=server.capability_policy,
)
out[stats_key] = {
"epoch": server.route_stats.epoch(stats_key),
@@ -6177,7 +6355,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
routes = []
remaining = list(candidates)
for _ in range(redundancy):
route, error = _select_route(remaining, required_start, required_end, contracts=server.contracts)
route, error = _select_route(
remaining, required_start, required_end,
contracts=server.contracts,
policy=server.capability_policy,
)
if error:
self._send_json(503, {"error": error})
return
@@ -6262,7 +6444,11 @@ class TrackerServer:
routing_config: RoutingConfig | None = None,
enable_test_runner: bool = False,
test_runner: TestRunManager | None = None,
capability_policy: str | None = None,
) -> None:
self._capability_policy = normalize_policy(
capability_policy if capability_policy is not None else policy_from_env()
)
self._host = host
self._requested_port = port
self._heartbeat_timeout = heartbeat_timeout
@@ -6475,6 +6661,7 @@ class TrackerServer:
route_stats=self._route_stats,
relay_status=http_relay_status,
test_runner=self._test_runner,
capability_policy=self._capability_policy,
)
self.port = self._server.server_address[1]
@@ -6708,6 +6895,15 @@ class TrackerServer:
else None
),
friendly_name=_normalize_friendly_name(payload.get("friendly_name")),
# A replicated registration carries its proof: without this, a proven
# node would be routable on the leader and dark on every follower.
capability=_capability_from_registration(
payload,
model=payload.get("model", "stub-model"),
hf_repo=payload.get("hf_repo"),
shard_start=shard_start,
shard_end=shard_end,
),
)
with self._lock:
self._registry[node_id] = entry