[verified] feat: complete Ralph task workstreams
This commit is contained in:
Dobromir Popov
415
packages/tracker/meshnet_tracker/capability.py
Normal file
415
packages/tracker/meshnet_tracker/capability.py
Normal file
@@ -0,0 +1,415 @@
|
||||
"""Tracker-side validation of the capability report a Node presents at registration.
|
||||
|
||||
A Node proves locally that it can execute one exact combination — model artifact,
|
||||
shard range, recipe, backend/device — and ships that proof with its registration
|
||||
(ADR-0023, NCA-001/002/003). The tracker does not re-run the forward; it decides
|
||||
whether the presented proof *covers what the node is advertising*, records the
|
||||
verdict as a small sanitized enum, and routes only to nodes whose verdict is
|
||||
`admitted`.
|
||||
|
||||
Two properties this module deliberately keeps:
|
||||
|
||||
* **No model knowledge.** Model ids, recipe ids, backend ids and device names are
|
||||
opaque labels. They are compared, never interpreted; no vendor string is a
|
||||
code path here.
|
||||
* **Evidence, not assertion.** A report is treated as a claim about identity, and
|
||||
the tracker only ever *narrows* what a node may serve with it. Nothing in a
|
||||
report can widen a node's eligibility or its routing weight — throughput
|
||||
routing stays measurement-driven (ADR-0013/0021).
|
||||
|
||||
Older nodes that predate the capability protocol present no report at all. They
|
||||
are handled by an explicit policy (`POLICY_COMPAT` vs `POLICY_ENFORCE`), never by
|
||||
silently treating "no proof" as "proven" — see `docs/adr/0023-…` for the rollout.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
# The capability report layout this tracker reads (meshnet_node.capability).
|
||||
SUPPORTED_SCHEMA_VERSION = 1
|
||||
|
||||
# The oldest recipe catalogue whose recipe semantics this tracker still trusts.
|
||||
# A node carrying an older catalogue may be running a recipe whose id has since
|
||||
# been redefined, so its proof cannot be matched to a name reliably.
|
||||
MIN_CATALOGUE_VERSION = "2026.07.1"
|
||||
|
||||
# How old a proof may be *at the moment it is presented*. Freshness after that is
|
||||
# carried by liveness: a registration is re-asserted on tracker restart and the
|
||||
# node is purged once heartbeats stop.
|
||||
DEFAULT_MAX_REPORT_AGE_SECONDS = 900.0
|
||||
|
||||
# A proof timestamped further ahead than this is not fresh, it is wrong.
|
||||
MAX_CLOCK_SKEW_SECONDS = 60.0
|
||||
|
||||
STATUS_PASSED = "passed"
|
||||
|
||||
# --- Admission verdicts. `admitted` is the only routable one under `enforce`. ---
|
||||
STATE_ADMITTED = "admitted"
|
||||
STATE_ABSENT = "absent"
|
||||
STATE_INVALID = "invalid"
|
||||
STATE_FAILED = "failed"
|
||||
STATE_STALE = "stale"
|
||||
STATE_MODEL_MISMATCH = "model-mismatch"
|
||||
STATE_SHARD_MISMATCH = "shard-mismatch"
|
||||
STATE_RECIPE_MISMATCH = "recipe-mismatch"
|
||||
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
|
||||
|
||||
ALL_STATES = (
|
||||
STATE_ADMITTED,
|
||||
STATE_ABSENT,
|
||||
STATE_INVALID,
|
||||
STATE_FAILED,
|
||||
STATE_STALE,
|
||||
STATE_MODEL_MISMATCH,
|
||||
STATE_SHARD_MISMATCH,
|
||||
STATE_RECIPE_MISMATCH,
|
||||
STATE_CATALOGUE_INCOMPATIBLE,
|
||||
)
|
||||
|
||||
# --- Compatibility policy for nodes that predate the capability protocol. ---
|
||||
# `compat` — a node presenting *no* proof still routes (legacy behaviour), but a
|
||||
# node presenting a *bad* proof never does. Presenting a broken or
|
||||
# mismatched proof is a stronger signal than presenting none.
|
||||
# `enforce` — only `admitted` routes. Absent proof is not routable.
|
||||
POLICY_COMPAT = "compat"
|
||||
POLICY_ENFORCE = "enforce"
|
||||
ALL_POLICIES = (POLICY_COMPAT, POLICY_ENFORCE)
|
||||
DEFAULT_POLICY = POLICY_COMPAT
|
||||
|
||||
POLICY_ENV_VAR = "MESHNET_TRACKER_CAPABILITY_POLICY"
|
||||
|
||||
# Operator-facing detail strings are short and never carry a raw exception.
|
||||
_MAX_DETAIL_CHARS = 240
|
||||
_MAX_DIAGNOSTICS = 3
|
||||
|
||||
_CREDENTIAL_PATTERNS = (
|
||||
re.compile(r"\b[A-Za-z0-9_]{2,6}_[A-Za-z0-9]{16,}\b"), # hf_…, ghp_…, sk_live_…
|
||||
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
|
||||
re.compile(r"(?i)\bbearer\s+\S+"),
|
||||
re.compile(r"(?i)\b(?:token|api[_-]?key|password|secret)\s*[=:]\s*\S+"),
|
||||
)
|
||||
_REDACTED = "[redacted]"
|
||||
|
||||
|
||||
def normalize_policy(value: Any) -> str:
|
||||
"""Return a known policy name, falling back to the default for anything else."""
|
||||
if isinstance(value, str) and value.strip().lower() in ALL_POLICIES:
|
||||
return value.strip().lower()
|
||||
return DEFAULT_POLICY
|
||||
|
||||
|
||||
def policy_from_env(environ: Mapping[str, str] | None = None) -> str:
|
||||
env = os.environ if environ is None else environ
|
||||
return normalize_policy(env.get(POLICY_ENV_VAR))
|
||||
|
||||
|
||||
def sanitize_detail(text: Any) -> str:
|
||||
"""Collapse, redact and clip a string bound for an operator view."""
|
||||
cleaned = " ".join(str(text).split())
|
||||
for pattern in _CREDENTIAL_PATTERNS:
|
||||
cleaned = pattern.sub(_REDACTED, cleaned)
|
||||
if len(cleaned) > _MAX_DETAIL_CHARS:
|
||||
cleaned = cleaned[: _MAX_DETAIL_CHARS - 1].rstrip() + "…"
|
||||
return cleaned
|
||||
|
||||
|
||||
def catalogue_is_compatible(version: Any) -> bool:
|
||||
"""True when `version` is at least `MIN_CATALOGUE_VERSION`.
|
||||
|
||||
Versions are dotted integer sequences (`2026.07.1`). Anything that does not
|
||||
parse is incompatible — an unparseable catalogue version cannot be shown to
|
||||
be new enough.
|
||||
"""
|
||||
parsed = _parse_version(version)
|
||||
if parsed is None:
|
||||
return False
|
||||
return parsed >= _parse_version(MIN_CATALOGUE_VERSION) # type: ignore[operator]
|
||||
|
||||
|
||||
def _parse_version(value: Any) -> tuple[int, ...] | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
parts = value.strip().split(".")
|
||||
try:
|
||||
return tuple(int(part) for part in parts)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityState:
|
||||
"""The tracker's sanitized verdict on one node's presented proof.
|
||||
|
||||
This is what the network map exposes and what route selection consults. It
|
||||
holds identity labels and a verdict — never a raw exception, a file path, or
|
||||
a credential.
|
||||
"""
|
||||
|
||||
state: str
|
||||
detail: str = ""
|
||||
model_id: str | None = None
|
||||
shard_start: int | None = None
|
||||
shard_end: int | None = None
|
||||
recipe_id: str | None = None
|
||||
recipe_version: str | None = None
|
||||
catalogue_version: str | None = None
|
||||
backend_id: str | None = None
|
||||
device: str | None = None
|
||||
quantization: str | None = None
|
||||
validated_at: float | None = None
|
||||
recorded_at: float = 0.0
|
||||
schema_version: int | None = None
|
||||
diagnostics: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def proven(self) -> bool:
|
||||
"""The presented proof covers exactly what the node advertised."""
|
||||
return self.state == STATE_ADMITTED
|
||||
|
||||
def routable_under(self, policy: str) -> bool:
|
||||
if self.proven:
|
||||
return True
|
||||
return self.state == STATE_ABSENT and normalize_policy(policy) == POLICY_COMPAT
|
||||
|
||||
def with_state(self, state: str, detail: str) -> CapabilityState:
|
||||
"""Re-verdict a recorded proof against what the node advertises *now*."""
|
||||
return replace(self, state=state, detail=sanitize_detail(detail))
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"state": self.state,
|
||||
"detail": self.detail,
|
||||
"model_id": self.model_id,
|
||||
"shard_start": self.shard_start,
|
||||
"shard_end": self.shard_end,
|
||||
"recipe_id": self.recipe_id,
|
||||
"recipe_version": self.recipe_version,
|
||||
"catalogue_version": self.catalogue_version,
|
||||
"backend_id": self.backend_id,
|
||||
"device": self.device,
|
||||
"quantization": self.quantization,
|
||||
"validated_at": self.validated_at,
|
||||
"recorded_at": self.recorded_at,
|
||||
"schema_version": self.schema_version,
|
||||
"diagnostics": list(self.diagnostics),
|
||||
}
|
||||
|
||||
|
||||
def absent_state(detail: str = "", *, now: float | None = None) -> CapabilityState:
|
||||
"""The verdict for a node that presented no proof at all (legacy node)."""
|
||||
return CapabilityState(
|
||||
state=STATE_ABSENT,
|
||||
detail=sanitize_detail(
|
||||
detail
|
||||
or "node registered without a capability report; it predates the "
|
||||
"capability protocol or ran with admission disabled"
|
||||
),
|
||||
recorded_at=time.time() if now is None else now,
|
||||
)
|
||||
|
||||
|
||||
def evaluate_report(
|
||||
report: Any,
|
||||
*,
|
||||
model_matches: Callable[[str], bool],
|
||||
advertised_model: str | None,
|
||||
shard_start: int | None,
|
||||
shard_end: int | None,
|
||||
declared_recipe_id: str | None = None,
|
||||
declared_recipe_version: str | None = None,
|
||||
now: float | None = None,
|
||||
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
|
||||
) -> CapabilityState:
|
||||
"""Judge the proof a node presented against what that node is advertising.
|
||||
|
||||
`model_matches` is the tracker's own alias-aware comparison against the
|
||||
node's registered model / hf_repo, so an opaque model id never has to be
|
||||
parsed here.
|
||||
|
||||
Returns a verdict for *every* input, including malformed ones: a bad proof is
|
||||
recorded and shown to the operator rather than dropped, so "why is my node not
|
||||
routing" has an answer in the network map.
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
|
||||
if report is None:
|
||||
return absent_state(now=now)
|
||||
|
||||
if not isinstance(report, Mapping):
|
||||
return CapabilityState(
|
||||
state=STATE_INVALID,
|
||||
detail=sanitize_detail(
|
||||
f"capability_report must be a JSON object, got "
|
||||
f"{type(report).__name__}"
|
||||
),
|
||||
recorded_at=now,
|
||||
)
|
||||
|
||||
try:
|
||||
parsed = _parse_report(report)
|
||||
except _ReportError as exc:
|
||||
return CapabilityState(
|
||||
state=STATE_INVALID,
|
||||
detail=sanitize_detail(str(exc)),
|
||||
recorded_at=now,
|
||||
schema_version=_maybe_int(report.get("schema_version")),
|
||||
)
|
||||
|
||||
status = parsed.pop("_status")
|
||||
base = CapabilityState(state=STATE_ADMITTED, recorded_at=now, **parsed)
|
||||
|
||||
if base.schema_version != SUPPORTED_SCHEMA_VERSION:
|
||||
return base.with_state(
|
||||
STATE_INVALID,
|
||||
f"capability report declares schema version {base.schema_version}; "
|
||||
f"this tracker reads version {SUPPORTED_SCHEMA_VERSION}",
|
||||
)
|
||||
|
||||
if not catalogue_is_compatible(base.catalogue_version):
|
||||
return base.with_state(
|
||||
STATE_CATALOGUE_INCOMPATIBLE,
|
||||
f"recipe catalogue {base.catalogue_version!r} is older than the "
|
||||
f"minimum this tracker trusts ({MIN_CATALOGUE_VERSION}); upgrade the node",
|
||||
)
|
||||
|
||||
if not model_matches(base.model_id or ""):
|
||||
return base.with_state(
|
||||
STATE_MODEL_MISMATCH,
|
||||
f"proof is for model {base.model_id!r}, but the node registered "
|
||||
f"{advertised_model!r}",
|
||||
)
|
||||
|
||||
if shard_start is not None and shard_end is not None:
|
||||
if (base.shard_start, base.shard_end) != (shard_start, shard_end):
|
||||
return base.with_state(
|
||||
STATE_SHARD_MISMATCH,
|
||||
f"proof is for layers {base.shard_start}–{base.shard_end}, but the "
|
||||
f"node registered layers {shard_start}–{shard_end}",
|
||||
)
|
||||
|
||||
if declared_recipe_id is not None and base.recipe_id != declared_recipe_id:
|
||||
return base.with_state(
|
||||
STATE_RECIPE_MISMATCH,
|
||||
f"proof is for recipe {base.recipe_id!r}, but the node declared it "
|
||||
f"serves with {declared_recipe_id!r}",
|
||||
)
|
||||
if (
|
||||
declared_recipe_version is not None
|
||||
and base.recipe_version != declared_recipe_version
|
||||
):
|
||||
return base.with_state(
|
||||
STATE_RECIPE_MISMATCH,
|
||||
f"proof is for recipe {base.recipe_id!r} v{base.recipe_version}, but "
|
||||
f"the node declared v{declared_recipe_version}",
|
||||
)
|
||||
|
||||
if status != STATUS_PASSED:
|
||||
return base.with_state(
|
||||
STATE_FAILED,
|
||||
f"capability validation {status} on the node"
|
||||
+ (f" — {' '.join(base.diagnostics)}" if base.diagnostics else ""),
|
||||
)
|
||||
|
||||
age = now - (base.validated_at or 0.0)
|
||||
if age > max_age_seconds:
|
||||
return base.with_state(
|
||||
STATE_STALE,
|
||||
f"proof is {age / 60:.0f} min old (limit {max_age_seconds / 60:.0f} min); "
|
||||
"the node must re-validate before it can be routed",
|
||||
)
|
||||
if age < -MAX_CLOCK_SKEW_SECONDS:
|
||||
return base.with_state(
|
||||
STATE_STALE,
|
||||
f"proof is timestamped {-age:.0f}s in the future; check the node's clock",
|
||||
)
|
||||
|
||||
return base.with_state(
|
||||
STATE_ADMITTED,
|
||||
f"{base.model_id} layers {base.shard_start}–{base.shard_end} proven on "
|
||||
f"{base.device} with recipe {base.recipe_id} (v{base.recipe_version})",
|
||||
)
|
||||
|
||||
|
||||
class _ReportError(ValueError):
|
||||
"""Malformed report input. Messages name the field, never echo a payload."""
|
||||
|
||||
|
||||
def _parse_report(doc: Mapping[str, Any]) -> dict:
|
||||
model = _object(doc.get("model"), "model")
|
||||
shard = _object(doc.get("shard"), "shard")
|
||||
recipe = _object(doc.get("recipe"), "recipe")
|
||||
backend = _object(doc.get("backend"), "backend")
|
||||
|
||||
validated_at = doc.get("validated_at")
|
||||
if isinstance(validated_at, bool) or not isinstance(validated_at, (int, float)):
|
||||
raise _ReportError("'validated_at' must be a Unix timestamp")
|
||||
|
||||
schema_version = doc.get("schema_version")
|
||||
if isinstance(schema_version, bool) or not isinstance(schema_version, int):
|
||||
raise _ReportError("'schema_version' must be an integer")
|
||||
|
||||
return {
|
||||
"model_id": _text(model.get("model_id"), "model.model_id"),
|
||||
"shard_start": _index(shard.get("start"), "shard.start"),
|
||||
"shard_end": _index(shard.get("end"), "shard.end"),
|
||||
"recipe_id": _text(recipe.get("recipe_id"), "recipe.recipe_id"),
|
||||
"recipe_version": _text(recipe.get("recipe_version"), "recipe.recipe_version"),
|
||||
"catalogue_version": _text(
|
||||
recipe.get("catalogue_version"), "recipe.catalogue_version"
|
||||
),
|
||||
"backend_id": _text(backend.get("backend_id"), "backend.backend_id"),
|
||||
"device": _text(backend.get("device"), "backend.device"),
|
||||
"quantization": _optional_text(
|
||||
backend.get("quantization"), "backend.quantization"
|
||||
),
|
||||
"validated_at": float(validated_at),
|
||||
"schema_version": schema_version,
|
||||
"diagnostics": _diagnostics(doc.get("diagnostics")),
|
||||
"_status": _text(doc.get("status"), "status"),
|
||||
}
|
||||
|
||||
|
||||
def _object(value: Any, field_name: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping):
|
||||
raise _ReportError(f"{field_name!r} must be a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def _text(value: Any, field_name: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise _ReportError(f"{field_name!r} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _optional_text(value: Any, field_name: str) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return _text(value, field_name)
|
||||
|
||||
|
||||
def _index(value: Any, field_name: str) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
||||
raise _ReportError(f"{field_name!r} must be a non-negative integer")
|
||||
return value
|
||||
|
||||
|
||||
def _maybe_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _diagnostics(value: Any) -> tuple[str, ...]:
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
out = [
|
||||
sanitize_detail(item)
|
||||
for item in value[:_MAX_DIAGNOSTICS]
|
||||
if isinstance(item, str) and item.strip()
|
||||
]
|
||||
return tuple(out)
|
||||
Reference in New Issue
Block a user