[verified] feat: complete Ralph task workstreams
This commit is contained in:
225
packages/node/meshnet_node/admission.py
Normal file
225
packages/node/meshnet_node/admission.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""Fail-closed admission: no routable registration without a fresh matching proof.
|
||||
|
||||
This module does not *produce* proof — `doctor` does that, by pushing a bounded
|
||||
real forward through the selected shard (NCA-002). This module *decides whether a
|
||||
proof covers what is about to be advertised*, and startup calls it immediately
|
||||
before it registers with the tracker.
|
||||
|
||||
A capability report proves one combination: model artifact, shard range, recipe,
|
||||
backend and device. Reusing it for anything else is the exact hole this closes —
|
||||
a report that failed, aged out, or describes a different model, shard, recipe or
|
||||
device is rejected here, and the node exits without ever registering an endpoint.
|
||||
|
||||
Nothing in here branches on a model, vendor or kernel name: identity fields are
|
||||
opaque labels that are compared, never interpreted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
from .capability import CapabilityReport
|
||||
from .doctor import DoctorSelection
|
||||
from .recipe_manifest import Recipe, RecipeManifest
|
||||
|
||||
# How long a passing report stays usable. Startup normally validates in-process
|
||||
# (age ≈ 0); this bounds how far a report written by an earlier `doctor` run can
|
||||
# be carried forward, after which the hardware, drivers or weights may have moved.
|
||||
DEFAULT_MAX_REPORT_AGE_SECONDS = 900.0
|
||||
|
||||
# A report timestamped this far in the future is not fresh, it is wrong.
|
||||
_MAX_CLOCK_SKEW_SECONDS = 60.0
|
||||
|
||||
REASON_NO_REPORT = "no-report"
|
||||
REASON_NOT_PASSED = "not-passed"
|
||||
REASON_STALE = "stale"
|
||||
REASON_MODEL_MISMATCH = "model-mismatch"
|
||||
REASON_SHARD_MISMATCH = "shard-mismatch"
|
||||
REASON_RECIPE_MISMATCH = "recipe-mismatch"
|
||||
REASON_BACKEND_MISMATCH = "backend-mismatch"
|
||||
|
||||
|
||||
class CapabilityAdmissionError(RuntimeError):
|
||||
"""This node may not advertise the selection: the proof does not cover it."""
|
||||
|
||||
def __init__(self, reason: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityContext:
|
||||
"""What is about to be advertised, and the loaded backend that would serve it."""
|
||||
|
||||
backend: Any
|
||||
selection: DoctorSelection
|
||||
recipe: Recipe
|
||||
manifest: RecipeManifest
|
||||
device: str
|
||||
|
||||
|
||||
# A validator turns the context into the report the gate then judges. Production
|
||||
# uses `probe_capability`; tests pass an explicit test-safe one (see
|
||||
# `meshnet_node.testing`) rather than switching this module into a lenient mode.
|
||||
CapabilityValidator = Callable[[CapabilityContext], CapabilityReport]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AdmissionRequirement:
|
||||
"""The one capability a report must prove for this node to register."""
|
||||
|
||||
model_id: str
|
||||
shard_start: int
|
||||
shard_end: int
|
||||
recipe_id: str
|
||||
recipe_version: str
|
||||
backend_id: str
|
||||
device: str
|
||||
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS
|
||||
|
||||
@classmethod
|
||||
def for_context(
|
||||
cls,
|
||||
context: CapabilityContext,
|
||||
*,
|
||||
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
|
||||
) -> AdmissionRequirement:
|
||||
return cls(
|
||||
model_id=context.selection.model_id,
|
||||
shard_start=context.selection.shard_start,
|
||||
shard_end=context.selection.shard_end,
|
||||
recipe_id=context.recipe.id,
|
||||
recipe_version=context.recipe.version,
|
||||
backend_id=context.recipe.backend_id,
|
||||
device=context.device,
|
||||
max_age_seconds=max_age_seconds,
|
||||
)
|
||||
|
||||
@property
|
||||
def shard_label(self) -> str:
|
||||
return f"layers {self.shard_start}–{self.shard_end}"
|
||||
|
||||
|
||||
def admit(
|
||||
requirement: AdmissionRequirement,
|
||||
report: CapabilityReport | None,
|
||||
*,
|
||||
now: float | None = None,
|
||||
) -> CapabilityReport:
|
||||
"""Return `report` if it admits `requirement`; otherwise refuse to register.
|
||||
|
||||
Checks run selection-first, so the operator is told the report is about the
|
||||
wrong thing before being told it is old.
|
||||
"""
|
||||
if report is None:
|
||||
raise CapabilityAdmissionError(
|
||||
REASON_NO_REPORT,
|
||||
f"no capability report for {requirement.model_id} "
|
||||
f"{requirement.shard_label}: this node has not proven it can serve it",
|
||||
)
|
||||
|
||||
if report.model.model_id != requirement.model_id:
|
||||
raise _mismatch(
|
||||
REASON_MODEL_MISMATCH,
|
||||
requirement,
|
||||
"model",
|
||||
report.model.model_id,
|
||||
requirement.model_id,
|
||||
)
|
||||
|
||||
if (report.shard.start, report.shard.end) != (
|
||||
requirement.shard_start,
|
||||
requirement.shard_end,
|
||||
):
|
||||
raise _mismatch(
|
||||
REASON_SHARD_MISMATCH,
|
||||
requirement,
|
||||
"shard",
|
||||
f"layers {report.shard.start}–{report.shard.end}",
|
||||
requirement.shard_label,
|
||||
)
|
||||
|
||||
if (report.recipe.recipe_id, report.recipe.recipe_version) != (
|
||||
requirement.recipe_id,
|
||||
requirement.recipe_version,
|
||||
):
|
||||
raise _mismatch(
|
||||
REASON_RECIPE_MISMATCH,
|
||||
requirement,
|
||||
"recipe",
|
||||
f"{report.recipe.recipe_id} (v{report.recipe.recipe_version})",
|
||||
f"{requirement.recipe_id} (v{requirement.recipe_version})",
|
||||
)
|
||||
|
||||
if (report.backend.backend_id, report.backend.device) != (
|
||||
requirement.backend_id,
|
||||
requirement.device,
|
||||
):
|
||||
raise _mismatch(
|
||||
REASON_BACKEND_MISMATCH,
|
||||
requirement,
|
||||
"backend",
|
||||
f"{report.backend.backend_id} on {report.backend.device}",
|
||||
f"{requirement.backend_id} on {requirement.device}",
|
||||
)
|
||||
|
||||
if not report.passed:
|
||||
raise CapabilityAdmissionError(
|
||||
REASON_NOT_PASSED,
|
||||
f"capability validation {report.status} for {requirement.model_id} "
|
||||
f"{requirement.shard_label} with recipe {requirement.recipe_id}"
|
||||
+ _diagnostics_suffix(report),
|
||||
)
|
||||
|
||||
now = time.time() if now is None else now
|
||||
age = now - report.validated_at
|
||||
if age > requirement.max_age_seconds:
|
||||
raise CapabilityAdmissionError(
|
||||
REASON_STALE,
|
||||
f"capability report for {requirement.model_id} {requirement.shard_label} "
|
||||
f"is {age / 60:.0f} min old (limit "
|
||||
f"{requirement.max_age_seconds / 60:.0f} min); re-run `meshnet-node doctor`",
|
||||
)
|
||||
if age < -_MAX_CLOCK_SKEW_SECONDS:
|
||||
raise CapabilityAdmissionError(
|
||||
REASON_STALE,
|
||||
f"capability report for {requirement.model_id} {requirement.shard_label} "
|
||||
f"is timestamped {-age:.0f}s in the future; check this host's clock",
|
||||
)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def _mismatch(
|
||||
reason: str,
|
||||
requirement: AdmissionRequirement,
|
||||
field_name: str,
|
||||
reported: str,
|
||||
required: str,
|
||||
) -> CapabilityAdmissionError:
|
||||
return CapabilityAdmissionError(
|
||||
reason,
|
||||
f"capability report proves a different {field_name}: it validated "
|
||||
f"{reported}, but this node would serve {required}. A report is only "
|
||||
"proof for the exact combination it ran.",
|
||||
)
|
||||
|
||||
|
||||
def _diagnostics_suffix(report: CapabilityReport) -> str:
|
||||
if not report.diagnostics:
|
||||
return ""
|
||||
return " — " + " ".join(report.diagnostics)
|
||||
|
||||
|
||||
def probe_capability(context: CapabilityContext) -> CapabilityReport:
|
||||
"""Production validator: one bounded real forward through the loaded shard."""
|
||||
from .doctor import validate_loaded_backend
|
||||
|
||||
return validate_loaded_backend(
|
||||
context.backend,
|
||||
context.selection,
|
||||
context.recipe,
|
||||
context.manifest,
|
||||
).report
|
||||
Reference in New Issue
Block a user