226 lines
7.4 KiB
Python
226 lines
7.4 KiB
Python
"""Fail-closed admission: no routable registration without a fresh matching proof.
|
||
|
||
This module does not *produce* proof — `doctor` does that, by pushing a bounded
|
||
real forward through the selected shard (NCA-002). This module *decides whether a
|
||
proof covers what is about to be advertised*, and startup calls it immediately
|
||
before it registers with the tracker.
|
||
|
||
A capability report proves one combination: model artifact, shard range, recipe,
|
||
backend and device. Reusing it for anything else is the exact hole this closes —
|
||
a report that failed, aged out, or describes a different model, shard, recipe or
|
||
device is rejected here, and the node exits without ever registering an endpoint.
|
||
|
||
Nothing in here branches on a model, vendor or kernel name: identity fields are
|
||
opaque labels that are compared, never interpreted.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import time
|
||
from dataclasses import dataclass
|
||
from typing import Any, Callable
|
||
|
||
from .capability import CapabilityReport
|
||
from .doctor import DoctorSelection
|
||
from .recipe_manifest import Recipe, RecipeManifest
|
||
|
||
# How long a passing report stays usable. Startup normally validates in-process
|
||
# (age ≈ 0); this bounds how far a report written by an earlier `doctor` run can
|
||
# be carried forward, after which the hardware, drivers or weights may have moved.
|
||
DEFAULT_MAX_REPORT_AGE_SECONDS = 900.0
|
||
|
||
# A report timestamped this far in the future is not fresh, it is wrong.
|
||
_MAX_CLOCK_SKEW_SECONDS = 60.0
|
||
|
||
REASON_NO_REPORT = "no-report"
|
||
REASON_NOT_PASSED = "not-passed"
|
||
REASON_STALE = "stale"
|
||
REASON_MODEL_MISMATCH = "model-mismatch"
|
||
REASON_SHARD_MISMATCH = "shard-mismatch"
|
||
REASON_RECIPE_MISMATCH = "recipe-mismatch"
|
||
REASON_BACKEND_MISMATCH = "backend-mismatch"
|
||
|
||
|
||
class CapabilityAdmissionError(RuntimeError):
|
||
"""This node may not advertise the selection: the proof does not cover it."""
|
||
|
||
def __init__(self, reason: str, message: str) -> None:
|
||
super().__init__(message)
|
||
self.reason = reason
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class CapabilityContext:
|
||
"""What is about to be advertised, and the loaded backend that would serve it."""
|
||
|
||
backend: Any
|
||
selection: DoctorSelection
|
||
recipe: Recipe
|
||
manifest: RecipeManifest
|
||
device: str
|
||
|
||
|
||
# A validator turns the context into the report the gate then judges. Production
|
||
# uses `probe_capability`; tests pass an explicit test-safe one (see
|
||
# `meshnet_node.testing`) rather than switching this module into a lenient mode.
|
||
CapabilityValidator = Callable[[CapabilityContext], CapabilityReport]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class AdmissionRequirement:
|
||
"""The one capability a report must prove for this node to register."""
|
||
|
||
model_id: str
|
||
shard_start: int
|
||
shard_end: int
|
||
recipe_id: str
|
||
recipe_version: str
|
||
backend_id: str
|
||
device: str
|
||
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS
|
||
|
||
@classmethod
|
||
def for_context(
|
||
cls,
|
||
context: CapabilityContext,
|
||
*,
|
||
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
|
||
) -> AdmissionRequirement:
|
||
return cls(
|
||
model_id=context.selection.model_id,
|
||
shard_start=context.selection.shard_start,
|
||
shard_end=context.selection.shard_end,
|
||
recipe_id=context.recipe.id,
|
||
recipe_version=context.recipe.version,
|
||
backend_id=context.recipe.backend_id,
|
||
device=context.device,
|
||
max_age_seconds=max_age_seconds,
|
||
)
|
||
|
||
@property
|
||
def shard_label(self) -> str:
|
||
return f"layers {self.shard_start}–{self.shard_end}"
|
||
|
||
|
||
def admit(
|
||
requirement: AdmissionRequirement,
|
||
report: CapabilityReport | None,
|
||
*,
|
||
now: float | None = None,
|
||
) -> CapabilityReport:
|
||
"""Return `report` if it admits `requirement`; otherwise refuse to register.
|
||
|
||
Checks run selection-first, so the operator is told the report is about the
|
||
wrong thing before being told it is old.
|
||
"""
|
||
if report is None:
|
||
raise CapabilityAdmissionError(
|
||
REASON_NO_REPORT,
|
||
f"no capability report for {requirement.model_id} "
|
||
f"{requirement.shard_label}: this node has not proven it can serve it",
|
||
)
|
||
|
||
if report.model.model_id != requirement.model_id:
|
||
raise _mismatch(
|
||
REASON_MODEL_MISMATCH,
|
||
requirement,
|
||
"model",
|
||
report.model.model_id,
|
||
requirement.model_id,
|
||
)
|
||
|
||
if (report.shard.start, report.shard.end) != (
|
||
requirement.shard_start,
|
||
requirement.shard_end,
|
||
):
|
||
raise _mismatch(
|
||
REASON_SHARD_MISMATCH,
|
||
requirement,
|
||
"shard",
|
||
f"layers {report.shard.start}–{report.shard.end}",
|
||
requirement.shard_label,
|
||
)
|
||
|
||
if (report.recipe.recipe_id, report.recipe.recipe_version) != (
|
||
requirement.recipe_id,
|
||
requirement.recipe_version,
|
||
):
|
||
raise _mismatch(
|
||
REASON_RECIPE_MISMATCH,
|
||
requirement,
|
||
"recipe",
|
||
f"{report.recipe.recipe_id} (v{report.recipe.recipe_version})",
|
||
f"{requirement.recipe_id} (v{requirement.recipe_version})",
|
||
)
|
||
|
||
if (report.backend.backend_id, report.backend.device) != (
|
||
requirement.backend_id,
|
||
requirement.device,
|
||
):
|
||
raise _mismatch(
|
||
REASON_BACKEND_MISMATCH,
|
||
requirement,
|
||
"backend",
|
||
f"{report.backend.backend_id} on {report.backend.device}",
|
||
f"{requirement.backend_id} on {requirement.device}",
|
||
)
|
||
|
||
if not report.passed:
|
||
raise CapabilityAdmissionError(
|
||
REASON_NOT_PASSED,
|
||
f"capability validation {report.status} for {requirement.model_id} "
|
||
f"{requirement.shard_label} with recipe {requirement.recipe_id}"
|
||
+ _diagnostics_suffix(report),
|
||
)
|
||
|
||
now = time.time() if now is None else now
|
||
age = now - report.validated_at
|
||
if age > requirement.max_age_seconds:
|
||
raise CapabilityAdmissionError(
|
||
REASON_STALE,
|
||
f"capability report for {requirement.model_id} {requirement.shard_label} "
|
||
f"is {age / 60:.0f} min old (limit "
|
||
f"{requirement.max_age_seconds / 60:.0f} min); re-run `meshnet-node doctor`",
|
||
)
|
||
if age < -_MAX_CLOCK_SKEW_SECONDS:
|
||
raise CapabilityAdmissionError(
|
||
REASON_STALE,
|
||
f"capability report for {requirement.model_id} {requirement.shard_label} "
|
||
f"is timestamped {-age:.0f}s in the future; check this host's clock",
|
||
)
|
||
|
||
return report
|
||
|
||
|
||
def _mismatch(
|
||
reason: str,
|
||
requirement: AdmissionRequirement,
|
||
field_name: str,
|
||
reported: str,
|
||
required: str,
|
||
) -> CapabilityAdmissionError:
|
||
return CapabilityAdmissionError(
|
||
reason,
|
||
f"capability report proves a different {field_name}: it validated "
|
||
f"{reported}, but this node would serve {required}. A report is only "
|
||
"proof for the exact combination it ran.",
|
||
)
|
||
|
||
|
||
def _diagnostics_suffix(report: CapabilityReport) -> str:
|
||
if not report.diagnostics:
|
||
return ""
|
||
return " — " + " ".join(report.diagnostics)
|
||
|
||
|
||
def probe_capability(context: CapabilityContext) -> CapabilityReport:
|
||
"""Production validator: one bounded real forward through the loaded shard."""
|
||
from .doctor import validate_loaded_backend
|
||
|
||
return validate_loaded_backend(
|
||
context.backend,
|
||
context.selection,
|
||
context.recipe,
|
||
context.manifest,
|
||
).report
|