popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden DGR-003 identity trust boundary

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-14 09:48:42 +03:00
parent 7364ed6731
commit 7b8e467c6b
11 changed files with 1325 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

412
packages/node/meshnet_node/runtime_recipe.py
View File

@@ -38,16 +38,17 @@ a recipe does not change a single number, so a rename must not partition a
route; changing `kv_dtype` changes every number, so it must. The digest commits
to what changes the numbers.
Recipes are **registered-but-dark** on arrival: known, visible to an operator,
and not routable for user traffic. A recipe leaves the dark only when a *real
distributed forward* over a route of at least two distinct physical nodes,
covering the whole model, has emitted real tokens — see
:class:`DistributedForwardEvidence`. Detected hardware is not a capability, a
single-host forward is not a distributed forward, and a synthetic worker
certifies nothing.
**Certification is not here.** Recipes are registered-but-dark on arrival, and a
recipe leaves the dark only when a real distributed forward over at least two
distinct physical nodes has emitted real tokens. That ledger lives in the
Tracker (:class:`meshnet_tracker.recipe.CertificationLedger`) and nowhere else —
a node that could decide it was certified would be marking its own homework, and
a second copy of the policy here would be a second thing to keep in step with it.
This module builds and compares identity; the Tracker decides what may serve.
The tracker re-derives all of this independently in
``meshnet_tracker.recipe`` — it does not import this package. The two
The tracker re-derives the digests independently in ``meshnet_tracker.recipe`` —
it does not import this package, because an admission gate that shares an
implementation with the thing it admits is not an independent check. The two
implementations are pinned together by a committed conformance vector
(``tests/data/recipe_fingerprint_vectors.json``); if they ever drift, a test
fails rather than a route silently forming.
@@ -58,8 +59,7 @@ from __future__ import annotations
import hashlib
import json
import re
import time
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import Any, Iterable, Mapping, Sequence
from .native_protocol import BUNDLE_VERSION, SCHEMA_VERSION, pb
@@ -72,6 +72,7 @@ RECIPE_IDENTITY_SCHEMA_VERSION = 1
# with — a recipe digest, even if their canonical JSON were somehow identical.
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
# The axes of a runtime recipe. Every one of these changes the numbers a Shard
# produces, so every one of them is part of identity and none of them may be
@@ -102,7 +103,8 @@ MISMATCH_RUNTIME = "runtime"
MISMATCH_BOUNDARY_SCHEMA = "boundary-schema"
MISMATCH_RANGE = "range"
MISMATCH_FINGERPRINT = "fingerprint"
MISMATCH_UNCERTIFIED = "uncertified"
MISMATCH_ROUTE_SESSION = "route-session"
MISMATCH_ROUTE_EPOCH = "route-epoch"
# Which fail-closed reason each axis reports. Several axes share a reason
# because they fail for the same operational cause: `activation_dtype` and
@@ -122,14 +124,6 @@ _AXIS_MISMATCH: Mapping[str, str] = {
"protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA,
}
# Certification states. `dark` is the arrival state, not an error state.
STATUS_UNKNOWN = "unknown"
STATUS_DARK = "dark"
STATUS_CERTIFIED = "certified"
# A route that proves a recipe must be a real distributed route.
MIN_CERTIFYING_NODES = 2
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
# A revision that can move is not a pin. DGR-017 learned this on the artifact;
@@ -141,10 +135,6 @@ class RecipeIdentityError(ValueError):
"""Malformed identity input. Messages name the field, never echo a payload."""
class RecipeNotCertified(RecipeIdentityError):
"""A recipe was asked to serve user traffic while still dark."""
class RouteIncompatible(RecipeIdentityError):
"""Two Shards cannot form an Inference Route.
@@ -618,6 +608,39 @@ class ShardIdentity:
catalogue_version=self.recipe.catalogue_version,
)
@property
def shard_binding_digest(self) -> str:
"""This Shard's own bytes and exact range, bound to the source artifact.
The fingerprint is range-independent on purpose — see the module
docstring — which means it cannot distinguish two *different splits of
the same source*. Both are the same recipe on the same model, and for
route compatibility that is exactly right. It is not right for
certification: a derivative could otherwise assert the correct source and
inherit a certification earned by bytes it does not hold.
So the derivative's own content hash and exact end-exclusive range get a
second digest, outside the fingerprint. The Tracker re-derives it at
admission and requires certification evidence to name the same binding
for every serving node.
"""
binding = self.artifact.derived_from
return _digest(
SHARD_BINDING_DIGEST_DOMAIN,
{
"source_digest": self.artifact.source_digest,
"content_digest": self.artifact.content_digest,
"architecture_digest": self.artifact.architecture_digest,
"derivative_range": (
None
if binding is None
else [binding.shard_start, binding.shard_end]
),
"shard_start": self.shard_start,
"shard_end": self.shard_end,
},
)
def to_dict(self) -> dict:
return {
"schema_version": RECIPE_IDENTITY_SCHEMA_VERSION,
@@ -747,57 +770,6 @@ def explain_mismatch(
return tuple(mismatches)
@dataclass(frozen=True)
class DistributedForwardEvidence:
"""Proof that a recipe really ran, distributed, end to end.
This is the only thing that takes a recipe out of the dark. The bar is
deliberately the product's bar, not a unit test's: at least two *distinct*
physical nodes, whose Shards cover the whole model with no hole, generating
real tokens. A synthetic worker is a unit fixture and certifies nothing —
the whole risk this guards against is a recipe that passes in a mock and
produces garbage on real weights.
"""
route_session_id: str
route_epoch: int
node_ids: tuple[str, ...]
shard_ranges: tuple[tuple[int, int], ...]
tokens_generated: int
layer_count: int
synthetic: bool = False
certified_at: float = field(default_factory=time.time)
def __post_init__(self) -> None:
_require_text(self.route_session_id, "evidence.route_session_id")
_require_int(self.route_epoch, "evidence.route_epoch", 0)
_require_int(self.tokens_generated, "evidence.tokens_generated", 0)
_require_int(self.layer_count, "evidence.layer_count", 1)
def rejection(self) -> str | None:
"""Why this evidence does not certify, or None when it does."""
if self.synthetic:
return (
"evidence comes from a synthetic worker; only a real distributed "
"forward certifies a recipe"
)
distinct = set(self.node_ids)
if len(distinct) < MIN_CERTIFYING_NODES:
return (
f"evidence covers {len(distinct)} distinct node(s); a distributed "
f"forward requires at least {MIN_CERTIFYING_NODES}"
)
if len(distinct) != len(self.node_ids):
return "the same node is counted more than once in the certifying route"
if self.tokens_generated < 1:
return "the certifying forward generated no tokens"
gap = _coverage_gap(self.shard_ranges, self.layer_count)
if gap is not None:
return gap
return None
def _coverage_gap(
ranges: Iterable[tuple[int, int]], layer_count: int
) -> str | None:
@@ -830,172 +802,20 @@ def _coverage_gap(
return None
@dataclass(frozen=True)
class RecipeStatus:
"""What the registry knows about one fingerprint."""
status: str
fingerprint: CompatibilityFingerprint | None = None
detail: str = ""
certified_at: float | None = None
@property
def may_serve(self) -> bool:
"""Only a certified recipe carries user traffic."""
return self.status == STATUS_CERTIFIED
@property
def may_certify(self) -> bool:
"""A dark recipe is eligible for a certification route — and only that.
Without this, certification is unreachable: serving requires
certification, certification requires a real distributed forward, and a
real distributed forward requires a route. A dark recipe may therefore
be routed *for the express purpose of certifying it*, and never for user
traffic.
"""
return self.status in (STATUS_DARK, STATUS_CERTIFIED)
def to_dict(self) -> dict:
return {
"status": self.status,
"detail": self.detail,
"certified_at": self.certified_at,
"fingerprint": (
self.fingerprint.to_dict() if self.fingerprint is not None else None
),
}
class RecipeRegistry:
"""Registered-but-dark recipes, and the ones a real forward has certified.
An unknown recipe is not routable. A known one is not routable either, until
it has been proven. This is the fail-closed default the roadmap asks for:
"unsupported architectures/backends remain registered-but-dark until real
certification passes."
"""
def __init__(self) -> None:
self._dark: dict[tuple[str, str], CompatibilityFingerprint] = {}
self._certified: dict[tuple[str, str], RecipeStatus] = {}
def register(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus:
"""Record a recipe as known. Known is not certified."""
fingerprint = _as_fingerprint(identity)
key = fingerprint.key
if key in self._certified:
return self._certified[key]
self._dark[key] = fingerprint
return RecipeStatus(
STATUS_DARK,
fingerprint,
"registered; dark until a real distributed forward certifies it",
)
def status(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus:
fingerprint = _as_fingerprint(identity)
key = fingerprint.key
if key in self._certified:
return self._certified[key]
if key in self._dark:
return RecipeStatus(
STATUS_DARK,
fingerprint,
"registered; dark until a real distributed forward certifies it",
)
return RecipeStatus(
STATUS_UNKNOWN,
fingerprint,
"this recipe has never been registered with the Tracker",
)
def certify(
self,
identity: ShardIdentity | CompatibilityFingerprint,
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote a dark recipe, if the evidence is a real distributed forward.
Raises rather than returning a failed status: certifying is a state
change, and a caller that ignores a returned failure would leave a
recipe it believes to be certified in the dark.
"""
fingerprint = _as_fingerprint(identity)
rejection = evidence.rejection()
if rejection is not None:
raise RecipeNotCertified(
f"this evidence does not certify {fingerprint.recipe_id!r}: {rejection}"
)
status = RecipeStatus(
STATUS_CERTIFIED,
fingerprint,
(
f"certified by route session {evidence.route_session_id} across "
f"{len(set(evidence.node_ids))} nodes"
),
certified_at=evidence.certified_at,
)
self._certified[fingerprint.key] = status
self._dark.pop(fingerprint.key, None)
return status
def require_serving(
self, identity: ShardIdentity | CompatibilityFingerprint
) -> RecipeStatus:
"""Admit a recipe for user traffic, or refuse and say why."""
status = self.status(identity)
if not status.may_serve:
raise RecipeNotCertified(
f"recipe {_as_fingerprint(identity).recipe_id!r} is {status.status}: "
f"{status.detail}"
)
return status
def certified_fingerprints(self) -> tuple[CompatibilityFingerprint, ...]:
return tuple(
status.fingerprint
for status in self._certified.values()
if status.fingerprint is not None
)
def to_dict(self) -> dict:
return {
"schema_version": RECIPE_IDENTITY_SCHEMA_VERSION,
"dark": [fp.to_dict() for fp in self._dark.values()],
"certified": [status.to_dict() for status in self._certified.values()],
}
def _as_fingerprint(
identity: ShardIdentity | CompatibilityFingerprint,
) -> CompatibilityFingerprint:
if isinstance(identity, CompatibilityFingerprint):
return identity
if isinstance(identity, ShardIdentity):
return identity.fingerprint
raise RecipeIdentityError(
f"expected a ShardIdentity or CompatibilityFingerprint, got "
f"{type(identity).__name__}"
)
def check_route(
shards: Sequence[ShardIdentity],
*,
registry: RecipeRegistry | None = None,
for_certification: bool = False,
) -> tuple[RouteMismatch, ...]:
"""Decide whether these Shards may form one Inference Route.
"""Why these Shards may not form one Inference Route; empty when they may.
Returns every reason they may not, empty when they may. Fail-closed by
construction: an empty route, an unknown recipe, a hole in the layer
coverage and a single differing dtype all produce a reason, and a caller that
only ever proceeds on an empty tuple cannot accidentally admit any of them.
This answers the *numerical* question only — do these Shards agree on every
axis that moves the numbers, and do they tile the model without a hole. It
deliberately does not answer "may this recipe carry user traffic": that is
certification, the Tracker owns it (`meshnet_tracker.recipe`), and a node
asking itself whether it is certified would be marking its own homework.
`for_certification` admits a dark recipe onto a route whose only purpose is
to certify it. It never admits an unknown one, and it is not the path user
traffic takes.
Fail-closed by construction: an empty route, a hole in the coverage and a
single differing dtype each produce a reason, so a caller that proceeds only
on an empty tuple cannot admit any of them.
"""
if not shards:
return (
@@ -1018,27 +838,12 @@ def check_route(
if gap is not None:
mismatches.append(RouteMismatch(MISMATCH_RANGE, gap))
if registry is not None:
status = registry.status(head.fingerprint)
allowed = status.may_certify if for_certification else status.may_serve
if not allowed:
mismatches.append(
RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}")
)
return tuple(mismatches)
def require_route(
shards: Sequence[ShardIdentity],
*,
registry: RecipeRegistry | None = None,
for_certification: bool = False,
) -> CompatibilityFingerprint:
def require_route(shards: Sequence[ShardIdentity]) -> CompatibilityFingerprint:
"""`check_route`, raising the structured reasons instead of returning them."""
mismatches = check_route(
shards, registry=registry, for_certification=for_certification
)
mismatches = check_route(shards)
if mismatches:
raise RouteIncompatible(mismatches)
return shards[0].fingerprint
@@ -1134,14 +939,111 @@ def check_handshake(
return tuple(reasons)
def check_session_open(
local: ShardIdentity,
remote: "pb.SessionOpen",
*,
expected_route_session_id: str | None = None,
expected_route_epoch: int | None = None,
) -> tuple[RouteMismatch, ...]:
"""Fail closed on the complete immutable portion of a ``SessionOpen``.
``SessionOpen`` is not just a fingerprint carrier: its route session and
epoch select the tracker-issued route whose identity was checked. A worker
that accepts a stale epoch can receive activations for a route that the
tracker has already replaced. Callers that have the tracker assignment
therefore pass both expected values; callers without it still reject the
protobuf defaults, which cannot identify a live route.
"""
reasons = list(check_handshake(local, remote.fingerprint))
if remote.schema_version != SCHEMA_VERSION:
reasons.append(
RouteMismatch(
MISMATCH_BOUNDARY_SCHEMA,
f"peer schema version {remote.schema_version} does not match "
f"{SCHEMA_VERSION}",
)
)
if not remote.route_session_id:
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_SESSION,
"peer omitted the tracker-issued route session id",
)
)
elif (
expected_route_session_id is not None
and remote.route_session_id != expected_route_session_id
):
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_SESSION,
"peer route session does not match the tracker assignment",
)
)
if remote.route_epoch < 1:
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_EPOCH,
"peer omitted a positive tracker-issued route epoch",
)
)
elif expected_route_epoch is not None and remote.route_epoch != expected_route_epoch:
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_EPOCH,
"peer route epoch does not match the tracker assignment",
)
)
advertised = (remote.shard_range.start_layer, remote.shard_range.end_layer)
expected = (local.shard_start, local.shard_end)
effective = remote.shard_range.effective_start_layer
if advertised != expected:
reasons.append(
RouteMismatch(
MISMATCH_RANGE,
f"peer advertised Shard range {advertised}, expected {expected}",
)
)
elif not local.shard_start <= effective < local.shard_end:
reasons.append(
RouteMismatch(
MISMATCH_RANGE,
f"effective start {effective} is outside Shard range {expected}",
)
)
return tuple(reasons)
def handshake_error(
mismatches: Sequence[RouteMismatch],
) -> "pb.ShardError | None":
"""The protocol status a rejected handshake closes the stream with."""
"""The protocol status a rejected handshake closes the stream with.
Each rejection maps to the most specific DGR-002 code the reasons allow:
a digest disagreement dominates (the peer is the wrong artifact or recipe,
whatever else is also wrong), then a schema disagreement (a peer on a
foreign schema cannot be trusted to have encoded anything else correctly),
and only a pure range disagreement reports the range code. Collapsing all
of these to one code would leave the initiator unable to tell "re-route me"
from "renegotiate the schema" from "wrong peer entirely".
"""
if not mismatches:
return None
reasons = {mismatch.reason for mismatch in mismatches}
if reasons == {MISMATCH_RANGE}:
code = pb.ERROR_CODE_SHARD_RANGE_MISMATCH
elif reasons <= {MISMATCH_ROUTE_SESSION, MISMATCH_ROUTE_EPOCH}:
code = pb.ERROR_CODE_EPOCH_STALE
elif reasons <= {MISMATCH_BOUNDARY_SCHEMA, MISMATCH_RANGE}:
code = pb.ERROR_CODE_SCHEMA_UNSUPPORTED
else:
code = pb.ERROR_CODE_FINGERPRINT_MISMATCH
return pb.ShardError(
code=pb.ERROR_CODE_FINGERPRINT_MISMATCH,
code=code,
detail="; ".join(m.describe() for m in mismatches),
retryable=False,
)