popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden DGR-003 identity trust boundary

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-14 09:48:42 +03:00
parent 7364ed6731
commit 7b8e467c6b
11 changed files with 1325 additions and 519 deletions
Download Patch File Download Diff File Expand all files Collapse all files

412
packages/node/meshnet_node/runtime_recipe.py
View File

@@ -38,16 +38,17 @@ a recipe does not change a single number, so a rename must not partition a
route; changing `kv_dtype` changes every number, so it must. The digest commits
to what changes the numbers.
Recipes are **registered-but-dark** on arrival: known, visible to an operator,
and not routable for user traffic. A recipe leaves the dark only when a *real
distributed forward* over a route of at least two distinct physical nodes,
covering the whole model, has emitted real tokens — see
:class:`DistributedForwardEvidence`. Detected hardware is not a capability, a
single-host forward is not a distributed forward, and a synthetic worker
certifies nothing.
**Certification is not here.** Recipes are registered-but-dark on arrival, and a
recipe leaves the dark only when a real distributed forward over at least two
distinct physical nodes has emitted real tokens. That ledger lives in the
Tracker (:class:`meshnet_tracker.recipe.CertificationLedger`) and nowhere else —
a node that could decide it was certified would be marking its own homework, and
a second copy of the policy here would be a second thing to keep in step with it.
This module builds and compares identity; the Tracker decides what may serve.
The tracker re-derives all of this independently in
``meshnet_tracker.recipe`` — it does not import this package. The two
The tracker re-derives the digests independently in ``meshnet_tracker.recipe`` —
it does not import this package, because an admission gate that shares an
implementation with the thing it admits is not an independent check. The two
implementations are pinned together by a committed conformance vector
(``tests/data/recipe_fingerprint_vectors.json``); if they ever drift, a test
fails rather than a route silently forming.
@@ -58,8 +59,7 @@ from __future__ import annotations
import hashlib
import json
import re
import time
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import Any, Iterable, Mapping, Sequence
from .native_protocol import BUNDLE_VERSION, SCHEMA_VERSION, pb
@@ -72,6 +72,7 @@ RECIPE_IDENTITY_SCHEMA_VERSION = 1
# with — a recipe digest, even if their canonical JSON were somehow identical.
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
# The axes of a runtime recipe. Every one of these changes the numbers a Shard
# produces, so every one of them is part of identity and none of them may be
@@ -102,7 +103,8 @@ MISMATCH_RUNTIME = "runtime"
MISMATCH_BOUNDARY_SCHEMA = "boundary-schema"
MISMATCH_RANGE = "range"
MISMATCH_FINGERPRINT = "fingerprint"
MISMATCH_UNCERTIFIED = "uncertified"
MISMATCH_ROUTE_SESSION = "route-session"
MISMATCH_ROUTE_EPOCH = "route-epoch"
# Which fail-closed reason each axis reports. Several axes share a reason
# because they fail for the same operational cause: `activation_dtype` and
@@ -122,14 +124,6 @@ _AXIS_MISMATCH: Mapping[str, str] = {
"protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA,
}
# Certification states. `dark` is the arrival state, not an error state.
STATUS_UNKNOWN = "unknown"
STATUS_DARK = "dark"
STATUS_CERTIFIED = "certified"
# A route that proves a recipe must be a real distributed route.
MIN_CERTIFYING_NODES = 2
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
# A revision that can move is not a pin. DGR-017 learned this on the artifact;
@@ -141,10 +135,6 @@ class RecipeIdentityError(ValueError):
"""Malformed identity input. Messages name the field, never echo a payload."""
class RecipeNotCertified(RecipeIdentityError):
"""A recipe was asked to serve user traffic while still dark."""
class RouteIncompatible(RecipeIdentityError):
"""Two Shards cannot form an Inference Route.
@@ -618,6 +608,39 @@ class ShardIdentity:
catalogue_version=self.recipe.catalogue_version,
)
@property
def shard_binding_digest(self) -> str:
"""This Shard's own bytes and exact range, bound to the source artifact.
The fingerprint is range-independent on purpose — see the module
docstring — which means it cannot distinguish two *different splits of
the same source*. Both are the same recipe on the same model, and for
route compatibility that is exactly right. It is not right for
certification: a derivative could otherwise assert the correct source and
inherit a certification earned by bytes it does not hold.
So the derivative's own content hash and exact end-exclusive range get a
second digest, outside the fingerprint. The Tracker re-derives it at
admission and requires certification evidence to name the same binding
for every serving node.
"""
binding = self.artifact.derived_from
return _digest(
SHARD_BINDING_DIGEST_DOMAIN,
{
"source_digest": self.artifact.source_digest,
"content_digest": self.artifact.content_digest,
"architecture_digest": self.artifact.architecture_digest,
"derivative_range": (
None
if binding is None
else [binding.shard_start, binding.shard_end]
),
"shard_start": self.shard_start,
"shard_end": self.shard_end,
},
)
def to_dict(self) -> dict:
return {
"schema_version": RECIPE_IDENTITY_SCHEMA_VERSION,
@@ -747,57 +770,6 @@ def explain_mismatch(
return tuple(mismatches)
@dataclass(frozen=True)
class DistributedForwardEvidence:
"""Proof that a recipe really ran, distributed, end to end.
This is the only thing that takes a recipe out of the dark. The bar is
deliberately the product's bar, not a unit test's: at least two *distinct*
physical nodes, whose Shards cover the whole model with no hole, generating
real tokens. A synthetic worker is a unit fixture and certifies nothing —
the whole risk this guards against is a recipe that passes in a mock and
produces garbage on real weights.
"""
route_session_id: str
route_epoch: int
node_ids: tuple[str, ...]
shard_ranges: tuple[tuple[int, int], ...]
tokens_generated: int
layer_count: int
synthetic: bool = False
certified_at: float = field(default_factory=time.time)
def __post_init__(self) -> None:
_require_text(self.route_session_id, "evidence.route_session_id")
_require_int(self.route_epoch, "evidence.route_epoch", 0)
_require_int(self.tokens_generated, "evidence.tokens_generated", 0)
_require_int(self.layer_count, "evidence.layer_count", 1)
def rejection(self) -> str | None:
"""Why this evidence does not certify, or None when it does."""
if self.synthetic:
return (
"evidence comes from a synthetic worker; only a real distributed "
"forward certifies a recipe"
)
distinct = set(self.node_ids)
if len(distinct) < MIN_CERTIFYING_NODES:
return (
f"evidence covers {len(distinct)} distinct node(s); a distributed "
f"forward requires at least {MIN_CERTIFYING_NODES}"
)
if len(distinct) != len(self.node_ids):
return "the same node is counted more than once in the certifying route"
if self.tokens_generated < 1:
return "the certifying forward generated no tokens"
gap = _coverage_gap(self.shard_ranges, self.layer_count)
if gap is not None:
return gap
return None
def _coverage_gap(
ranges: Iterable[tuple[int, int]], layer_count: int
) -> str | None:
@@ -830,172 +802,20 @@ def _coverage_gap(
return None
@dataclass(frozen=True)
class RecipeStatus:
"""What the registry knows about one fingerprint."""
status: str
fingerprint: CompatibilityFingerprint | None = None
detail: str = ""
certified_at: float | None = None
@property
def may_serve(self) -> bool:
"""Only a certified recipe carries user traffic."""
return self.status == STATUS_CERTIFIED
@property
def may_certify(self) -> bool:
"""A dark recipe is eligible for a certification route — and only that.
Without this, certification is unreachable: serving requires
certification, certification requires a real distributed forward, and a
real distributed forward requires a route. A dark recipe may therefore
be routed *for the express purpose of certifying it*, and never for user
traffic.
"""
return self.status in (STATUS_DARK, STATUS_CERTIFIED)
def to_dict(self) -> dict:
return {
"status": self.status,
"detail": self.detail,
"certified_at": self.certified_at,
"fingerprint": (
self.fingerprint.to_dict() if self.fingerprint is not None else None
),
}
class RecipeRegistry:
"""Registered-but-dark recipes, and the ones a real forward has certified.
An unknown recipe is not routable. A known one is not routable either, until
it has been proven. This is the fail-closed default the roadmap asks for:
"unsupported architectures/backends remain registered-but-dark until real
certification passes."
"""
def __init__(self) -> None:
self._dark: dict[tuple[str, str], CompatibilityFingerprint] = {}
self._certified: dict[tuple[str, str], RecipeStatus] = {}
def register(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus:
"""Record a recipe as known. Known is not certified."""
fingerprint = _as_fingerprint(identity)
key = fingerprint.key
if key in self._certified:
return self._certified[key]
self._dark[key] = fingerprint
return RecipeStatus(
STATUS_DARK,
fingerprint,
"registered; dark until a real distributed forward certifies it",
)
def status(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus:
fingerprint = _as_fingerprint(identity)
key = fingerprint.key
if key in self._certified:
return self._certified[key]
if key in self._dark:
return RecipeStatus(
STATUS_DARK,
fingerprint,
"registered; dark until a real distributed forward certifies it",
)
return RecipeStatus(
STATUS_UNKNOWN,
fingerprint,
"this recipe has never been registered with the Tracker",
)
def certify(
self,
identity: ShardIdentity | CompatibilityFingerprint,
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote a dark recipe, if the evidence is a real distributed forward.
Raises rather than returning a failed status: certifying is a state
change, and a caller that ignores a returned failure would leave a
recipe it believes to be certified in the dark.
"""
fingerprint = _as_fingerprint(identity)
rejection = evidence.rejection()
if rejection is not None:
raise RecipeNotCertified(
f"this evidence does not certify {fingerprint.recipe_id!r}: {rejection}"
)
status = RecipeStatus(
STATUS_CERTIFIED,
fingerprint,
(
f"certified by route session {evidence.route_session_id} across "
f"{len(set(evidence.node_ids))} nodes"
),
certified_at=evidence.certified_at,
)
self._certified[fingerprint.key] = status
self._dark.pop(fingerprint.key, None)
return status
def require_serving(
self, identity: ShardIdentity | CompatibilityFingerprint
) -> RecipeStatus:
"""Admit a recipe for user traffic, or refuse and say why."""
status = self.status(identity)
if not status.may_serve:
raise RecipeNotCertified(
f"recipe {_as_fingerprint(identity).recipe_id!r} is {status.status}: "
f"{status.detail}"
)
return status
def certified_fingerprints(self) -> tuple[CompatibilityFingerprint, ...]:
return tuple(
status.fingerprint
for status in self._certified.values()
if status.fingerprint is not None
)
def to_dict(self) -> dict:
return {
"schema_version": RECIPE_IDENTITY_SCHEMA_VERSION,
"dark": [fp.to_dict() for fp in self._dark.values()],
"certified": [status.to_dict() for status in self._certified.values()],
}
def _as_fingerprint(
identity: ShardIdentity | CompatibilityFingerprint,
) -> CompatibilityFingerprint:
if isinstance(identity, CompatibilityFingerprint):
return identity
if isinstance(identity, ShardIdentity):
return identity.fingerprint
raise RecipeIdentityError(
f"expected a ShardIdentity or CompatibilityFingerprint, got "
f"{type(identity).__name__}"
)
def check_route(
shards: Sequence[ShardIdentity],
*,
registry: RecipeRegistry | None = None,
for_certification: bool = False,
) -> tuple[RouteMismatch, ...]:
"""Decide whether these Shards may form one Inference Route.
"""Why these Shards may not form one Inference Route; empty when they may.
Returns every reason they may not, empty when they may. Fail-closed by
construction: an empty route, an unknown recipe, a hole in the layer
coverage and a single differing dtype all produce a reason, and a caller that
only ever proceeds on an empty tuple cannot accidentally admit any of them.
This answers the *numerical* question only — do these Shards agree on every
axis that moves the numbers, and do they tile the model without a hole. It
deliberately does not answer "may this recipe carry user traffic": that is
certification, the Tracker owns it (`meshnet_tracker.recipe`), and a node
asking itself whether it is certified would be marking its own homework.
`for_certification` admits a dark recipe onto a route whose only purpose is
to certify it. It never admits an unknown one, and it is not the path user
traffic takes.
Fail-closed by construction: an empty route, a hole in the coverage and a
single differing dtype each produce a reason, so a caller that proceeds only
on an empty tuple cannot admit any of them.
"""
if not shards:
return (
@@ -1018,27 +838,12 @@ def check_route(
if gap is not None:
mismatches.append(RouteMismatch(MISMATCH_RANGE, gap))
if registry is not None:
status = registry.status(head.fingerprint)
allowed = status.may_certify if for_certification else status.may_serve
if not allowed:
mismatches.append(
RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}")
)
return tuple(mismatches)
def require_route(
shards: Sequence[ShardIdentity],
*,
registry: RecipeRegistry | None = None,
for_certification: bool = False,
) -> CompatibilityFingerprint:
def require_route(shards: Sequence[ShardIdentity]) -> CompatibilityFingerprint:
"""`check_route`, raising the structured reasons instead of returning them."""
mismatches = check_route(
shards, registry=registry, for_certification=for_certification
)
mismatches = check_route(shards)
if mismatches:
raise RouteIncompatible(mismatches)
return shards[0].fingerprint
@@ -1134,14 +939,111 @@ def check_handshake(
return tuple(reasons)
def check_session_open(
local: ShardIdentity,
remote: "pb.SessionOpen",
*,
expected_route_session_id: str | None = None,
expected_route_epoch: int | None = None,
) -> tuple[RouteMismatch, ...]:
"""Fail closed on the complete immutable portion of a ``SessionOpen``.
``SessionOpen`` is not just a fingerprint carrier: its route session and
epoch select the tracker-issued route whose identity was checked. A worker
that accepts a stale epoch can receive activations for a route that the
tracker has already replaced. Callers that have the tracker assignment
therefore pass both expected values; callers without it still reject the
protobuf defaults, which cannot identify a live route.
"""
reasons = list(check_handshake(local, remote.fingerprint))
if remote.schema_version != SCHEMA_VERSION:
reasons.append(
RouteMismatch(
MISMATCH_BOUNDARY_SCHEMA,
f"peer schema version {remote.schema_version} does not match "
f"{SCHEMA_VERSION}",
)
)
if not remote.route_session_id:
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_SESSION,
"peer omitted the tracker-issued route session id",
)
)
elif (
expected_route_session_id is not None
and remote.route_session_id != expected_route_session_id
):
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_SESSION,
"peer route session does not match the tracker assignment",
)
)
if remote.route_epoch < 1:
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_EPOCH,
"peer omitted a positive tracker-issued route epoch",
)
)
elif expected_route_epoch is not None and remote.route_epoch != expected_route_epoch:
reasons.append(
RouteMismatch(
MISMATCH_ROUTE_EPOCH,
"peer route epoch does not match the tracker assignment",
)
)
advertised = (remote.shard_range.start_layer, remote.shard_range.end_layer)
expected = (local.shard_start, local.shard_end)
effective = remote.shard_range.effective_start_layer
if advertised != expected:
reasons.append(
RouteMismatch(
MISMATCH_RANGE,
f"peer advertised Shard range {advertised}, expected {expected}",
)
)
elif not local.shard_start <= effective < local.shard_end:
reasons.append(
RouteMismatch(
MISMATCH_RANGE,
f"effective start {effective} is outside Shard range {expected}",
)
)
return tuple(reasons)
def handshake_error(
mismatches: Sequence[RouteMismatch],
) -> "pb.ShardError | None":
"""The protocol status a rejected handshake closes the stream with."""
"""The protocol status a rejected handshake closes the stream with.
Each rejection maps to the most specific DGR-002 code the reasons allow:
a digest disagreement dominates (the peer is the wrong artifact or recipe,
whatever else is also wrong), then a schema disagreement (a peer on a
foreign schema cannot be trusted to have encoded anything else correctly),
and only a pure range disagreement reports the range code. Collapsing all
of these to one code would leave the initiator unable to tell "re-route me"
from "renegotiate the schema" from "wrong peer entirely".
"""
if not mismatches:
return None
reasons = {mismatch.reason for mismatch in mismatches}
if reasons == {MISMATCH_RANGE}:
code = pb.ERROR_CODE_SHARD_RANGE_MISMATCH
elif reasons <= {MISMATCH_ROUTE_SESSION, MISMATCH_ROUTE_EPOCH}:
code = pb.ERROR_CODE_EPOCH_STALE
elif reasons <= {MISMATCH_BOUNDARY_SCHEMA, MISMATCH_RANGE}:
code = pb.ERROR_CODE_SCHEMA_UNSUPPORTED
else:
code = pb.ERROR_CODE_FINGERPRINT_MISMATCH
return pb.ShardError(
code=pb.ERROR_CODE_FINGERPRINT_MISMATCH,
code=code,
detail="; ".join(m.describe() for m in mismatches),
retryable=False,
)

38
packages/tracker/meshnet_tracker/capability.py
View File

@@ -185,6 +185,10 @@ class CapabilityState:
# Absent for a node that predates DGR-003.
model_artifact_digest: str | None = None
runtime_recipe_digest: str | None = None
shard_binding_digest: str | None = None
# The tracker ledger's verdict on that fingerprint at evaluation time
# ("dark"/"certified"), so the network map answers "why is this exact node
# not routing" without a second query. None when no identity was presented.
certification: str | None = None
@property
@@ -227,6 +231,7 @@ class CapabilityState:
"diagnostics": list(self.diagnostics),
"model_artifact_digest": self.model_artifact_digest,
"runtime_recipe_digest": self.runtime_recipe_digest,
"shard_binding_digest": self.shard_binding_digest,
"certification": self.certification,
}
@@ -372,6 +377,22 @@ def evaluate_report(
f"identity is for artifact {identity.artifact_id!r}, but the node "
f"registered {advertised_model!r}",
)
model_claim = report["model"]
if model_claim.get("revision") != identity.revision:
return base.with_state(
STATE_MODEL_MISMATCH,
"identity revision does not match the capability proof",
)
config_fingerprint = model_claim.get("config_fingerprint")
if isinstance(config_fingerprint, str) and config_fingerprint.startswith(
"sha256:"
):
config_fingerprint = config_fingerprint.removeprefix("sha256:")
if config_fingerprint != identity.architecture_digest:
return base.with_state(
STATE_MODEL_MISMATCH,
"identity architecture/config digest does not match the capability proof",
)
if (
identity.recipe_id != base.recipe_id
or identity.recipe_version != base.recipe_version
@@ -399,6 +420,7 @@ def evaluate_report(
base,
model_artifact_digest=identity.model_artifact_digest,
runtime_recipe_digest=identity.runtime_recipe_digest,
shard_binding_digest=identity.shard_binding_digest,
)
if status != STATUS_PASSED:
@@ -425,8 +447,22 @@ def evaluate_report(
# not authenticate a node or prove a distributed forward. Exact recipes are
# therefore registered-but-dark until tracker-owned certification records
# that forward.
#
# `ledger is None` means the caller owns no certification authority. It is
# not "certify anything" and it is not "make one up": a disposable ledger
# would register the recipe into state that is discarded on return, which
# reads like certification is wired when nothing is recording it. The only
# safe reading is that nothing here has certified this recipe, so it stays
# dark. `TrackerServer` owns the real ledger and passes it in.
if identity is not None:
recipe_status = (ledger or CertificationLedger()).register(identity)
if ledger is None:
return base.with_state(
STATE_UNCERTIFIED,
"no certification ledger; an exact recipe is dark until a "
"tracker-owned distributed forward certifies it",
)
recipe_status = ledger.register(identity)
base = replace(base, certification=recipe_status.status)
if not recipe_status.may_serve:
return base.with_state(STATE_UNCERTIFIED, recipe_status.detail)

181
packages/tracker/meshnet_tracker/recipe.py
View File

@@ -32,7 +32,7 @@ import json
import re
import time
from dataclasses import dataclass, field
from typing import Any, Iterable, Mapping, Sequence
from typing import Any, Iterable, Mapping
# Layout of the identity block this tracker reads (meshnet_node.runtime_recipe).
RECIPE_IDENTITY_SCHEMA_VERSION = 1
@@ -42,6 +42,7 @@ RECIPE_IDENTITY_SCHEMA_VERSION = 1
# without changing it there silently partitions every route.
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
# The axes a recipe digest commits to. Order is irrelevant (the canonical JSON
# sorts keys); membership is not — an axis missing here is an axis the tracker
@@ -62,16 +63,6 @@ RECIPE_AXES: tuple[str, ...] = (
_INT_AXES = frozenset({"boundary_schema_version", "protocol_schema_version"})
MISMATCH_ARTIFACT = "artifact"
MISMATCH_TOKENIZER = "tokenizer"
MISMATCH_ARCHITECTURE = "architecture"
MISMATCH_RANGE = "range"
MISMATCH_BOUNDARY_SCHEMA = "boundary-schema"
MISMATCH_ACTIVATION_RECIPE = "activation-recipe"
MISMATCH_CACHE_LAYOUT = "cache-layout"
MISMATCH_FINGERPRINT = "fingerprint"
MISMATCH_UNCERTIFIED = "uncertified"
STATUS_UNKNOWN = "unknown"
STATUS_DARK = "dark"
STATUS_CERTIFIED = "certified"
@@ -183,6 +174,8 @@ class PresentedIdentity:
artifact_id: str
revision: str
source_digest: str
content_digest: str
derivative_range: tuple[int, int] | None
architecture: str
architecture_digest: str
layer_count: int
@@ -211,6 +204,36 @@ class PresentedIdentity:
def key(self) -> tuple[str, str]:
return (self.model_artifact_digest, self.runtime_recipe_digest)
@property
def shard_binding_digest(self) -> str:
"""This participant's own bytes and exact range, bound to the source.
The route fingerprint (:attr:`key`) is deliberately range-independent —
Shards on one route own different ranges, so a range-sensitive digest
would stop any two of them from ever agreeing. The consequence is that
the fingerprint alone cannot tell two *different splits of the same
source* apart: a derivative can assert the right source and recipe and
inherit certification earned by bytes it does not hold.
This digest is what closes that. It commits to the derivative's own
content hash and its exact end-exclusive range, so certification can be
recipe-wide while every serving node is still separately pinned to the
blob it was admitted on (`TrackerServer.certify_recipe`).
"""
return _digest(
SHARD_BINDING_DIGEST_DOMAIN,
{
"source_digest": self.source_digest,
"content_digest": self.content_digest,
"architecture_digest": self.architecture_digest,
"derivative_range": list(self.derivative_range)
if self.derivative_range is not None
else None,
"shard_start": self.shard_start,
"shard_end": self.shard_end,
},
)
@property
def tokenizer_revision(self) -> str:
return str(self.axes["tokenizer_revision"])
@@ -315,6 +338,8 @@ def parse_identity(data: Any) -> PresentedIdentity:
artifact_id=_text(artifact.get("artifact_id"), "artifact.artifact_id"),
revision=_pin(artifact.get("revision"), "artifact.revision"),
source_digest=source_digest,
content_digest=content_digest,
derivative_range=binding,
architecture=_text(artifact.get("architecture"), "artifact.architecture"),
architecture_digest=_hex64(
artifact.get("architecture_digest"), "artifact.architecture_digest"
@@ -349,78 +374,6 @@ def parse_identity(data: Any) -> PresentedIdentity:
return identity
@dataclass(frozen=True)
class RouteMismatch:
reason: str
detail: str
def describe(self) -> str:
return f"{self.reason}: {self.detail}"
def compare(
route: PresentedIdentity, candidate: PresentedIdentity
) -> tuple[RouteMismatch, ...]:
"""Why `candidate` may not join a route already running `route`."""
if route.key == candidate.key:
return ()
reasons: list[RouteMismatch] = []
if route.source_digest != candidate.source_digest:
reasons.append(
RouteMismatch(
MISMATCH_ARTIFACT,
f"serves Model Artifact {candidate.source_digest[:12]}…, the route "
f"runs {route.source_digest[:12]}…",
)
)
if route.architecture_digest != candidate.architecture_digest:
reasons.append(
RouteMismatch(
MISMATCH_ARCHITECTURE,
"architecture/config snapshot differs from the route's",
)
)
if route.layer_count != candidate.layer_count:
reasons.append(
RouteMismatch(
MISMATCH_ARTIFACT,
f"artifact has {candidate.layer_count} layers, the route's has "
f"{route.layer_count}",
)
)
axis_reason = {
"tokenizer_revision": MISMATCH_TOKENIZER,
"architecture_adapter": MISMATCH_ARCHITECTURE,
"activation_dtype": MISMATCH_ACTIVATION_RECIPE,
"compute_dtype": MISMATCH_ACTIVATION_RECIPE,
"kv_dtype": MISMATCH_CACHE_LAYOUT,
"kv_layout": MISMATCH_CACHE_LAYOUT,
"boundary_schema_version": MISMATCH_BOUNDARY_SCHEMA,
"protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA,
}
for axis in RECIPE_AXES:
mine = route.axes.get(axis)
theirs = candidate.axes.get(axis)
if mine != theirs:
reasons.append(
RouteMismatch(
axis_reason.get(axis, MISMATCH_FINGERPRINT),
f"{axis} is {theirs!r}, the route runs {mine!r}",
)
)
if not reasons:
reasons.append(
RouteMismatch(
MISMATCH_FINGERPRINT,
"fingerprints differ on a field this comparison does not cover",
)
)
return tuple(reasons)
def coverage_gap(
ranges: Iterable[tuple[int, int]], layer_count: int
) -> str | None:
@@ -462,6 +415,7 @@ class DistributedForwardEvidence:
tokens_generated: int
layer_count: int
fingerprint: tuple[str, str]
participants: tuple[PresentedIdentity, ...]
synthetic: bool = False
certified_at: float = field(default_factory=time.time)
@@ -471,6 +425,19 @@ class DistributedForwardEvidence:
"evidence comes from a synthetic worker; only a real distributed "
"forward certifies a recipe"
)
if len(self.participants) != len(self.node_ids):
return "participant identities do not match the certifying node list"
if len(self.shard_ranges) != len(self.participants):
return "participant identities do not match the recorded effective ranges"
for participant, shard_range in zip(
self.participants, self.shard_ranges, strict=True
):
if participant.key != self.fingerprint:
return "a participant fingerprint differs from the certifying route"
if participant.layer_count != self.layer_count:
return "a participant artifact layer count differs from the certifying route"
if (participant.shard_start, participant.shard_end) != shard_range:
return "a participant identity does not match its recorded effective range"
distinct = set(self.node_ids)
if len(distinct) < MIN_CERTIFYING_NODES:
return (
@@ -540,11 +507,11 @@ class CertificationLedger:
def certify(
self,
identity: PresentedIdentity | tuple[str, str],
identity: PresentedIdentity,
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote a recipe out of the dark, or raise saying why the evidence is short."""
key = identity if isinstance(identity, tuple) else identity.key
key = identity.key
if key not in self._dark and key not in self._certified:
raise RecipeIdentityError(
"this fingerprint is not registered; unknown recipes cannot be certified"
@@ -553,6 +520,10 @@ class CertificationLedger:
raise RecipeIdentityError(
"certification evidence fingerprint does not match the recipe being promoted"
)
if evidence.layer_count != identity.layer_count:
raise RecipeIdentityError(
"certification evidence layer count does not match the artifact being promoted"
)
rejection = evidence.rejection()
if rejection is not None:
raise RecipeIdentityError(f"this evidence does not certify: {rejection}")
@@ -578,38 +549,8 @@ class CertificationLedger:
}
def admit_route(
identities: Sequence[PresentedIdentity],
*,
ledger: CertificationLedger | None = None,
for_certification: bool = False,
) -> tuple[RouteMismatch, ...]:
"""Every reason these Shards may not form one Inference Route; empty when they may.
Fail-closed by construction: an empty route, a hole in the coverage, one
differing dtype, and an uncertified recipe each produce a reason, so a caller
that proceeds only on an empty result cannot admit any of them.
"""
if not identities:
return (RouteMismatch(MISMATCH_RANGE, "a route needs at least one Shard"),)
head = identities[0]
reasons: list[RouteMismatch] = []
for candidate in identities[1:]:
reasons.extend(compare(head, candidate))
gap = coverage_gap(
[(i.shard_start, i.shard_end) for i in identities], head.layer_count
)
if gap is not None:
reasons.append(RouteMismatch(MISMATCH_RANGE, gap))
if ledger is not None:
status = ledger.status(head)
allowed = status.may_certify if for_certification else status.may_serve
if not allowed:
reasons.append(
RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}")
)
return tuple(reasons)
# Route formation itself lives in `server.py` (`_select_route`, `_enumerate_routes`,
# `_find_pinned_route`): candidates are partitioned by the exact fingerprint this
# module derives, so a route can only ever be assembled inside one identity. A
# second route gate here would be a second copy of that policy to keep in step —
# the same reason the node module holds no certification authority.

171
packages/tracker/meshnet_tracker/server.py
View File

@@ -45,7 +45,7 @@ import urllib.parse
import urllib.request
import uuid
from collections import deque
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from importlib.resources import files
from pathlib import Path
from typing import Any
@@ -60,6 +60,7 @@ from .capability import (
STATE_ADMITTED,
STATE_MODEL_MISMATCH,
STATE_SHARD_MISMATCH,
STATE_UNCERTIFIED,
CapabilityState,
absent_state,
evaluate_report,
@@ -84,7 +85,13 @@ from .routing_stats import (
)
from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode
from .recipe import CertificationLedger
from .recipe import (
CertificationLedger,
DistributedForwardEvidence,
PresentedIdentity,
RecipeIdentityError,
RecipeStatus,
)
_CONSOLE_LIMIT = 300
@@ -833,6 +840,11 @@ def _admitted_nodes(nodes: list["_NodeEntry"], policy: str | None) -> list["_Nod
return [node for node in nodes if _capability_routable(node, effective)]
def _route_identity_partition(node: "_NodeEntry") -> tuple[str, ...]:
fingerprint = _node_admission(node).fingerprint
return ("legacy",) if fingerprint is None else ("exact", *fingerprint)
def _select_route(
nodes: list[_NodeEntry],
required_start: int,
@@ -856,29 +868,51 @@ def _select_route(
],
key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
)
route: list[_NodeEntry] = []
covered_up_to = required_start - 1
def _routing_score(node: "_NodeEntry") -> float:
return _effective_throughput(node, model) * _reputation_multiplier(node, contracts)
while covered_up_to < required_end:
best: _NodeEntry | None = None
for node in candidates:
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
if best is None:
best = node
elif node.shard_end > best.shard_end:
best = node
elif node.shard_end == best.shard_end and _routing_score(node) > _routing_score(best):
best = node
if best is None:
missing = covered_up_to + 1
return [], f"no route available: no registered node covers layer {missing}"
route.append(best)
covered_up_to = best.shard_end
candidates = [n for n in candidates if n is not best]
partitions: dict[tuple[str, ...], list[_NodeEntry]] = {}
for node in candidates:
partitions.setdefault(_route_identity_partition(node), []).append(node)
complete: list[list[_NodeEntry]] = []
furthest = required_start - 1
for partition in partitions.values():
pool = list(partition)
route: list[_NodeEntry] = []
covered_up_to = required_start - 1
while covered_up_to < required_end:
best: _NodeEntry | None = None
for node in pool:
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
if best is None:
best = node
elif node.shard_end > best.shard_end:
best = node
elif (
node.shard_end == best.shard_end
and _routing_score(node) > _routing_score(best)
):
best = node
if best is None:
break
route.append(best)
covered_up_to = best.shard_end
pool = [node for node in pool if node is not best]
furthest = max(furthest, covered_up_to)
if covered_up_to >= required_end:
complete.append(route)
if not complete:
return [], f"no route available: no registered node covers layer {furthest + 1}"
route = max(
complete,
key=lambda candidate: (
min(_routing_score(node) for node in candidate),
-len(candidate),
),
)
return route, ""
@@ -914,7 +948,12 @@ def _enumerate_routes(
for head in heads:
route = [head]
covered_up_to = head.shard_end
pool = [n for n in sharded if n is not head]
head_partition = _route_identity_partition(head)
pool = [
node
for node in sharded
if node is not head and _route_identity_partition(node) == head_partition
]
while covered_up_to < required_end:
best = None
for n in pool:
@@ -2059,14 +2098,24 @@ def _find_pinned_route(
hop_count: int,
) -> list[_NodeEntry] | None:
"""First combination of exactly ``hop_count`` distinct nodes covering the
layer range, where every node extends coverage (US-030 benchmark routes)."""
layer range, where every node extends coverage (US-030 benchmark routes).
Benchmark combos run real inference, so they obey the same DGR-003 rule as
every other route builder: one route, one exact identity. A combo that mixed
two fingerprints — or an exact Shard with a legacy one — would measure a
numerically incoherent route and record the garbage as a benchmark.
"""
for combo in itertools.permutations(nodes, hop_count):
covered = required_start - 1
valid = True
partition = _route_identity_partition(combo[0])
for candidate in combo:
if candidate.shard_start is None or candidate.shard_end is None:
valid = False
break
if _route_identity_partition(candidate) != partition:
valid = False
break
if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
valid = False
break
@@ -6647,6 +6696,84 @@ class TrackerServer:
self._test_runner: TestRunManager | None = test_runner
self.port: int | None = None
def certify_recipe(
self,
identity: PresentedIdentity,
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote one exact recipe from tracker-recorded distributed evidence.
This is the tracker's only certification authority. The evidence names
the nodes that served the forward; this method refuses to take any of
them on the evidence's word. For each one it goes back to what *this
tracker* re-derived at admission and requires the evidence to describe
the same Shard: same route fingerprint, same registered range, and the
same shard binding digest — the derivative's own bytes and exact range.
Without the binding check, a route fingerprint is range-independent by
design (:mod:`meshnet_tracker.recipe`), so evidence could name the right
recipe while describing splits nobody on the route was actually serving,
and the certification would attach to blobs that never ran.
"""
with self._lock:
for node_id, participant in zip(
evidence.node_ids, evidence.participants, strict=True
):
node = self._registry.get(node_id)
if node is None:
raise RecipeIdentityError(
f"certification participant {node_id!r} is not registered"
)
admitted = node.capability
if admitted.fingerprint != identity.key:
raise RecipeIdentityError(
f"certification participant {node_id!r} is not admitted under "
"the promoted fingerprint"
)
if admitted.state not in (STATE_UNCERTIFIED, STATE_ADMITTED):
raise RecipeIdentityError(
f"certification participant {node_id!r} has capability state "
f"{admitted.state!r}"
)
registered_range = (
node.shard_start,
None if node.shard_end is None else node.shard_end + 1,
)
if registered_range != (
participant.shard_start,
participant.shard_end,
):
raise RecipeIdentityError(
f"certification participant {node_id!r} range differs from its "
"registered Shard range"
)
# The exact bytes and range this node was admitted on. A node that
# registered without an identity has no binding, and cannot be a
# participant in an exact certification at all.
if admitted.shard_binding_digest is None:
raise RecipeIdentityError(
f"certification participant {node_id!r} registered no exact "
"Shard binding; it cannot certify a recipe"
)
if admitted.shard_binding_digest != participant.shard_binding_digest:
raise RecipeIdentityError(
f"certification participant {node_id!r} presents a Shard "
"binding this tracker did not admit it on; the evidence "
"describes different derivative bytes or a different range"
)
status = self._recipe_certifications.certify(identity, evidence)
for node in self._registry.values():
if (
node.capability.state == STATE_UNCERTIFIED
and node.capability.fingerprint == identity.key
):
node.capability = replace(
node.capability.with_state(STATE_ADMITTED, status.detail),
certification=status.status,
)
return status
def _start_embedded_relay(self) -> dict:
"""Start the shared RelayServer class in-process for tracker+relay deployments."""
if not self._embedded_relay_enabled: