"""Exact Model Artifact and runtime recipe identity (DGR-003). A route is a chain of Shards that together compute one forward pass. If two Shards disagree about *anything* that moves the numbers — the weights, the quantization, the dtype the seam is serialized in, the dtype the kernels accumulate in, the KV layout, the tokenizer, the architecture adapter, the backend, or the runtime build — the route still runs and still emits tokens. They are simply the wrong tokens, and nothing upstream notices. Identity is therefore a digest over the executable semantics, not a model name. Four decisions shape this module. **The axes are separate, and each one is separately fatal.** `Q4_K_M` weights are not a `bfloat16` seam, which is not an `fp32` accumulate, which is not a `q8_0` KV cache. Collapsing them into one "precision" label is how a route ends up numerically split while every node reports agreement. :data:`RECIPE_AXES` is the full list; the digest commits to each under its own key, so swapping two axis *values* changes the digest even though the multiset of values did not. **The fingerprint deliberately excludes the Shard range.** Shards on one route own *different* ranges; if the range were digested, no two Shards on a route could ever agree. Range compatibility is a route-level coverage question (:func:`check_route`), not an identity question. **A split artifact routes under its source's identity.** A node holding layers 32–63 of a GGUF holds a *different file* with a *different content hash* than a node holding layers 0–31, and neither equals the whole model. So the digest binds :attr:`ArtifactIdentity.source_digest` — the whole-model artifact every Shard descends from — and the derivative's own hash and range are bound to that source locally (:class:`DerivativeBinding`). A derivative that cannot name the exact artifact it was cut from is not admissible: it is an anonymous blob that happens to have the right layer count. **Labels are not digested.** `recipe_id`, `recipe_version` and `catalogue_version` ride alongside the digests for diagnosis, mirroring the capability report an admitted node already registers with (ADR-0023). Renaming a recipe does not change a single number, so a rename must not partition a route; changing `kv_dtype` changes every number, so it must. The digest commits to what changes the numbers. Recipes are **registered-but-dark** on arrival: known, visible to an operator, and not routable for user traffic. A recipe leaves the dark only when a *real distributed forward* over a route of at least two distinct physical nodes, covering the whole model, has emitted real tokens — see :class:`DistributedForwardEvidence`. Detected hardware is not a capability, a single-host forward is not a distributed forward, and a synthetic worker certifies nothing. The tracker re-derives all of this independently in ``meshnet_tracker.recipe`` — it does not import this package. The two implementations are pinned together by a committed conformance vector (``tests/data/recipe_fingerprint_vectors.json``); if they ever drift, a test fails rather than a route silently forming. """ from __future__ import annotations import hashlib import json import re import time from dataclasses import dataclass, field from typing import Any, Iterable, Mapping, Sequence from .native_protocol import BUNDLE_VERSION, SCHEMA_VERSION, pb # Layout of the serialized identity block. Bump when the JSON shape changes. RECIPE_IDENTITY_SCHEMA_VERSION = 1 # Domain separation. A digest means nothing outside the structure it commits to; # the prefix guarantees an artifact digest can never be mistaken for — or collide # with — a recipe digest, even if their canonical JSON were somehow identical. ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1" RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1" # The axes of a runtime recipe. Every one of these changes the numbers a Shard # produces, so every one of them is part of identity and none of them may be # folded into another. RECIPE_AXES: tuple[str, ...] = ( "weight_quantization", "activation_dtype", "compute_dtype", "kv_dtype", "kv_layout", "tokenizer_revision", "architecture_adapter", "backend_id", "runtime_version", "boundary_schema_version", "protocol_schema_version", ) # --- Why two identities failed to match. These are the fail-closed reasons. --- MISMATCH_ARTIFACT = "artifact" MISMATCH_WEIGHT_QUANTIZATION = "weight-quantization" MISMATCH_ACTIVATION_RECIPE = "activation-recipe" MISMATCH_CACHE_LAYOUT = "cache-layout" MISMATCH_TOKENIZER = "tokenizer" MISMATCH_ARCHITECTURE = "architecture" MISMATCH_BACKEND = "backend" MISMATCH_RUNTIME = "runtime" MISMATCH_BOUNDARY_SCHEMA = "boundary-schema" MISMATCH_RANGE = "range" MISMATCH_FINGERPRINT = "fingerprint" MISMATCH_UNCERTIFIED = "uncertified" # Which fail-closed reason each axis reports. Several axes share a reason # because they fail for the same operational cause: `activation_dtype` and # `compute_dtype` are both "the activation recipe disagrees", and `kv_dtype` # and `kv_layout` are both "the cache layout disagrees". _AXIS_MISMATCH: Mapping[str, str] = { "weight_quantization": MISMATCH_WEIGHT_QUANTIZATION, "activation_dtype": MISMATCH_ACTIVATION_RECIPE, "compute_dtype": MISMATCH_ACTIVATION_RECIPE, "kv_dtype": MISMATCH_CACHE_LAYOUT, "kv_layout": MISMATCH_CACHE_LAYOUT, "tokenizer_revision": MISMATCH_TOKENIZER, "architecture_adapter": MISMATCH_ARCHITECTURE, "backend_id": MISMATCH_BACKEND, "runtime_version": MISMATCH_RUNTIME, "boundary_schema_version": MISMATCH_BOUNDARY_SCHEMA, "protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA, } # Certification states. `dark` is the arrival state, not an error state. STATUS_UNKNOWN = "unknown" STATUS_DARK = "dark" STATUS_CERTIFIED = "certified" # A route that proves a recipe must be a real distributed route. MIN_CERTIFYING_NODES = 2 _HEX64 = re.compile(r"^[0-9a-f]{64}$") # A revision that can move is not a pin. DGR-017 learned this on the artifact; # it is just as true of a tokenizer. _MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"}) class RecipeIdentityError(ValueError): """Malformed identity input. Messages name the field, never echo a payload.""" class RecipeNotCertified(RecipeIdentityError): """A recipe was asked to serve user traffic while still dark.""" class RouteIncompatible(RecipeIdentityError): """Two Shards cannot form an Inference Route. Carries the structured reasons, so a caller can map them to a gRPC status rather than re-parsing a sentence. """ def __init__(self, mismatches: Sequence["RouteMismatch"]) -> None: self.mismatches = tuple(mismatches) super().__init__("; ".join(m.describe() for m in self.mismatches)) def canonical_sha256(value: Any) -> str: """SHA-256 over canonical JSON — the repository's digest convention.""" payload = json.dumps( value, sort_keys=True, separators=(",", ":"), ensure_ascii=False ) return hashlib.sha256(payload.encode("utf-8")).hexdigest() def _digest(domain: str, body: Mapping[str, Any]) -> str: return canonical_sha256({"domain": domain, "body": dict(body)}) def _require_text(value: Any, what: str) -> str: if not isinstance(value, str) or not value.strip(): raise RecipeIdentityError(f"{what!r} must be a non-empty string") return value def _require_hex64(value: Any, what: str) -> str: text = _require_text(value, what) if not _HEX64.match(text): raise RecipeIdentityError( f"{what!r} must be a lowercase 64-character SHA-256 hex digest" ) return text def _require_int(value: Any, what: str, minimum: int) -> int: if isinstance(value, bool) or not isinstance(value, int): raise RecipeIdentityError(f"{what!r} must be an integer") if value < minimum: raise RecipeIdentityError(f"{what!r} must be >= {minimum}, got {value}") return value def _require_pin(value: Any, what: str) -> str: """A revision must identify one immutable thing, not a ref that moves.""" text = _require_text(value, what) lowered = text.strip().lower() if lowered in _MOVING_REFS or lowered.startswith("refs/"): raise RecipeIdentityError( f"{what!r} is {text!r}, which is a moving reference, not a pin; " "identity requires an exact immutable revision" ) return text def _as_mapping(value: Any, what: str) -> Mapping[str, Any]: if not isinstance(value, Mapping): raise RecipeIdentityError( f"{what!r} must be a JSON object, got {type(value).__name__}" ) return value @dataclass(frozen=True) class DerivativeBinding: """A split or derived artifact, bound to the exact artifact it was cut from. Without this binding a split GGUF is an anonymous blob with a plausible layer count. With it, the Shard's own bytes are checkable *and* its identity on the route is the source model's, which is what lets a whole-model oracle and a five-way split agree on a fingerprint. The range is inclusive start, exclusive end, matching the protocol (``ShardRange`` in ``shard_runtime.proto``). """ source_artifact_digest: str shard_start: int shard_end: int def __post_init__(self) -> None: _require_hex64( self.source_artifact_digest, "derived_from.source_artifact_digest" ) _require_int(self.shard_start, "derived_from.shard_start", 0) _require_int(self.shard_end, "derived_from.shard_end", 1) if self.shard_end <= self.shard_start: raise RecipeIdentityError( f"'derived_from.shard_end' ({self.shard_end}) must be greater than " f"'derived_from.shard_start' ({self.shard_start}); an empty split " "artifact covers no layers and can prove nothing" ) def covers(self, start: int, end: int) -> bool: return self.shard_start <= start and end <= self.shard_end def to_dict(self) -> dict: return { "source_artifact_digest": self.source_artifact_digest, "shard_start": self.shard_start, "shard_end": self.shard_end, } @classmethod def from_dict(cls, data: Any) -> DerivativeBinding: doc = _as_mapping(data, "derived_from") return cls( source_artifact_digest=_require_hex64( doc.get("source_artifact_digest"), "derived_from.source_artifact_digest", ), shard_start=_require_int( doc.get("shard_start"), "derived_from.shard_start", 0 ), shard_end=_require_int(doc.get("shard_end"), "derived_from.shard_end", 1), ) @dataclass(frozen=True) class ArtifactIdentity: """Exactly which weights, config and architecture a Shard loaded. `content_digest` is *this* file's hash — the whole model for an undivided artifact, one split's bytes for a derivative. `source_digest` is what the route compares. For an undivided artifact they are the same value; that identity is the whole point, because it is what makes the DGR-018 whole-model oracle and the DGR-020 distributed route the same artifact. """ artifact_id: str revision: str content_digest: str architecture: str architecture_digest: str layer_count: int derived_from: DerivativeBinding | None = None def __post_init__(self) -> None: _require_text(self.artifact_id, "artifact.artifact_id") _require_pin(self.revision, "artifact.revision") _require_hex64(self.content_digest, "artifact.content_digest") _require_text(self.architecture, "artifact.architecture") _require_hex64(self.architecture_digest, "artifact.architecture_digest") _require_int(self.layer_count, "artifact.layer_count", 1) binding = self.derived_from if binding is None: return if binding.source_artifact_digest == self.content_digest: raise RecipeIdentityError( "'artifact.derived_from.source_artifact_digest' equals this " "artifact's own content digest; an artifact is not a split of itself" ) if binding.shard_end > self.layer_count: raise RecipeIdentityError( f"'artifact.derived_from' claims layers " f"{binding.shard_start}–{binding.shard_end}, but the source model " f"has only {self.layer_count} layers" ) @property def is_derivative(self) -> bool: return self.derived_from is not None @property def source_digest(self) -> str: """The whole-model artifact this Shard descends from — what a route compares.""" if self.derived_from is None: return self.content_digest return self.derived_from.source_artifact_digest @property def model_artifact_digest(self) -> str: """The artifact half of the compatibility fingerprint. Commits to the source weights, the architecture and its config/tokenizer snapshot, and the layer count — and deliberately *not* to the repository the bytes were fetched from. A mirror of identical bytes is the same artifact; the repo name buys diagnosis, not numerical identity. """ return _digest( ARTIFACT_DIGEST_DOMAIN, { "source_digest": self.source_digest, "architecture": self.architecture, "architecture_digest": self.architecture_digest, "layer_count": self.layer_count, }, ) def to_dict(self) -> dict: return { "artifact_id": self.artifact_id, "revision": self.revision, "content_digest": self.content_digest, "architecture": self.architecture, "architecture_digest": self.architecture_digest, "layer_count": self.layer_count, "derived_from": ( self.derived_from.to_dict() if self.derived_from else None ), } @classmethod def from_dict(cls, data: Any) -> ArtifactIdentity: doc = _as_mapping(data, "artifact") raw_binding = doc.get("derived_from") return cls( artifact_id=_require_text(doc.get("artifact_id"), "artifact.artifact_id"), revision=_require_pin(doc.get("revision"), "artifact.revision"), content_digest=_require_hex64( doc.get("content_digest"), "artifact.content_digest" ), architecture=_require_text( doc.get("architecture"), "artifact.architecture" ), architecture_digest=_require_hex64( doc.get("architecture_digest"), "artifact.architecture_digest" ), layer_count=_require_int(doc.get("layer_count"), "artifact.layer_count", 1), derived_from=( None if raw_binding is None else DerivativeBinding.from_dict(raw_binding) ), ) @dataclass(frozen=True) class RuntimeRecipe: """How a Shard executes the artifact — one field per numerically live axis. Weight quantization is not activation dtype is not compute dtype is not KV dtype. Two nodes running `UD-IQ1_S` weights, one accumulating in fp32 and one in fp16, produce different logits from the same bytes. Keeping the axes apart is the entire safety property; see :data:`RECIPE_AXES`. The three label fields are diagnosis only and are not digested. """ weight_quantization: str activation_dtype: str compute_dtype: str kv_dtype: str kv_layout: str tokenizer_revision: str architecture_adapter: str backend_id: str runtime_version: str boundary_schema_version: int = BUNDLE_VERSION protocol_schema_version: int = int(SCHEMA_VERSION) recipe_id: str = "unnamed" recipe_version: str = "0" catalogue_version: str = "0" def __post_init__(self) -> None: for axis in RECIPE_AXES: value = getattr(self, axis) if axis.endswith("_schema_version"): _require_int(value, f"recipe.{axis}", 1) else: _require_text(value, f"recipe.{axis}") _require_pin(self.tokenizer_revision, "recipe.tokenizer_revision") _require_text(self.recipe_id, "recipe.recipe_id") _require_text(self.recipe_version, "recipe.recipe_version") _require_text(self.catalogue_version, "recipe.catalogue_version") @property def axes(self) -> dict[str, Any]: """The digested fields, and only those.""" return {axis: getattr(self, axis) for axis in RECIPE_AXES} @property def runtime_recipe_digest(self) -> str: """The recipe half of the compatibility fingerprint.""" return _digest(RECIPE_DIGEST_DOMAIN, self.axes) def to_dict(self) -> dict: doc = self.axes doc.update( { "recipe_id": self.recipe_id, "recipe_version": self.recipe_version, "catalogue_version": self.catalogue_version, } ) return doc @classmethod def from_dict(cls, data: Any) -> RuntimeRecipe: doc = _as_mapping(data, "recipe") missing = [axis for axis in RECIPE_AXES if axis not in doc] if missing: raise RecipeIdentityError( "recipe is missing required axes: " + ", ".join(missing) + "; an unstated axis is an unproven one, so it cannot default" ) kwargs: dict[str, Any] = {} for axis in RECIPE_AXES: value = doc[axis] if axis.endswith("_schema_version"): kwargs[axis] = _require_int(value, f"recipe.{axis}", 1) else: kwargs[axis] = _require_text(value, f"recipe.{axis}") return cls( **kwargs, recipe_id=_require_text(doc.get("recipe_id"), "recipe.recipe_id"), recipe_version=_require_text( doc.get("recipe_version"), "recipe.recipe_version" ), catalogue_version=_require_text( doc.get("catalogue_version"), "recipe.catalogue_version" ), ) @dataclass(frozen=True) class CompatibilityFingerprint: """The stable identity two peers actually compare. This is the Python face of the protocol's ``Fingerprint`` message (DGR-002). Do not invent a second identity struct: capability admission and the gRPC handshake compare *this*, and they must compare the same thing or a route can pass one gate and fail the other. """ model_artifact_digest: str runtime_recipe_digest: str recipe_id: str recipe_version: str catalogue_version: str def __post_init__(self) -> None: _require_hex64(self.model_artifact_digest, "fingerprint.model_artifact_digest") _require_hex64( self.runtime_recipe_digest, "fingerprint.runtime_recipe_digest" ) @property def key(self) -> tuple[str, str]: """What equality means: the digests, never the labels.""" return (self.model_artifact_digest, self.runtime_recipe_digest) def matches(self, other: CompatibilityFingerprint) -> bool: return self.key == other.key def to_dict(self) -> dict: return { "model_artifact_digest": self.model_artifact_digest, "runtime_recipe_digest": self.runtime_recipe_digest, "recipe_id": self.recipe_id, "recipe_version": self.recipe_version, "catalogue_version": self.catalogue_version, } @classmethod def from_dict(cls, data: Any) -> CompatibilityFingerprint: doc = _as_mapping(data, "fingerprint") return cls( model_artifact_digest=_require_hex64( doc.get("model_artifact_digest"), "fingerprint.model_artifact_digest" ), runtime_recipe_digest=_require_hex64( doc.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest" ), recipe_id=_require_text(doc.get("recipe_id"), "fingerprint.recipe_id"), recipe_version=_require_text( doc.get("recipe_version"), "fingerprint.recipe_version" ), catalogue_version=_require_text( doc.get("catalogue_version"), "fingerprint.catalogue_version" ), ) def to_proto(self) -> "pb.Fingerprint": """Populate the DGR-002 ``Fingerprint`` message for the gRPC handshake.""" return pb.Fingerprint( model_artifact_digest=self.model_artifact_digest, runtime_recipe_digest=self.runtime_recipe_digest, recipe_id=self.recipe_id, recipe_version=self.recipe_version, catalogue_version=self.catalogue_version, ) @classmethod def from_proto(cls, message: "pb.Fingerprint") -> CompatibilityFingerprint: return cls( model_artifact_digest=_require_hex64( message.model_artifact_digest, "fingerprint.model_artifact_digest" ), runtime_recipe_digest=_require_hex64( message.runtime_recipe_digest, "fingerprint.runtime_recipe_digest" ), recipe_id=_require_text(message.recipe_id, "fingerprint.recipe_id"), recipe_version=_require_text( message.recipe_version, "fingerprint.recipe_version" ), catalogue_version=_require_text( message.catalogue_version, "fingerprint.catalogue_version" ), ) @dataclass(frozen=True) class RouteMismatch: """One structured, fail-closed reason two identities do not compose.""" reason: str detail: str shard_index: int | None = None def describe(self) -> str: where = "" if self.shard_index is None else f"shard {self.shard_index}: " return f"{where}{self.reason}: {self.detail}" def to_dict(self) -> dict: return { "reason": self.reason, "detail": self.detail, "shard_index": self.shard_index, } @dataclass(frozen=True) class ShardIdentity: """What one Shard is: an artifact, a recipe, and the layers it owns. The owned range is inclusive start, exclusive end (protocol convention). It is *not* part of the fingerprint — see the module docstring — but it is validated against the artifact, because a Shard cannot serve layers its artifact does not contain. """ artifact: ArtifactIdentity recipe: RuntimeRecipe shard_start: int shard_end: int def __post_init__(self) -> None: _require_int(self.shard_start, "shard_start", 0) _require_int(self.shard_end, "shard_end", 1) if self.shard_end <= self.shard_start: raise RecipeIdentityError( f"'shard_end' ({self.shard_end}) must be greater than 'shard_start' " f"({self.shard_start}); a Shard owning no layer computes nothing" ) if self.shard_end > self.artifact.layer_count: raise RecipeIdentityError( f"Shard owns layers {self.shard_start}–{self.shard_end}, but the " f"artifact has only {self.artifact.layer_count} layers" ) binding = self.artifact.derived_from if binding is not None and not binding.covers(self.shard_start, self.shard_end): raise RecipeIdentityError( f"Shard advertises layers {self.shard_start}–{self.shard_end}, but its " f"split artifact only contains layers {binding.shard_start}–" f"{binding.shard_end}; it cannot serve weights it does not hold" ) @property def fingerprint(self) -> CompatibilityFingerprint: """The derived identity. Never stored, never trusted from the wire — derived.""" return CompatibilityFingerprint( model_artifact_digest=self.artifact.model_artifact_digest, runtime_recipe_digest=self.recipe.runtime_recipe_digest, recipe_id=self.recipe.recipe_id, recipe_version=self.recipe.recipe_version, catalogue_version=self.recipe.catalogue_version, ) def to_dict(self) -> dict: return { "schema_version": RECIPE_IDENTITY_SCHEMA_VERSION, "artifact": self.artifact.to_dict(), "recipe": self.recipe.to_dict(), "shard_start": self.shard_start, "shard_end": self.shard_end, "fingerprint": self.fingerprint.to_dict(), } @classmethod def from_dict(cls, data: Any) -> ShardIdentity: """Parse an identity block, re-deriving — never trusting — its fingerprint. A declared fingerprint is a compatibility claim, not authentication. Recomputing it catches an inconsistent declaration; only tracker-owned distributed certification decides whether a recipe may serve. """ doc = _as_mapping(data, "identity") declared_schema = doc.get("schema_version") if declared_schema != RECIPE_IDENTITY_SCHEMA_VERSION: raise RecipeIdentityError( f"identity block declares schema version {declared_schema!r}, but this " f"node reads version {RECIPE_IDENTITY_SCHEMA_VERSION}" ) identity = cls( artifact=ArtifactIdentity.from_dict(doc.get("artifact")), recipe=RuntimeRecipe.from_dict(doc.get("recipe")), shard_start=_require_int(doc.get("shard_start"), "shard_start", 0), shard_end=_require_int(doc.get("shard_end"), "shard_end", 1), ) declared = doc.get("fingerprint") if declared is not None: claim = CompatibilityFingerprint.from_dict(declared) derived = identity.fingerprint if not claim.matches(derived): raise RouteIncompatible( [ RouteMismatch( MISMATCH_FINGERPRINT, "declared fingerprint does not match the identity it " "claims to describe; the digest is derived from the " "artifact and recipe, it is not an assertion", ) ] ) return identity def explain_mismatch( left: ShardIdentity, right: ShardIdentity, *, shard_index: int | None = None, ) -> tuple[RouteMismatch, ...]: """Name every axis on which two Shards disagree. The fingerprint comparison already answers *whether* they compose. This answers *why not*, which is the difference between an operator fixing their node in a minute and filing a bug. """ mismatches: list[RouteMismatch] = [] if left.artifact.source_digest != right.artifact.source_digest: mismatches.append( RouteMismatch( MISMATCH_ARTIFACT, f"source Model Artifact {right.artifact.source_digest[:12]}… does not " f"match the route's {left.artifact.source_digest[:12]}…", shard_index, ) ) if left.artifact.architecture != right.artifact.architecture: mismatches.append( RouteMismatch( MISMATCH_ARCHITECTURE, f"artifact architecture {right.artifact.architecture!r} does not match " f"the route's {left.artifact.architecture!r}", shard_index, ) ) if left.artifact.architecture_digest != right.artifact.architecture_digest: mismatches.append( RouteMismatch( MISMATCH_ARCHITECTURE, "architecture/config snapshot differs from the route's; the same " "architecture name with different config is a different model", shard_index, ) ) if left.artifact.layer_count != right.artifact.layer_count: mismatches.append( RouteMismatch( MISMATCH_ARTIFACT, f"artifact has {right.artifact.layer_count} layers, but the route's " f"has {left.artifact.layer_count}", shard_index, ) ) for axis in RECIPE_AXES: mine = getattr(left.recipe, axis) theirs = getattr(right.recipe, axis) if mine != theirs: mismatches.append( RouteMismatch( _AXIS_MISMATCH[axis], f"{axis} is {theirs!r}, but the route runs {mine!r}", shard_index, ) ) if not mismatches and not left.fingerprint.matches(right.fingerprint): # Unreachable via the fields above; if it ever fires, identity has grown a # digested field that `explain_mismatch` does not know about, and silently # reporting "compatible" would be far worse than an unhelpful message. mismatches.append( RouteMismatch( MISMATCH_FINGERPRINT, "fingerprints differ on a field this comparison does not cover", shard_index, ) ) return tuple(mismatches) @dataclass(frozen=True) class DistributedForwardEvidence: """Proof that a recipe really ran, distributed, end to end. This is the only thing that takes a recipe out of the dark. The bar is deliberately the product's bar, not a unit test's: at least two *distinct* physical nodes, whose Shards cover the whole model with no hole, generating real tokens. A synthetic worker is a unit fixture and certifies nothing — the whole risk this guards against is a recipe that passes in a mock and produces garbage on real weights. """ route_session_id: str route_epoch: int node_ids: tuple[str, ...] shard_ranges: tuple[tuple[int, int], ...] tokens_generated: int layer_count: int synthetic: bool = False certified_at: float = field(default_factory=time.time) def __post_init__(self) -> None: _require_text(self.route_session_id, "evidence.route_session_id") _require_int(self.route_epoch, "evidence.route_epoch", 0) _require_int(self.tokens_generated, "evidence.tokens_generated", 0) _require_int(self.layer_count, "evidence.layer_count", 1) def rejection(self) -> str | None: """Why this evidence does not certify, or None when it does.""" if self.synthetic: return ( "evidence comes from a synthetic worker; only a real distributed " "forward certifies a recipe" ) distinct = set(self.node_ids) if len(distinct) < MIN_CERTIFYING_NODES: return ( f"evidence covers {len(distinct)} distinct node(s); a distributed " f"forward requires at least {MIN_CERTIFYING_NODES}" ) if len(distinct) != len(self.node_ids): return "the same node is counted more than once in the certifying route" if self.tokens_generated < 1: return "the certifying forward generated no tokens" gap = _coverage_gap(self.shard_ranges, self.layer_count) if gap is not None: return gap return None def _coverage_gap( ranges: Iterable[tuple[int, int]], layer_count: int ) -> str | None: """Return why `ranges` fail to tile ``[0, layer_count)``, or None if they do. Ranges may overlap: ADR-0012 lets the Tracker resolve an overlap by telling a hop where the previous hop stopped, so an overlap is a routing decision, not a defect. A *hole* is a defect — the layers in it are never computed, and the route would silently emit tokens from a truncated model. """ ordered = sorted(ranges) if not ordered: return "the certifying route owns no layers" if ordered[0][0] != 0: return f"layers 0–{ordered[0][0]} are owned by no Shard on the route" covered = 0 for start, end in ordered: if start > covered: return f"layers {covered}–{start} are owned by no Shard on the route" covered = max(covered, end) if covered < layer_count: return f"layers {covered}–{layer_count} are owned by no Shard on the route" if covered > layer_count: return ( f"the route claims to cover {covered} layers, but the model has only " f"{layer_count}" ) return None @dataclass(frozen=True) class RecipeStatus: """What the registry knows about one fingerprint.""" status: str fingerprint: CompatibilityFingerprint | None = None detail: str = "" certified_at: float | None = None @property def may_serve(self) -> bool: """Only a certified recipe carries user traffic.""" return self.status == STATUS_CERTIFIED @property def may_certify(self) -> bool: """A dark recipe is eligible for a certification route — and only that. Without this, certification is unreachable: serving requires certification, certification requires a real distributed forward, and a real distributed forward requires a route. A dark recipe may therefore be routed *for the express purpose of certifying it*, and never for user traffic. """ return self.status in (STATUS_DARK, STATUS_CERTIFIED) def to_dict(self) -> dict: return { "status": self.status, "detail": self.detail, "certified_at": self.certified_at, "fingerprint": ( self.fingerprint.to_dict() if self.fingerprint is not None else None ), } class RecipeRegistry: """Registered-but-dark recipes, and the ones a real forward has certified. An unknown recipe is not routable. A known one is not routable either, until it has been proven. This is the fail-closed default the roadmap asks for: "unsupported architectures/backends remain registered-but-dark until real certification passes." """ def __init__(self) -> None: self._dark: dict[tuple[str, str], CompatibilityFingerprint] = {} self._certified: dict[tuple[str, str], RecipeStatus] = {} def register(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus: """Record a recipe as known. Known is not certified.""" fingerprint = _as_fingerprint(identity) key = fingerprint.key if key in self._certified: return self._certified[key] self._dark[key] = fingerprint return RecipeStatus( STATUS_DARK, fingerprint, "registered; dark until a real distributed forward certifies it", ) def status(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus: fingerprint = _as_fingerprint(identity) key = fingerprint.key if key in self._certified: return self._certified[key] if key in self._dark: return RecipeStatus( STATUS_DARK, fingerprint, "registered; dark until a real distributed forward certifies it", ) return RecipeStatus( STATUS_UNKNOWN, fingerprint, "this recipe has never been registered with the Tracker", ) def certify( self, identity: ShardIdentity | CompatibilityFingerprint, evidence: DistributedForwardEvidence, ) -> RecipeStatus: """Promote a dark recipe, if the evidence is a real distributed forward. Raises rather than returning a failed status: certifying is a state change, and a caller that ignores a returned failure would leave a recipe it believes to be certified in the dark. """ fingerprint = _as_fingerprint(identity) rejection = evidence.rejection() if rejection is not None: raise RecipeNotCertified( f"this evidence does not certify {fingerprint.recipe_id!r}: {rejection}" ) status = RecipeStatus( STATUS_CERTIFIED, fingerprint, ( f"certified by route session {evidence.route_session_id} across " f"{len(set(evidence.node_ids))} nodes" ), certified_at=evidence.certified_at, ) self._certified[fingerprint.key] = status self._dark.pop(fingerprint.key, None) return status def require_serving( self, identity: ShardIdentity | CompatibilityFingerprint ) -> RecipeStatus: """Admit a recipe for user traffic, or refuse and say why.""" status = self.status(identity) if not status.may_serve: raise RecipeNotCertified( f"recipe {_as_fingerprint(identity).recipe_id!r} is {status.status}: " f"{status.detail}" ) return status def certified_fingerprints(self) -> tuple[CompatibilityFingerprint, ...]: return tuple( status.fingerprint for status in self._certified.values() if status.fingerprint is not None ) def to_dict(self) -> dict: return { "schema_version": RECIPE_IDENTITY_SCHEMA_VERSION, "dark": [fp.to_dict() for fp in self._dark.values()], "certified": [status.to_dict() for status in self._certified.values()], } def _as_fingerprint( identity: ShardIdentity | CompatibilityFingerprint, ) -> CompatibilityFingerprint: if isinstance(identity, CompatibilityFingerprint): return identity if isinstance(identity, ShardIdentity): return identity.fingerprint raise RecipeIdentityError( f"expected a ShardIdentity or CompatibilityFingerprint, got " f"{type(identity).__name__}" ) def check_route( shards: Sequence[ShardIdentity], *, registry: RecipeRegistry | None = None, for_certification: bool = False, ) -> tuple[RouteMismatch, ...]: """Decide whether these Shards may form one Inference Route. Returns every reason they may not, empty when they may. Fail-closed by construction: an empty route, an unknown recipe, a hole in the layer coverage and a single differing dtype all produce a reason, and a caller that only ever proceeds on an empty tuple cannot accidentally admit any of them. `for_certification` admits a dark recipe onto a route whose only purpose is to certify it. It never admits an unknown one, and it is not the path user traffic takes. """ if not shards: return ( RouteMismatch( MISMATCH_RANGE, "a route needs at least one Shard; this one has none" ), ) mismatches: list[RouteMismatch] = [] head = shards[0] for index, shard in enumerate(shards[1:], start=1): if not head.fingerprint.matches(shard.fingerprint): mismatches.extend(explain_mismatch(head, shard, shard_index=index)) gap = _coverage_gap( [(shard.shard_start, shard.shard_end) for shard in shards], head.artifact.layer_count, ) if gap is not None: mismatches.append(RouteMismatch(MISMATCH_RANGE, gap)) if registry is not None: status = registry.status(head.fingerprint) allowed = status.may_certify if for_certification else status.may_serve if not allowed: mismatches.append( RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}") ) return tuple(mismatches) def require_route( shards: Sequence[ShardIdentity], *, registry: RecipeRegistry | None = None, for_certification: bool = False, ) -> CompatibilityFingerprint: """`check_route`, raising the structured reasons instead of returning them.""" mismatches = check_route( shards, registry=registry, for_certification=for_certification ) if mismatches: raise RouteIncompatible(mismatches) return shards[0].fingerprint # --------------------------------------------------------------------------- # The alpha target (DGR-017) # --------------------------------------------------------------------------- def glm_alpha_artifact( derived_from: DerivativeBinding | None = None, *, content_digest: str | None = None, ) -> ArtifactIdentity: """Artifact identity for the pinned GLM-5.2 ``UD-IQ1_S`` alpha target. Reads the locked manifest and architecture snapshot DGR-017 sealed, rather than restating their digests here — a second copy of a pinned digest is a second thing that can drift from the target it claims to pin. `layer_count` is `num_hidden_layers` (78), *not* `total_artifact_layers` (79). The 79th is the MTP/next-token-prediction layer, which is not part of the transformer stack a route tiles and which alpha explicitly defers (roadmap decision 17). Tiling 79 layers would leave every route one layer short of "covered" forever. Pass `derived_from` when this node holds a split of the target, and `content_digest` for that split's own bytes. """ from .glm_alpha import load_locked_target _contract, manifest, snapshot = load_locked_target() return ArtifactIdentity( artifact_id=manifest.gguf_repo_id, revision=manifest.gguf_revision, content_digest=content_digest or manifest.digest, architecture=str(snapshot["model_type"]), architecture_digest=snapshot.digest, layer_count=int(snapshot["num_hidden_layers"]), derived_from=derived_from, ) # --------------------------------------------------------------------------- # gRPC handshake (DGR-002 `Fingerprint`, `ERROR_CODE_FINGERPRINT_MISMATCH`) # --------------------------------------------------------------------------- def check_handshake( local: ShardIdentity, remote: "pb.Fingerprint", ) -> tuple[RouteMismatch, ...]: """Compare the fingerprint a peer opened the stream with against our own. An incompatible peer must fail at `SessionOpen`, not mid-generation: by the time a wrong activation has been folded into a KV cache, the session is already producing wrong tokens and there is nothing to salvage. """ try: presented = CompatibilityFingerprint.from_proto(remote) except RecipeIdentityError as exc: return ( RouteMismatch( MISMATCH_FINGERPRINT, f"peer presented an unusable fingerprint: {exc}", ), ) mine = local.fingerprint if presented.matches(mine): return () reasons: list[RouteMismatch] = [] if presented.model_artifact_digest != mine.model_artifact_digest: reasons.append( RouteMismatch( MISMATCH_ARTIFACT, f"peer serves Model Artifact {presented.model_artifact_digest[:12]}…, " f"this Shard serves {mine.model_artifact_digest[:12]}…", ) ) if presented.runtime_recipe_digest != mine.runtime_recipe_digest: reasons.append( RouteMismatch( MISMATCH_FINGERPRINT, f"peer runs recipe {presented.runtime_recipe_digest[:12]}… " f"({presented.recipe_id} v{presented.recipe_version}), this Shard runs " f"{mine.runtime_recipe_digest[:12]}… ({mine.recipe_id} " f"v{mine.recipe_version})", ) ) return tuple(reasons) def handshake_error( mismatches: Sequence[RouteMismatch], ) -> "pb.ShardError | None": """The protocol status a rejected handshake closes the stream with.""" if not mismatches: return None return pb.ShardError( code=pb.ERROR_CODE_FINGERPRINT_MISMATCH, detail="; ".join(m.describe() for m in mismatches), retryable=False, )