557 lines
21 KiB
Python
557 lines
21 KiB
Python
"""Tracker-side artifact and runtime recipe identity (DGR-003).
|
||
|
||
The node computes a compatibility fingerprint in `meshnet_node.runtime_recipe`.
|
||
This module recomputes it, and the recomputation is the entire point.
|
||
|
||
A fingerprint that arrives on the wire is a **claim**. If the tracker stored
|
||
what a node asserted, a node could assert the digest of a certified recipe while
|
||
running an uncertified one — and admission, route selection, and the gRPC
|
||
handshake would all wave it through, because they all compare the digest it
|
||
handed them. So the tracker derives the digest from the *axes* the node
|
||
declared, and refuses any report whose claim does not match the derivation. A
|
||
node can lie about its axes, and a real forward will catch that; it cannot lie
|
||
about the digest of the axes it declared.
|
||
|
||
This module deliberately does **not** import `meshnet_node`: the tracker package
|
||
does not depend on the node package, and an admission gate that shares an
|
||
implementation with the thing it admits is not an independent check. The two
|
||
implementations are pinned together by a committed conformance vector
|
||
(``tests/data/recipe_fingerprint_vectors.json``). If they drift, a test fails —
|
||
rather than a route quietly failing to form, or worse, quietly forming.
|
||
|
||
Recipes arrive **dark**: registered, visible to an operator, and not routable for
|
||
user traffic. Only a real distributed forward — at least two distinct nodes,
|
||
covering the whole model, emitting real tokens — takes a recipe out of the dark
|
||
(:class:`CertificationLedger`).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from typing import Any, Iterable, Mapping
|
||
|
||
# Layout of the identity block this tracker reads (meshnet_node.runtime_recipe).
|
||
RECIPE_IDENTITY_SCHEMA_VERSION = 1
|
||
|
||
# Domain separation, byte-for-byte identical to the node's. These strings are
|
||
# part of the wire contract, not an implementation detail: changing one here
|
||
# without changing it there silently partitions every route.
|
||
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
|
||
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
|
||
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
|
||
|
||
# The axes a recipe digest commits to. Order is irrelevant (the canonical JSON
|
||
# sorts keys); membership is not — an axis missing here is an axis the tracker
|
||
# would let a node change without changing its identity.
|
||
RECIPE_AXES: tuple[str, ...] = (
|
||
"weight_quantization",
|
||
"activation_dtype",
|
||
"compute_dtype",
|
||
"kv_dtype",
|
||
"kv_layout",
|
||
"tokenizer_revision",
|
||
"architecture_adapter",
|
||
"backend_id",
|
||
"runtime_version",
|
||
"boundary_schema_version",
|
||
"protocol_schema_version",
|
||
)
|
||
|
||
_INT_AXES = frozenset({"boundary_schema_version", "protocol_schema_version"})
|
||
|
||
STATUS_UNKNOWN = "unknown"
|
||
STATUS_DARK = "dark"
|
||
STATUS_CERTIFIED = "certified"
|
||
|
||
MIN_CERTIFYING_NODES = 2
|
||
|
||
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
||
_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"})
|
||
|
||
|
||
class RecipeIdentityError(ValueError):
|
||
"""A presented identity block is malformed or internally inconsistent."""
|
||
|
||
|
||
class FingerprintMismatch(RecipeIdentityError):
|
||
"""A supplied digest is inconsistent with its identity declaration.
|
||
|
||
A digest is an integrity and compatibility claim, not an authenticated
|
||
statement about what a node is actually executing. Distributed
|
||
certification remains the trust boundary.
|
||
"""
|
||
|
||
|
||
def canonical_sha256(value: Any) -> str:
|
||
payload = json.dumps(
|
||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||
)
|
||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _digest(domain: str, body: Mapping[str, Any]) -> str:
|
||
return canonical_sha256({"domain": domain, "body": dict(body)})
|
||
|
||
|
||
def _text(value: Any, what: str) -> str:
|
||
if not isinstance(value, str) or not value.strip():
|
||
raise RecipeIdentityError(f"{what!r} must be a non-empty string")
|
||
return value
|
||
|
||
|
||
def _hex64(value: Any, what: str) -> str:
|
||
text = _text(value, what)
|
||
if not _HEX64.match(text):
|
||
raise RecipeIdentityError(f"{what!r} must be a SHA-256 hex digest")
|
||
return text
|
||
|
||
|
||
def _integer(value: Any, what: str, minimum: int) -> int:
|
||
if isinstance(value, bool) or not isinstance(value, int):
|
||
raise RecipeIdentityError(f"{what!r} must be an integer")
|
||
if value < minimum:
|
||
raise RecipeIdentityError(f"{what!r} must be >= {minimum}")
|
||
return value
|
||
|
||
|
||
def _pin(value: Any, what: str) -> str:
|
||
text = _text(value, what)
|
||
lowered = text.strip().lower()
|
||
if lowered in _MOVING_REFS or lowered.startswith("refs/"):
|
||
raise RecipeIdentityError(
|
||
f"{what!r} is a moving reference, not an exact revision pin"
|
||
)
|
||
return text
|
||
|
||
|
||
def _mapping(value: Any, what: str) -> Mapping[str, Any]:
|
||
if not isinstance(value, Mapping):
|
||
raise RecipeIdentityError(f"{what!r} must be a JSON object")
|
||
return value
|
||
|
||
|
||
def artifact_digest(
|
||
*,
|
||
source_digest: str,
|
||
architecture: str,
|
||
architecture_digest: str,
|
||
layer_count: int,
|
||
) -> str:
|
||
"""The artifact half of the fingerprint, derived exactly as the node derives it."""
|
||
return _digest(
|
||
ARTIFACT_DIGEST_DOMAIN,
|
||
{
|
||
"source_digest": source_digest,
|
||
"architecture": architecture,
|
||
"architecture_digest": architecture_digest,
|
||
"layer_count": layer_count,
|
||
},
|
||
)
|
||
|
||
|
||
def recipe_digest(axes: Mapping[str, Any]) -> str:
|
||
"""The recipe half of the fingerprint, derived exactly as the node derives it."""
|
||
missing = [axis for axis in RECIPE_AXES if axis not in axes]
|
||
if missing:
|
||
raise RecipeIdentityError(
|
||
"recipe is missing required axes: " + ", ".join(missing)
|
||
)
|
||
return _digest(RECIPE_DIGEST_DOMAIN, {axis: axes[axis] for axis in RECIPE_AXES})
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class PresentedIdentity:
|
||
"""One node's declared artifact/recipe identity, with digests re-derived here.
|
||
|
||
`model_artifact_digest` and `runtime_recipe_digest` are computed by this
|
||
tracker from the declared axes. They are never copied from the report.
|
||
"""
|
||
|
||
artifact_id: str
|
||
revision: str
|
||
source_digest: str
|
||
content_digest: str
|
||
derivative_range: tuple[int, int] | None
|
||
architecture: str
|
||
architecture_digest: str
|
||
layer_count: int
|
||
is_derivative: bool
|
||
shard_start: int
|
||
shard_end: int
|
||
axes: Mapping[str, Any]
|
||
recipe_id: str
|
||
recipe_version: str
|
||
catalogue_version: str
|
||
|
||
@property
|
||
def model_artifact_digest(self) -> str:
|
||
return artifact_digest(
|
||
source_digest=self.source_digest,
|
||
architecture=self.architecture,
|
||
architecture_digest=self.architecture_digest,
|
||
layer_count=self.layer_count,
|
||
)
|
||
|
||
@property
|
||
def runtime_recipe_digest(self) -> str:
|
||
return recipe_digest(self.axes)
|
||
|
||
@property
|
||
def key(self) -> tuple[str, str]:
|
||
return (self.model_artifact_digest, self.runtime_recipe_digest)
|
||
|
||
@property
|
||
def shard_binding_digest(self) -> str:
|
||
"""This participant's own bytes and exact range, bound to the source.
|
||
|
||
The route fingerprint (:attr:`key`) is deliberately range-independent —
|
||
Shards on one route own different ranges, so a range-sensitive digest
|
||
would stop any two of them from ever agreeing. The consequence is that
|
||
the fingerprint alone cannot tell two *different splits of the same
|
||
source* apart: a derivative can assert the right source and recipe and
|
||
inherit certification earned by bytes it does not hold.
|
||
|
||
This digest is what closes that. It commits to the derivative's own
|
||
content hash and its exact end-exclusive range, so certification can be
|
||
recipe-wide while every serving node is still separately pinned to the
|
||
blob it was admitted on (`TrackerServer.certify_recipe`).
|
||
"""
|
||
return _digest(
|
||
SHARD_BINDING_DIGEST_DOMAIN,
|
||
{
|
||
"source_digest": self.source_digest,
|
||
"content_digest": self.content_digest,
|
||
"architecture_digest": self.architecture_digest,
|
||
"derivative_range": list(self.derivative_range)
|
||
if self.derivative_range is not None
|
||
else None,
|
||
"shard_start": self.shard_start,
|
||
"shard_end": self.shard_end,
|
||
},
|
||
)
|
||
|
||
@property
|
||
def tokenizer_revision(self) -> str:
|
||
return str(self.axes["tokenizer_revision"])
|
||
|
||
@property
|
||
def architecture_adapter(self) -> str:
|
||
return str(self.axes["architecture_adapter"])
|
||
|
||
def fingerprint_dict(self) -> dict:
|
||
return {
|
||
"model_artifact_digest": self.model_artifact_digest,
|
||
"runtime_recipe_digest": self.runtime_recipe_digest,
|
||
"recipe_id": self.recipe_id,
|
||
"recipe_version": self.recipe_version,
|
||
"catalogue_version": self.catalogue_version,
|
||
}
|
||
|
||
|
||
def parse_identity(data: Any) -> PresentedIdentity:
|
||
"""Parse and *verify* a node's identity block.
|
||
|
||
Raises when the block is malformed, when a split artifact does not name the
|
||
exact source it was cut from, when a Shard advertises layers its artifact
|
||
does not contain, or when the declared fingerprint does not match the digest
|
||
of the axes it was declared with.
|
||
"""
|
||
doc = _mapping(data, "identity")
|
||
|
||
schema_version = doc.get("schema_version")
|
||
if schema_version != RECIPE_IDENTITY_SCHEMA_VERSION:
|
||
raise RecipeIdentityError(
|
||
f"identity block declares schema version {schema_version!r}; this tracker "
|
||
f"reads version {RECIPE_IDENTITY_SCHEMA_VERSION}"
|
||
)
|
||
|
||
artifact = _mapping(doc.get("artifact"), "identity.artifact")
|
||
recipe = _mapping(doc.get("recipe"), "identity.recipe")
|
||
|
||
layer_count = _integer(artifact.get("layer_count"), "artifact.layer_count", 1)
|
||
content_digest = _hex64(artifact.get("content_digest"), "artifact.content_digest")
|
||
|
||
raw_binding = artifact.get("derived_from")
|
||
if raw_binding is None:
|
||
source_digest = content_digest
|
||
binding: tuple[int, int] | None = None
|
||
else:
|
||
derived = _mapping(raw_binding, "artifact.derived_from")
|
||
source_digest = _hex64(
|
||
derived.get("source_artifact_digest"),
|
||
"artifact.derived_from.source_artifact_digest",
|
||
)
|
||
binding = (
|
||
_integer(derived.get("shard_start"), "derived_from.shard_start", 0),
|
||
_integer(derived.get("shard_end"), "derived_from.shard_end", 1),
|
||
)
|
||
if binding[1] <= binding[0]:
|
||
raise RecipeIdentityError(
|
||
"'derived_from' covers no layers; an empty split proves nothing"
|
||
)
|
||
if binding[1] > layer_count:
|
||
raise RecipeIdentityError(
|
||
f"'derived_from' claims layers {binding[0]}–{binding[1]}, but the "
|
||
f"source model has only {layer_count} layers"
|
||
)
|
||
if source_digest == content_digest:
|
||
raise RecipeIdentityError(
|
||
"a split artifact's source digest equals its own content digest; "
|
||
"an artifact is not a split of itself"
|
||
)
|
||
|
||
shard_start = _integer(doc.get("shard_start"), "shard_start", 0)
|
||
shard_end = _integer(doc.get("shard_end"), "shard_end", 1)
|
||
if shard_end <= shard_start:
|
||
raise RecipeIdentityError("a Shard owning no layer computes nothing")
|
||
if shard_end > layer_count:
|
||
raise RecipeIdentityError(
|
||
f"Shard owns layers {shard_start}–{shard_end}, but the artifact has only "
|
||
f"{layer_count} layers"
|
||
)
|
||
if binding is not None and not (
|
||
binding[0] <= shard_start and shard_end <= binding[1]
|
||
):
|
||
raise RecipeIdentityError(
|
||
f"Shard advertises layers {shard_start}–{shard_end}, but its split "
|
||
f"artifact only contains layers {binding[0]}–{binding[1]}"
|
||
)
|
||
|
||
axes: dict[str, Any] = {}
|
||
for axis in RECIPE_AXES:
|
||
if axis not in recipe:
|
||
raise RecipeIdentityError(
|
||
f"recipe is missing axis {axis!r}; an unstated axis cannot default"
|
||
)
|
||
value = recipe[axis]
|
||
if axis in _INT_AXES:
|
||
axes[axis] = _integer(value, f"recipe.{axis}", 1)
|
||
else:
|
||
axes[axis] = _text(value, f"recipe.{axis}")
|
||
_pin(axes["tokenizer_revision"], "recipe.tokenizer_revision")
|
||
|
||
identity = PresentedIdentity(
|
||
artifact_id=_text(artifact.get("artifact_id"), "artifact.artifact_id"),
|
||
revision=_pin(artifact.get("revision"), "artifact.revision"),
|
||
source_digest=source_digest,
|
||
content_digest=content_digest,
|
||
derivative_range=binding,
|
||
architecture=_text(artifact.get("architecture"), "artifact.architecture"),
|
||
architecture_digest=_hex64(
|
||
artifact.get("architecture_digest"), "artifact.architecture_digest"
|
||
),
|
||
layer_count=layer_count,
|
||
is_derivative=binding is not None,
|
||
shard_start=shard_start,
|
||
shard_end=shard_end,
|
||
axes=axes,
|
||
recipe_id=_text(recipe.get("recipe_id"), "recipe.recipe_id"),
|
||
recipe_version=_text(recipe.get("recipe_version"), "recipe.recipe_version"),
|
||
catalogue_version=_text(
|
||
recipe.get("catalogue_version"), "recipe.catalogue_version"
|
||
),
|
||
)
|
||
|
||
declared = doc.get("fingerprint")
|
||
if declared is not None:
|
||
claim = _mapping(declared, "identity.fingerprint")
|
||
claimed_artifact = _hex64(
|
||
claim.get("model_artifact_digest"), "fingerprint.model_artifact_digest"
|
||
)
|
||
claimed_recipe = _hex64(
|
||
claim.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest"
|
||
)
|
||
if (claimed_artifact, claimed_recipe) != identity.key:
|
||
raise FingerprintMismatch(
|
||
"declared fingerprint is inconsistent with the artifact and recipe "
|
||
"claim; the tracker recomputes compatibility digests"
|
||
)
|
||
|
||
return identity
|
||
|
||
|
||
def coverage_gap(
|
||
ranges: Iterable[tuple[int, int]], layer_count: int
|
||
) -> str | None:
|
||
"""Why `ranges` fail to tile ``[0, layer_count)``, or None when they do.
|
||
|
||
Overlaps are legal — ADR-0012 lets the Tracker resolve one by telling a hop
|
||
where the previous hop stopped. A hole is not: its layers are never computed,
|
||
and the route emits fluent tokens from a truncated model.
|
||
"""
|
||
ordered = sorted(ranges)
|
||
if not ordered:
|
||
return "the route owns no layers"
|
||
if ordered[0][0] != 0:
|
||
return f"layers 0–{ordered[0][0]} are owned by no Shard on the route"
|
||
|
||
covered = 0
|
||
for start, end in ordered:
|
||
if start > covered:
|
||
return f"layers {covered}–{start} are owned by no Shard on the route"
|
||
covered = max(covered, end)
|
||
|
||
if covered < layer_count:
|
||
return f"layers {covered}–{layer_count} are owned by no Shard on the route"
|
||
if covered > layer_count:
|
||
return (
|
||
f"the route claims {covered} layers, but the model has only {layer_count}"
|
||
)
|
||
return None
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class DistributedForwardEvidence:
|
||
"""A real distributed forward — the only thing that certifies a recipe."""
|
||
|
||
route_session_id: str
|
||
route_epoch: int
|
||
node_ids: tuple[str, ...]
|
||
shard_ranges: tuple[tuple[int, int], ...]
|
||
tokens_generated: int
|
||
layer_count: int
|
||
fingerprint: tuple[str, str]
|
||
participants: tuple[PresentedIdentity, ...]
|
||
synthetic: bool = False
|
||
certified_at: float = field(default_factory=time.time)
|
||
|
||
def rejection(self) -> str | None:
|
||
if self.synthetic:
|
||
return (
|
||
"evidence comes from a synthetic worker; only a real distributed "
|
||
"forward certifies a recipe"
|
||
)
|
||
if len(self.participants) != len(self.node_ids):
|
||
return "participant identities do not match the certifying node list"
|
||
if len(self.shard_ranges) != len(self.participants):
|
||
return "participant identities do not match the recorded effective ranges"
|
||
for participant, shard_range in zip(
|
||
self.participants, self.shard_ranges, strict=True
|
||
):
|
||
if participant.key != self.fingerprint:
|
||
return "a participant fingerprint differs from the certifying route"
|
||
if participant.layer_count != self.layer_count:
|
||
return "a participant artifact layer count differs from the certifying route"
|
||
if (participant.shard_start, participant.shard_end) != shard_range:
|
||
return "a participant identity does not match its recorded effective range"
|
||
distinct = set(self.node_ids)
|
||
if len(distinct) < MIN_CERTIFYING_NODES:
|
||
return (
|
||
f"evidence covers {len(distinct)} distinct node(s); a distributed "
|
||
f"forward requires at least {MIN_CERTIFYING_NODES}"
|
||
)
|
||
if len(distinct) != len(self.node_ids):
|
||
return "the same node is counted more than once in the certifying route"
|
||
if self.tokens_generated < 1:
|
||
return "the certifying forward generated no tokens"
|
||
return coverage_gap(self.shard_ranges, self.layer_count)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RecipeStatus:
|
||
status: str
|
||
detail: str = ""
|
||
certified_at: float | None = None
|
||
|
||
@property
|
||
def may_serve(self) -> bool:
|
||
return self.status == STATUS_CERTIFIED
|
||
|
||
@property
|
||
def may_certify(self) -> bool:
|
||
"""A dark recipe may be routed to *certify* it, and for nothing else.
|
||
|
||
Without this, certification is unreachable by construction: serving needs
|
||
certification, certification needs a real distributed forward, and a real
|
||
distributed forward needs a route.
|
||
"""
|
||
return self.status in (STATUS_DARK, STATUS_CERTIFIED)
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"status": self.status,
|
||
"detail": self.detail,
|
||
"certified_at": self.certified_at,
|
||
}
|
||
|
||
|
||
_DARK_DETAIL = "registered; dark until a real distributed forward certifies it"
|
||
_UNKNOWN_DETAIL = "this recipe has never been registered with the tracker"
|
||
|
||
|
||
class CertificationLedger:
|
||
"""Which recipes the tracker has seen, and which a real forward has proven."""
|
||
|
||
def __init__(self) -> None:
|
||
self._dark: set[tuple[str, str]] = set()
|
||
self._certified: dict[tuple[str, str], RecipeStatus] = {}
|
||
|
||
def register(self, identity: PresentedIdentity) -> RecipeStatus:
|
||
key = identity.key
|
||
if key in self._certified:
|
||
return self._certified[key]
|
||
self._dark.add(key)
|
||
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
|
||
|
||
def status(self, identity: PresentedIdentity | tuple[str, str]) -> RecipeStatus:
|
||
key = identity if isinstance(identity, tuple) else identity.key
|
||
if key in self._certified:
|
||
return self._certified[key]
|
||
if key in self._dark:
|
||
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
|
||
return RecipeStatus(STATUS_UNKNOWN, _UNKNOWN_DETAIL)
|
||
|
||
def certify(
|
||
self,
|
||
identity: PresentedIdentity,
|
||
evidence: DistributedForwardEvidence,
|
||
) -> RecipeStatus:
|
||
"""Promote a recipe out of the dark, or raise saying why the evidence is short."""
|
||
key = identity.key
|
||
if key not in self._dark and key not in self._certified:
|
||
raise RecipeIdentityError(
|
||
"this fingerprint is not registered; unknown recipes cannot be certified"
|
||
)
|
||
if evidence.fingerprint != key:
|
||
raise RecipeIdentityError(
|
||
"certification evidence fingerprint does not match the recipe being promoted"
|
||
)
|
||
if evidence.layer_count != identity.layer_count:
|
||
raise RecipeIdentityError(
|
||
"certification evidence layer count does not match the artifact being promoted"
|
||
)
|
||
rejection = evidence.rejection()
|
||
if rejection is not None:
|
||
raise RecipeIdentityError(f"this evidence does not certify: {rejection}")
|
||
status = RecipeStatus(
|
||
STATUS_CERTIFIED,
|
||
(
|
||
f"certified by route session {evidence.route_session_id} across "
|
||
f"{len(set(evidence.node_ids))} nodes"
|
||
),
|
||
certified_at=evidence.certified_at,
|
||
)
|
||
self._certified[key] = status
|
||
self._dark.discard(key)
|
||
return status
|
||
|
||
def to_dict(self) -> dict:
|
||
return {
|
||
"dark": [list(key) for key in sorted(self._dark)],
|
||
"certified": {
|
||
"/".join(key): status.to_dict()
|
||
for key, status in sorted(self._certified.items())
|
||
},
|
||
}
|
||
|
||
|
||
# Route formation itself lives in `server.py` (`_select_route`, `_enumerate_routes`,
|
||
# `_find_pinned_route`): candidates are partitioned by the exact fingerprint this
|
||
# module derives, so a route can only ever be assembled inside one identity. A
|
||
# second route gate here would be a second copy of that policy to keep in step —
|
||
# the same reason the node module holds no certification authority.
|