popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: DGR-003 - Define exact Artifact and runtime recipe identity

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-14 00:31:14 +03:00
parent e7c780a623
commit ad2d17541c
11 changed files with 2395 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
packages/node/meshnet_node/capability.py
View File

@@ -20,6 +20,8 @@ import time
from dataclasses import dataclass, field
from typing import Any, Mapping
from .runtime_recipe import CompatibilityFingerprint, ShardIdentity
# Layout of the serialized report. Bump when the JSON shape changes.
CAPABILITY_SCHEMA_VERSION = 1
@@ -330,7 +332,16 @@ def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]:
@dataclass(frozen=True)
class CapabilityReport:
"""One node's validated (or failed) model/shard/recipe/backend combination."""
"""One node's validated (or failed) model/shard/recipe/backend combination.
`identity` is the exact DGR-003 artifact/runtime-recipe block: the separated
numerical axes and the compatibility fingerprint derived from them. It is
optional and additive — a node that predates DGR-003 presents none, and the
tracker falls back to the coarse label comparison it has always done
(ADR-0023's compat rollout). A node that *does* present one is held to it:
the tracker re-derives the fingerprint and refuses a report whose claim does
not match its own derivation.
"""
model: ModelIdentity
shard: ShardRange
@@ -341,6 +352,7 @@ class CapabilityReport:
duration_ms: int
diagnostics: tuple[str, ...] = ()
schema_version: int = CAPABILITY_SCHEMA_VERSION
identity: ShardIdentity | None = None
def __post_init__(self) -> None:
if self.status not in VALID_STATUSES:
@@ -360,6 +372,11 @@ class CapabilityReport:
def passed(self) -> bool:
return self.status == STATUS_PASSED
@property
def fingerprint(self) -> CompatibilityFingerprint | None:
"""The exact compatibility fingerprint, when this node declares one."""
return None if self.identity is None else self.identity.fingerprint
def identity_key(self) -> tuple[str, int, int, str, str, str, str]:
"""The tuple a consumer must match to reuse this proof.
@@ -380,7 +397,7 @@ class CapabilityReport:
return max(0.0, (time.time() if now is None else now) - self.validated_at)
def to_dict(self) -> dict:
return {
doc = {
"schema_version": self.schema_version,
"model": self.model.to_dict(),
"shard": self.shard.to_dict(),
@@ -391,6 +408,9 @@ class CapabilityReport:
"duration_ms": self.duration_ms,
"diagnostics": list(self.diagnostics),
}
if self.identity is not None:
doc["identity"] = self.identity.to_dict()
return doc
def to_json(self, indent: int | None = None) -> str:
return json.dumps(self.to_dict(), indent=indent, sort_keys=True)
@@ -417,6 +437,7 @@ class CapabilityReport:
):
raise CapabilityReportError("'validated_at' must be a Unix timestamp")
raw_identity = doc.get("identity")
return cls(
schema_version=schema_version,
model=ModelIdentity.from_dict(doc.get("model")),
@@ -427,6 +448,9 @@ class CapabilityReport:
validated_at=float(validated_at),
duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0),
diagnostics=sanitize_diagnostics(doc.get("diagnostics")),
identity=(
None if raw_identity is None else ShardIdentity.from_dict(raw_identity)
),
)
@classmethod
@@ -461,12 +485,14 @@ def build_capability_report(
diagnostics: Any = None,
validated_at: float | None = None,
environ: Mapping[str, str] | None = None,
identity: ShardIdentity | None = None,
) -> CapabilityReport:
"""Assemble a report from flat validation results.
`model_config` may be the loaded config mapping (hashed into a fingerprint)
or an already-computed ``sha256:…`` string. `validated_at` defaults to now,
so callers that need determinism pass it explicitly.
so callers that need determinism pass it explicitly. `identity` is the exact
DGR-003 artifact/recipe block, when the backend can state one.
"""
return CapabilityReport(
model=ModelIdentity(
@@ -491,4 +517,5 @@ def build_capability_report(
validated_at=time.time() if validated_at is None else validated_at,
duration_ms=duration_ms,
diagnostics=sanitize_diagnostics(diagnostics, environ),
identity=identity,
)

1147
packages/node/meshnet_node/runtime_recipe.py Normal file
View File

File diff suppressed because it is too large Load Diff

102
packages/tracker/meshnet_tracker/capability.py
View File

@@ -30,6 +30,13 @@ import time
from dataclasses import dataclass, replace
from typing import Any, Callable, Mapping
from .recipe import (
CertificationLedger,
FingerprintMismatch,
RecipeIdentityError,
parse_identity,
)
# The capability report layout this tracker reads (meshnet_node.capability).
SUPPORTED_SCHEMA_VERSION = 1
@@ -58,6 +65,12 @@ STATE_MODEL_MISMATCH = "model-mismatch"
STATE_SHARD_MISMATCH = "shard-mismatch"
STATE_RECIPE_MISMATCH = "recipe-mismatch"
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
# The node presented a DGR-003 identity block whose declared fingerprint is not
# the digest of the axes it declared. Identity is derived, never asserted.
STATE_FINGERPRINT_MISMATCH = "fingerprint-mismatch"
# Registered-but-dark: a known recipe no real distributed forward has certified.
# Visible to an operator, never routable for user traffic.
STATE_UNCERTIFIED = "uncertified"
ALL_STATES = (
STATE_ADMITTED,
@@ -69,6 +82,8 @@ ALL_STATES = (
STATE_SHARD_MISMATCH,
STATE_RECIPE_MISMATCH,
STATE_CATALOGUE_INCOMPATIBLE,
STATE_FINGERPRINT_MISMATCH,
STATE_UNCERTIFIED,
)
# --- Compatibility policy for nodes that predate the capability protocol. ---
@@ -165,12 +180,25 @@ class CapabilityState:
recorded_at: float = 0.0
schema_version: int | None = None
diagnostics: tuple[str, ...] = ()
# The DGR-003 compatibility fingerprint, *re-derived* by this tracker from
# the axes the node declared — never copied from what the node claimed.
# Absent for a node that predates DGR-003.
model_artifact_digest: str | None = None
runtime_recipe_digest: str | None = None
certification: str | None = None
@property
def proven(self) -> bool:
"""The presented proof covers exactly what the node advertised."""
return self.state == STATE_ADMITTED
@property
def fingerprint(self) -> tuple[str, str] | None:
"""What route formation compares. `None` when the node declares no identity."""
if self.model_artifact_digest is None or self.runtime_recipe_digest is None:
return None
return (self.model_artifact_digest, self.runtime_recipe_digest)
def routable_under(self, policy: str) -> bool:
if self.proven:
return True
@@ -197,6 +225,9 @@ class CapabilityState:
"recorded_at": self.recorded_at,
"schema_version": self.schema_version,
"diagnostics": list(self.diagnostics),
"model_artifact_digest": self.model_artifact_digest,
"runtime_recipe_digest": self.runtime_recipe_digest,
"certification": self.certification,
}
@@ -224,6 +255,7 @@ def evaluate_report(
declared_recipe_version: str | None = None,
now: float | None = None,
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
ledger: CertificationLedger | None = None,
) -> CapabilityState:
"""Judge the proof a node presented against what that node is advertising.
@@ -308,6 +340,67 @@ def evaluate_report(
f"the node declared v{declared_recipe_version}",
)
identity = None
if report.get("identity") is not None:
try:
identity = parse_identity(report["identity"])
except FingerprintMismatch as exc:
return base.with_state(STATE_FINGERPRINT_MISMATCH, str(exc))
except RecipeIdentityError as exc:
return base.with_state(
STATE_INVALID, f"capability identity block is unusable: {exc}"
)
# The report's `shard` range is inclusive/inclusive (the CLI and backend
# convention); the identity's is inclusive/exclusive (the protocol's, and
# ADR-0012's). A node whose two halves disagree has not proven the range
# it claims, whichever one is right.
if (identity.shard_start, identity.shard_end) != (
base.shard_start,
(base.shard_end or 0) + 1,
):
return base.with_state(
STATE_SHARD_MISMATCH,
f"identity covers layers {identity.shard_start}–{identity.shard_end} "
f"(end-exclusive), but the proof is for layers {base.shard_start}–"
f"{base.shard_end} (end-inclusive)",
)
if not model_matches(identity.artifact_id):
return base.with_state(
STATE_MODEL_MISMATCH,
f"identity is for artifact {identity.artifact_id!r}, but the node "
f"registered {advertised_model!r}",
)
if (
identity.recipe_id != base.recipe_id
or identity.recipe_version != base.recipe_version
or identity.catalogue_version != base.catalogue_version
):
return base.with_state(
STATE_RECIPE_MISMATCH,
"identity recipe labels do not match the capability proof",
)
if identity.axes["backend_id"] != base.backend_id:
return base.with_state(
STATE_RECIPE_MISMATCH,
"identity backend does not match the capability proof",
)
if (
base.quantization is not None
and identity.axes["weight_quantization"] != base.quantization
):
return base.with_state(
STATE_RECIPE_MISMATCH,
"identity weight quantization does not match the capability proof",
)
base = replace(
base,
model_artifact_digest=identity.model_artifact_digest,
runtime_recipe_digest=identity.runtime_recipe_digest,
)
if status != STATUS_PASSED:
return base.with_state(
STATE_FAILED,
@@ -328,6 +421,15 @@ def evaluate_report(
f"proof is timestamped {-age:.0f}s in the future; check the node's clock",
)
# The digest only establishes that this report is self-consistent. It does
# not authenticate a node or prove a distributed forward. Exact recipes are
# therefore registered-but-dark until tracker-owned certification records
# that forward.
if identity is not None:
recipe_status = (ledger or CertificationLedger()).register(identity)
if not recipe_status.may_serve:
return base.with_state(STATE_UNCERTIFIED, recipe_status.detail)
return base.with_state(
STATE_ADMITTED,
f"{base.model_id} layers {base.shard_start}–{base.shard_end} proven on "

615
packages/tracker/meshnet_tracker/recipe.py Normal file
View File

@@ -0,0 +1,615 @@
"""Tracker-side artifact and runtime recipe identity (DGR-003).
The node computes a compatibility fingerprint in `meshnet_node.runtime_recipe`.
This module recomputes it, and the recomputation is the entire point.
A fingerprint that arrives on the wire is a **claim**. If the tracker stored
what a node asserted, a node could assert the digest of a certified recipe while
running an uncertified one — and admission, route selection, and the gRPC
handshake would all wave it through, because they all compare the digest it
handed them. So the tracker derives the digest from the *axes* the node
declared, and refuses any report whose claim does not match the derivation. A
node can lie about its axes, and a real forward will catch that; it cannot lie
about the digest of the axes it declared.
This module deliberately does **not** import `meshnet_node`: the tracker package
does not depend on the node package, and an admission gate that shares an
implementation with the thing it admits is not an independent check. The two
implementations are pinned together by a committed conformance vector
(``tests/data/recipe_fingerprint_vectors.json``). If they drift, a test fails —
rather than a route quietly failing to form, or worse, quietly forming.
Recipes arrive **dark**: registered, visible to an operator, and not routable for
user traffic. Only a real distributed forward — at least two distinct nodes,
covering the whole model, emitting real tokens — takes a recipe out of the dark
(:class:`CertificationLedger`).
"""
from __future__ import annotations
import hashlib
import json
import re
import time
from dataclasses import dataclass, field
from typing import Any, Iterable, Mapping, Sequence
# Layout of the identity block this tracker reads (meshnet_node.runtime_recipe).
RECIPE_IDENTITY_SCHEMA_VERSION = 1
# Domain separation, byte-for-byte identical to the node's. These strings are
# part of the wire contract, not an implementation detail: changing one here
# without changing it there silently partitions every route.
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
# The axes a recipe digest commits to. Order is irrelevant (the canonical JSON
# sorts keys); membership is not — an axis missing here is an axis the tracker
# would let a node change without changing its identity.
RECIPE_AXES: tuple[str, ...] = (
"weight_quantization",
"activation_dtype",
"compute_dtype",
"kv_dtype",
"kv_layout",
"tokenizer_revision",
"architecture_adapter",
"backend_id",
"runtime_version",
"boundary_schema_version",
"protocol_schema_version",
)
_INT_AXES = frozenset({"boundary_schema_version", "protocol_schema_version"})
MISMATCH_ARTIFACT = "artifact"
MISMATCH_TOKENIZER = "tokenizer"
MISMATCH_ARCHITECTURE = "architecture"
MISMATCH_RANGE = "range"
MISMATCH_BOUNDARY_SCHEMA = "boundary-schema"
MISMATCH_ACTIVATION_RECIPE = "activation-recipe"
MISMATCH_CACHE_LAYOUT = "cache-layout"
MISMATCH_FINGERPRINT = "fingerprint"
MISMATCH_UNCERTIFIED = "uncertified"
STATUS_UNKNOWN = "unknown"
STATUS_DARK = "dark"
STATUS_CERTIFIED = "certified"
MIN_CERTIFYING_NODES = 2
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"})
class RecipeIdentityError(ValueError):
"""A presented identity block is malformed or internally inconsistent."""
class FingerprintMismatch(RecipeIdentityError):
"""A supplied digest is inconsistent with its identity declaration.
A digest is an integrity and compatibility claim, not an authenticated
statement about what a node is actually executing. Distributed
certification remains the trust boundary.
"""
def canonical_sha256(value: Any) -> str:
payload = json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _digest(domain: str, body: Mapping[str, Any]) -> str:
return canonical_sha256({"domain": domain, "body": dict(body)})
def _text(value: Any, what: str) -> str:
if not isinstance(value, str) or not value.strip():
raise RecipeIdentityError(f"{what!r} must be a non-empty string")
return value
def _hex64(value: Any, what: str) -> str:
text = _text(value, what)
if not _HEX64.match(text):
raise RecipeIdentityError(f"{what!r} must be a SHA-256 hex digest")
return text
def _integer(value: Any, what: str, minimum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise RecipeIdentityError(f"{what!r} must be an integer")
if value < minimum:
raise RecipeIdentityError(f"{what!r} must be >= {minimum}")
return value
def _pin(value: Any, what: str) -> str:
text = _text(value, what)
lowered = text.strip().lower()
if lowered in _MOVING_REFS or lowered.startswith("refs/"):
raise RecipeIdentityError(
f"{what!r} is a moving reference, not an exact revision pin"
)
return text
def _mapping(value: Any, what: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise RecipeIdentityError(f"{what!r} must be a JSON object")
return value
def artifact_digest(
*,
source_digest: str,
architecture: str,
architecture_digest: str,
layer_count: int,
) -> str:
"""The artifact half of the fingerprint, derived exactly as the node derives it."""
return _digest(
ARTIFACT_DIGEST_DOMAIN,
{
"source_digest": source_digest,
"architecture": architecture,
"architecture_digest": architecture_digest,
"layer_count": layer_count,
},
)
def recipe_digest(axes: Mapping[str, Any]) -> str:
"""The recipe half of the fingerprint, derived exactly as the node derives it."""
missing = [axis for axis in RECIPE_AXES if axis not in axes]
if missing:
raise RecipeIdentityError(
"recipe is missing required axes: " + ", ".join(missing)
)
return _digest(RECIPE_DIGEST_DOMAIN, {axis: axes[axis] for axis in RECIPE_AXES})
@dataclass(frozen=True)
class PresentedIdentity:
"""One node's declared artifact/recipe identity, with digests re-derived here.
`model_artifact_digest` and `runtime_recipe_digest` are computed by this
tracker from the declared axes. They are never copied from the report.
"""
artifact_id: str
revision: str
source_digest: str
architecture: str
architecture_digest: str
layer_count: int
is_derivative: bool
shard_start: int
shard_end: int
axes: Mapping[str, Any]
recipe_id: str
recipe_version: str
catalogue_version: str
@property
def model_artifact_digest(self) -> str:
return artifact_digest(
source_digest=self.source_digest,
architecture=self.architecture,
architecture_digest=self.architecture_digest,
layer_count=self.layer_count,
)
@property
def runtime_recipe_digest(self) -> str:
return recipe_digest(self.axes)
@property
def key(self) -> tuple[str, str]:
return (self.model_artifact_digest, self.runtime_recipe_digest)
@property
def tokenizer_revision(self) -> str:
return str(self.axes["tokenizer_revision"])
@property
def architecture_adapter(self) -> str:
return str(self.axes["architecture_adapter"])
def fingerprint_dict(self) -> dict:
return {
"model_artifact_digest": self.model_artifact_digest,
"runtime_recipe_digest": self.runtime_recipe_digest,
"recipe_id": self.recipe_id,
"recipe_version": self.recipe_version,
"catalogue_version": self.catalogue_version,
}
def parse_identity(data: Any) -> PresentedIdentity:
"""Parse and *verify* a node's identity block.
Raises when the block is malformed, when a split artifact does not name the
exact source it was cut from, when a Shard advertises layers its artifact
does not contain, or when the declared fingerprint does not match the digest
of the axes it was declared with.
"""
doc = _mapping(data, "identity")
schema_version = doc.get("schema_version")
if schema_version != RECIPE_IDENTITY_SCHEMA_VERSION:
raise RecipeIdentityError(
f"identity block declares schema version {schema_version!r}; this tracker "
f"reads version {RECIPE_IDENTITY_SCHEMA_VERSION}"
)
artifact = _mapping(doc.get("artifact"), "identity.artifact")
recipe = _mapping(doc.get("recipe"), "identity.recipe")
layer_count = _integer(artifact.get("layer_count"), "artifact.layer_count", 1)
content_digest = _hex64(artifact.get("content_digest"), "artifact.content_digest")
raw_binding = artifact.get("derived_from")
if raw_binding is None:
source_digest = content_digest
binding: tuple[int, int] | None = None
else:
derived = _mapping(raw_binding, "artifact.derived_from")
source_digest = _hex64(
derived.get("source_artifact_digest"),
"artifact.derived_from.source_artifact_digest",
)
binding = (
_integer(derived.get("shard_start"), "derived_from.shard_start", 0),
_integer(derived.get("shard_end"), "derived_from.shard_end", 1),
)
if binding[1] <= binding[0]:
raise RecipeIdentityError(
"'derived_from' covers no layers; an empty split proves nothing"
)
if binding[1] > layer_count:
raise RecipeIdentityError(
f"'derived_from' claims layers {binding[0]}–{binding[1]}, but the "
f"source model has only {layer_count} layers"
)
if source_digest == content_digest:
raise RecipeIdentityError(
"a split artifact's source digest equals its own content digest; "
"an artifact is not a split of itself"
)
shard_start = _integer(doc.get("shard_start"), "shard_start", 0)
shard_end = _integer(doc.get("shard_end"), "shard_end", 1)
if shard_end <= shard_start:
raise RecipeIdentityError("a Shard owning no layer computes nothing")
if shard_end > layer_count:
raise RecipeIdentityError(
f"Shard owns layers {shard_start}–{shard_end}, but the artifact has only "
f"{layer_count} layers"
)
if binding is not None and not (
binding[0] <= shard_start and shard_end <= binding[1]
):
raise RecipeIdentityError(
f"Shard advertises layers {shard_start}–{shard_end}, but its split "
f"artifact only contains layers {binding[0]}–{binding[1]}"
)
axes: dict[str, Any] = {}
for axis in RECIPE_AXES:
if axis not in recipe:
raise RecipeIdentityError(
f"recipe is missing axis {axis!r}; an unstated axis cannot default"
)
value = recipe[axis]
if axis in _INT_AXES:
axes[axis] = _integer(value, f"recipe.{axis}", 1)
else:
axes[axis] = _text(value, f"recipe.{axis}")
_pin(axes["tokenizer_revision"], "recipe.tokenizer_revision")
identity = PresentedIdentity(
artifact_id=_text(artifact.get("artifact_id"), "artifact.artifact_id"),
revision=_pin(artifact.get("revision"), "artifact.revision"),
source_digest=source_digest,
architecture=_text(artifact.get("architecture"), "artifact.architecture"),
architecture_digest=_hex64(
artifact.get("architecture_digest"), "artifact.architecture_digest"
),
layer_count=layer_count,
is_derivative=binding is not None,
shard_start=shard_start,
shard_end=shard_end,
axes=axes,
recipe_id=_text(recipe.get("recipe_id"), "recipe.recipe_id"),
recipe_version=_text(recipe.get("recipe_version"), "recipe.recipe_version"),
catalogue_version=_text(
recipe.get("catalogue_version"), "recipe.catalogue_version"
),
)
declared = doc.get("fingerprint")
if declared is not None:
claim = _mapping(declared, "identity.fingerprint")
claimed_artifact = _hex64(
claim.get("model_artifact_digest"), "fingerprint.model_artifact_digest"
)
claimed_recipe = _hex64(
claim.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest"
)
if (claimed_artifact, claimed_recipe) != identity.key:
raise FingerprintMismatch(
"declared fingerprint is inconsistent with the artifact and recipe "
"claim; the tracker recomputes compatibility digests"
)
return identity
@dataclass(frozen=True)
class RouteMismatch:
reason: str
detail: str
def describe(self) -> str:
return f"{self.reason}: {self.detail}"
def compare(
route: PresentedIdentity, candidate: PresentedIdentity
) -> tuple[RouteMismatch, ...]:
"""Why `candidate` may not join a route already running `route`."""
if route.key == candidate.key:
return ()
reasons: list[RouteMismatch] = []
if route.source_digest != candidate.source_digest:
reasons.append(
RouteMismatch(
MISMATCH_ARTIFACT,
f"serves Model Artifact {candidate.source_digest[:12]}…, the route "
f"runs {route.source_digest[:12]}…",
)
)
if route.architecture_digest != candidate.architecture_digest:
reasons.append(
RouteMismatch(
MISMATCH_ARCHITECTURE,
"architecture/config snapshot differs from the route's",
)
)
if route.layer_count != candidate.layer_count:
reasons.append(
RouteMismatch(
MISMATCH_ARTIFACT,
f"artifact has {candidate.layer_count} layers, the route's has "
f"{route.layer_count}",
)
)
axis_reason = {
"tokenizer_revision": MISMATCH_TOKENIZER,
"architecture_adapter": MISMATCH_ARCHITECTURE,
"activation_dtype": MISMATCH_ACTIVATION_RECIPE,
"compute_dtype": MISMATCH_ACTIVATION_RECIPE,
"kv_dtype": MISMATCH_CACHE_LAYOUT,
"kv_layout": MISMATCH_CACHE_LAYOUT,
"boundary_schema_version": MISMATCH_BOUNDARY_SCHEMA,
"protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA,
}
for axis in RECIPE_AXES:
mine = route.axes.get(axis)
theirs = candidate.axes.get(axis)
if mine != theirs:
reasons.append(
RouteMismatch(
axis_reason.get(axis, MISMATCH_FINGERPRINT),
f"{axis} is {theirs!r}, the route runs {mine!r}",
)
)
if not reasons:
reasons.append(
RouteMismatch(
MISMATCH_FINGERPRINT,
"fingerprints differ on a field this comparison does not cover",
)
)
return tuple(reasons)
def coverage_gap(
ranges: Iterable[tuple[int, int]], layer_count: int
) -> str | None:
"""Why `ranges` fail to tile ``[0, layer_count)``, or None when they do.
Overlaps are legal — ADR-0012 lets the Tracker resolve one by telling a hop
where the previous hop stopped. A hole is not: its layers are never computed,
and the route emits fluent tokens from a truncated model.
"""
ordered = sorted(ranges)
if not ordered:
return "the route owns no layers"
if ordered[0][0] != 0:
return f"layers 0–{ordered[0][0]} are owned by no Shard on the route"
covered = 0
for start, end in ordered:
if start > covered:
return f"layers {covered}–{start} are owned by no Shard on the route"
covered = max(covered, end)
if covered < layer_count:
return f"layers {covered}–{layer_count} are owned by no Shard on the route"
if covered > layer_count:
return (
f"the route claims {covered} layers, but the model has only {layer_count}"
)
return None
@dataclass(frozen=True)
class DistributedForwardEvidence:
"""A real distributed forward — the only thing that certifies a recipe."""
route_session_id: str
route_epoch: int
node_ids: tuple[str, ...]
shard_ranges: tuple[tuple[int, int], ...]
tokens_generated: int
layer_count: int
fingerprint: tuple[str, str]
synthetic: bool = False
certified_at: float = field(default_factory=time.time)
def rejection(self) -> str | None:
if self.synthetic:
return (
"evidence comes from a synthetic worker; only a real distributed "
"forward certifies a recipe"
)
distinct = set(self.node_ids)
if len(distinct) < MIN_CERTIFYING_NODES:
return (
f"evidence covers {len(distinct)} distinct node(s); a distributed "
f"forward requires at least {MIN_CERTIFYING_NODES}"
)
if len(distinct) != len(self.node_ids):
return "the same node is counted more than once in the certifying route"
if self.tokens_generated < 1:
return "the certifying forward generated no tokens"
return coverage_gap(self.shard_ranges, self.layer_count)
@dataclass(frozen=True)
class RecipeStatus:
status: str
detail: str = ""
certified_at: float | None = None
@property
def may_serve(self) -> bool:
return self.status == STATUS_CERTIFIED
@property
def may_certify(self) -> bool:
"""A dark recipe may be routed to *certify* it, and for nothing else.
Without this, certification is unreachable by construction: serving needs
certification, certification needs a real distributed forward, and a real
distributed forward needs a route.
"""
return self.status in (STATUS_DARK, STATUS_CERTIFIED)
def to_dict(self) -> dict:
return {
"status": self.status,
"detail": self.detail,
"certified_at": self.certified_at,
}
_DARK_DETAIL = "registered; dark until a real distributed forward certifies it"
_UNKNOWN_DETAIL = "this recipe has never been registered with the tracker"
class CertificationLedger:
"""Which recipes the tracker has seen, and which a real forward has proven."""
def __init__(self) -> None:
self._dark: set[tuple[str, str]] = set()
self._certified: dict[tuple[str, str], RecipeStatus] = {}
def register(self, identity: PresentedIdentity) -> RecipeStatus:
key = identity.key
if key in self._certified:
return self._certified[key]
self._dark.add(key)
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
def status(self, identity: PresentedIdentity | tuple[str, str]) -> RecipeStatus:
key = identity if isinstance(identity, tuple) else identity.key
if key in self._certified:
return self._certified[key]
if key in self._dark:
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
return RecipeStatus(STATUS_UNKNOWN, _UNKNOWN_DETAIL)
def certify(
self,
identity: PresentedIdentity | tuple[str, str],
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote a recipe out of the dark, or raise saying why the evidence is short."""
key = identity if isinstance(identity, tuple) else identity.key
if key not in self._dark and key not in self._certified:
raise RecipeIdentityError(
"this fingerprint is not registered; unknown recipes cannot be certified"
)
if evidence.fingerprint != key:
raise RecipeIdentityError(
"certification evidence fingerprint does not match the recipe being promoted"
)
rejection = evidence.rejection()
if rejection is not None:
raise RecipeIdentityError(f"this evidence does not certify: {rejection}")
status = RecipeStatus(
STATUS_CERTIFIED,
(
f"certified by route session {evidence.route_session_id} across "
f"{len(set(evidence.node_ids))} nodes"
),
certified_at=evidence.certified_at,
)
self._certified[key] = status
self._dark.discard(key)
return status
def to_dict(self) -> dict:
return {
"dark": [list(key) for key in sorted(self._dark)],
"certified": {
"/".join(key): status.to_dict()
for key, status in sorted(self._certified.items())
},
}
def admit_route(
identities: Sequence[PresentedIdentity],
*,
ledger: CertificationLedger | None = None,
for_certification: bool = False,
) -> tuple[RouteMismatch, ...]:
"""Every reason these Shards may not form one Inference Route; empty when they may.
Fail-closed by construction: an empty route, a hole in the coverage, one
differing dtype, and an uncertified recipe each produce a reason, so a caller
that proceeds only on an empty result cannot admit any of them.
"""
if not identities:
return (RouteMismatch(MISMATCH_RANGE, "a route needs at least one Shard"),)
head = identities[0]
reasons: list[RouteMismatch] = []
for candidate in identities[1:]:
reasons.extend(compare(head, candidate))
gap = coverage_gap(
[(i.shard_start, i.shard_end) for i in identities], head.layer_count
)
if gap is not None:
reasons.append(RouteMismatch(MISMATCH_RANGE, gap))
if ledger is not None:
status = ledger.status(head)
allowed = status.may_certify if for_certification else status.may_serve
if not allowed:
reasons.append(
RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}")
)
return tuple(reasons)

9
packages/tracker/meshnet_tracker/server.py
View File

@@ -84,6 +84,7 @@ from .routing_stats import (
)
from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode
from .recipe import CertificationLedger
_CONSOLE_LIMIT = 300
@@ -792,6 +793,7 @@ def _capability_from_registration(
hf_repo: str | None,
shard_start: int | None,
shard_end: int | None,
recipe_certifications: CertificationLedger,
) -> CapabilityState:
"""The tracker's verdict on the proof carried by one registration payload.
@@ -811,6 +813,7 @@ def _capability_from_registration(
declared_recipe_version=(
recipe_version if isinstance(recipe_version, str) else None
),
ledger=recipe_certifications,
)
@@ -2847,9 +2850,11 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
relay_status: dict | None = None,
test_runner: "TestRunManager | None" = None,
capability_policy: str | None = None,
recipe_certifications: CertificationLedger | None = None,
) -> None:
super().__init__(addr, handler)
self.registry = registry
self.recipe_certifications = recipe_certifications or CertificationLedger()
self.capability_policy = normalize_policy(
capability_policy if capability_policy is not None else policy_from_env()
)
@@ -4555,6 +4560,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
hf_repo=hf_repo,
shard_start=shard_start,
shard_end=shard_end,
recipe_certifications=server.recipe_certifications,
)
node_id = _node_id_for_registration(
@@ -6531,6 +6537,7 @@ class TrackerServer:
self._embedded_relay: Any | None = None
self._embedded_relay_actual_port: int | None = None
self._registry: dict[str, _NodeEntry] = {}
self._recipe_certifications = CertificationLedger()
self._lock = threading.Lock()
self._server: _TrackerHTTPServer | None = None
self._thread: threading.Thread | None = None
@@ -6725,6 +6732,7 @@ class TrackerServer:
relay_status=http_relay_status,
test_runner=self._test_runner,
capability_policy=self._capability_policy,
recipe_certifications=self._recipe_certifications,
)
self.port = self._server.server_address[1]
@@ -6983,6 +6991,7 @@ class TrackerServer:
hf_repo=payload.get("hf_repo"),
shard_start=shard_start,
shard_end=shard_end,
recipe_certifications=self._recipe_certifications,
),
)
with self._lock: