Merge branch 'archived_ralph/dgr-001-performance-contract' into merge/all-branches-into-master

# Conflicts:
#	.claude/memory/MEMORY.md
#	.scratch/distributed-gguf-runtime/PRD.md
#	.scratch/distributed-gguf-runtime/RALPH-CONTEXT.md
#	.scratch/distributed-gguf-runtime/README.md
#	.scratch/distributed-gguf-runtime/architecture.md
#	.scratch/distributed-gguf-runtime/evidence/DGR-017/README.md
#	.scratch/distributed-gguf-runtime/implementation-strategy.md
#	.scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md
#	.scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md
#	.scratch/distributed-gguf-runtime/milestones.md
#	.scratch/distributed-gguf-runtime/prd.json
#	docs/issues/distributed-gguf-runtime/01-lock-the-safetensors-versus-gguf-performance-contract.md
#	docs/issues/distributed-gguf-runtime/02-adopt-the-versioned-grpc-shard-protocol.md
#	docs/issues/distributed-gguf-runtime/03-define-exact-artifact-and-runtime-recipe-identity.md
#	docs/issues/distributed-gguf-runtime/05-implement-dense-llama-range-aware-gguf-ownership.md
#	docs/issues/distributed-gguf-runtime/06-implement-architecture-defined-boundary-input-output.md
This commit is contained in:
Dobromir Popov
2026-07-17 13:44:52 +03:00
124 changed files with 24939 additions and 91 deletions

View File

@@ -30,6 +30,13 @@ import time
from dataclasses import dataclass, replace
from typing import Any, Callable, Mapping
from .recipe import (
CertificationLedger,
FingerprintMismatch,
RecipeIdentityError,
parse_identity,
)
# The capability report layout this tracker reads (meshnet_node.capability).
SUPPORTED_SCHEMA_VERSION = 1
@@ -58,6 +65,12 @@ STATE_MODEL_MISMATCH = "model-mismatch"
STATE_SHARD_MISMATCH = "shard-mismatch"
STATE_RECIPE_MISMATCH = "recipe-mismatch"
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
# The node presented a DGR-003 identity block whose declared fingerprint is not
# the digest of the axes it declared. Identity is derived, never asserted.
STATE_FINGERPRINT_MISMATCH = "fingerprint-mismatch"
# Registered-but-dark: a known recipe no real distributed forward has certified.
# Visible to an operator, never routable for user traffic.
STATE_UNCERTIFIED = "uncertified"
ALL_STATES = (
STATE_ADMITTED,
@@ -69,6 +82,8 @@ ALL_STATES = (
STATE_SHARD_MISMATCH,
STATE_RECIPE_MISMATCH,
STATE_CATALOGUE_INCOMPATIBLE,
STATE_FINGERPRINT_MISMATCH,
STATE_UNCERTIFIED,
)
# --- Compatibility policy for nodes that predate the capability protocol. ---
@@ -165,12 +180,29 @@ class CapabilityState:
recorded_at: float = 0.0
schema_version: int | None = None
diagnostics: tuple[str, ...] = ()
# The DGR-003 compatibility fingerprint, *re-derived* by this tracker from
# the axes the node declared — never copied from what the node claimed.
# Absent for a node that predates DGR-003.
model_artifact_digest: str | None = None
runtime_recipe_digest: str | None = None
shard_binding_digest: str | None = None
# The tracker ledger's verdict on that fingerprint at evaluation time
# ("dark"/"certified"), so the network map answers "why is this exact node
# not routing" without a second query. None when no identity was presented.
certification: str | None = None
@property
def proven(self) -> bool:
"""The presented proof covers exactly what the node advertised."""
return self.state == STATE_ADMITTED
@property
def fingerprint(self) -> tuple[str, str] | None:
"""What route formation compares. `None` when the node declares no identity."""
if self.model_artifact_digest is None or self.runtime_recipe_digest is None:
return None
return (self.model_artifact_digest, self.runtime_recipe_digest)
def routable_under(self, policy: str) -> bool:
if self.proven:
return True
@@ -197,6 +229,10 @@ class CapabilityState:
"recorded_at": self.recorded_at,
"schema_version": self.schema_version,
"diagnostics": list(self.diagnostics),
"model_artifact_digest": self.model_artifact_digest,
"runtime_recipe_digest": self.runtime_recipe_digest,
"shard_binding_digest": self.shard_binding_digest,
"certification": self.certification,
}
@@ -224,6 +260,7 @@ def evaluate_report(
declared_recipe_version: str | None = None,
now: float | None = None,
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
ledger: CertificationLedger | None = None,
) -> CapabilityState:
"""Judge the proof a node presented against what that node is advertising.
@@ -308,6 +345,84 @@ def evaluate_report(
f"the node declared v{declared_recipe_version}",
)
identity = None
if report.get("identity") is not None:
try:
identity = parse_identity(report["identity"])
except FingerprintMismatch as exc:
return base.with_state(STATE_FINGERPRINT_MISMATCH, str(exc))
except RecipeIdentityError as exc:
return base.with_state(
STATE_INVALID, f"capability identity block is unusable: {exc}"
)
# The report's `shard` range is inclusive/inclusive (the CLI and backend
# convention); the identity's is inclusive/exclusive (the protocol's, and
# ADR-0012's). A node whose two halves disagree has not proven the range
# it claims, whichever one is right.
if (identity.shard_start, identity.shard_end) != (
base.shard_start,
(base.shard_end or 0) + 1,
):
return base.with_state(
STATE_SHARD_MISMATCH,
f"identity covers layers {identity.shard_start}{identity.shard_end} "
f"(end-exclusive), but the proof is for layers {base.shard_start}"
f"{base.shard_end} (end-inclusive)",
)
if not model_matches(identity.artifact_id):
return base.with_state(
STATE_MODEL_MISMATCH,
f"identity is for artifact {identity.artifact_id!r}, but the node "
f"registered {advertised_model!r}",
)
model_claim = report["model"]
if model_claim.get("revision") != identity.revision:
return base.with_state(
STATE_MODEL_MISMATCH,
"identity revision does not match the capability proof",
)
config_fingerprint = model_claim.get("config_fingerprint")
if isinstance(config_fingerprint, str) and config_fingerprint.startswith(
"sha256:"
):
config_fingerprint = config_fingerprint.removeprefix("sha256:")
if config_fingerprint != identity.architecture_digest:
return base.with_state(
STATE_MODEL_MISMATCH,
"identity architecture/config digest does not match the capability proof",
)
if (
identity.recipe_id != base.recipe_id
or identity.recipe_version != base.recipe_version
or identity.catalogue_version != base.catalogue_version
):
return base.with_state(
STATE_RECIPE_MISMATCH,
"identity recipe labels do not match the capability proof",
)
if identity.axes["backend_id"] != base.backend_id:
return base.with_state(
STATE_RECIPE_MISMATCH,
"identity backend does not match the capability proof",
)
if (
base.quantization is not None
and identity.axes["weight_quantization"] != base.quantization
):
return base.with_state(
STATE_RECIPE_MISMATCH,
"identity weight quantization does not match the capability proof",
)
base = replace(
base,
model_artifact_digest=identity.model_artifact_digest,
runtime_recipe_digest=identity.runtime_recipe_digest,
shard_binding_digest=identity.shard_binding_digest,
)
if status != STATUS_PASSED:
return base.with_state(
STATE_FAILED,
@@ -328,6 +443,29 @@ def evaluate_report(
f"proof is timestamped {-age:.0f}s in the future; check the node's clock",
)
# The digest only establishes that this report is self-consistent. It does
# not authenticate a node or prove a distributed forward. Exact recipes are
# therefore registered-but-dark until tracker-owned certification records
# that forward.
#
# `ledger is None` means the caller owns no certification authority. It is
# not "certify anything" and it is not "make one up": a disposable ledger
# would register the recipe into state that is discarded on return, which
# reads like certification is wired when nothing is recording it. The only
# safe reading is that nothing here has certified this recipe, so it stays
# dark. `TrackerServer` owns the real ledger and passes it in.
if identity is not None:
if ledger is None:
return base.with_state(
STATE_UNCERTIFIED,
"no certification ledger; an exact recipe is dark until a "
"tracker-owned distributed forward certifies it",
)
recipe_status = ledger.register(identity)
base = replace(base, certification=recipe_status.status)
if not recipe_status.may_serve:
return base.with_state(STATE_UNCERTIFIED, recipe_status.detail)
return base.with_state(
STATE_ADMITTED,
f"{base.model_id} layers {base.shard_start}{base.shard_end} proven on "

View File

@@ -16,8 +16,8 @@ import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any, Callable
from dataclasses import dataclass
from typing import Callable
@dataclass

View File

@@ -0,0 +1,556 @@
"""Tracker-side artifact and runtime recipe identity (DGR-003).
The node computes a compatibility fingerprint in `meshnet_node.runtime_recipe`.
This module recomputes it, and the recomputation is the entire point.
A fingerprint that arrives on the wire is a **claim**. If the tracker stored
what a node asserted, a node could assert the digest of a certified recipe while
running an uncertified one — and admission, route selection, and the gRPC
handshake would all wave it through, because they all compare the digest it
handed them. So the tracker derives the digest from the *axes* the node
declared, and refuses any report whose claim does not match the derivation. A
node can lie about its axes, and a real forward will catch that; it cannot lie
about the digest of the axes it declared.
This module deliberately does **not** import `meshnet_node`: the tracker package
does not depend on the node package, and an admission gate that shares an
implementation with the thing it admits is not an independent check. The two
implementations are pinned together by a committed conformance vector
(``tests/data/recipe_fingerprint_vectors.json``). If they drift, a test fails —
rather than a route quietly failing to form, or worse, quietly forming.
Recipes arrive **dark**: registered, visible to an operator, and not routable for
user traffic. Only a real distributed forward — at least two distinct nodes,
covering the whole model, emitting real tokens — takes a recipe out of the dark
(:class:`CertificationLedger`).
"""
from __future__ import annotations
import hashlib
import json
import re
import time
from dataclasses import dataclass, field
from typing import Any, Iterable, Mapping
# Layout of the identity block this tracker reads (meshnet_node.runtime_recipe).
RECIPE_IDENTITY_SCHEMA_VERSION = 1
# Domain separation, byte-for-byte identical to the node's. These strings are
# part of the wire contract, not an implementation detail: changing one here
# without changing it there silently partitions every route.
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
# The axes a recipe digest commits to. Order is irrelevant (the canonical JSON
# sorts keys); membership is not — an axis missing here is an axis the tracker
# would let a node change without changing its identity.
RECIPE_AXES: tuple[str, ...] = (
"weight_quantization",
"activation_dtype",
"compute_dtype",
"kv_dtype",
"kv_layout",
"tokenizer_revision",
"architecture_adapter",
"backend_id",
"runtime_version",
"boundary_schema_version",
"protocol_schema_version",
)
_INT_AXES = frozenset({"boundary_schema_version", "protocol_schema_version"})
STATUS_UNKNOWN = "unknown"
STATUS_DARK = "dark"
STATUS_CERTIFIED = "certified"
MIN_CERTIFYING_NODES = 2
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"})
class RecipeIdentityError(ValueError):
"""A presented identity block is malformed or internally inconsistent."""
class FingerprintMismatch(RecipeIdentityError):
"""A supplied digest is inconsistent with its identity declaration.
A digest is an integrity and compatibility claim, not an authenticated
statement about what a node is actually executing. Distributed
certification remains the trust boundary.
"""
def canonical_sha256(value: Any) -> str:
payload = json.dumps(
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def _digest(domain: str, body: Mapping[str, Any]) -> str:
return canonical_sha256({"domain": domain, "body": dict(body)})
def _text(value: Any, what: str) -> str:
if not isinstance(value, str) or not value.strip():
raise RecipeIdentityError(f"{what!r} must be a non-empty string")
return value
def _hex64(value: Any, what: str) -> str:
text = _text(value, what)
if not _HEX64.match(text):
raise RecipeIdentityError(f"{what!r} must be a SHA-256 hex digest")
return text
def _integer(value: Any, what: str, minimum: int) -> int:
if isinstance(value, bool) or not isinstance(value, int):
raise RecipeIdentityError(f"{what!r} must be an integer")
if value < minimum:
raise RecipeIdentityError(f"{what!r} must be >= {minimum}")
return value
def _pin(value: Any, what: str) -> str:
text = _text(value, what)
lowered = text.strip().lower()
if lowered in _MOVING_REFS or lowered.startswith("refs/"):
raise RecipeIdentityError(
f"{what!r} is a moving reference, not an exact revision pin"
)
return text
def _mapping(value: Any, what: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise RecipeIdentityError(f"{what!r} must be a JSON object")
return value
def artifact_digest(
*,
source_digest: str,
architecture: str,
architecture_digest: str,
layer_count: int,
) -> str:
"""The artifact half of the fingerprint, derived exactly as the node derives it."""
return _digest(
ARTIFACT_DIGEST_DOMAIN,
{
"source_digest": source_digest,
"architecture": architecture,
"architecture_digest": architecture_digest,
"layer_count": layer_count,
},
)
def recipe_digest(axes: Mapping[str, Any]) -> str:
"""The recipe half of the fingerprint, derived exactly as the node derives it."""
missing = [axis for axis in RECIPE_AXES if axis not in axes]
if missing:
raise RecipeIdentityError(
"recipe is missing required axes: " + ", ".join(missing)
)
return _digest(RECIPE_DIGEST_DOMAIN, {axis: axes[axis] for axis in RECIPE_AXES})
@dataclass(frozen=True)
class PresentedIdentity:
"""One node's declared artifact/recipe identity, with digests re-derived here.
`model_artifact_digest` and `runtime_recipe_digest` are computed by this
tracker from the declared axes. They are never copied from the report.
"""
artifact_id: str
revision: str
source_digest: str
content_digest: str
derivative_range: tuple[int, int] | None
architecture: str
architecture_digest: str
layer_count: int
is_derivative: bool
shard_start: int
shard_end: int
axes: Mapping[str, Any]
recipe_id: str
recipe_version: str
catalogue_version: str
@property
def model_artifact_digest(self) -> str:
return artifact_digest(
source_digest=self.source_digest,
architecture=self.architecture,
architecture_digest=self.architecture_digest,
layer_count=self.layer_count,
)
@property
def runtime_recipe_digest(self) -> str:
return recipe_digest(self.axes)
@property
def key(self) -> tuple[str, str]:
return (self.model_artifact_digest, self.runtime_recipe_digest)
@property
def shard_binding_digest(self) -> str:
"""This participant's own bytes and exact range, bound to the source.
The route fingerprint (:attr:`key`) is deliberately range-independent —
Shards on one route own different ranges, so a range-sensitive digest
would stop any two of them from ever agreeing. The consequence is that
the fingerprint alone cannot tell two *different splits of the same
source* apart: a derivative can assert the right source and recipe and
inherit certification earned by bytes it does not hold.
This digest is what closes that. It commits to the derivative's own
content hash and its exact end-exclusive range, so certification can be
recipe-wide while every serving node is still separately pinned to the
blob it was admitted on (`TrackerServer.certify_recipe`).
"""
return _digest(
SHARD_BINDING_DIGEST_DOMAIN,
{
"source_digest": self.source_digest,
"content_digest": self.content_digest,
"architecture_digest": self.architecture_digest,
"derivative_range": list(self.derivative_range)
if self.derivative_range is not None
else None,
"shard_start": self.shard_start,
"shard_end": self.shard_end,
},
)
@property
def tokenizer_revision(self) -> str:
return str(self.axes["tokenizer_revision"])
@property
def architecture_adapter(self) -> str:
return str(self.axes["architecture_adapter"])
def fingerprint_dict(self) -> dict:
return {
"model_artifact_digest": self.model_artifact_digest,
"runtime_recipe_digest": self.runtime_recipe_digest,
"recipe_id": self.recipe_id,
"recipe_version": self.recipe_version,
"catalogue_version": self.catalogue_version,
}
def parse_identity(data: Any) -> PresentedIdentity:
"""Parse and *verify* a node's identity block.
Raises when the block is malformed, when a split artifact does not name the
exact source it was cut from, when a Shard advertises layers its artifact
does not contain, or when the declared fingerprint does not match the digest
of the axes it was declared with.
"""
doc = _mapping(data, "identity")
schema_version = doc.get("schema_version")
if schema_version != RECIPE_IDENTITY_SCHEMA_VERSION:
raise RecipeIdentityError(
f"identity block declares schema version {schema_version!r}; this tracker "
f"reads version {RECIPE_IDENTITY_SCHEMA_VERSION}"
)
artifact = _mapping(doc.get("artifact"), "identity.artifact")
recipe = _mapping(doc.get("recipe"), "identity.recipe")
layer_count = _integer(artifact.get("layer_count"), "artifact.layer_count", 1)
content_digest = _hex64(artifact.get("content_digest"), "artifact.content_digest")
raw_binding = artifact.get("derived_from")
if raw_binding is None:
source_digest = content_digest
binding: tuple[int, int] | None = None
else:
derived = _mapping(raw_binding, "artifact.derived_from")
source_digest = _hex64(
derived.get("source_artifact_digest"),
"artifact.derived_from.source_artifact_digest",
)
binding = (
_integer(derived.get("shard_start"), "derived_from.shard_start", 0),
_integer(derived.get("shard_end"), "derived_from.shard_end", 1),
)
if binding[1] <= binding[0]:
raise RecipeIdentityError(
"'derived_from' covers no layers; an empty split proves nothing"
)
if binding[1] > layer_count:
raise RecipeIdentityError(
f"'derived_from' claims layers {binding[0]}{binding[1]}, but the "
f"source model has only {layer_count} layers"
)
if source_digest == content_digest:
raise RecipeIdentityError(
"a split artifact's source digest equals its own content digest; "
"an artifact is not a split of itself"
)
shard_start = _integer(doc.get("shard_start"), "shard_start", 0)
shard_end = _integer(doc.get("shard_end"), "shard_end", 1)
if shard_end <= shard_start:
raise RecipeIdentityError("a Shard owning no layer computes nothing")
if shard_end > layer_count:
raise RecipeIdentityError(
f"Shard owns layers {shard_start}{shard_end}, but the artifact has only "
f"{layer_count} layers"
)
if binding is not None and not (
binding[0] <= shard_start and shard_end <= binding[1]
):
raise RecipeIdentityError(
f"Shard advertises layers {shard_start}{shard_end}, but its split "
f"artifact only contains layers {binding[0]}{binding[1]}"
)
axes: dict[str, Any] = {}
for axis in RECIPE_AXES:
if axis not in recipe:
raise RecipeIdentityError(
f"recipe is missing axis {axis!r}; an unstated axis cannot default"
)
value = recipe[axis]
if axis in _INT_AXES:
axes[axis] = _integer(value, f"recipe.{axis}", 1)
else:
axes[axis] = _text(value, f"recipe.{axis}")
_pin(axes["tokenizer_revision"], "recipe.tokenizer_revision")
identity = PresentedIdentity(
artifact_id=_text(artifact.get("artifact_id"), "artifact.artifact_id"),
revision=_pin(artifact.get("revision"), "artifact.revision"),
source_digest=source_digest,
content_digest=content_digest,
derivative_range=binding,
architecture=_text(artifact.get("architecture"), "artifact.architecture"),
architecture_digest=_hex64(
artifact.get("architecture_digest"), "artifact.architecture_digest"
),
layer_count=layer_count,
is_derivative=binding is not None,
shard_start=shard_start,
shard_end=shard_end,
axes=axes,
recipe_id=_text(recipe.get("recipe_id"), "recipe.recipe_id"),
recipe_version=_text(recipe.get("recipe_version"), "recipe.recipe_version"),
catalogue_version=_text(
recipe.get("catalogue_version"), "recipe.catalogue_version"
),
)
declared = doc.get("fingerprint")
if declared is not None:
claim = _mapping(declared, "identity.fingerprint")
claimed_artifact = _hex64(
claim.get("model_artifact_digest"), "fingerprint.model_artifact_digest"
)
claimed_recipe = _hex64(
claim.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest"
)
if (claimed_artifact, claimed_recipe) != identity.key:
raise FingerprintMismatch(
"declared fingerprint is inconsistent with the artifact and recipe "
"claim; the tracker recomputes compatibility digests"
)
return identity
def coverage_gap(
ranges: Iterable[tuple[int, int]], layer_count: int
) -> str | None:
"""Why `ranges` fail to tile ``[0, layer_count)``, or None when they do.
Overlaps are legal — ADR-0012 lets the Tracker resolve one by telling a hop
where the previous hop stopped. A hole is not: its layers are never computed,
and the route emits fluent tokens from a truncated model.
"""
ordered = sorted(ranges)
if not ordered:
return "the route owns no layers"
if ordered[0][0] != 0:
return f"layers 0{ordered[0][0]} are owned by no Shard on the route"
covered = 0
for start, end in ordered:
if start > covered:
return f"layers {covered}{start} are owned by no Shard on the route"
covered = max(covered, end)
if covered < layer_count:
return f"layers {covered}{layer_count} are owned by no Shard on the route"
if covered > layer_count:
return (
f"the route claims {covered} layers, but the model has only {layer_count}"
)
return None
@dataclass(frozen=True)
class DistributedForwardEvidence:
"""A real distributed forward — the only thing that certifies a recipe."""
route_session_id: str
route_epoch: int
node_ids: tuple[str, ...]
shard_ranges: tuple[tuple[int, int], ...]
tokens_generated: int
layer_count: int
fingerprint: tuple[str, str]
participants: tuple[PresentedIdentity, ...]
synthetic: bool = False
certified_at: float = field(default_factory=time.time)
def rejection(self) -> str | None:
if self.synthetic:
return (
"evidence comes from a synthetic worker; only a real distributed "
"forward certifies a recipe"
)
if len(self.participants) != len(self.node_ids):
return "participant identities do not match the certifying node list"
if len(self.shard_ranges) != len(self.participants):
return "participant identities do not match the recorded effective ranges"
for participant, shard_range in zip(
self.participants, self.shard_ranges, strict=True
):
if participant.key != self.fingerprint:
return "a participant fingerprint differs from the certifying route"
if participant.layer_count != self.layer_count:
return "a participant artifact layer count differs from the certifying route"
if (participant.shard_start, participant.shard_end) != shard_range:
return "a participant identity does not match its recorded effective range"
distinct = set(self.node_ids)
if len(distinct) < MIN_CERTIFYING_NODES:
return (
f"evidence covers {len(distinct)} distinct node(s); a distributed "
f"forward requires at least {MIN_CERTIFYING_NODES}"
)
if len(distinct) != len(self.node_ids):
return "the same node is counted more than once in the certifying route"
if self.tokens_generated < 1:
return "the certifying forward generated no tokens"
return coverage_gap(self.shard_ranges, self.layer_count)
@dataclass(frozen=True)
class RecipeStatus:
status: str
detail: str = ""
certified_at: float | None = None
@property
def may_serve(self) -> bool:
return self.status == STATUS_CERTIFIED
@property
def may_certify(self) -> bool:
"""A dark recipe may be routed to *certify* it, and for nothing else.
Without this, certification is unreachable by construction: serving needs
certification, certification needs a real distributed forward, and a real
distributed forward needs a route.
"""
return self.status in (STATUS_DARK, STATUS_CERTIFIED)
def to_dict(self) -> dict:
return {
"status": self.status,
"detail": self.detail,
"certified_at": self.certified_at,
}
_DARK_DETAIL = "registered; dark until a real distributed forward certifies it"
_UNKNOWN_DETAIL = "this recipe has never been registered with the tracker"
class CertificationLedger:
"""Which recipes the tracker has seen, and which a real forward has proven."""
def __init__(self) -> None:
self._dark: set[tuple[str, str]] = set()
self._certified: dict[tuple[str, str], RecipeStatus] = {}
def register(self, identity: PresentedIdentity) -> RecipeStatus:
key = identity.key
if key in self._certified:
return self._certified[key]
self._dark.add(key)
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
def status(self, identity: PresentedIdentity | tuple[str, str]) -> RecipeStatus:
key = identity if isinstance(identity, tuple) else identity.key
if key in self._certified:
return self._certified[key]
if key in self._dark:
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
return RecipeStatus(STATUS_UNKNOWN, _UNKNOWN_DETAIL)
def certify(
self,
identity: PresentedIdentity,
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote a recipe out of the dark, or raise saying why the evidence is short."""
key = identity.key
if key not in self._dark and key not in self._certified:
raise RecipeIdentityError(
"this fingerprint is not registered; unknown recipes cannot be certified"
)
if evidence.fingerprint != key:
raise RecipeIdentityError(
"certification evidence fingerprint does not match the recipe being promoted"
)
if evidence.layer_count != identity.layer_count:
raise RecipeIdentityError(
"certification evidence layer count does not match the artifact being promoted"
)
rejection = evidence.rejection()
if rejection is not None:
raise RecipeIdentityError(f"this evidence does not certify: {rejection}")
status = RecipeStatus(
STATUS_CERTIFIED,
(
f"certified by route session {evidence.route_session_id} across "
f"{len(set(evidence.node_ids))} nodes"
),
certified_at=evidence.certified_at,
)
self._certified[key] = status
self._dark.discard(key)
return status
def to_dict(self) -> dict:
return {
"dark": [list(key) for key in sorted(self._dark)],
"certified": {
"/".join(key): status.to_dict()
for key, status in sorted(self._certified.items())
},
}
# Route formation itself lives in `server.py` (`_select_route`, `_enumerate_routes`,
# `_find_pinned_route`): candidates are partitioned by the exact fingerprint this
# module derives, so a route can only ever be assembled inside one identity. A
# second route gate here would be a second copy of that policy to keep in step —
# the same reason the node module holds no certification authority.

View File

@@ -26,7 +26,7 @@ import random
import sqlite3
import threading
import time
from dataclasses import dataclass, field
from dataclasses import dataclass
from typing import Any, Iterable

View File

@@ -45,7 +45,7 @@ import urllib.parse
import urllib.request
import uuid
from collections import deque
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from importlib.resources import files
from pathlib import Path
from typing import Any
@@ -53,13 +53,10 @@ from typing import Any
from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
from .auth import is_validator_token, sign_hive_request, verify_hive_request
from .capability import (
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
POLICY_COMPAT,
POLICY_ENFORCE,
STATE_ABSENT,
STATE_ADMITTED,
STATE_MODEL_MISMATCH,
STATE_SHARD_MISMATCH,
STATE_UNCERTIFIED,
CapabilityState,
absent_state,
evaluate_report,
@@ -68,7 +65,7 @@ from .capability import (
)
from .wallet_proof import binding_message, verify_wallet_signature
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
from .calibration import ToplocCalibrationStore
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
from .gossip import NodeGossip
from .logging_setup import tracker_logger
@@ -84,6 +81,13 @@ from .routing_stats import (
)
from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode
from .recipe import (
CertificationLedger,
DistributedForwardEvidence,
PresentedIdentity,
RecipeIdentityError,
RecipeStatus,
)
_CONSOLE_LIMIT = 1000
@@ -792,6 +796,7 @@ def _capability_from_registration(
hf_repo: str | None,
shard_start: int | None,
shard_end: int | None,
recipe_certifications: CertificationLedger,
) -> CapabilityState:
"""The tracker's verdict on the proof carried by one registration payload.
@@ -811,6 +816,7 @@ def _capability_from_registration(
declared_recipe_version=(
recipe_version if isinstance(recipe_version, str) else None
),
ledger=recipe_certifications,
)
@@ -830,6 +836,11 @@ def _admitted_nodes(nodes: list["_NodeEntry"], policy: str | None) -> list["_Nod
return [node for node in nodes if _capability_routable(node, effective)]
def _route_identity_partition(node: "_NodeEntry") -> tuple[str, ...]:
fingerprint = _node_admission(node).fingerprint
return ("legacy",) if fingerprint is None else ("exact", *fingerprint)
def _select_route(
nodes: list[_NodeEntry],
required_start: int,
@@ -853,29 +864,51 @@ def _select_route(
],
key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
)
route: list[_NodeEntry] = []
covered_up_to = required_start - 1
def _routing_score(node: "_NodeEntry") -> float:
return _effective_throughput(node, model) * _reputation_multiplier(node, contracts)
while covered_up_to < required_end:
best: _NodeEntry | None = None
for node in candidates:
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
if best is None:
best = node
elif node.shard_end > best.shard_end:
best = node
elif node.shard_end == best.shard_end and _routing_score(node) > _routing_score(best):
best = node
if best is None:
missing = covered_up_to + 1
return [], f"no route available: no registered node covers layer {missing}"
route.append(best)
covered_up_to = best.shard_end
candidates = [n for n in candidates if n is not best]
partitions: dict[tuple[str, ...], list[_NodeEntry]] = {}
for node in candidates:
partitions.setdefault(_route_identity_partition(node), []).append(node)
complete: list[list[_NodeEntry]] = []
furthest = required_start - 1
for partition in partitions.values():
pool = list(partition)
route: list[_NodeEntry] = []
covered_up_to = required_start - 1
while covered_up_to < required_end:
best: _NodeEntry | None = None
for node in pool:
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
if best is None:
best = node
elif node.shard_end > best.shard_end:
best = node
elif (
node.shard_end == best.shard_end
and _routing_score(node) > _routing_score(best)
):
best = node
if best is None:
break
route.append(best)
covered_up_to = best.shard_end
pool = [node for node in pool if node is not best]
furthest = max(furthest, covered_up_to)
if covered_up_to >= required_end:
complete.append(route)
if not complete:
return [], f"no route available: no registered node covers layer {furthest + 1}"
route = max(
complete,
key=lambda candidate: (
min(_routing_score(node) for node in candidate),
-len(candidate),
),
)
return route, ""
@@ -911,7 +944,12 @@ def _enumerate_routes(
for head in heads:
route = [head]
covered_up_to = head.shard_end
pool = [n for n in sharded if n is not head]
head_partition = _route_identity_partition(head)
pool = [
node
for node in sharded
if node is not head and _route_identity_partition(node) == head_partition
]
while covered_up_to < required_end:
best = None
for n in pool:
@@ -2126,14 +2164,24 @@ def _find_pinned_route(
hop_count: int,
) -> list[_NodeEntry] | None:
"""First combination of exactly ``hop_count`` distinct nodes covering the
layer range, where every node extends coverage (US-030 benchmark routes)."""
layer range, where every node extends coverage (US-030 benchmark routes).
Benchmark combos run real inference, so they obey the same DGR-003 rule as
every other route builder: one route, one exact identity. A combo that mixed
two fingerprints — or an exact Shard with a legacy one — would measure a
numerically incoherent route and record the garbage as a benchmark.
"""
for combo in itertools.permutations(nodes, hop_count):
covered = required_start - 1
valid = True
partition = _route_identity_partition(combo[0])
for candidate in combo:
if candidate.shard_start is None or candidate.shard_end is None:
valid = False
break
if _route_identity_partition(candidate) != partition:
valid = False
break
if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
valid = False
break
@@ -2584,8 +2632,8 @@ def _estimate_prompt_tokens(body: dict) -> int | None:
def _requested_completion_token_limit(body: dict) -> int | None:
for field in ("max_completion_tokens", "max_tokens"):
value = body.get(field)
for key in ("max_completion_tokens", "max_tokens"):
value = body.get(key)
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
@@ -2917,9 +2965,11 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
relay_status: dict | None = None,
test_runner: "TestRunManager | None" = None,
capability_policy: str | None = None,
recipe_certifications: CertificationLedger | None = None,
) -> None:
super().__init__(addr, handler)
self.registry = registry
self.recipe_certifications = recipe_certifications or CertificationLedger()
self.capability_policy = normalize_policy(
capability_policy if capability_policy is not None else policy_from_env()
)
@@ -2964,7 +3014,7 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
def log_message(self, fmt, *args): # suppress request logs in tests
pass
def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
@@ -4646,6 +4696,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
hf_repo=hf_repo,
shard_start=shard_start,
shard_end=shard_end,
recipe_certifications=server.recipe_certifications,
)
node_id = _node_id_for_registration(
@@ -6690,6 +6741,7 @@ class TrackerServer:
self._embedded_relay: Any | None = None
self._embedded_relay_actual_port: int | None = None
self._registry: dict[str, _NodeEntry] = {}
self._recipe_certifications = CertificationLedger()
self._lock = threading.Lock()
self._server: _TrackerHTTPServer | None = None
self._thread: threading.Thread | None = None
@@ -6799,6 +6851,84 @@ class TrackerServer:
self._test_runner: TestRunManager | None = test_runner
self.port: int | None = None
def certify_recipe(
self,
identity: PresentedIdentity,
evidence: DistributedForwardEvidence,
) -> RecipeStatus:
"""Promote one exact recipe from tracker-recorded distributed evidence.
This is the tracker's only certification authority. The evidence names
the nodes that served the forward; this method refuses to take any of
them on the evidence's word. For each one it goes back to what *this
tracker* re-derived at admission and requires the evidence to describe
the same Shard: same route fingerprint, same registered range, and the
same shard binding digest — the derivative's own bytes and exact range.
Without the binding check, a route fingerprint is range-independent by
design (:mod:`meshnet_tracker.recipe`), so evidence could name the right
recipe while describing splits nobody on the route was actually serving,
and the certification would attach to blobs that never ran.
"""
with self._lock:
for node_id, participant in zip(
evidence.node_ids, evidence.participants, strict=True
):
node = self._registry.get(node_id)
if node is None:
raise RecipeIdentityError(
f"certification participant {node_id!r} is not registered"
)
admitted = node.capability
if admitted.fingerprint != identity.key:
raise RecipeIdentityError(
f"certification participant {node_id!r} is not admitted under "
"the promoted fingerprint"
)
if admitted.state not in (STATE_UNCERTIFIED, STATE_ADMITTED):
raise RecipeIdentityError(
f"certification participant {node_id!r} has capability state "
f"{admitted.state!r}"
)
registered_range = (
node.shard_start,
None if node.shard_end is None else node.shard_end + 1,
)
if registered_range != (
participant.shard_start,
participant.shard_end,
):
raise RecipeIdentityError(
f"certification participant {node_id!r} range differs from its "
"registered Shard range"
)
# The exact bytes and range this node was admitted on. A node that
# registered without an identity has no binding, and cannot be a
# participant in an exact certification at all.
if admitted.shard_binding_digest is None:
raise RecipeIdentityError(
f"certification participant {node_id!r} registered no exact "
"Shard binding; it cannot certify a recipe"
)
if admitted.shard_binding_digest != participant.shard_binding_digest:
raise RecipeIdentityError(
f"certification participant {node_id!r} presents a Shard "
"binding this tracker did not admit it on; the evidence "
"describes different derivative bytes or a different range"
)
status = self._recipe_certifications.certify(identity, evidence)
for node in self._registry.values():
if (
node.capability.state == STATE_UNCERTIFIED
and node.capability.fingerprint == identity.key
):
node.capability = replace(
node.capability.with_state(STATE_ADMITTED, status.detail),
certification=status.status,
)
return status
def _start_embedded_relay(self) -> dict:
"""Start the shared RelayServer class in-process for tracker+relay deployments."""
if not self._embedded_relay_enabled:
@@ -6884,6 +7014,7 @@ class TrackerServer:
relay_status=http_relay_status,
test_runner=self._test_runner,
capability_policy=self._capability_policy,
recipe_certifications=self._recipe_certifications,
)
self.port = self._server.server_address[1]
@@ -7093,10 +7224,6 @@ class TrackerServer:
shard_end = int(payload["shard_end"]) if payload.get("shard_end") is not None else None
except (TypeError, ValueError):
return
try:
friendly_name = _normalize_friendly_name(payload.get("friendly_name"))
except ValueError:
friendly_name = None
# The replicated payload is the raw registration body, so the follower can
# resolve precision exactly as the leader did -- including telling a legacy
# absent `quantization` from a declared one. Dropping these fields here
@@ -7142,6 +7269,7 @@ class TrackerServer:
hf_repo=payload.get("hf_repo"),
shard_start=shard_start,
shard_end=shard_end,
recipe_certifications=self._recipe_certifications,
),
)
with self._lock: