Merge branch 'archived_ralph/dgr-001-performance-contract' into merge/all-branches-into-master
# Conflicts: # .claude/memory/MEMORY.md # .scratch/distributed-gguf-runtime/PRD.md # .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md # .scratch/distributed-gguf-runtime/README.md # .scratch/distributed-gguf-runtime/architecture.md # .scratch/distributed-gguf-runtime/evidence/DGR-017/README.md # .scratch/distributed-gguf-runtime/implementation-strategy.md # .scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md # .scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md # .scratch/distributed-gguf-runtime/milestones.md # .scratch/distributed-gguf-runtime/prd.json # docs/issues/distributed-gguf-runtime/01-lock-the-safetensors-versus-gguf-performance-contract.md # docs/issues/distributed-gguf-runtime/02-adopt-the-versioned-grpc-shard-protocol.md # docs/issues/distributed-gguf-runtime/03-define-exact-artifact-and-runtime-recipe-identity.md # docs/issues/distributed-gguf-runtime/05-implement-dense-llama-range-aware-gguf-ownership.md # docs/issues/distributed-gguf-runtime/06-implement-architecture-defined-boundary-input-output.md
This commit is contained in:
@@ -30,6 +30,13 @@ import time
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from .recipe import (
|
||||
CertificationLedger,
|
||||
FingerprintMismatch,
|
||||
RecipeIdentityError,
|
||||
parse_identity,
|
||||
)
|
||||
|
||||
# The capability report layout this tracker reads (meshnet_node.capability).
|
||||
SUPPORTED_SCHEMA_VERSION = 1
|
||||
|
||||
@@ -58,6 +65,12 @@ STATE_MODEL_MISMATCH = "model-mismatch"
|
||||
STATE_SHARD_MISMATCH = "shard-mismatch"
|
||||
STATE_RECIPE_MISMATCH = "recipe-mismatch"
|
||||
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
|
||||
# The node presented a DGR-003 identity block whose declared fingerprint is not
|
||||
# the digest of the axes it declared. Identity is derived, never asserted.
|
||||
STATE_FINGERPRINT_MISMATCH = "fingerprint-mismatch"
|
||||
# Registered-but-dark: a known recipe no real distributed forward has certified.
|
||||
# Visible to an operator, never routable for user traffic.
|
||||
STATE_UNCERTIFIED = "uncertified"
|
||||
|
||||
ALL_STATES = (
|
||||
STATE_ADMITTED,
|
||||
@@ -69,6 +82,8 @@ ALL_STATES = (
|
||||
STATE_SHARD_MISMATCH,
|
||||
STATE_RECIPE_MISMATCH,
|
||||
STATE_CATALOGUE_INCOMPATIBLE,
|
||||
STATE_FINGERPRINT_MISMATCH,
|
||||
STATE_UNCERTIFIED,
|
||||
)
|
||||
|
||||
# --- Compatibility policy for nodes that predate the capability protocol. ---
|
||||
@@ -165,12 +180,29 @@ class CapabilityState:
|
||||
recorded_at: float = 0.0
|
||||
schema_version: int | None = None
|
||||
diagnostics: tuple[str, ...] = ()
|
||||
# The DGR-003 compatibility fingerprint, *re-derived* by this tracker from
|
||||
# the axes the node declared — never copied from what the node claimed.
|
||||
# Absent for a node that predates DGR-003.
|
||||
model_artifact_digest: str | None = None
|
||||
runtime_recipe_digest: str | None = None
|
||||
shard_binding_digest: str | None = None
|
||||
# The tracker ledger's verdict on that fingerprint at evaluation time
|
||||
# ("dark"/"certified"), so the network map answers "why is this exact node
|
||||
# not routing" without a second query. None when no identity was presented.
|
||||
certification: str | None = None
|
||||
|
||||
@property
|
||||
def proven(self) -> bool:
|
||||
"""The presented proof covers exactly what the node advertised."""
|
||||
return self.state == STATE_ADMITTED
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> tuple[str, str] | None:
|
||||
"""What route formation compares. `None` when the node declares no identity."""
|
||||
if self.model_artifact_digest is None or self.runtime_recipe_digest is None:
|
||||
return None
|
||||
return (self.model_artifact_digest, self.runtime_recipe_digest)
|
||||
|
||||
def routable_under(self, policy: str) -> bool:
|
||||
if self.proven:
|
||||
return True
|
||||
@@ -197,6 +229,10 @@ class CapabilityState:
|
||||
"recorded_at": self.recorded_at,
|
||||
"schema_version": self.schema_version,
|
||||
"diagnostics": list(self.diagnostics),
|
||||
"model_artifact_digest": self.model_artifact_digest,
|
||||
"runtime_recipe_digest": self.runtime_recipe_digest,
|
||||
"shard_binding_digest": self.shard_binding_digest,
|
||||
"certification": self.certification,
|
||||
}
|
||||
|
||||
|
||||
@@ -224,6 +260,7 @@ def evaluate_report(
|
||||
declared_recipe_version: str | None = None,
|
||||
now: float | None = None,
|
||||
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
|
||||
ledger: CertificationLedger | None = None,
|
||||
) -> CapabilityState:
|
||||
"""Judge the proof a node presented against what that node is advertising.
|
||||
|
||||
@@ -308,6 +345,84 @@ def evaluate_report(
|
||||
f"the node declared v{declared_recipe_version}",
|
||||
)
|
||||
|
||||
identity = None
|
||||
if report.get("identity") is not None:
|
||||
try:
|
||||
identity = parse_identity(report["identity"])
|
||||
except FingerprintMismatch as exc:
|
||||
return base.with_state(STATE_FINGERPRINT_MISMATCH, str(exc))
|
||||
except RecipeIdentityError as exc:
|
||||
return base.with_state(
|
||||
STATE_INVALID, f"capability identity block is unusable: {exc}"
|
||||
)
|
||||
|
||||
# The report's `shard` range is inclusive/inclusive (the CLI and backend
|
||||
# convention); the identity's is inclusive/exclusive (the protocol's, and
|
||||
# ADR-0012's). A node whose two halves disagree has not proven the range
|
||||
# it claims, whichever one is right.
|
||||
if (identity.shard_start, identity.shard_end) != (
|
||||
base.shard_start,
|
||||
(base.shard_end or 0) + 1,
|
||||
):
|
||||
return base.with_state(
|
||||
STATE_SHARD_MISMATCH,
|
||||
f"identity covers layers {identity.shard_start}–{identity.shard_end} "
|
||||
f"(end-exclusive), but the proof is for layers {base.shard_start}–"
|
||||
f"{base.shard_end} (end-inclusive)",
|
||||
)
|
||||
|
||||
if not model_matches(identity.artifact_id):
|
||||
return base.with_state(
|
||||
STATE_MODEL_MISMATCH,
|
||||
f"identity is for artifact {identity.artifact_id!r}, but the node "
|
||||
f"registered {advertised_model!r}",
|
||||
)
|
||||
model_claim = report["model"]
|
||||
if model_claim.get("revision") != identity.revision:
|
||||
return base.with_state(
|
||||
STATE_MODEL_MISMATCH,
|
||||
"identity revision does not match the capability proof",
|
||||
)
|
||||
config_fingerprint = model_claim.get("config_fingerprint")
|
||||
if isinstance(config_fingerprint, str) and config_fingerprint.startswith(
|
||||
"sha256:"
|
||||
):
|
||||
config_fingerprint = config_fingerprint.removeprefix("sha256:")
|
||||
if config_fingerprint != identity.architecture_digest:
|
||||
return base.with_state(
|
||||
STATE_MODEL_MISMATCH,
|
||||
"identity architecture/config digest does not match the capability proof",
|
||||
)
|
||||
if (
|
||||
identity.recipe_id != base.recipe_id
|
||||
or identity.recipe_version != base.recipe_version
|
||||
or identity.catalogue_version != base.catalogue_version
|
||||
):
|
||||
return base.with_state(
|
||||
STATE_RECIPE_MISMATCH,
|
||||
"identity recipe labels do not match the capability proof",
|
||||
)
|
||||
if identity.axes["backend_id"] != base.backend_id:
|
||||
return base.with_state(
|
||||
STATE_RECIPE_MISMATCH,
|
||||
"identity backend does not match the capability proof",
|
||||
)
|
||||
if (
|
||||
base.quantization is not None
|
||||
and identity.axes["weight_quantization"] != base.quantization
|
||||
):
|
||||
return base.with_state(
|
||||
STATE_RECIPE_MISMATCH,
|
||||
"identity weight quantization does not match the capability proof",
|
||||
)
|
||||
|
||||
base = replace(
|
||||
base,
|
||||
model_artifact_digest=identity.model_artifact_digest,
|
||||
runtime_recipe_digest=identity.runtime_recipe_digest,
|
||||
shard_binding_digest=identity.shard_binding_digest,
|
||||
)
|
||||
|
||||
if status != STATUS_PASSED:
|
||||
return base.with_state(
|
||||
STATE_FAILED,
|
||||
@@ -328,6 +443,29 @@ def evaluate_report(
|
||||
f"proof is timestamped {-age:.0f}s in the future; check the node's clock",
|
||||
)
|
||||
|
||||
# The digest only establishes that this report is self-consistent. It does
|
||||
# not authenticate a node or prove a distributed forward. Exact recipes are
|
||||
# therefore registered-but-dark until tracker-owned certification records
|
||||
# that forward.
|
||||
#
|
||||
# `ledger is None` means the caller owns no certification authority. It is
|
||||
# not "certify anything" and it is not "make one up": a disposable ledger
|
||||
# would register the recipe into state that is discarded on return, which
|
||||
# reads like certification is wired when nothing is recording it. The only
|
||||
# safe reading is that nothing here has certified this recipe, so it stays
|
||||
# dark. `TrackerServer` owns the real ledger and passes it in.
|
||||
if identity is not None:
|
||||
if ledger is None:
|
||||
return base.with_state(
|
||||
STATE_UNCERTIFIED,
|
||||
"no certification ledger; an exact recipe is dark until a "
|
||||
"tracker-owned distributed forward certifies it",
|
||||
)
|
||||
recipe_status = ledger.register(identity)
|
||||
base = replace(base, certification=recipe_status.status)
|
||||
if not recipe_status.may_serve:
|
||||
return base.with_state(STATE_UNCERTIFIED, recipe_status.detail)
|
||||
|
||||
return base.with_state(
|
||||
STATE_ADMITTED,
|
||||
f"{base.model_id} layers {base.shard_start}–{base.shard_end} proven on "
|
||||
|
||||
@@ -16,8 +16,8 @@ import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
556
packages/tracker/meshnet_tracker/recipe.py
Normal file
556
packages/tracker/meshnet_tracker/recipe.py
Normal file
@@ -0,0 +1,556 @@
|
||||
"""Tracker-side artifact and runtime recipe identity (DGR-003).
|
||||
|
||||
The node computes a compatibility fingerprint in `meshnet_node.runtime_recipe`.
|
||||
This module recomputes it, and the recomputation is the entire point.
|
||||
|
||||
A fingerprint that arrives on the wire is a **claim**. If the tracker stored
|
||||
what a node asserted, a node could assert the digest of a certified recipe while
|
||||
running an uncertified one — and admission, route selection, and the gRPC
|
||||
handshake would all wave it through, because they all compare the digest it
|
||||
handed them. So the tracker derives the digest from the *axes* the node
|
||||
declared, and refuses any report whose claim does not match the derivation. A
|
||||
node can lie about its axes, and a real forward will catch that; it cannot lie
|
||||
about the digest of the axes it declared.
|
||||
|
||||
This module deliberately does **not** import `meshnet_node`: the tracker package
|
||||
does not depend on the node package, and an admission gate that shares an
|
||||
implementation with the thing it admits is not an independent check. The two
|
||||
implementations are pinned together by a committed conformance vector
|
||||
(``tests/data/recipe_fingerprint_vectors.json``). If they drift, a test fails —
|
||||
rather than a route quietly failing to form, or worse, quietly forming.
|
||||
|
||||
Recipes arrive **dark**: registered, visible to an operator, and not routable for
|
||||
user traffic. Only a real distributed forward — at least two distinct nodes,
|
||||
covering the whole model, emitting real tokens — takes a recipe out of the dark
|
||||
(:class:`CertificationLedger`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
# Layout of the identity block this tracker reads (meshnet_node.runtime_recipe).
|
||||
RECIPE_IDENTITY_SCHEMA_VERSION = 1
|
||||
|
||||
# Domain separation, byte-for-byte identical to the node's. These strings are
|
||||
# part of the wire contract, not an implementation detail: changing one here
|
||||
# without changing it there silently partitions every route.
|
||||
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
|
||||
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
|
||||
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
|
||||
|
||||
# The axes a recipe digest commits to. Order is irrelevant (the canonical JSON
|
||||
# sorts keys); membership is not — an axis missing here is an axis the tracker
|
||||
# would let a node change without changing its identity.
|
||||
RECIPE_AXES: tuple[str, ...] = (
|
||||
"weight_quantization",
|
||||
"activation_dtype",
|
||||
"compute_dtype",
|
||||
"kv_dtype",
|
||||
"kv_layout",
|
||||
"tokenizer_revision",
|
||||
"architecture_adapter",
|
||||
"backend_id",
|
||||
"runtime_version",
|
||||
"boundary_schema_version",
|
||||
"protocol_schema_version",
|
||||
)
|
||||
|
||||
_INT_AXES = frozenset({"boundary_schema_version", "protocol_schema_version"})
|
||||
|
||||
STATUS_UNKNOWN = "unknown"
|
||||
STATUS_DARK = "dark"
|
||||
STATUS_CERTIFIED = "certified"
|
||||
|
||||
MIN_CERTIFYING_NODES = 2
|
||||
|
||||
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"})
|
||||
|
||||
|
||||
class RecipeIdentityError(ValueError):
|
||||
"""A presented identity block is malformed or internally inconsistent."""
|
||||
|
||||
|
||||
class FingerprintMismatch(RecipeIdentityError):
|
||||
"""A supplied digest is inconsistent with its identity declaration.
|
||||
|
||||
A digest is an integrity and compatibility claim, not an authenticated
|
||||
statement about what a node is actually executing. Distributed
|
||||
certification remains the trust boundary.
|
||||
"""
|
||||
|
||||
|
||||
def canonical_sha256(value: Any) -> str:
|
||||
payload = json.dumps(
|
||||
value, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _digest(domain: str, body: Mapping[str, Any]) -> str:
|
||||
return canonical_sha256({"domain": domain, "body": dict(body)})
|
||||
|
||||
|
||||
def _text(value: Any, what: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise RecipeIdentityError(f"{what!r} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _hex64(value: Any, what: str) -> str:
|
||||
text = _text(value, what)
|
||||
if not _HEX64.match(text):
|
||||
raise RecipeIdentityError(f"{what!r} must be a SHA-256 hex digest")
|
||||
return text
|
||||
|
||||
|
||||
def _integer(value: Any, what: str, minimum: int) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
raise RecipeIdentityError(f"{what!r} must be an integer")
|
||||
if value < minimum:
|
||||
raise RecipeIdentityError(f"{what!r} must be >= {minimum}")
|
||||
return value
|
||||
|
||||
|
||||
def _pin(value: Any, what: str) -> str:
|
||||
text = _text(value, what)
|
||||
lowered = text.strip().lower()
|
||||
if lowered in _MOVING_REFS or lowered.startswith("refs/"):
|
||||
raise RecipeIdentityError(
|
||||
f"{what!r} is a moving reference, not an exact revision pin"
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def _mapping(value: Any, what: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping):
|
||||
raise RecipeIdentityError(f"{what!r} must be a JSON object")
|
||||
return value
|
||||
|
||||
|
||||
def artifact_digest(
|
||||
*,
|
||||
source_digest: str,
|
||||
architecture: str,
|
||||
architecture_digest: str,
|
||||
layer_count: int,
|
||||
) -> str:
|
||||
"""The artifact half of the fingerprint, derived exactly as the node derives it."""
|
||||
return _digest(
|
||||
ARTIFACT_DIGEST_DOMAIN,
|
||||
{
|
||||
"source_digest": source_digest,
|
||||
"architecture": architecture,
|
||||
"architecture_digest": architecture_digest,
|
||||
"layer_count": layer_count,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def recipe_digest(axes: Mapping[str, Any]) -> str:
|
||||
"""The recipe half of the fingerprint, derived exactly as the node derives it."""
|
||||
missing = [axis for axis in RECIPE_AXES if axis not in axes]
|
||||
if missing:
|
||||
raise RecipeIdentityError(
|
||||
"recipe is missing required axes: " + ", ".join(missing)
|
||||
)
|
||||
return _digest(RECIPE_DIGEST_DOMAIN, {axis: axes[axis] for axis in RECIPE_AXES})
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PresentedIdentity:
|
||||
"""One node's declared artifact/recipe identity, with digests re-derived here.
|
||||
|
||||
`model_artifact_digest` and `runtime_recipe_digest` are computed by this
|
||||
tracker from the declared axes. They are never copied from the report.
|
||||
"""
|
||||
|
||||
artifact_id: str
|
||||
revision: str
|
||||
source_digest: str
|
||||
content_digest: str
|
||||
derivative_range: tuple[int, int] | None
|
||||
architecture: str
|
||||
architecture_digest: str
|
||||
layer_count: int
|
||||
is_derivative: bool
|
||||
shard_start: int
|
||||
shard_end: int
|
||||
axes: Mapping[str, Any]
|
||||
recipe_id: str
|
||||
recipe_version: str
|
||||
catalogue_version: str
|
||||
|
||||
@property
|
||||
def model_artifact_digest(self) -> str:
|
||||
return artifact_digest(
|
||||
source_digest=self.source_digest,
|
||||
architecture=self.architecture,
|
||||
architecture_digest=self.architecture_digest,
|
||||
layer_count=self.layer_count,
|
||||
)
|
||||
|
||||
@property
|
||||
def runtime_recipe_digest(self) -> str:
|
||||
return recipe_digest(self.axes)
|
||||
|
||||
@property
|
||||
def key(self) -> tuple[str, str]:
|
||||
return (self.model_artifact_digest, self.runtime_recipe_digest)
|
||||
|
||||
@property
|
||||
def shard_binding_digest(self) -> str:
|
||||
"""This participant's own bytes and exact range, bound to the source.
|
||||
|
||||
The route fingerprint (:attr:`key`) is deliberately range-independent —
|
||||
Shards on one route own different ranges, so a range-sensitive digest
|
||||
would stop any two of them from ever agreeing. The consequence is that
|
||||
the fingerprint alone cannot tell two *different splits of the same
|
||||
source* apart: a derivative can assert the right source and recipe and
|
||||
inherit certification earned by bytes it does not hold.
|
||||
|
||||
This digest is what closes that. It commits to the derivative's own
|
||||
content hash and its exact end-exclusive range, so certification can be
|
||||
recipe-wide while every serving node is still separately pinned to the
|
||||
blob it was admitted on (`TrackerServer.certify_recipe`).
|
||||
"""
|
||||
return _digest(
|
||||
SHARD_BINDING_DIGEST_DOMAIN,
|
||||
{
|
||||
"source_digest": self.source_digest,
|
||||
"content_digest": self.content_digest,
|
||||
"architecture_digest": self.architecture_digest,
|
||||
"derivative_range": list(self.derivative_range)
|
||||
if self.derivative_range is not None
|
||||
else None,
|
||||
"shard_start": self.shard_start,
|
||||
"shard_end": self.shard_end,
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def tokenizer_revision(self) -> str:
|
||||
return str(self.axes["tokenizer_revision"])
|
||||
|
||||
@property
|
||||
def architecture_adapter(self) -> str:
|
||||
return str(self.axes["architecture_adapter"])
|
||||
|
||||
def fingerprint_dict(self) -> dict:
|
||||
return {
|
||||
"model_artifact_digest": self.model_artifact_digest,
|
||||
"runtime_recipe_digest": self.runtime_recipe_digest,
|
||||
"recipe_id": self.recipe_id,
|
||||
"recipe_version": self.recipe_version,
|
||||
"catalogue_version": self.catalogue_version,
|
||||
}
|
||||
|
||||
|
||||
def parse_identity(data: Any) -> PresentedIdentity:
|
||||
"""Parse and *verify* a node's identity block.
|
||||
|
||||
Raises when the block is malformed, when a split artifact does not name the
|
||||
exact source it was cut from, when a Shard advertises layers its artifact
|
||||
does not contain, or when the declared fingerprint does not match the digest
|
||||
of the axes it was declared with.
|
||||
"""
|
||||
doc = _mapping(data, "identity")
|
||||
|
||||
schema_version = doc.get("schema_version")
|
||||
if schema_version != RECIPE_IDENTITY_SCHEMA_VERSION:
|
||||
raise RecipeIdentityError(
|
||||
f"identity block declares schema version {schema_version!r}; this tracker "
|
||||
f"reads version {RECIPE_IDENTITY_SCHEMA_VERSION}"
|
||||
)
|
||||
|
||||
artifact = _mapping(doc.get("artifact"), "identity.artifact")
|
||||
recipe = _mapping(doc.get("recipe"), "identity.recipe")
|
||||
|
||||
layer_count = _integer(artifact.get("layer_count"), "artifact.layer_count", 1)
|
||||
content_digest = _hex64(artifact.get("content_digest"), "artifact.content_digest")
|
||||
|
||||
raw_binding = artifact.get("derived_from")
|
||||
if raw_binding is None:
|
||||
source_digest = content_digest
|
||||
binding: tuple[int, int] | None = None
|
||||
else:
|
||||
derived = _mapping(raw_binding, "artifact.derived_from")
|
||||
source_digest = _hex64(
|
||||
derived.get("source_artifact_digest"),
|
||||
"artifact.derived_from.source_artifact_digest",
|
||||
)
|
||||
binding = (
|
||||
_integer(derived.get("shard_start"), "derived_from.shard_start", 0),
|
||||
_integer(derived.get("shard_end"), "derived_from.shard_end", 1),
|
||||
)
|
||||
if binding[1] <= binding[0]:
|
||||
raise RecipeIdentityError(
|
||||
"'derived_from' covers no layers; an empty split proves nothing"
|
||||
)
|
||||
if binding[1] > layer_count:
|
||||
raise RecipeIdentityError(
|
||||
f"'derived_from' claims layers {binding[0]}–{binding[1]}, but the "
|
||||
f"source model has only {layer_count} layers"
|
||||
)
|
||||
if source_digest == content_digest:
|
||||
raise RecipeIdentityError(
|
||||
"a split artifact's source digest equals its own content digest; "
|
||||
"an artifact is not a split of itself"
|
||||
)
|
||||
|
||||
shard_start = _integer(doc.get("shard_start"), "shard_start", 0)
|
||||
shard_end = _integer(doc.get("shard_end"), "shard_end", 1)
|
||||
if shard_end <= shard_start:
|
||||
raise RecipeIdentityError("a Shard owning no layer computes nothing")
|
||||
if shard_end > layer_count:
|
||||
raise RecipeIdentityError(
|
||||
f"Shard owns layers {shard_start}–{shard_end}, but the artifact has only "
|
||||
f"{layer_count} layers"
|
||||
)
|
||||
if binding is not None and not (
|
||||
binding[0] <= shard_start and shard_end <= binding[1]
|
||||
):
|
||||
raise RecipeIdentityError(
|
||||
f"Shard advertises layers {shard_start}–{shard_end}, but its split "
|
||||
f"artifact only contains layers {binding[0]}–{binding[1]}"
|
||||
)
|
||||
|
||||
axes: dict[str, Any] = {}
|
||||
for axis in RECIPE_AXES:
|
||||
if axis not in recipe:
|
||||
raise RecipeIdentityError(
|
||||
f"recipe is missing axis {axis!r}; an unstated axis cannot default"
|
||||
)
|
||||
value = recipe[axis]
|
||||
if axis in _INT_AXES:
|
||||
axes[axis] = _integer(value, f"recipe.{axis}", 1)
|
||||
else:
|
||||
axes[axis] = _text(value, f"recipe.{axis}")
|
||||
_pin(axes["tokenizer_revision"], "recipe.tokenizer_revision")
|
||||
|
||||
identity = PresentedIdentity(
|
||||
artifact_id=_text(artifact.get("artifact_id"), "artifact.artifact_id"),
|
||||
revision=_pin(artifact.get("revision"), "artifact.revision"),
|
||||
source_digest=source_digest,
|
||||
content_digest=content_digest,
|
||||
derivative_range=binding,
|
||||
architecture=_text(artifact.get("architecture"), "artifact.architecture"),
|
||||
architecture_digest=_hex64(
|
||||
artifact.get("architecture_digest"), "artifact.architecture_digest"
|
||||
),
|
||||
layer_count=layer_count,
|
||||
is_derivative=binding is not None,
|
||||
shard_start=shard_start,
|
||||
shard_end=shard_end,
|
||||
axes=axes,
|
||||
recipe_id=_text(recipe.get("recipe_id"), "recipe.recipe_id"),
|
||||
recipe_version=_text(recipe.get("recipe_version"), "recipe.recipe_version"),
|
||||
catalogue_version=_text(
|
||||
recipe.get("catalogue_version"), "recipe.catalogue_version"
|
||||
),
|
||||
)
|
||||
|
||||
declared = doc.get("fingerprint")
|
||||
if declared is not None:
|
||||
claim = _mapping(declared, "identity.fingerprint")
|
||||
claimed_artifact = _hex64(
|
||||
claim.get("model_artifact_digest"), "fingerprint.model_artifact_digest"
|
||||
)
|
||||
claimed_recipe = _hex64(
|
||||
claim.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest"
|
||||
)
|
||||
if (claimed_artifact, claimed_recipe) != identity.key:
|
||||
raise FingerprintMismatch(
|
||||
"declared fingerprint is inconsistent with the artifact and recipe "
|
||||
"claim; the tracker recomputes compatibility digests"
|
||||
)
|
||||
|
||||
return identity
|
||||
|
||||
|
||||
def coverage_gap(
|
||||
ranges: Iterable[tuple[int, int]], layer_count: int
|
||||
) -> str | None:
|
||||
"""Why `ranges` fail to tile ``[0, layer_count)``, or None when they do.
|
||||
|
||||
Overlaps are legal — ADR-0012 lets the Tracker resolve one by telling a hop
|
||||
where the previous hop stopped. A hole is not: its layers are never computed,
|
||||
and the route emits fluent tokens from a truncated model.
|
||||
"""
|
||||
ordered = sorted(ranges)
|
||||
if not ordered:
|
||||
return "the route owns no layers"
|
||||
if ordered[0][0] != 0:
|
||||
return f"layers 0–{ordered[0][0]} are owned by no Shard on the route"
|
||||
|
||||
covered = 0
|
||||
for start, end in ordered:
|
||||
if start > covered:
|
||||
return f"layers {covered}–{start} are owned by no Shard on the route"
|
||||
covered = max(covered, end)
|
||||
|
||||
if covered < layer_count:
|
||||
return f"layers {covered}–{layer_count} are owned by no Shard on the route"
|
||||
if covered > layer_count:
|
||||
return (
|
||||
f"the route claims {covered} layers, but the model has only {layer_count}"
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DistributedForwardEvidence:
|
||||
"""A real distributed forward — the only thing that certifies a recipe."""
|
||||
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
node_ids: tuple[str, ...]
|
||||
shard_ranges: tuple[tuple[int, int], ...]
|
||||
tokens_generated: int
|
||||
layer_count: int
|
||||
fingerprint: tuple[str, str]
|
||||
participants: tuple[PresentedIdentity, ...]
|
||||
synthetic: bool = False
|
||||
certified_at: float = field(default_factory=time.time)
|
||||
|
||||
def rejection(self) -> str | None:
|
||||
if self.synthetic:
|
||||
return (
|
||||
"evidence comes from a synthetic worker; only a real distributed "
|
||||
"forward certifies a recipe"
|
||||
)
|
||||
if len(self.participants) != len(self.node_ids):
|
||||
return "participant identities do not match the certifying node list"
|
||||
if len(self.shard_ranges) != len(self.participants):
|
||||
return "participant identities do not match the recorded effective ranges"
|
||||
for participant, shard_range in zip(
|
||||
self.participants, self.shard_ranges, strict=True
|
||||
):
|
||||
if participant.key != self.fingerprint:
|
||||
return "a participant fingerprint differs from the certifying route"
|
||||
if participant.layer_count != self.layer_count:
|
||||
return "a participant artifact layer count differs from the certifying route"
|
||||
if (participant.shard_start, participant.shard_end) != shard_range:
|
||||
return "a participant identity does not match its recorded effective range"
|
||||
distinct = set(self.node_ids)
|
||||
if len(distinct) < MIN_CERTIFYING_NODES:
|
||||
return (
|
||||
f"evidence covers {len(distinct)} distinct node(s); a distributed "
|
||||
f"forward requires at least {MIN_CERTIFYING_NODES}"
|
||||
)
|
||||
if len(distinct) != len(self.node_ids):
|
||||
return "the same node is counted more than once in the certifying route"
|
||||
if self.tokens_generated < 1:
|
||||
return "the certifying forward generated no tokens"
|
||||
return coverage_gap(self.shard_ranges, self.layer_count)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecipeStatus:
|
||||
status: str
|
||||
detail: str = ""
|
||||
certified_at: float | None = None
|
||||
|
||||
@property
|
||||
def may_serve(self) -> bool:
|
||||
return self.status == STATUS_CERTIFIED
|
||||
|
||||
@property
|
||||
def may_certify(self) -> bool:
|
||||
"""A dark recipe may be routed to *certify* it, and for nothing else.
|
||||
|
||||
Without this, certification is unreachable by construction: serving needs
|
||||
certification, certification needs a real distributed forward, and a real
|
||||
distributed forward needs a route.
|
||||
"""
|
||||
return self.status in (STATUS_DARK, STATUS_CERTIFIED)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"status": self.status,
|
||||
"detail": self.detail,
|
||||
"certified_at": self.certified_at,
|
||||
}
|
||||
|
||||
|
||||
_DARK_DETAIL = "registered; dark until a real distributed forward certifies it"
|
||||
_UNKNOWN_DETAIL = "this recipe has never been registered with the tracker"
|
||||
|
||||
|
||||
class CertificationLedger:
|
||||
"""Which recipes the tracker has seen, and which a real forward has proven."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._dark: set[tuple[str, str]] = set()
|
||||
self._certified: dict[tuple[str, str], RecipeStatus] = {}
|
||||
|
||||
def register(self, identity: PresentedIdentity) -> RecipeStatus:
|
||||
key = identity.key
|
||||
if key in self._certified:
|
||||
return self._certified[key]
|
||||
self._dark.add(key)
|
||||
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
|
||||
|
||||
def status(self, identity: PresentedIdentity | tuple[str, str]) -> RecipeStatus:
|
||||
key = identity if isinstance(identity, tuple) else identity.key
|
||||
if key in self._certified:
|
||||
return self._certified[key]
|
||||
if key in self._dark:
|
||||
return RecipeStatus(STATUS_DARK, _DARK_DETAIL)
|
||||
return RecipeStatus(STATUS_UNKNOWN, _UNKNOWN_DETAIL)
|
||||
|
||||
def certify(
|
||||
self,
|
||||
identity: PresentedIdentity,
|
||||
evidence: DistributedForwardEvidence,
|
||||
) -> RecipeStatus:
|
||||
"""Promote a recipe out of the dark, or raise saying why the evidence is short."""
|
||||
key = identity.key
|
||||
if key not in self._dark and key not in self._certified:
|
||||
raise RecipeIdentityError(
|
||||
"this fingerprint is not registered; unknown recipes cannot be certified"
|
||||
)
|
||||
if evidence.fingerprint != key:
|
||||
raise RecipeIdentityError(
|
||||
"certification evidence fingerprint does not match the recipe being promoted"
|
||||
)
|
||||
if evidence.layer_count != identity.layer_count:
|
||||
raise RecipeIdentityError(
|
||||
"certification evidence layer count does not match the artifact being promoted"
|
||||
)
|
||||
rejection = evidence.rejection()
|
||||
if rejection is not None:
|
||||
raise RecipeIdentityError(f"this evidence does not certify: {rejection}")
|
||||
status = RecipeStatus(
|
||||
STATUS_CERTIFIED,
|
||||
(
|
||||
f"certified by route session {evidence.route_session_id} across "
|
||||
f"{len(set(evidence.node_ids))} nodes"
|
||||
),
|
||||
certified_at=evidence.certified_at,
|
||||
)
|
||||
self._certified[key] = status
|
||||
self._dark.discard(key)
|
||||
return status
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"dark": [list(key) for key in sorted(self._dark)],
|
||||
"certified": {
|
||||
"/".join(key): status.to_dict()
|
||||
for key, status in sorted(self._certified.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Route formation itself lives in `server.py` (`_select_route`, `_enumerate_routes`,
|
||||
# `_find_pinned_route`): candidates are partitioned by the exact fingerprint this
|
||||
# module derives, so a route can only ever be assembled inside one identity. A
|
||||
# second route gate here would be a second copy of that policy to keep in step —
|
||||
# the same reason the node module holds no certification authority.
|
||||
@@ -26,7 +26,7 @@ import random
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -53,13 +53,10 @@ from typing import Any
|
||||
from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
|
||||
from .auth import is_validator_token, sign_hive_request, verify_hive_request
|
||||
from .capability import (
|
||||
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
|
||||
POLICY_COMPAT,
|
||||
POLICY_ENFORCE,
|
||||
STATE_ABSENT,
|
||||
STATE_ADMITTED,
|
||||
STATE_MODEL_MISMATCH,
|
||||
STATE_SHARD_MISMATCH,
|
||||
STATE_UNCERTIFIED,
|
||||
CapabilityState,
|
||||
absent_state,
|
||||
evaluate_report,
|
||||
@@ -68,7 +65,7 @@ from .capability import (
|
||||
)
|
||||
from .wallet_proof import binding_message, verify_wallet_signature
|
||||
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
|
||||
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
||||
from .calibration import ToplocCalibrationStore
|
||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||||
from .gossip import NodeGossip
|
||||
from .logging_setup import tracker_logger
|
||||
@@ -84,6 +81,13 @@ from .routing_stats import (
|
||||
)
|
||||
from .model_files import files_for_layer_range, snapshot_dir_for_repo
|
||||
from .raft import RaftNode
|
||||
from .recipe import (
|
||||
CertificationLedger,
|
||||
DistributedForwardEvidence,
|
||||
PresentedIdentity,
|
||||
RecipeIdentityError,
|
||||
RecipeStatus,
|
||||
)
|
||||
|
||||
|
||||
_CONSOLE_LIMIT = 1000
|
||||
@@ -792,6 +796,7 @@ def _capability_from_registration(
|
||||
hf_repo: str | None,
|
||||
shard_start: int | None,
|
||||
shard_end: int | None,
|
||||
recipe_certifications: CertificationLedger,
|
||||
) -> CapabilityState:
|
||||
"""The tracker's verdict on the proof carried by one registration payload.
|
||||
|
||||
@@ -811,6 +816,7 @@ def _capability_from_registration(
|
||||
declared_recipe_version=(
|
||||
recipe_version if isinstance(recipe_version, str) else None
|
||||
),
|
||||
ledger=recipe_certifications,
|
||||
)
|
||||
|
||||
|
||||
@@ -830,6 +836,11 @@ def _admitted_nodes(nodes: list["_NodeEntry"], policy: str | None) -> list["_Nod
|
||||
return [node for node in nodes if _capability_routable(node, effective)]
|
||||
|
||||
|
||||
def _route_identity_partition(node: "_NodeEntry") -> tuple[str, ...]:
|
||||
fingerprint = _node_admission(node).fingerprint
|
||||
return ("legacy",) if fingerprint is None else ("exact", *fingerprint)
|
||||
|
||||
|
||||
def _select_route(
|
||||
nodes: list[_NodeEntry],
|
||||
required_start: int,
|
||||
@@ -853,29 +864,51 @@ def _select_route(
|
||||
],
|
||||
key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
|
||||
)
|
||||
route: list[_NodeEntry] = []
|
||||
covered_up_to = required_start - 1
|
||||
|
||||
def _routing_score(node: "_NodeEntry") -> float:
|
||||
return _effective_throughput(node, model) * _reputation_multiplier(node, contracts)
|
||||
|
||||
while covered_up_to < required_end:
|
||||
best: _NodeEntry | None = None
|
||||
for node in candidates:
|
||||
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
|
||||
if best is None:
|
||||
best = node
|
||||
elif node.shard_end > best.shard_end:
|
||||
best = node
|
||||
elif node.shard_end == best.shard_end and _routing_score(node) > _routing_score(best):
|
||||
best = node
|
||||
if best is None:
|
||||
missing = covered_up_to + 1
|
||||
return [], f"no route available: no registered node covers layer {missing}"
|
||||
route.append(best)
|
||||
covered_up_to = best.shard_end
|
||||
candidates = [n for n in candidates if n is not best]
|
||||
partitions: dict[tuple[str, ...], list[_NodeEntry]] = {}
|
||||
for node in candidates:
|
||||
partitions.setdefault(_route_identity_partition(node), []).append(node)
|
||||
|
||||
complete: list[list[_NodeEntry]] = []
|
||||
furthest = required_start - 1
|
||||
for partition in partitions.values():
|
||||
pool = list(partition)
|
||||
route: list[_NodeEntry] = []
|
||||
covered_up_to = required_start - 1
|
||||
while covered_up_to < required_end:
|
||||
best: _NodeEntry | None = None
|
||||
for node in pool:
|
||||
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
|
||||
if best is None:
|
||||
best = node
|
||||
elif node.shard_end > best.shard_end:
|
||||
best = node
|
||||
elif (
|
||||
node.shard_end == best.shard_end
|
||||
and _routing_score(node) > _routing_score(best)
|
||||
):
|
||||
best = node
|
||||
if best is None:
|
||||
break
|
||||
route.append(best)
|
||||
covered_up_to = best.shard_end
|
||||
pool = [node for node in pool if node is not best]
|
||||
furthest = max(furthest, covered_up_to)
|
||||
if covered_up_to >= required_end:
|
||||
complete.append(route)
|
||||
|
||||
if not complete:
|
||||
return [], f"no route available: no registered node covers layer {furthest + 1}"
|
||||
route = max(
|
||||
complete,
|
||||
key=lambda candidate: (
|
||||
min(_routing_score(node) for node in candidate),
|
||||
-len(candidate),
|
||||
),
|
||||
)
|
||||
return route, ""
|
||||
|
||||
|
||||
@@ -911,7 +944,12 @@ def _enumerate_routes(
|
||||
for head in heads:
|
||||
route = [head]
|
||||
covered_up_to = head.shard_end
|
||||
pool = [n for n in sharded if n is not head]
|
||||
head_partition = _route_identity_partition(head)
|
||||
pool = [
|
||||
node
|
||||
for node in sharded
|
||||
if node is not head and _route_identity_partition(node) == head_partition
|
||||
]
|
||||
while covered_up_to < required_end:
|
||||
best = None
|
||||
for n in pool:
|
||||
@@ -2126,14 +2164,24 @@ def _find_pinned_route(
|
||||
hop_count: int,
|
||||
) -> list[_NodeEntry] | None:
|
||||
"""First combination of exactly ``hop_count`` distinct nodes covering the
|
||||
layer range, where every node extends coverage (US-030 benchmark routes)."""
|
||||
layer range, where every node extends coverage (US-030 benchmark routes).
|
||||
|
||||
Benchmark combos run real inference, so they obey the same DGR-003 rule as
|
||||
every other route builder: one route, one exact identity. A combo that mixed
|
||||
two fingerprints — or an exact Shard with a legacy one — would measure a
|
||||
numerically incoherent route and record the garbage as a benchmark.
|
||||
"""
|
||||
for combo in itertools.permutations(nodes, hop_count):
|
||||
covered = required_start - 1
|
||||
valid = True
|
||||
partition = _route_identity_partition(combo[0])
|
||||
for candidate in combo:
|
||||
if candidate.shard_start is None or candidate.shard_end is None:
|
||||
valid = False
|
||||
break
|
||||
if _route_identity_partition(candidate) != partition:
|
||||
valid = False
|
||||
break
|
||||
if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
|
||||
valid = False
|
||||
break
|
||||
@@ -2584,8 +2632,8 @@ def _estimate_prompt_tokens(body: dict) -> int | None:
|
||||
|
||||
|
||||
def _requested_completion_token_limit(body: dict) -> int | None:
|
||||
for field in ("max_completion_tokens", "max_tokens"):
|
||||
value = body.get(field)
|
||||
for key in ("max_completion_tokens", "max_tokens"):
|
||||
value = body.get(key)
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
@@ -2917,9 +2965,11 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
relay_status: dict | None = None,
|
||||
test_runner: "TestRunManager | None" = None,
|
||||
capability_policy: str | None = None,
|
||||
recipe_certifications: CertificationLedger | None = None,
|
||||
) -> None:
|
||||
super().__init__(addr, handler)
|
||||
self.registry = registry
|
||||
self.recipe_certifications = recipe_certifications or CertificationLedger()
|
||||
self.capability_policy = normalize_policy(
|
||||
capability_policy if capability_policy is not None else policy_from_env()
|
||||
)
|
||||
@@ -2964,7 +3014,7 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||||
|
||||
|
||||
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
||||
def log_message(self, fmt, *args): # suppress request logs in tests
|
||||
pass
|
||||
|
||||
def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
|
||||
@@ -4646,6 +4696,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
hf_repo=hf_repo,
|
||||
shard_start=shard_start,
|
||||
shard_end=shard_end,
|
||||
recipe_certifications=server.recipe_certifications,
|
||||
)
|
||||
|
||||
node_id = _node_id_for_registration(
|
||||
@@ -6690,6 +6741,7 @@ class TrackerServer:
|
||||
self._embedded_relay: Any | None = None
|
||||
self._embedded_relay_actual_port: int | None = None
|
||||
self._registry: dict[str, _NodeEntry] = {}
|
||||
self._recipe_certifications = CertificationLedger()
|
||||
self._lock = threading.Lock()
|
||||
self._server: _TrackerHTTPServer | None = None
|
||||
self._thread: threading.Thread | None = None
|
||||
@@ -6799,6 +6851,84 @@ class TrackerServer:
|
||||
self._test_runner: TestRunManager | None = test_runner
|
||||
self.port: int | None = None
|
||||
|
||||
def certify_recipe(
|
||||
self,
|
||||
identity: PresentedIdentity,
|
||||
evidence: DistributedForwardEvidence,
|
||||
) -> RecipeStatus:
|
||||
"""Promote one exact recipe from tracker-recorded distributed evidence.
|
||||
|
||||
This is the tracker's only certification authority. The evidence names
|
||||
the nodes that served the forward; this method refuses to take any of
|
||||
them on the evidence's word. For each one it goes back to what *this
|
||||
tracker* re-derived at admission and requires the evidence to describe
|
||||
the same Shard: same route fingerprint, same registered range, and the
|
||||
same shard binding digest — the derivative's own bytes and exact range.
|
||||
|
||||
Without the binding check, a route fingerprint is range-independent by
|
||||
design (:mod:`meshnet_tracker.recipe`), so evidence could name the right
|
||||
recipe while describing splits nobody on the route was actually serving,
|
||||
and the certification would attach to blobs that never ran.
|
||||
"""
|
||||
with self._lock:
|
||||
for node_id, participant in zip(
|
||||
evidence.node_ids, evidence.participants, strict=True
|
||||
):
|
||||
node = self._registry.get(node_id)
|
||||
if node is None:
|
||||
raise RecipeIdentityError(
|
||||
f"certification participant {node_id!r} is not registered"
|
||||
)
|
||||
admitted = node.capability
|
||||
if admitted.fingerprint != identity.key:
|
||||
raise RecipeIdentityError(
|
||||
f"certification participant {node_id!r} is not admitted under "
|
||||
"the promoted fingerprint"
|
||||
)
|
||||
if admitted.state not in (STATE_UNCERTIFIED, STATE_ADMITTED):
|
||||
raise RecipeIdentityError(
|
||||
f"certification participant {node_id!r} has capability state "
|
||||
f"{admitted.state!r}"
|
||||
)
|
||||
registered_range = (
|
||||
node.shard_start,
|
||||
None if node.shard_end is None else node.shard_end + 1,
|
||||
)
|
||||
if registered_range != (
|
||||
participant.shard_start,
|
||||
participant.shard_end,
|
||||
):
|
||||
raise RecipeIdentityError(
|
||||
f"certification participant {node_id!r} range differs from its "
|
||||
"registered Shard range"
|
||||
)
|
||||
# The exact bytes and range this node was admitted on. A node that
|
||||
# registered without an identity has no binding, and cannot be a
|
||||
# participant in an exact certification at all.
|
||||
if admitted.shard_binding_digest is None:
|
||||
raise RecipeIdentityError(
|
||||
f"certification participant {node_id!r} registered no exact "
|
||||
"Shard binding; it cannot certify a recipe"
|
||||
)
|
||||
if admitted.shard_binding_digest != participant.shard_binding_digest:
|
||||
raise RecipeIdentityError(
|
||||
f"certification participant {node_id!r} presents a Shard "
|
||||
"binding this tracker did not admit it on; the evidence "
|
||||
"describes different derivative bytes or a different range"
|
||||
)
|
||||
|
||||
status = self._recipe_certifications.certify(identity, evidence)
|
||||
for node in self._registry.values():
|
||||
if (
|
||||
node.capability.state == STATE_UNCERTIFIED
|
||||
and node.capability.fingerprint == identity.key
|
||||
):
|
||||
node.capability = replace(
|
||||
node.capability.with_state(STATE_ADMITTED, status.detail),
|
||||
certification=status.status,
|
||||
)
|
||||
return status
|
||||
|
||||
def _start_embedded_relay(self) -> dict:
|
||||
"""Start the shared RelayServer class in-process for tracker+relay deployments."""
|
||||
if not self._embedded_relay_enabled:
|
||||
@@ -6884,6 +7014,7 @@ class TrackerServer:
|
||||
relay_status=http_relay_status,
|
||||
test_runner=self._test_runner,
|
||||
capability_policy=self._capability_policy,
|
||||
recipe_certifications=self._recipe_certifications,
|
||||
)
|
||||
self.port = self._server.server_address[1]
|
||||
|
||||
@@ -7093,10 +7224,6 @@ class TrackerServer:
|
||||
shard_end = int(payload["shard_end"]) if payload.get("shard_end") is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
try:
|
||||
friendly_name = _normalize_friendly_name(payload.get("friendly_name"))
|
||||
except ValueError:
|
||||
friendly_name = None
|
||||
# The replicated payload is the raw registration body, so the follower can
|
||||
# resolve precision exactly as the leader did -- including telling a legacy
|
||||
# absent `quantization` from a declared one. Dropping these fields here
|
||||
@@ -7142,6 +7269,7 @@ class TrackerServer:
|
||||
hf_repo=payload.get("hf_repo"),
|
||||
shard_start=shard_start,
|
||||
shard_end=shard_end,
|
||||
recipe_certifications=self._recipe_certifications,
|
||||
),
|
||||
)
|
||||
with self._lock:
|
||||
|
||||
Reference in New Issue
Block a user