From ad2d17541ccc1f4d4829b0f3a94d5f3b0394eeb8 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 00:31:14 +0300 Subject: [PATCH] feat: DGR-003 - Define exact Artifact and runtime recipe identity --- .../evidence/DGR-003/README.md | 119 ++ .../evidence/DGR-003/commands.txt | 34 + ...ct-artifact-and-runtime-recipe-identity.md | 32 +- .scratch/distributed-gguf-runtime/prd.json | 2 +- packages/node/meshnet_node/capability.py | 33 +- packages/node/meshnet_node/runtime_recipe.py | 1147 +++++++++++++++++ .../tracker/meshnet_tracker/capability.py | 102 ++ packages/tracker/meshnet_tracker/recipe.py | 615 +++++++++ packages/tracker/meshnet_tracker/server.py | 9 + tests/data/recipe_fingerprint_vectors.json | 52 + tests/test_runtime_recipe_identity.py | 270 ++++ 11 files changed, 2395 insertions(+), 20 deletions(-) create mode 100644 .scratch/distributed-gguf-runtime/evidence/DGR-003/README.md create mode 100644 .scratch/distributed-gguf-runtime/evidence/DGR-003/commands.txt create mode 100644 packages/node/meshnet_node/runtime_recipe.py create mode 100644 packages/tracker/meshnet_tracker/recipe.py create mode 100644 tests/data/recipe_fingerprint_vectors.json create mode 100644 tests/test_runtime_recipe_identity.py diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-003/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-003/README.md new file mode 100644 index 0000000..9a6fc7d --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-003/README.md @@ -0,0 +1,119 @@ +# DGR-003 — exact Artifact and runtime recipe identity + +Evidence class: deterministic offline/unit. No model payload, GPU, external API, +network node, or API credit is required or claimed. + +## Result + +DGR-003 defines an exact, model-agnostic compatibility identity and connects it +to both DGR-002's gRPC `Fingerprint` and tracker capability admission. + +A matching digest proves canonical consistency, **not node authenticity or real +execution**. Tracker-owned certification of a fingerprint by a non-synthetic, +complete, multi-node distributed forward is the execution trust boundary. + +## Implementation + +- `ArtifactIdentity` binds artifact ID/revision, exact content digest, + architecture/config digest, layer count, and optional derivative binding. +- `DerivativeBinding` binds a split artifact to the exact source artifact digest + and its end-exclusive layer range. A Shard cannot advertise outside that range. +- `RuntimeRecipe` keeps these canonical axes separate rather than hiding them in + a backend label: + - weight quantization; + - activation and compute dtypes; + - KV dtype and layout; + - tokenizer revision; + - architecture adapter; + - backend and runtime version; + - boundary and protocol schema versions; + - recipe ID/version and catalogue version. +- `CompatibilityFingerprint` populates the existing DGR-002 Protobuf + `Fingerprint`; `check_handshake()` returns DGR-002's structured fingerprint + mismatch error. +- Node and tracker implementations independently canonicalize the declaration. + This is intentional: the tracker must not trust a digest copied from a node, + and future native/C++ workers also need an independent implementation. Their + behavior is pinned by `tests/data/recipe_fingerprint_vectors.json`. +- Tracker admission cross-checks the exact identity against the capability + proof's model, range, recipe labels, backend, and weight quantization. Any + disagreement fails closed. +- `TrackerServer` owns one certification ledger and passes it through direct and + replicated registration paths. A known exact recipe is `uncertified` and dark + for user traffic until the same exact fingerprint is certified. +- Certification evidence is bound to the promoted fingerprint, requires at + least two distinct nodes, complete layer coverage, generated tokens, and + `synthetic=false`. Unknown or mismatched fingerprints cannot be promoted. + +## Files changed + +- `packages/node/meshnet_node/runtime_recipe.py` +- `packages/node/meshnet_node/capability.py` +- `packages/tracker/meshnet_tracker/recipe.py` +- `packages/tracker/meshnet_tracker/capability.py` +- `packages/tracker/meshnet_tracker/server.py` +- `tests/data/recipe_fingerprint_vectors.json` +- `tests/test_runtime_recipe_identity.py` +- this evidence directory, issue state, and DGR-003 PRD state + +A late review of dependency DGR-017 also found and fixed two genuine contract +continuity defects before DGR-003 was accepted: v1 now has an independently +trusted digest and recursively immutable parsed state. Those changes and tests +are recorded in DGR-017 evidence rather than claimed as DGR-003 functionality. + +## Verification + +Exact commands and outcomes are in `commands.txt`. + +Observed final results: + +- DGR-003 identity + node/tracker capability suites: **99 passed**. +- DGR-017 focused dependency repair suite: **99 passed**. +- Full deterministic suite: **872 passed, 13 skipped**. +- `python -m compileall -q packages tests`: pass. +- `git diff --check`: pass. +- Ruff on the changed identity, capability, contract, and test modules: pass. +- `server.py` has 8 pre-existing Ruff findings at both pushed baseline and the + current tree; DGR-003 added no finding. + +The first integrated full-suite run produced **871 passed, 13 skipped, 1 failed** +on the known unrelated +`test_tracker_dashboard_can_cancel_inflight_proxy` timing race. Its fixture +completed after three seconds just before cancellation, so the cancel endpoint +returned 404. The same test then passed **5/5 in isolation**, and the complete +integrated rerun passed **872/872** tests. No cancellation-test code was changed. + +## Limitations + +- Certification state is process-local in this story. The same running tracker + reuses it across registrations, but durable/cluster-wide certification-event + persistence belongs with the later real distributed-forward control path. + Restart or failover therefore returns exact recipes to the safe dark state; + it never makes an unsupported recipe routable. +- The node module still contains a local registry helper from the interrupted + partial implementation. It has no call sites and is not used by admission; + tracker remains the live certification authority. Removing that unpushed + helper is safe cleanup, not an acceptance dependency. +- This story proves identity and admission behavior with deterministic fixtures. + It does not claim a real GLM forward or hardware certification. + +## Compatibility + +- Capability report identity is additive. Legacy reports without the new block + retain ADR-0023's explicit compatibility-policy behavior. +- Reports that opt into exact identity are held to it and fail closed on malformed, + inconsistent, unknown, dark, or mismatched declarations. +- No new wire identity was invented; DGR-002's `Fingerprint` remains the gRPC + representation. + +## Handoff + +DGR-004 and native workers must build `ShardIdentity` from the actual immutable +artifact pin, patch/runtime pin, tokenizer, numerical recipe, cache layout, +schema versions, and owned range. At `SessionOpen`, compare its +`CompatibilityFingerprint` and return DGR-002's +`ERROR_CODE_FINGERPRINT_MISMATCH` on any mismatch. + +A digest match is not certification. Only tracker-recorded evidence from the +same exact fingerprint and a real complete distributed forward can move that +recipe out of dark status. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-003/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-003/commands.txt new file mode 100644 index 0000000..f09d998 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-003/commands.txt @@ -0,0 +1,34 @@ +# DGR-003 final verification — 2026-07-14 + +PYTHONPATH=packages/node:packages/tracker:packages/contracts /run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python -m pytest -q tests/test_runtime_recipe_identity.py tests/test_node_capability.py tests/test_tracker_capability_admission.py +# result: 99 passed in 4.76s + +PYTHONPATH=packages/node /run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python -m pytest -q tests/test_glm_alpha_target.py +# result: 99 passed in 0.15s + +/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python -m pytest -q +# first integrated result: 871 passed, 13 skipped, 1 failed in 258.18s +# sole failure: tests/test_tracker_routing.py::test_tracker_dashboard_can_cancel_inflight_proxy +# fixture completed at ~3s before cancellation; cancel endpoint returned 404 + +for i in 1 2 3 4 5; do + /run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python -m pytest -q tests/test_tracker_routing.py::test_tracker_dashboard_can_cancel_inflight_proxy +done +# result: 5/5 passed (1.14s, 1.14s, 1.26s, 1.14s, 1.64s) + +/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python -m pytest -q +# final integrated result: 872 passed, 13 skipped in 253.46s + +/run/media/popov/d/DEV/repos/d-popov.com/AI/.venv/bin/python -m compileall -q packages tests +# result: pass + +git diff --check +# result: pass + +ruff check packages/node/meshnet_node/glm_alpha/contract.py packages/node/meshnet_node/runtime_recipe.py packages/tracker/meshnet_tracker/recipe.py packages/tracker/meshnet_tracker/capability.py tests/test_glm_alpha_target.py tests/test_runtime_recipe_identity.py +# result: All checks passed! + +git show e7c780a:packages/tracker/meshnet_tracker/server.py > /tmp/dgr003-server-base.py +ruff check /tmp/dgr003-server-base.py +ruff check packages/tracker/meshnet_tracker/server.py +# result: both baseline and current server.py report the same 8 pre-existing findings diff --git a/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md b/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md index 80a90f5..e48a504 100644 --- a/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md +++ b/.scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md @@ -1,6 +1,6 @@ # 03 — Define exact Artifact and runtime recipe identity -Status: ready-for-agent +Status: done ## Mandatory fresh-session context @@ -21,21 +21,21 @@ As the Tracker, I need exact compatibility identity so that only numerically and ## Acceptance criteria -- [ ] Separate weight quantization, activation dtype, compute dtype, KV dtype/layout, tokenizer revision, architecture adapter, backend, and runtime version. -- [ ] Bind derivative or split artifacts to an exact source Model Artifact hash and Shard range. -- [ ] Produce a stable compatibility fingerprint used by capability admission and the gRPC handshake. -- [ ] Fail closed on mismatched artifact, tokenizer, architecture, range, boundary schema, activation recipe, or cache layout. -- [ ] Keep unsupported recipes registered-but-dark until a real distributed forward certifies them. -- [ ] Targeted pytest tests pass -- [ ] python -m compileall packages tests passes for Python changes -- [ ] git diff --check passes -- [ ] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free -- [ ] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction -- [ ] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code -- [ ] Read and verify every dependency evidence README before relying on dependency behavior -- [ ] Preserve all pre-existing working-tree changes and stage only files belonging to this story -- [ ] Write .scratch/distributed-gguf-runtime/evidence/DGR-003/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff -- [ ] Update only this story issue to Status: done after every acceptance criterion and quality gate passes +- [x] Separate weight quantization, activation dtype, compute dtype, KV dtype/layout, tokenizer revision, architecture adapter, backend, and runtime version. +- [x] Bind derivative or split artifacts to an exact source Model Artifact hash and Shard range. +- [x] Produce a stable compatibility fingerprint used by capability admission and the gRPC handshake. +- [x] Fail closed on mismatched artifact, tokenizer, architecture, range, boundary schema, activation recipe, or cache layout. +- [x] Keep unsupported recipes registered-but-dark until a real distributed forward certifies them. +- [x] Targeted pytest tests pass +- [x] python -m compileall packages tests passes for Python changes +- [x] git diff --check passes +- [x] Default tests remain deterministic, model-download-free, API-credit-free, and GPU-free +- [x] Full deterministic pytest -q passes, or the exact pre-existing unrelated failure is recorded with a clean-tree reproduction +- [x] Read .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md and this story issue completely before changing code +- [x] Read and verify every dependency evidence README before relying on dependency behavior +- [x] Preserve all pre-existing working-tree changes and stage only files belonging to this story +- [x] Write .scratch/distributed-gguf-runtime/evidence/DGR-003/README.md with files changed, exact commands and real results, limitations, compatibility notes, and dependent-story handoff +- [x] Update only this story issue to Status: done after every acceptance criterion and quality gate passes ## Dependency handoff diff --git a/.scratch/distributed-gguf-runtime/prd.json b/.scratch/distributed-gguf-runtime/prd.json index d774044..b5d4a21 100644 --- a/.scratch/distributed-gguf-runtime/prd.json +++ b/.scratch/distributed-gguf-runtime/prd.json @@ -80,7 +80,7 @@ "Update only this story issue to Status: done after every acceptance criterion and quality gate passes" ], "priority": 4, - "passes": false, + "passes": true, "notes": "Source issue: .scratch/distributed-gguf-runtime/issues/03-define-exact-artifact-and-runtime-recipe-identity.md", "dependsOn": [ "DGR-002", diff --git a/packages/node/meshnet_node/capability.py b/packages/node/meshnet_node/capability.py index eac587b..8986b6a 100644 --- a/packages/node/meshnet_node/capability.py +++ b/packages/node/meshnet_node/capability.py @@ -20,6 +20,8 @@ import time from dataclasses import dataclass, field from typing import Any, Mapping +from .runtime_recipe import CompatibilityFingerprint, ShardIdentity + # Layout of the serialized report. Bump when the JSON shape changes. CAPABILITY_SCHEMA_VERSION = 1 @@ -330,7 +332,16 @@ def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]: @dataclass(frozen=True) class CapabilityReport: - """One node's validated (or failed) model/shard/recipe/backend combination.""" + """One node's validated (or failed) model/shard/recipe/backend combination. + + `identity` is the exact DGR-003 artifact/runtime-recipe block: the separated + numerical axes and the compatibility fingerprint derived from them. It is + optional and additive — a node that predates DGR-003 presents none, and the + tracker falls back to the coarse label comparison it has always done + (ADR-0023's compat rollout). A node that *does* present one is held to it: + the tracker re-derives the fingerprint and refuses a report whose claim does + not match its own derivation. + """ model: ModelIdentity shard: ShardRange @@ -341,6 +352,7 @@ class CapabilityReport: duration_ms: int diagnostics: tuple[str, ...] = () schema_version: int = CAPABILITY_SCHEMA_VERSION + identity: ShardIdentity | None = None def __post_init__(self) -> None: if self.status not in VALID_STATUSES: @@ -360,6 +372,11 @@ class CapabilityReport: def passed(self) -> bool: return self.status == STATUS_PASSED + @property + def fingerprint(self) -> CompatibilityFingerprint | None: + """The exact compatibility fingerprint, when this node declares one.""" + return None if self.identity is None else self.identity.fingerprint + def identity_key(self) -> tuple[str, int, int, str, str, str, str]: """The tuple a consumer must match to reuse this proof. @@ -380,7 +397,7 @@ class CapabilityReport: return max(0.0, (time.time() if now is None else now) - self.validated_at) def to_dict(self) -> dict: - return { + doc = { "schema_version": self.schema_version, "model": self.model.to_dict(), "shard": self.shard.to_dict(), @@ -391,6 +408,9 @@ class CapabilityReport: "duration_ms": self.duration_ms, "diagnostics": list(self.diagnostics), } + if self.identity is not None: + doc["identity"] = self.identity.to_dict() + return doc def to_json(self, indent: int | None = None) -> str: return json.dumps(self.to_dict(), indent=indent, sort_keys=True) @@ -417,6 +437,7 @@ class CapabilityReport: ): raise CapabilityReportError("'validated_at' must be a Unix timestamp") + raw_identity = doc.get("identity") return cls( schema_version=schema_version, model=ModelIdentity.from_dict(doc.get("model")), @@ -427,6 +448,9 @@ class CapabilityReport: validated_at=float(validated_at), duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0), diagnostics=sanitize_diagnostics(doc.get("diagnostics")), + identity=( + None if raw_identity is None else ShardIdentity.from_dict(raw_identity) + ), ) @classmethod @@ -461,12 +485,14 @@ def build_capability_report( diagnostics: Any = None, validated_at: float | None = None, environ: Mapping[str, str] | None = None, + identity: ShardIdentity | None = None, ) -> CapabilityReport: """Assemble a report from flat validation results. `model_config` may be the loaded config mapping (hashed into a fingerprint) or an already-computed ``sha256:…`` string. `validated_at` defaults to now, - so callers that need determinism pass it explicitly. + so callers that need determinism pass it explicitly. `identity` is the exact + DGR-003 artifact/recipe block, when the backend can state one. """ return CapabilityReport( model=ModelIdentity( @@ -491,4 +517,5 @@ def build_capability_report( validated_at=time.time() if validated_at is None else validated_at, duration_ms=duration_ms, diagnostics=sanitize_diagnostics(diagnostics, environ), + identity=identity, ) diff --git a/packages/node/meshnet_node/runtime_recipe.py b/packages/node/meshnet_node/runtime_recipe.py new file mode 100644 index 0000000..1745377 --- /dev/null +++ b/packages/node/meshnet_node/runtime_recipe.py @@ -0,0 +1,1147 @@ +"""Exact Model Artifact and runtime recipe identity (DGR-003). + +A route is a chain of Shards that together compute one forward pass. If two +Shards disagree about *anything* that moves the numbers — the weights, the +quantization, the dtype the seam is serialized in, the dtype the kernels +accumulate in, the KV layout, the tokenizer, the architecture adapter, the +backend, or the runtime build — the route still runs and still emits tokens. +They are simply the wrong tokens, and nothing upstream notices. Identity is +therefore a digest over the executable semantics, not a model name. + +Four decisions shape this module. + +**The axes are separate, and each one is separately fatal.** `Q4_K_M` weights +are not a `bfloat16` seam, which is not an `fp32` accumulate, which is not a +`q8_0` KV cache. Collapsing them into one "precision" label is how a route ends +up numerically split while every node reports agreement. :data:`RECIPE_AXES` is +the full list; the digest commits to each under its own key, so swapping two +axis *values* changes the digest even though the multiset of values did not. + +**The fingerprint deliberately excludes the Shard range.** Shards on one route +own *different* ranges; if the range were digested, no two Shards on a route +could ever agree. Range compatibility is a route-level coverage question +(:func:`check_route`), not an identity question. + +**A split artifact routes under its source's identity.** A node holding layers +32–63 of a GGUF holds a *different file* with a *different content hash* than a +node holding layers 0–31, and neither equals the whole model. So the digest +binds :attr:`ArtifactIdentity.source_digest` — the whole-model artifact every +Shard descends from — and the derivative's own hash and range are bound to that +source locally (:class:`DerivativeBinding`). A derivative that cannot name the +exact artifact it was cut from is not admissible: it is an anonymous blob that +happens to have the right layer count. + +**Labels are not digested.** `recipe_id`, `recipe_version` and +`catalogue_version` ride alongside the digests for diagnosis, mirroring the +capability report an admitted node already registers with (ADR-0023). Renaming +a recipe does not change a single number, so a rename must not partition a +route; changing `kv_dtype` changes every number, so it must. The digest commits +to what changes the numbers. + +Recipes are **registered-but-dark** on arrival: known, visible to an operator, +and not routable for user traffic. A recipe leaves the dark only when a *real +distributed forward* over a route of at least two distinct physical nodes, +covering the whole model, has emitted real tokens — see +:class:`DistributedForwardEvidence`. Detected hardware is not a capability, a +single-host forward is not a distributed forward, and a synthetic worker +certifies nothing. + +The tracker re-derives all of this independently in +``meshnet_tracker.recipe`` — it does not import this package. The two +implementations are pinned together by a committed conformance vector +(``tests/data/recipe_fingerprint_vectors.json``); if they ever drift, a test +fails rather than a route silently forming. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import time +from dataclasses import dataclass, field +from typing import Any, Iterable, Mapping, Sequence + +from .native_protocol import BUNDLE_VERSION, SCHEMA_VERSION, pb + +# Layout of the serialized identity block. Bump when the JSON shape changes. +RECIPE_IDENTITY_SCHEMA_VERSION = 1 + +# Domain separation. A digest means nothing outside the structure it commits to; +# the prefix guarantees an artifact digest can never be mistaken for — or collide +# with — a recipe digest, even if their canonical JSON were somehow identical. +ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1" +RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1" + +# The axes of a runtime recipe. Every one of these changes the numbers a Shard +# produces, so every one of them is part of identity and none of them may be +# folded into another. +RECIPE_AXES: tuple[str, ...] = ( + "weight_quantization", + "activation_dtype", + "compute_dtype", + "kv_dtype", + "kv_layout", + "tokenizer_revision", + "architecture_adapter", + "backend_id", + "runtime_version", + "boundary_schema_version", + "protocol_schema_version", +) + +# --- Why two identities failed to match. These are the fail-closed reasons. --- +MISMATCH_ARTIFACT = "artifact" +MISMATCH_WEIGHT_QUANTIZATION = "weight-quantization" +MISMATCH_ACTIVATION_RECIPE = "activation-recipe" +MISMATCH_CACHE_LAYOUT = "cache-layout" +MISMATCH_TOKENIZER = "tokenizer" +MISMATCH_ARCHITECTURE = "architecture" +MISMATCH_BACKEND = "backend" +MISMATCH_RUNTIME = "runtime" +MISMATCH_BOUNDARY_SCHEMA = "boundary-schema" +MISMATCH_RANGE = "range" +MISMATCH_FINGERPRINT = "fingerprint" +MISMATCH_UNCERTIFIED = "uncertified" + +# Which fail-closed reason each axis reports. Several axes share a reason +# because they fail for the same operational cause: `activation_dtype` and +# `compute_dtype` are both "the activation recipe disagrees", and `kv_dtype` +# and `kv_layout` are both "the cache layout disagrees". +_AXIS_MISMATCH: Mapping[str, str] = { + "weight_quantization": MISMATCH_WEIGHT_QUANTIZATION, + "activation_dtype": MISMATCH_ACTIVATION_RECIPE, + "compute_dtype": MISMATCH_ACTIVATION_RECIPE, + "kv_dtype": MISMATCH_CACHE_LAYOUT, + "kv_layout": MISMATCH_CACHE_LAYOUT, + "tokenizer_revision": MISMATCH_TOKENIZER, + "architecture_adapter": MISMATCH_ARCHITECTURE, + "backend_id": MISMATCH_BACKEND, + "runtime_version": MISMATCH_RUNTIME, + "boundary_schema_version": MISMATCH_BOUNDARY_SCHEMA, + "protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA, +} + +# Certification states. `dark` is the arrival state, not an error state. +STATUS_UNKNOWN = "unknown" +STATUS_DARK = "dark" +STATUS_CERTIFIED = "certified" + +# A route that proves a recipe must be a real distributed route. +MIN_CERTIFYING_NODES = 2 + +_HEX64 = re.compile(r"^[0-9a-f]{64}$") + +# A revision that can move is not a pin. DGR-017 learned this on the artifact; +# it is just as true of a tokenizer. +_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"}) + + +class RecipeIdentityError(ValueError): + """Malformed identity input. Messages name the field, never echo a payload.""" + + +class RecipeNotCertified(RecipeIdentityError): + """A recipe was asked to serve user traffic while still dark.""" + + +class RouteIncompatible(RecipeIdentityError): + """Two Shards cannot form an Inference Route. + + Carries the structured reasons, so a caller can map them to a gRPC status + rather than re-parsing a sentence. + """ + + def __init__(self, mismatches: Sequence["RouteMismatch"]) -> None: + self.mismatches = tuple(mismatches) + super().__init__("; ".join(m.describe() for m in self.mismatches)) + + +def canonical_sha256(value: Any) -> str: + """SHA-256 over canonical JSON — the repository's digest convention.""" + payload = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _digest(domain: str, body: Mapping[str, Any]) -> str: + return canonical_sha256({"domain": domain, "body": dict(body)}) + + +def _require_text(value: Any, what: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise RecipeIdentityError(f"{what!r} must be a non-empty string") + return value + + +def _require_hex64(value: Any, what: str) -> str: + text = _require_text(value, what) + if not _HEX64.match(text): + raise RecipeIdentityError( + f"{what!r} must be a lowercase 64-character SHA-256 hex digest" + ) + return text + + +def _require_int(value: Any, what: str, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise RecipeIdentityError(f"{what!r} must be an integer") + if value < minimum: + raise RecipeIdentityError(f"{what!r} must be >= {minimum}, got {value}") + return value + + +def _require_pin(value: Any, what: str) -> str: + """A revision must identify one immutable thing, not a ref that moves.""" + text = _require_text(value, what) + lowered = text.strip().lower() + if lowered in _MOVING_REFS or lowered.startswith("refs/"): + raise RecipeIdentityError( + f"{what!r} is {text!r}, which is a moving reference, not a pin; " + "identity requires an exact immutable revision" + ) + return text + + +def _as_mapping(value: Any, what: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise RecipeIdentityError( + f"{what!r} must be a JSON object, got {type(value).__name__}" + ) + return value + + +@dataclass(frozen=True) +class DerivativeBinding: + """A split or derived artifact, bound to the exact artifact it was cut from. + + Without this binding a split GGUF is an anonymous blob with a plausible + layer count. With it, the Shard's own bytes are checkable *and* its identity + on the route is the source model's, which is what lets a whole-model oracle + and a five-way split agree on a fingerprint. + + The range is inclusive start, exclusive end, matching the protocol + (``ShardRange`` in ``shard_runtime.proto``). + """ + + source_artifact_digest: str + shard_start: int + shard_end: int + + def __post_init__(self) -> None: + _require_hex64( + self.source_artifact_digest, "derived_from.source_artifact_digest" + ) + _require_int(self.shard_start, "derived_from.shard_start", 0) + _require_int(self.shard_end, "derived_from.shard_end", 1) + if self.shard_end <= self.shard_start: + raise RecipeIdentityError( + f"'derived_from.shard_end' ({self.shard_end}) must be greater than " + f"'derived_from.shard_start' ({self.shard_start}); an empty split " + "artifact covers no layers and can prove nothing" + ) + + def covers(self, start: int, end: int) -> bool: + return self.shard_start <= start and end <= self.shard_end + + def to_dict(self) -> dict: + return { + "source_artifact_digest": self.source_artifact_digest, + "shard_start": self.shard_start, + "shard_end": self.shard_end, + } + + @classmethod + def from_dict(cls, data: Any) -> DerivativeBinding: + doc = _as_mapping(data, "derived_from") + return cls( + source_artifact_digest=_require_hex64( + doc.get("source_artifact_digest"), + "derived_from.source_artifact_digest", + ), + shard_start=_require_int( + doc.get("shard_start"), "derived_from.shard_start", 0 + ), + shard_end=_require_int(doc.get("shard_end"), "derived_from.shard_end", 1), + ) + + +@dataclass(frozen=True) +class ArtifactIdentity: + """Exactly which weights, config and architecture a Shard loaded. + + `content_digest` is *this* file's hash — the whole model for an undivided + artifact, one split's bytes for a derivative. `source_digest` is what the + route compares. For an undivided artifact they are the same value; that + identity is the whole point, because it is what makes the DGR-018 + whole-model oracle and the DGR-020 distributed route the same artifact. + """ + + artifact_id: str + revision: str + content_digest: str + architecture: str + architecture_digest: str + layer_count: int + derived_from: DerivativeBinding | None = None + + def __post_init__(self) -> None: + _require_text(self.artifact_id, "artifact.artifact_id") + _require_pin(self.revision, "artifact.revision") + _require_hex64(self.content_digest, "artifact.content_digest") + _require_text(self.architecture, "artifact.architecture") + _require_hex64(self.architecture_digest, "artifact.architecture_digest") + _require_int(self.layer_count, "artifact.layer_count", 1) + + binding = self.derived_from + if binding is None: + return + if binding.source_artifact_digest == self.content_digest: + raise RecipeIdentityError( + "'artifact.derived_from.source_artifact_digest' equals this " + "artifact's own content digest; an artifact is not a split of itself" + ) + if binding.shard_end > self.layer_count: + raise RecipeIdentityError( + f"'artifact.derived_from' claims layers " + f"{binding.shard_start}–{binding.shard_end}, but the source model " + f"has only {self.layer_count} layers" + ) + + @property + def is_derivative(self) -> bool: + return self.derived_from is not None + + @property + def source_digest(self) -> str: + """The whole-model artifact this Shard descends from — what a route compares.""" + if self.derived_from is None: + return self.content_digest + return self.derived_from.source_artifact_digest + + @property + def model_artifact_digest(self) -> str: + """The artifact half of the compatibility fingerprint. + + Commits to the source weights, the architecture and its config/tokenizer + snapshot, and the layer count — and deliberately *not* to the repository + the bytes were fetched from. A mirror of identical bytes is the same + artifact; the repo name buys diagnosis, not numerical identity. + """ + return _digest( + ARTIFACT_DIGEST_DOMAIN, + { + "source_digest": self.source_digest, + "architecture": self.architecture, + "architecture_digest": self.architecture_digest, + "layer_count": self.layer_count, + }, + ) + + def to_dict(self) -> dict: + return { + "artifact_id": self.artifact_id, + "revision": self.revision, + "content_digest": self.content_digest, + "architecture": self.architecture, + "architecture_digest": self.architecture_digest, + "layer_count": self.layer_count, + "derived_from": ( + self.derived_from.to_dict() if self.derived_from else None + ), + } + + @classmethod + def from_dict(cls, data: Any) -> ArtifactIdentity: + doc = _as_mapping(data, "artifact") + raw_binding = doc.get("derived_from") + return cls( + artifact_id=_require_text(doc.get("artifact_id"), "artifact.artifact_id"), + revision=_require_pin(doc.get("revision"), "artifact.revision"), + content_digest=_require_hex64( + doc.get("content_digest"), "artifact.content_digest" + ), + architecture=_require_text( + doc.get("architecture"), "artifact.architecture" + ), + architecture_digest=_require_hex64( + doc.get("architecture_digest"), "artifact.architecture_digest" + ), + layer_count=_require_int(doc.get("layer_count"), "artifact.layer_count", 1), + derived_from=( + None if raw_binding is None else DerivativeBinding.from_dict(raw_binding) + ), + ) + + +@dataclass(frozen=True) +class RuntimeRecipe: + """How a Shard executes the artifact — one field per numerically live axis. + + Weight quantization is not activation dtype is not compute dtype is not KV + dtype. Two nodes running `UD-IQ1_S` weights, one accumulating in fp32 and + one in fp16, produce different logits from the same bytes. Keeping the axes + apart is the entire safety property; see :data:`RECIPE_AXES`. + + The three label fields are diagnosis only and are not digested. + """ + + weight_quantization: str + activation_dtype: str + compute_dtype: str + kv_dtype: str + kv_layout: str + tokenizer_revision: str + architecture_adapter: str + backend_id: str + runtime_version: str + boundary_schema_version: int = BUNDLE_VERSION + protocol_schema_version: int = int(SCHEMA_VERSION) + recipe_id: str = "unnamed" + recipe_version: str = "0" + catalogue_version: str = "0" + + def __post_init__(self) -> None: + for axis in RECIPE_AXES: + value = getattr(self, axis) + if axis.endswith("_schema_version"): + _require_int(value, f"recipe.{axis}", 1) + else: + _require_text(value, f"recipe.{axis}") + _require_pin(self.tokenizer_revision, "recipe.tokenizer_revision") + _require_text(self.recipe_id, "recipe.recipe_id") + _require_text(self.recipe_version, "recipe.recipe_version") + _require_text(self.catalogue_version, "recipe.catalogue_version") + + @property + def axes(self) -> dict[str, Any]: + """The digested fields, and only those.""" + return {axis: getattr(self, axis) for axis in RECIPE_AXES} + + @property + def runtime_recipe_digest(self) -> str: + """The recipe half of the compatibility fingerprint.""" + return _digest(RECIPE_DIGEST_DOMAIN, self.axes) + + def to_dict(self) -> dict: + doc = self.axes + doc.update( + { + "recipe_id": self.recipe_id, + "recipe_version": self.recipe_version, + "catalogue_version": self.catalogue_version, + } + ) + return doc + + @classmethod + def from_dict(cls, data: Any) -> RuntimeRecipe: + doc = _as_mapping(data, "recipe") + missing = [axis for axis in RECIPE_AXES if axis not in doc] + if missing: + raise RecipeIdentityError( + "recipe is missing required axes: " + + ", ".join(missing) + + "; an unstated axis is an unproven one, so it cannot default" + ) + kwargs: dict[str, Any] = {} + for axis in RECIPE_AXES: + value = doc[axis] + if axis.endswith("_schema_version"): + kwargs[axis] = _require_int(value, f"recipe.{axis}", 1) + else: + kwargs[axis] = _require_text(value, f"recipe.{axis}") + return cls( + **kwargs, + recipe_id=_require_text(doc.get("recipe_id"), "recipe.recipe_id"), + recipe_version=_require_text( + doc.get("recipe_version"), "recipe.recipe_version" + ), + catalogue_version=_require_text( + doc.get("catalogue_version"), "recipe.catalogue_version" + ), + ) + + +@dataclass(frozen=True) +class CompatibilityFingerprint: + """The stable identity two peers actually compare. + + This is the Python face of the protocol's ``Fingerprint`` message + (DGR-002). Do not invent a second identity struct: capability admission and + the gRPC handshake compare *this*, and they must compare the same thing or + a route can pass one gate and fail the other. + """ + + model_artifact_digest: str + runtime_recipe_digest: str + recipe_id: str + recipe_version: str + catalogue_version: str + + def __post_init__(self) -> None: + _require_hex64(self.model_artifact_digest, "fingerprint.model_artifact_digest") + _require_hex64( + self.runtime_recipe_digest, "fingerprint.runtime_recipe_digest" + ) + + @property + def key(self) -> tuple[str, str]: + """What equality means: the digests, never the labels.""" + return (self.model_artifact_digest, self.runtime_recipe_digest) + + def matches(self, other: CompatibilityFingerprint) -> bool: + return self.key == other.key + + def to_dict(self) -> dict: + return { + "model_artifact_digest": self.model_artifact_digest, + "runtime_recipe_digest": self.runtime_recipe_digest, + "recipe_id": self.recipe_id, + "recipe_version": self.recipe_version, + "catalogue_version": self.catalogue_version, + } + + @classmethod + def from_dict(cls, data: Any) -> CompatibilityFingerprint: + doc = _as_mapping(data, "fingerprint") + return cls( + model_artifact_digest=_require_hex64( + doc.get("model_artifact_digest"), "fingerprint.model_artifact_digest" + ), + runtime_recipe_digest=_require_hex64( + doc.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest" + ), + recipe_id=_require_text(doc.get("recipe_id"), "fingerprint.recipe_id"), + recipe_version=_require_text( + doc.get("recipe_version"), "fingerprint.recipe_version" + ), + catalogue_version=_require_text( + doc.get("catalogue_version"), "fingerprint.catalogue_version" + ), + ) + + def to_proto(self) -> "pb.Fingerprint": + """Populate the DGR-002 ``Fingerprint`` message for the gRPC handshake.""" + return pb.Fingerprint( + model_artifact_digest=self.model_artifact_digest, + runtime_recipe_digest=self.runtime_recipe_digest, + recipe_id=self.recipe_id, + recipe_version=self.recipe_version, + catalogue_version=self.catalogue_version, + ) + + @classmethod + def from_proto(cls, message: "pb.Fingerprint") -> CompatibilityFingerprint: + return cls( + model_artifact_digest=_require_hex64( + message.model_artifact_digest, "fingerprint.model_artifact_digest" + ), + runtime_recipe_digest=_require_hex64( + message.runtime_recipe_digest, "fingerprint.runtime_recipe_digest" + ), + recipe_id=_require_text(message.recipe_id, "fingerprint.recipe_id"), + recipe_version=_require_text( + message.recipe_version, "fingerprint.recipe_version" + ), + catalogue_version=_require_text( + message.catalogue_version, "fingerprint.catalogue_version" + ), + ) + + +@dataclass(frozen=True) +class RouteMismatch: + """One structured, fail-closed reason two identities do not compose.""" + + reason: str + detail: str + shard_index: int | None = None + + def describe(self) -> str: + where = "" if self.shard_index is None else f"shard {self.shard_index}: " + return f"{where}{self.reason}: {self.detail}" + + def to_dict(self) -> dict: + return { + "reason": self.reason, + "detail": self.detail, + "shard_index": self.shard_index, + } + + +@dataclass(frozen=True) +class ShardIdentity: + """What one Shard is: an artifact, a recipe, and the layers it owns. + + The owned range is inclusive start, exclusive end (protocol convention). It + is *not* part of the fingerprint — see the module docstring — but it is + validated against the artifact, because a Shard cannot serve layers its + artifact does not contain. + """ + + artifact: ArtifactIdentity + recipe: RuntimeRecipe + shard_start: int + shard_end: int + + def __post_init__(self) -> None: + _require_int(self.shard_start, "shard_start", 0) + _require_int(self.shard_end, "shard_end", 1) + if self.shard_end <= self.shard_start: + raise RecipeIdentityError( + f"'shard_end' ({self.shard_end}) must be greater than 'shard_start' " + f"({self.shard_start}); a Shard owning no layer computes nothing" + ) + if self.shard_end > self.artifact.layer_count: + raise RecipeIdentityError( + f"Shard owns layers {self.shard_start}–{self.shard_end}, but the " + f"artifact has only {self.artifact.layer_count} layers" + ) + binding = self.artifact.derived_from + if binding is not None and not binding.covers(self.shard_start, self.shard_end): + raise RecipeIdentityError( + f"Shard advertises layers {self.shard_start}–{self.shard_end}, but its " + f"split artifact only contains layers {binding.shard_start}–" + f"{binding.shard_end}; it cannot serve weights it does not hold" + ) + + @property + def fingerprint(self) -> CompatibilityFingerprint: + """The derived identity. Never stored, never trusted from the wire — derived.""" + return CompatibilityFingerprint( + model_artifact_digest=self.artifact.model_artifact_digest, + runtime_recipe_digest=self.recipe.runtime_recipe_digest, + recipe_id=self.recipe.recipe_id, + recipe_version=self.recipe.recipe_version, + catalogue_version=self.recipe.catalogue_version, + ) + + def to_dict(self) -> dict: + return { + "schema_version": RECIPE_IDENTITY_SCHEMA_VERSION, + "artifact": self.artifact.to_dict(), + "recipe": self.recipe.to_dict(), + "shard_start": self.shard_start, + "shard_end": self.shard_end, + "fingerprint": self.fingerprint.to_dict(), + } + + @classmethod + def from_dict(cls, data: Any) -> ShardIdentity: + """Parse an identity block, re-deriving — never trusting — its fingerprint. + + A declared fingerprint is a compatibility claim, not authentication. + Recomputing it catches an inconsistent declaration; only tracker-owned + distributed certification decides whether a recipe may serve. + """ + doc = _as_mapping(data, "identity") + + declared_schema = doc.get("schema_version") + if declared_schema != RECIPE_IDENTITY_SCHEMA_VERSION: + raise RecipeIdentityError( + f"identity block declares schema version {declared_schema!r}, but this " + f"node reads version {RECIPE_IDENTITY_SCHEMA_VERSION}" + ) + + identity = cls( + artifact=ArtifactIdentity.from_dict(doc.get("artifact")), + recipe=RuntimeRecipe.from_dict(doc.get("recipe")), + shard_start=_require_int(doc.get("shard_start"), "shard_start", 0), + shard_end=_require_int(doc.get("shard_end"), "shard_end", 1), + ) + + declared = doc.get("fingerprint") + if declared is not None: + claim = CompatibilityFingerprint.from_dict(declared) + derived = identity.fingerprint + if not claim.matches(derived): + raise RouteIncompatible( + [ + RouteMismatch( + MISMATCH_FINGERPRINT, + "declared fingerprint does not match the identity it " + "claims to describe; the digest is derived from the " + "artifact and recipe, it is not an assertion", + ) + ] + ) + return identity + + +def explain_mismatch( + left: ShardIdentity, + right: ShardIdentity, + *, + shard_index: int | None = None, +) -> tuple[RouteMismatch, ...]: + """Name every axis on which two Shards disagree. + + The fingerprint comparison already answers *whether* they compose. This + answers *why not*, which is the difference between an operator fixing their + node in a minute and filing a bug. + """ + mismatches: list[RouteMismatch] = [] + + if left.artifact.source_digest != right.artifact.source_digest: + mismatches.append( + RouteMismatch( + MISMATCH_ARTIFACT, + f"source Model Artifact {right.artifact.source_digest[:12]}… does not " + f"match the route's {left.artifact.source_digest[:12]}…", + shard_index, + ) + ) + if left.artifact.architecture != right.artifact.architecture: + mismatches.append( + RouteMismatch( + MISMATCH_ARCHITECTURE, + f"artifact architecture {right.artifact.architecture!r} does not match " + f"the route's {left.artifact.architecture!r}", + shard_index, + ) + ) + if left.artifact.architecture_digest != right.artifact.architecture_digest: + mismatches.append( + RouteMismatch( + MISMATCH_ARCHITECTURE, + "architecture/config snapshot differs from the route's; the same " + "architecture name with different config is a different model", + shard_index, + ) + ) + if left.artifact.layer_count != right.artifact.layer_count: + mismatches.append( + RouteMismatch( + MISMATCH_ARTIFACT, + f"artifact has {right.artifact.layer_count} layers, but the route's " + f"has {left.artifact.layer_count}", + shard_index, + ) + ) + + for axis in RECIPE_AXES: + mine = getattr(left.recipe, axis) + theirs = getattr(right.recipe, axis) + if mine != theirs: + mismatches.append( + RouteMismatch( + _AXIS_MISMATCH[axis], + f"{axis} is {theirs!r}, but the route runs {mine!r}", + shard_index, + ) + ) + + if not mismatches and not left.fingerprint.matches(right.fingerprint): + # Unreachable via the fields above; if it ever fires, identity has grown a + # digested field that `explain_mismatch` does not know about, and silently + # reporting "compatible" would be far worse than an unhelpful message. + mismatches.append( + RouteMismatch( + MISMATCH_FINGERPRINT, + "fingerprints differ on a field this comparison does not cover", + shard_index, + ) + ) + return tuple(mismatches) + + +@dataclass(frozen=True) +class DistributedForwardEvidence: + """Proof that a recipe really ran, distributed, end to end. + + This is the only thing that takes a recipe out of the dark. The bar is + deliberately the product's bar, not a unit test's: at least two *distinct* + physical nodes, whose Shards cover the whole model with no hole, generating + real tokens. A synthetic worker is a unit fixture and certifies nothing — + the whole risk this guards against is a recipe that passes in a mock and + produces garbage on real weights. + """ + + route_session_id: str + route_epoch: int + node_ids: tuple[str, ...] + shard_ranges: tuple[tuple[int, int], ...] + tokens_generated: int + layer_count: int + synthetic: bool = False + certified_at: float = field(default_factory=time.time) + + def __post_init__(self) -> None: + _require_text(self.route_session_id, "evidence.route_session_id") + _require_int(self.route_epoch, "evidence.route_epoch", 0) + _require_int(self.tokens_generated, "evidence.tokens_generated", 0) + _require_int(self.layer_count, "evidence.layer_count", 1) + + def rejection(self) -> str | None: + """Why this evidence does not certify, or None when it does.""" + if self.synthetic: + return ( + "evidence comes from a synthetic worker; only a real distributed " + "forward certifies a recipe" + ) + distinct = set(self.node_ids) + if len(distinct) < MIN_CERTIFYING_NODES: + return ( + f"evidence covers {len(distinct)} distinct node(s); a distributed " + f"forward requires at least {MIN_CERTIFYING_NODES}" + ) + if len(distinct) != len(self.node_ids): + return "the same node is counted more than once in the certifying route" + if self.tokens_generated < 1: + return "the certifying forward generated no tokens" + + gap = _coverage_gap(self.shard_ranges, self.layer_count) + if gap is not None: + return gap + return None + + +def _coverage_gap( + ranges: Iterable[tuple[int, int]], layer_count: int +) -> str | None: + """Return why `ranges` fail to tile ``[0, layer_count)``, or None if they do. + + Ranges may overlap: ADR-0012 lets the Tracker resolve an overlap by telling a + hop where the previous hop stopped, so an overlap is a routing decision, not + a defect. A *hole* is a defect — the layers in it are never computed, and the + route would silently emit tokens from a truncated model. + """ + ordered = sorted(ranges) + if not ordered: + return "the certifying route owns no layers" + if ordered[0][0] != 0: + return f"layers 0–{ordered[0][0]} are owned by no Shard on the route" + + covered = 0 + for start, end in ordered: + if start > covered: + return f"layers {covered}–{start} are owned by no Shard on the route" + covered = max(covered, end) + + if covered < layer_count: + return f"layers {covered}–{layer_count} are owned by no Shard on the route" + if covered > layer_count: + return ( + f"the route claims to cover {covered} layers, but the model has only " + f"{layer_count}" + ) + return None + + +@dataclass(frozen=True) +class RecipeStatus: + """What the registry knows about one fingerprint.""" + + status: str + fingerprint: CompatibilityFingerprint | None = None + detail: str = "" + certified_at: float | None = None + + @property + def may_serve(self) -> bool: + """Only a certified recipe carries user traffic.""" + return self.status == STATUS_CERTIFIED + + @property + def may_certify(self) -> bool: + """A dark recipe is eligible for a certification route — and only that. + + Without this, certification is unreachable: serving requires + certification, certification requires a real distributed forward, and a + real distributed forward requires a route. A dark recipe may therefore + be routed *for the express purpose of certifying it*, and never for user + traffic. + """ + return self.status in (STATUS_DARK, STATUS_CERTIFIED) + + def to_dict(self) -> dict: + return { + "status": self.status, + "detail": self.detail, + "certified_at": self.certified_at, + "fingerprint": ( + self.fingerprint.to_dict() if self.fingerprint is not None else None + ), + } + + +class RecipeRegistry: + """Registered-but-dark recipes, and the ones a real forward has certified. + + An unknown recipe is not routable. A known one is not routable either, until + it has been proven. This is the fail-closed default the roadmap asks for: + "unsupported architectures/backends remain registered-but-dark until real + certification passes." + """ + + def __init__(self) -> None: + self._dark: dict[tuple[str, str], CompatibilityFingerprint] = {} + self._certified: dict[tuple[str, str], RecipeStatus] = {} + + def register(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus: + """Record a recipe as known. Known is not certified.""" + fingerprint = _as_fingerprint(identity) + key = fingerprint.key + if key in self._certified: + return self._certified[key] + self._dark[key] = fingerprint + return RecipeStatus( + STATUS_DARK, + fingerprint, + "registered; dark until a real distributed forward certifies it", + ) + + def status(self, identity: ShardIdentity | CompatibilityFingerprint) -> RecipeStatus: + fingerprint = _as_fingerprint(identity) + key = fingerprint.key + if key in self._certified: + return self._certified[key] + if key in self._dark: + return RecipeStatus( + STATUS_DARK, + fingerprint, + "registered; dark until a real distributed forward certifies it", + ) + return RecipeStatus( + STATUS_UNKNOWN, + fingerprint, + "this recipe has never been registered with the Tracker", + ) + + def certify( + self, + identity: ShardIdentity | CompatibilityFingerprint, + evidence: DistributedForwardEvidence, + ) -> RecipeStatus: + """Promote a dark recipe, if the evidence is a real distributed forward. + + Raises rather than returning a failed status: certifying is a state + change, and a caller that ignores a returned failure would leave a + recipe it believes to be certified in the dark. + """ + fingerprint = _as_fingerprint(identity) + rejection = evidence.rejection() + if rejection is not None: + raise RecipeNotCertified( + f"this evidence does not certify {fingerprint.recipe_id!r}: {rejection}" + ) + status = RecipeStatus( + STATUS_CERTIFIED, + fingerprint, + ( + f"certified by route session {evidence.route_session_id} across " + f"{len(set(evidence.node_ids))} nodes" + ), + certified_at=evidence.certified_at, + ) + self._certified[fingerprint.key] = status + self._dark.pop(fingerprint.key, None) + return status + + def require_serving( + self, identity: ShardIdentity | CompatibilityFingerprint + ) -> RecipeStatus: + """Admit a recipe for user traffic, or refuse and say why.""" + status = self.status(identity) + if not status.may_serve: + raise RecipeNotCertified( + f"recipe {_as_fingerprint(identity).recipe_id!r} is {status.status}: " + f"{status.detail}" + ) + return status + + def certified_fingerprints(self) -> tuple[CompatibilityFingerprint, ...]: + return tuple( + status.fingerprint + for status in self._certified.values() + if status.fingerprint is not None + ) + + def to_dict(self) -> dict: + return { + "schema_version": RECIPE_IDENTITY_SCHEMA_VERSION, + "dark": [fp.to_dict() for fp in self._dark.values()], + "certified": [status.to_dict() for status in self._certified.values()], + } + + +def _as_fingerprint( + identity: ShardIdentity | CompatibilityFingerprint, +) -> CompatibilityFingerprint: + if isinstance(identity, CompatibilityFingerprint): + return identity + if isinstance(identity, ShardIdentity): + return identity.fingerprint + raise RecipeIdentityError( + f"expected a ShardIdentity or CompatibilityFingerprint, got " + f"{type(identity).__name__}" + ) + + +def check_route( + shards: Sequence[ShardIdentity], + *, + registry: RecipeRegistry | None = None, + for_certification: bool = False, +) -> tuple[RouteMismatch, ...]: + """Decide whether these Shards may form one Inference Route. + + Returns every reason they may not, empty when they may. Fail-closed by + construction: an empty route, an unknown recipe, a hole in the layer + coverage and a single differing dtype all produce a reason, and a caller that + only ever proceeds on an empty tuple cannot accidentally admit any of them. + + `for_certification` admits a dark recipe onto a route whose only purpose is + to certify it. It never admits an unknown one, and it is not the path user + traffic takes. + """ + if not shards: + return ( + RouteMismatch( + MISMATCH_RANGE, "a route needs at least one Shard; this one has none" + ), + ) + + mismatches: list[RouteMismatch] = [] + + head = shards[0] + for index, shard in enumerate(shards[1:], start=1): + if not head.fingerprint.matches(shard.fingerprint): + mismatches.extend(explain_mismatch(head, shard, shard_index=index)) + + gap = _coverage_gap( + [(shard.shard_start, shard.shard_end) for shard in shards], + head.artifact.layer_count, + ) + if gap is not None: + mismatches.append(RouteMismatch(MISMATCH_RANGE, gap)) + + if registry is not None: + status = registry.status(head.fingerprint) + allowed = status.may_certify if for_certification else status.may_serve + if not allowed: + mismatches.append( + RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}") + ) + + return tuple(mismatches) + + +def require_route( + shards: Sequence[ShardIdentity], + *, + registry: RecipeRegistry | None = None, + for_certification: bool = False, +) -> CompatibilityFingerprint: + """`check_route`, raising the structured reasons instead of returning them.""" + mismatches = check_route( + shards, registry=registry, for_certification=for_certification + ) + if mismatches: + raise RouteIncompatible(mismatches) + return shards[0].fingerprint + + +# --------------------------------------------------------------------------- +# The alpha target (DGR-017) +# --------------------------------------------------------------------------- + + +def glm_alpha_artifact( + derived_from: DerivativeBinding | None = None, + *, + content_digest: str | None = None, +) -> ArtifactIdentity: + """Artifact identity for the pinned GLM-5.2 ``UD-IQ1_S`` alpha target. + + Reads the locked manifest and architecture snapshot DGR-017 sealed, rather + than restating their digests here — a second copy of a pinned digest is a + second thing that can drift from the target it claims to pin. + + `layer_count` is `num_hidden_layers` (78), *not* `total_artifact_layers` + (79). The 79th is the MTP/next-token-prediction layer, which is not part of + the transformer stack a route tiles and which alpha explicitly defers + (roadmap decision 17). Tiling 79 layers would leave every route one layer + short of "covered" forever. + + Pass `derived_from` when this node holds a split of the target, and + `content_digest` for that split's own bytes. + """ + from .glm_alpha import load_locked_target + + _contract, manifest, snapshot = load_locked_target() + return ArtifactIdentity( + artifact_id=manifest.gguf_repo_id, + revision=manifest.gguf_revision, + content_digest=content_digest or manifest.digest, + architecture=str(snapshot["model_type"]), + architecture_digest=snapshot.digest, + layer_count=int(snapshot["num_hidden_layers"]), + derived_from=derived_from, + ) + + +# --------------------------------------------------------------------------- +# gRPC handshake (DGR-002 `Fingerprint`, `ERROR_CODE_FINGERPRINT_MISMATCH`) +# --------------------------------------------------------------------------- + + +def check_handshake( + local: ShardIdentity, + remote: "pb.Fingerprint", +) -> tuple[RouteMismatch, ...]: + """Compare the fingerprint a peer opened the stream with against our own. + + An incompatible peer must fail at `SessionOpen`, not mid-generation: by the + time a wrong activation has been folded into a KV cache, the session is + already producing wrong tokens and there is nothing to salvage. + """ + try: + presented = CompatibilityFingerprint.from_proto(remote) + except RecipeIdentityError as exc: + return ( + RouteMismatch( + MISMATCH_FINGERPRINT, + f"peer presented an unusable fingerprint: {exc}", + ), + ) + + mine = local.fingerprint + if presented.matches(mine): + return () + + reasons: list[RouteMismatch] = [] + if presented.model_artifact_digest != mine.model_artifact_digest: + reasons.append( + RouteMismatch( + MISMATCH_ARTIFACT, + f"peer serves Model Artifact {presented.model_artifact_digest[:12]}…, " + f"this Shard serves {mine.model_artifact_digest[:12]}…", + ) + ) + if presented.runtime_recipe_digest != mine.runtime_recipe_digest: + reasons.append( + RouteMismatch( + MISMATCH_FINGERPRINT, + f"peer runs recipe {presented.runtime_recipe_digest[:12]}… " + f"({presented.recipe_id} v{presented.recipe_version}), this Shard runs " + f"{mine.runtime_recipe_digest[:12]}… ({mine.recipe_id} " + f"v{mine.recipe_version})", + ) + ) + return tuple(reasons) + + +def handshake_error( + mismatches: Sequence[RouteMismatch], +) -> "pb.ShardError | None": + """The protocol status a rejected handshake closes the stream with.""" + if not mismatches: + return None + return pb.ShardError( + code=pb.ERROR_CODE_FINGERPRINT_MISMATCH, + detail="; ".join(m.describe() for m in mismatches), + retryable=False, + ) diff --git a/packages/tracker/meshnet_tracker/capability.py b/packages/tracker/meshnet_tracker/capability.py index c60364f..c4c384e 100644 --- a/packages/tracker/meshnet_tracker/capability.py +++ b/packages/tracker/meshnet_tracker/capability.py @@ -30,6 +30,13 @@ import time from dataclasses import dataclass, replace from typing import Any, Callable, Mapping +from .recipe import ( + CertificationLedger, + FingerprintMismatch, + RecipeIdentityError, + parse_identity, +) + # The capability report layout this tracker reads (meshnet_node.capability). SUPPORTED_SCHEMA_VERSION = 1 @@ -58,6 +65,12 @@ STATE_MODEL_MISMATCH = "model-mismatch" STATE_SHARD_MISMATCH = "shard-mismatch" STATE_RECIPE_MISMATCH = "recipe-mismatch" STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible" +# The node presented a DGR-003 identity block whose declared fingerprint is not +# the digest of the axes it declared. Identity is derived, never asserted. +STATE_FINGERPRINT_MISMATCH = "fingerprint-mismatch" +# Registered-but-dark: a known recipe no real distributed forward has certified. +# Visible to an operator, never routable for user traffic. +STATE_UNCERTIFIED = "uncertified" ALL_STATES = ( STATE_ADMITTED, @@ -69,6 +82,8 @@ ALL_STATES = ( STATE_SHARD_MISMATCH, STATE_RECIPE_MISMATCH, STATE_CATALOGUE_INCOMPATIBLE, + STATE_FINGERPRINT_MISMATCH, + STATE_UNCERTIFIED, ) # --- Compatibility policy for nodes that predate the capability protocol. --- @@ -165,12 +180,25 @@ class CapabilityState: recorded_at: float = 0.0 schema_version: int | None = None diagnostics: tuple[str, ...] = () + # The DGR-003 compatibility fingerprint, *re-derived* by this tracker from + # the axes the node declared — never copied from what the node claimed. + # Absent for a node that predates DGR-003. + model_artifact_digest: str | None = None + runtime_recipe_digest: str | None = None + certification: str | None = None @property def proven(self) -> bool: """The presented proof covers exactly what the node advertised.""" return self.state == STATE_ADMITTED + @property + def fingerprint(self) -> tuple[str, str] | None: + """What route formation compares. `None` when the node declares no identity.""" + if self.model_artifact_digest is None or self.runtime_recipe_digest is None: + return None + return (self.model_artifact_digest, self.runtime_recipe_digest) + def routable_under(self, policy: str) -> bool: if self.proven: return True @@ -197,6 +225,9 @@ class CapabilityState: "recorded_at": self.recorded_at, "schema_version": self.schema_version, "diagnostics": list(self.diagnostics), + "model_artifact_digest": self.model_artifact_digest, + "runtime_recipe_digest": self.runtime_recipe_digest, + "certification": self.certification, } @@ -224,6 +255,7 @@ def evaluate_report( declared_recipe_version: str | None = None, now: float | None = None, max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS, + ledger: CertificationLedger | None = None, ) -> CapabilityState: """Judge the proof a node presented against what that node is advertising. @@ -308,6 +340,67 @@ def evaluate_report( f"the node declared v{declared_recipe_version}", ) + identity = None + if report.get("identity") is not None: + try: + identity = parse_identity(report["identity"]) + except FingerprintMismatch as exc: + return base.with_state(STATE_FINGERPRINT_MISMATCH, str(exc)) + except RecipeIdentityError as exc: + return base.with_state( + STATE_INVALID, f"capability identity block is unusable: {exc}" + ) + + # The report's `shard` range is inclusive/inclusive (the CLI and backend + # convention); the identity's is inclusive/exclusive (the protocol's, and + # ADR-0012's). A node whose two halves disagree has not proven the range + # it claims, whichever one is right. + if (identity.shard_start, identity.shard_end) != ( + base.shard_start, + (base.shard_end or 0) + 1, + ): + return base.with_state( + STATE_SHARD_MISMATCH, + f"identity covers layers {identity.shard_start}–{identity.shard_end} " + f"(end-exclusive), but the proof is for layers {base.shard_start}–" + f"{base.shard_end} (end-inclusive)", + ) + + if not model_matches(identity.artifact_id): + return base.with_state( + STATE_MODEL_MISMATCH, + f"identity is for artifact {identity.artifact_id!r}, but the node " + f"registered {advertised_model!r}", + ) + if ( + identity.recipe_id != base.recipe_id + or identity.recipe_version != base.recipe_version + or identity.catalogue_version != base.catalogue_version + ): + return base.with_state( + STATE_RECIPE_MISMATCH, + "identity recipe labels do not match the capability proof", + ) + if identity.axes["backend_id"] != base.backend_id: + return base.with_state( + STATE_RECIPE_MISMATCH, + "identity backend does not match the capability proof", + ) + if ( + base.quantization is not None + and identity.axes["weight_quantization"] != base.quantization + ): + return base.with_state( + STATE_RECIPE_MISMATCH, + "identity weight quantization does not match the capability proof", + ) + + base = replace( + base, + model_artifact_digest=identity.model_artifact_digest, + runtime_recipe_digest=identity.runtime_recipe_digest, + ) + if status != STATUS_PASSED: return base.with_state( STATE_FAILED, @@ -328,6 +421,15 @@ def evaluate_report( f"proof is timestamped {-age:.0f}s in the future; check the node's clock", ) + # The digest only establishes that this report is self-consistent. It does + # not authenticate a node or prove a distributed forward. Exact recipes are + # therefore registered-but-dark until tracker-owned certification records + # that forward. + if identity is not None: + recipe_status = (ledger or CertificationLedger()).register(identity) + if not recipe_status.may_serve: + return base.with_state(STATE_UNCERTIFIED, recipe_status.detail) + return base.with_state( STATE_ADMITTED, f"{base.model_id} layers {base.shard_start}–{base.shard_end} proven on " diff --git a/packages/tracker/meshnet_tracker/recipe.py b/packages/tracker/meshnet_tracker/recipe.py new file mode 100644 index 0000000..5547e3a --- /dev/null +++ b/packages/tracker/meshnet_tracker/recipe.py @@ -0,0 +1,615 @@ +"""Tracker-side artifact and runtime recipe identity (DGR-003). + +The node computes a compatibility fingerprint in `meshnet_node.runtime_recipe`. +This module recomputes it, and the recomputation is the entire point. + +A fingerprint that arrives on the wire is a **claim**. If the tracker stored +what a node asserted, a node could assert the digest of a certified recipe while +running an uncertified one — and admission, route selection, and the gRPC +handshake would all wave it through, because they all compare the digest it +handed them. So the tracker derives the digest from the *axes* the node +declared, and refuses any report whose claim does not match the derivation. A +node can lie about its axes, and a real forward will catch that; it cannot lie +about the digest of the axes it declared. + +This module deliberately does **not** import `meshnet_node`: the tracker package +does not depend on the node package, and an admission gate that shares an +implementation with the thing it admits is not an independent check. The two +implementations are pinned together by a committed conformance vector +(``tests/data/recipe_fingerprint_vectors.json``). If they drift, a test fails — +rather than a route quietly failing to form, or worse, quietly forming. + +Recipes arrive **dark**: registered, visible to an operator, and not routable for +user traffic. Only a real distributed forward — at least two distinct nodes, +covering the whole model, emitting real tokens — takes a recipe out of the dark +(:class:`CertificationLedger`). +""" + +from __future__ import annotations + +import hashlib +import json +import re +import time +from dataclasses import dataclass, field +from typing import Any, Iterable, Mapping, Sequence + +# Layout of the identity block this tracker reads (meshnet_node.runtime_recipe). +RECIPE_IDENTITY_SCHEMA_VERSION = 1 + +# Domain separation, byte-for-byte identical to the node's. These strings are +# part of the wire contract, not an implementation detail: changing one here +# without changing it there silently partitions every route. +ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1" +RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1" + +# The axes a recipe digest commits to. Order is irrelevant (the canonical JSON +# sorts keys); membership is not — an axis missing here is an axis the tracker +# would let a node change without changing its identity. +RECIPE_AXES: tuple[str, ...] = ( + "weight_quantization", + "activation_dtype", + "compute_dtype", + "kv_dtype", + "kv_layout", + "tokenizer_revision", + "architecture_adapter", + "backend_id", + "runtime_version", + "boundary_schema_version", + "protocol_schema_version", +) + +_INT_AXES = frozenset({"boundary_schema_version", "protocol_schema_version"}) + +MISMATCH_ARTIFACT = "artifact" +MISMATCH_TOKENIZER = "tokenizer" +MISMATCH_ARCHITECTURE = "architecture" +MISMATCH_RANGE = "range" +MISMATCH_BOUNDARY_SCHEMA = "boundary-schema" +MISMATCH_ACTIVATION_RECIPE = "activation-recipe" +MISMATCH_CACHE_LAYOUT = "cache-layout" +MISMATCH_FINGERPRINT = "fingerprint" +MISMATCH_UNCERTIFIED = "uncertified" + +STATUS_UNKNOWN = "unknown" +STATUS_DARK = "dark" +STATUS_CERTIFIED = "certified" + +MIN_CERTIFYING_NODES = 2 + +_HEX64 = re.compile(r"^[0-9a-f]{64}$") +_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"}) + + +class RecipeIdentityError(ValueError): + """A presented identity block is malformed or internally inconsistent.""" + + +class FingerprintMismatch(RecipeIdentityError): + """A supplied digest is inconsistent with its identity declaration. + + A digest is an integrity and compatibility claim, not an authenticated + statement about what a node is actually executing. Distributed + certification remains the trust boundary. + """ + + +def canonical_sha256(value: Any) -> str: + payload = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _digest(domain: str, body: Mapping[str, Any]) -> str: + return canonical_sha256({"domain": domain, "body": dict(body)}) + + +def _text(value: Any, what: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise RecipeIdentityError(f"{what!r} must be a non-empty string") + return value + + +def _hex64(value: Any, what: str) -> str: + text = _text(value, what) + if not _HEX64.match(text): + raise RecipeIdentityError(f"{what!r} must be a SHA-256 hex digest") + return text + + +def _integer(value: Any, what: str, minimum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise RecipeIdentityError(f"{what!r} must be an integer") + if value < minimum: + raise RecipeIdentityError(f"{what!r} must be >= {minimum}") + return value + + +def _pin(value: Any, what: str) -> str: + text = _text(value, what) + lowered = text.strip().lower() + if lowered in _MOVING_REFS or lowered.startswith("refs/"): + raise RecipeIdentityError( + f"{what!r} is a moving reference, not an exact revision pin" + ) + return text + + +def _mapping(value: Any, what: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise RecipeIdentityError(f"{what!r} must be a JSON object") + return value + + +def artifact_digest( + *, + source_digest: str, + architecture: str, + architecture_digest: str, + layer_count: int, +) -> str: + """The artifact half of the fingerprint, derived exactly as the node derives it.""" + return _digest( + ARTIFACT_DIGEST_DOMAIN, + { + "source_digest": source_digest, + "architecture": architecture, + "architecture_digest": architecture_digest, + "layer_count": layer_count, + }, + ) + + +def recipe_digest(axes: Mapping[str, Any]) -> str: + """The recipe half of the fingerprint, derived exactly as the node derives it.""" + missing = [axis for axis in RECIPE_AXES if axis not in axes] + if missing: + raise RecipeIdentityError( + "recipe is missing required axes: " + ", ".join(missing) + ) + return _digest(RECIPE_DIGEST_DOMAIN, {axis: axes[axis] for axis in RECIPE_AXES}) + + +@dataclass(frozen=True) +class PresentedIdentity: + """One node's declared artifact/recipe identity, with digests re-derived here. + + `model_artifact_digest` and `runtime_recipe_digest` are computed by this + tracker from the declared axes. They are never copied from the report. + """ + + artifact_id: str + revision: str + source_digest: str + architecture: str + architecture_digest: str + layer_count: int + is_derivative: bool + shard_start: int + shard_end: int + axes: Mapping[str, Any] + recipe_id: str + recipe_version: str + catalogue_version: str + + @property + def model_artifact_digest(self) -> str: + return artifact_digest( + source_digest=self.source_digest, + architecture=self.architecture, + architecture_digest=self.architecture_digest, + layer_count=self.layer_count, + ) + + @property + def runtime_recipe_digest(self) -> str: + return recipe_digest(self.axes) + + @property + def key(self) -> tuple[str, str]: + return (self.model_artifact_digest, self.runtime_recipe_digest) + + @property + def tokenizer_revision(self) -> str: + return str(self.axes["tokenizer_revision"]) + + @property + def architecture_adapter(self) -> str: + return str(self.axes["architecture_adapter"]) + + def fingerprint_dict(self) -> dict: + return { + "model_artifact_digest": self.model_artifact_digest, + "runtime_recipe_digest": self.runtime_recipe_digest, + "recipe_id": self.recipe_id, + "recipe_version": self.recipe_version, + "catalogue_version": self.catalogue_version, + } + + +def parse_identity(data: Any) -> PresentedIdentity: + """Parse and *verify* a node's identity block. + + Raises when the block is malformed, when a split artifact does not name the + exact source it was cut from, when a Shard advertises layers its artifact + does not contain, or when the declared fingerprint does not match the digest + of the axes it was declared with. + """ + doc = _mapping(data, "identity") + + schema_version = doc.get("schema_version") + if schema_version != RECIPE_IDENTITY_SCHEMA_VERSION: + raise RecipeIdentityError( + f"identity block declares schema version {schema_version!r}; this tracker " + f"reads version {RECIPE_IDENTITY_SCHEMA_VERSION}" + ) + + artifact = _mapping(doc.get("artifact"), "identity.artifact") + recipe = _mapping(doc.get("recipe"), "identity.recipe") + + layer_count = _integer(artifact.get("layer_count"), "artifact.layer_count", 1) + content_digest = _hex64(artifact.get("content_digest"), "artifact.content_digest") + + raw_binding = artifact.get("derived_from") + if raw_binding is None: + source_digest = content_digest + binding: tuple[int, int] | None = None + else: + derived = _mapping(raw_binding, "artifact.derived_from") + source_digest = _hex64( + derived.get("source_artifact_digest"), + "artifact.derived_from.source_artifact_digest", + ) + binding = ( + _integer(derived.get("shard_start"), "derived_from.shard_start", 0), + _integer(derived.get("shard_end"), "derived_from.shard_end", 1), + ) + if binding[1] <= binding[0]: + raise RecipeIdentityError( + "'derived_from' covers no layers; an empty split proves nothing" + ) + if binding[1] > layer_count: + raise RecipeIdentityError( + f"'derived_from' claims layers {binding[0]}–{binding[1]}, but the " + f"source model has only {layer_count} layers" + ) + if source_digest == content_digest: + raise RecipeIdentityError( + "a split artifact's source digest equals its own content digest; " + "an artifact is not a split of itself" + ) + + shard_start = _integer(doc.get("shard_start"), "shard_start", 0) + shard_end = _integer(doc.get("shard_end"), "shard_end", 1) + if shard_end <= shard_start: + raise RecipeIdentityError("a Shard owning no layer computes nothing") + if shard_end > layer_count: + raise RecipeIdentityError( + f"Shard owns layers {shard_start}–{shard_end}, but the artifact has only " + f"{layer_count} layers" + ) + if binding is not None and not ( + binding[0] <= shard_start and shard_end <= binding[1] + ): + raise RecipeIdentityError( + f"Shard advertises layers {shard_start}–{shard_end}, but its split " + f"artifact only contains layers {binding[0]}–{binding[1]}" + ) + + axes: dict[str, Any] = {} + for axis in RECIPE_AXES: + if axis not in recipe: + raise RecipeIdentityError( + f"recipe is missing axis {axis!r}; an unstated axis cannot default" + ) + value = recipe[axis] + if axis in _INT_AXES: + axes[axis] = _integer(value, f"recipe.{axis}", 1) + else: + axes[axis] = _text(value, f"recipe.{axis}") + _pin(axes["tokenizer_revision"], "recipe.tokenizer_revision") + + identity = PresentedIdentity( + artifact_id=_text(artifact.get("artifact_id"), "artifact.artifact_id"), + revision=_pin(artifact.get("revision"), "artifact.revision"), + source_digest=source_digest, + architecture=_text(artifact.get("architecture"), "artifact.architecture"), + architecture_digest=_hex64( + artifact.get("architecture_digest"), "artifact.architecture_digest" + ), + layer_count=layer_count, + is_derivative=binding is not None, + shard_start=shard_start, + shard_end=shard_end, + axes=axes, + recipe_id=_text(recipe.get("recipe_id"), "recipe.recipe_id"), + recipe_version=_text(recipe.get("recipe_version"), "recipe.recipe_version"), + catalogue_version=_text( + recipe.get("catalogue_version"), "recipe.catalogue_version" + ), + ) + + declared = doc.get("fingerprint") + if declared is not None: + claim = _mapping(declared, "identity.fingerprint") + claimed_artifact = _hex64( + claim.get("model_artifact_digest"), "fingerprint.model_artifact_digest" + ) + claimed_recipe = _hex64( + claim.get("runtime_recipe_digest"), "fingerprint.runtime_recipe_digest" + ) + if (claimed_artifact, claimed_recipe) != identity.key: + raise FingerprintMismatch( + "declared fingerprint is inconsistent with the artifact and recipe " + "claim; the tracker recomputes compatibility digests" + ) + + return identity + + +@dataclass(frozen=True) +class RouteMismatch: + reason: str + detail: str + + def describe(self) -> str: + return f"{self.reason}: {self.detail}" + + +def compare( + route: PresentedIdentity, candidate: PresentedIdentity +) -> tuple[RouteMismatch, ...]: + """Why `candidate` may not join a route already running `route`.""" + if route.key == candidate.key: + return () + + reasons: list[RouteMismatch] = [] + if route.source_digest != candidate.source_digest: + reasons.append( + RouteMismatch( + MISMATCH_ARTIFACT, + f"serves Model Artifact {candidate.source_digest[:12]}…, the route " + f"runs {route.source_digest[:12]}…", + ) + ) + if route.architecture_digest != candidate.architecture_digest: + reasons.append( + RouteMismatch( + MISMATCH_ARCHITECTURE, + "architecture/config snapshot differs from the route's", + ) + ) + if route.layer_count != candidate.layer_count: + reasons.append( + RouteMismatch( + MISMATCH_ARTIFACT, + f"artifact has {candidate.layer_count} layers, the route's has " + f"{route.layer_count}", + ) + ) + + axis_reason = { + "tokenizer_revision": MISMATCH_TOKENIZER, + "architecture_adapter": MISMATCH_ARCHITECTURE, + "activation_dtype": MISMATCH_ACTIVATION_RECIPE, + "compute_dtype": MISMATCH_ACTIVATION_RECIPE, + "kv_dtype": MISMATCH_CACHE_LAYOUT, + "kv_layout": MISMATCH_CACHE_LAYOUT, + "boundary_schema_version": MISMATCH_BOUNDARY_SCHEMA, + "protocol_schema_version": MISMATCH_BOUNDARY_SCHEMA, + } + for axis in RECIPE_AXES: + mine = route.axes.get(axis) + theirs = candidate.axes.get(axis) + if mine != theirs: + reasons.append( + RouteMismatch( + axis_reason.get(axis, MISMATCH_FINGERPRINT), + f"{axis} is {theirs!r}, the route runs {mine!r}", + ) + ) + + if not reasons: + reasons.append( + RouteMismatch( + MISMATCH_FINGERPRINT, + "fingerprints differ on a field this comparison does not cover", + ) + ) + return tuple(reasons) + + +def coverage_gap( + ranges: Iterable[tuple[int, int]], layer_count: int +) -> str | None: + """Why `ranges` fail to tile ``[0, layer_count)``, or None when they do. + + Overlaps are legal — ADR-0012 lets the Tracker resolve one by telling a hop + where the previous hop stopped. A hole is not: its layers are never computed, + and the route emits fluent tokens from a truncated model. + """ + ordered = sorted(ranges) + if not ordered: + return "the route owns no layers" + if ordered[0][0] != 0: + return f"layers 0–{ordered[0][0]} are owned by no Shard on the route" + + covered = 0 + for start, end in ordered: + if start > covered: + return f"layers {covered}–{start} are owned by no Shard on the route" + covered = max(covered, end) + + if covered < layer_count: + return f"layers {covered}–{layer_count} are owned by no Shard on the route" + if covered > layer_count: + return ( + f"the route claims {covered} layers, but the model has only {layer_count}" + ) + return None + + +@dataclass(frozen=True) +class DistributedForwardEvidence: + """A real distributed forward — the only thing that certifies a recipe.""" + + route_session_id: str + route_epoch: int + node_ids: tuple[str, ...] + shard_ranges: tuple[tuple[int, int], ...] + tokens_generated: int + layer_count: int + fingerprint: tuple[str, str] + synthetic: bool = False + certified_at: float = field(default_factory=time.time) + + def rejection(self) -> str | None: + if self.synthetic: + return ( + "evidence comes from a synthetic worker; only a real distributed " + "forward certifies a recipe" + ) + distinct = set(self.node_ids) + if len(distinct) < MIN_CERTIFYING_NODES: + return ( + f"evidence covers {len(distinct)} distinct node(s); a distributed " + f"forward requires at least {MIN_CERTIFYING_NODES}" + ) + if len(distinct) != len(self.node_ids): + return "the same node is counted more than once in the certifying route" + if self.tokens_generated < 1: + return "the certifying forward generated no tokens" + return coverage_gap(self.shard_ranges, self.layer_count) + + +@dataclass(frozen=True) +class RecipeStatus: + status: str + detail: str = "" + certified_at: float | None = None + + @property + def may_serve(self) -> bool: + return self.status == STATUS_CERTIFIED + + @property + def may_certify(self) -> bool: + """A dark recipe may be routed to *certify* it, and for nothing else. + + Without this, certification is unreachable by construction: serving needs + certification, certification needs a real distributed forward, and a real + distributed forward needs a route. + """ + return self.status in (STATUS_DARK, STATUS_CERTIFIED) + + def to_dict(self) -> dict: + return { + "status": self.status, + "detail": self.detail, + "certified_at": self.certified_at, + } + + +_DARK_DETAIL = "registered; dark until a real distributed forward certifies it" +_UNKNOWN_DETAIL = "this recipe has never been registered with the tracker" + + +class CertificationLedger: + """Which recipes the tracker has seen, and which a real forward has proven.""" + + def __init__(self) -> None: + self._dark: set[tuple[str, str]] = set() + self._certified: dict[tuple[str, str], RecipeStatus] = {} + + def register(self, identity: PresentedIdentity) -> RecipeStatus: + key = identity.key + if key in self._certified: + return self._certified[key] + self._dark.add(key) + return RecipeStatus(STATUS_DARK, _DARK_DETAIL) + + def status(self, identity: PresentedIdentity | tuple[str, str]) -> RecipeStatus: + key = identity if isinstance(identity, tuple) else identity.key + if key in self._certified: + return self._certified[key] + if key in self._dark: + return RecipeStatus(STATUS_DARK, _DARK_DETAIL) + return RecipeStatus(STATUS_UNKNOWN, _UNKNOWN_DETAIL) + + def certify( + self, + identity: PresentedIdentity | tuple[str, str], + evidence: DistributedForwardEvidence, + ) -> RecipeStatus: + """Promote a recipe out of the dark, or raise saying why the evidence is short.""" + key = identity if isinstance(identity, tuple) else identity.key + if key not in self._dark and key not in self._certified: + raise RecipeIdentityError( + "this fingerprint is not registered; unknown recipes cannot be certified" + ) + if evidence.fingerprint != key: + raise RecipeIdentityError( + "certification evidence fingerprint does not match the recipe being promoted" + ) + rejection = evidence.rejection() + if rejection is not None: + raise RecipeIdentityError(f"this evidence does not certify: {rejection}") + status = RecipeStatus( + STATUS_CERTIFIED, + ( + f"certified by route session {evidence.route_session_id} across " + f"{len(set(evidence.node_ids))} nodes" + ), + certified_at=evidence.certified_at, + ) + self._certified[key] = status + self._dark.discard(key) + return status + + def to_dict(self) -> dict: + return { + "dark": [list(key) for key in sorted(self._dark)], + "certified": { + "/".join(key): status.to_dict() + for key, status in sorted(self._certified.items()) + }, + } + + +def admit_route( + identities: Sequence[PresentedIdentity], + *, + ledger: CertificationLedger | None = None, + for_certification: bool = False, +) -> tuple[RouteMismatch, ...]: + """Every reason these Shards may not form one Inference Route; empty when they may. + + Fail-closed by construction: an empty route, a hole in the coverage, one + differing dtype, and an uncertified recipe each produce a reason, so a caller + that proceeds only on an empty result cannot admit any of them. + """ + if not identities: + return (RouteMismatch(MISMATCH_RANGE, "a route needs at least one Shard"),) + + head = identities[0] + reasons: list[RouteMismatch] = [] + for candidate in identities[1:]: + reasons.extend(compare(head, candidate)) + + gap = coverage_gap( + [(i.shard_start, i.shard_end) for i in identities], head.layer_count + ) + if gap is not None: + reasons.append(RouteMismatch(MISMATCH_RANGE, gap)) + + if ledger is not None: + status = ledger.status(head) + allowed = status.may_certify if for_certification else status.may_serve + if not allowed: + reasons.append( + RouteMismatch(MISMATCH_UNCERTIFIED, f"{status.status}: {status.detail}") + ) + + return tuple(reasons) diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py index 2683036..77def85 100644 --- a/packages/tracker/meshnet_tracker/server.py +++ b/packages/tracker/meshnet_tracker/server.py @@ -84,6 +84,7 @@ from .routing_stats import ( ) from .model_files import files_for_layer_range, snapshot_dir_for_repo from .raft import RaftNode +from .recipe import CertificationLedger _CONSOLE_LIMIT = 300 @@ -792,6 +793,7 @@ def _capability_from_registration( hf_repo: str | None, shard_start: int | None, shard_end: int | None, + recipe_certifications: CertificationLedger, ) -> CapabilityState: """The tracker's verdict on the proof carried by one registration payload. @@ -811,6 +813,7 @@ def _capability_from_registration( declared_recipe_version=( recipe_version if isinstance(recipe_version, str) else None ), + ledger=recipe_certifications, ) @@ -2847,9 +2850,11 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer): relay_status: dict | None = None, test_runner: "TestRunManager | None" = None, capability_policy: str | None = None, + recipe_certifications: CertificationLedger | None = None, ) -> None: super().__init__(addr, handler) self.registry = registry + self.recipe_certifications = recipe_certifications or CertificationLedger() self.capability_policy = normalize_policy( capability_policy if capability_policy is not None else policy_from_env() ) @@ -4555,6 +4560,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler): hf_repo=hf_repo, shard_start=shard_start, shard_end=shard_end, + recipe_certifications=server.recipe_certifications, ) node_id = _node_id_for_registration( @@ -6531,6 +6537,7 @@ class TrackerServer: self._embedded_relay: Any | None = None self._embedded_relay_actual_port: int | None = None self._registry: dict[str, _NodeEntry] = {} + self._recipe_certifications = CertificationLedger() self._lock = threading.Lock() self._server: _TrackerHTTPServer | None = None self._thread: threading.Thread | None = None @@ -6725,6 +6732,7 @@ class TrackerServer: relay_status=http_relay_status, test_runner=self._test_runner, capability_policy=self._capability_policy, + recipe_certifications=self._recipe_certifications, ) self.port = self._server.server_address[1] @@ -6983,6 +6991,7 @@ class TrackerServer: hf_repo=payload.get("hf_repo"), shard_start=shard_start, shard_end=shard_end, + recipe_certifications=self._recipe_certifications, ), ) with self._lock: diff --git a/tests/data/recipe_fingerprint_vectors.json b/tests/data/recipe_fingerprint_vectors.json new file mode 100644 index 0000000..6fcfe37 --- /dev/null +++ b/tests/data/recipe_fingerprint_vectors.json @@ -0,0 +1,52 @@ +{ + "schema_version": 1, + "vectors": [ + { + "fingerprint": { + "catalogue_version": "2026.07.1", + "model_artifact_digest": "8a0f43d6aa49d77834bdb47bcae9f42c886b7ccfe0ac014932b2a2b38697a47b", + "recipe_id": "example-gguf", + "recipe_version": "1", + "runtime_recipe_digest": "9b14d70b0835a6428457e4888d453649dd0d2e41fc8ac9d84d232c8c237e68fa" + }, + "identity": { + "artifact": { + "architecture": "dense-llama", + "architecture_digest": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "artifact_id": "example/model", + "content_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "derived_from": null, + "layer_count": 8, + "revision": "0123456789abcdef" + }, + "fingerprint": { + "catalogue_version": "2026.07.1", + "model_artifact_digest": "8a0f43d6aa49d77834bdb47bcae9f42c886b7ccfe0ac014932b2a2b38697a47b", + "recipe_id": "example-gguf", + "recipe_version": "1", + "runtime_recipe_digest": "9b14d70b0835a6428457e4888d453649dd0d2e41fc8ac9d84d232c8c237e68fa" + }, + "recipe": { + "activation_dtype": "bfloat16", + "architecture_adapter": "llama/range-v1", + "backend_id": "llama.cpp", + "boundary_schema_version": 1, + "catalogue_version": "2026.07.1", + "compute_dtype": "float32", + "kv_dtype": "q8_0", + "kv_layout": "paged-v1", + "protocol_schema_version": 1, + "recipe_id": "example-gguf", + "recipe_version": "1", + "runtime_version": "llama.cpp@deadbeef+meshnet.1", + "tokenizer_revision": "0123456789abcdef", + "weight_quantization": "Q4_K_M" + }, + "schema_version": 1, + "shard_end": 4, + "shard_start": 0 + }, + "name": "example-v1" + } + ] +} diff --git a/tests/test_runtime_recipe_identity.py b/tests/test_runtime_recipe_identity.py new file mode 100644 index 0000000..9cd753a --- /dev/null +++ b/tests/test_runtime_recipe_identity.py @@ -0,0 +1,270 @@ +"""Deterministic DGR-003 identity and tracker-admission conformance tests.""" + +from __future__ import annotations + +from dataclasses import replace +import json +from pathlib import Path +import time + +import pytest + +from meshnet_node.runtime_recipe import ( + ArtifactIdentity, + CompatibilityFingerprint, + DerivativeBinding, + RecipeIdentityError, + RuntimeRecipe, + ShardIdentity, + check_handshake, + check_route, + handshake_error, +) +from meshnet_tracker.capability import ( + STATE_ADMITTED, + STATE_FINGERPRINT_MISMATCH, + STATE_MODEL_MISMATCH, + STATE_RECIPE_MISMATCH, + STATE_UNCERTIFIED, + evaluate_report, +) +from meshnet_tracker.server import TrackerServer, _capability_from_registration +from meshnet_tracker.recipe import ( + CertificationLedger, + DistributedForwardEvidence, + parse_identity, +) + + +def _digest(char: str) -> str: + return char * 64 + + +def _identity(start: int = 0, end: int = 4, **recipe_changes: object) -> ShardIdentity: + recipe_fields: dict[str, object] = { + "weight_quantization": "Q4_K_M", + "activation_dtype": "bfloat16", + "compute_dtype": "float32", + "kv_dtype": "q8_0", + "kv_layout": "paged-v1", + "tokenizer_revision": "0123456789abcdef", + "architecture_adapter": "llama/range-v1", + "backend_id": "llama.cpp", + "runtime_version": "llama.cpp@deadbeef+meshnet.1", + "recipe_id": "example-gguf", + "recipe_version": "1", + "catalogue_version": "2026.07.1", + } + recipe_fields.update(recipe_changes) + recipe = RuntimeRecipe(**recipe_fields) # type: ignore[arg-type] + return ShardIdentity( + ArtifactIdentity( + "example/model", + "0123456789abcdef", + _digest("a"), + "dense-llama", + _digest("b"), + 8, + ), + recipe, + start, + end, + ) + + +def _report(identity: ShardIdentity) -> dict: + return { + "schema_version": 1, + "model": {"model_id": "example/model"}, + "shard": {"start": identity.shard_start, "end": identity.shard_end - 1}, + "recipe": identity.recipe.to_dict() | { + "recipe_id": identity.recipe.recipe_id, + "recipe_version": identity.recipe.recipe_version, + "catalogue_version": identity.recipe.catalogue_version, + }, + "backend": {"backend_id": "llama.cpp", "device": "test"}, + "status": "passed", + "validated_at": 100.0, + "duration_ms": 1, + "diagnostics": [], + "identity": identity.to_dict(), + } + + +def _evaluate(report: dict, **kwargs: object): + return evaluate_report( + report, + model_matches=lambda model: model == "example/model", + advertised_model="example/model", + shard_start=0, + shard_end=3, + now=100.0, + **kwargs, + ) + + +def test_node_and_tracker_share_the_committed_canonical_fingerprint_vector(): + vector_path = Path(__file__).parent / "data" / "recipe_fingerprint_vectors.json" + vector = json.loads(vector_path.read_text(encoding="utf-8"))["vectors"][0] + expected = vector["fingerprint"] + identity = ShardIdentity.from_dict(vector["identity"]) + + assert identity.fingerprint.to_dict() == expected + assert parse_identity(vector["identity"]).fingerprint_dict() == expected + assert ( + CompatibilityFingerprint.from_proto(identity.fingerprint.to_proto()).to_dict() + == expected + ) + + +@pytest.mark.parametrize( + "axis,value", + [ + ("weight_quantization", "Q5_K_M"), + ("activation_dtype", "float16"), + ("compute_dtype", "float16"), + ("kv_dtype", "float16"), + ("kv_layout", "contiguous-v2"), + ("tokenizer_revision", "fedcba9876543210"), + ("architecture_adapter", "llama/range-v2"), + ("backend_id", "other-backend"), + ("runtime_version", "llama.cpp@other+meshnet.1"), + ("boundary_schema_version", 2), + ("protocol_schema_version", 2), + ], +) +def test_every_recipe_axis_changes_the_fingerprint_and_blocks_route(axis, value): + left = _identity() + right = _identity(4, 8, **{axis: value}) + + assert left.fingerprint.key != right.fingerprint.key + assert check_route([left, right]) + + +def test_split_artifact_is_bound_to_exact_source_and_owned_range(): + source = _identity() + split = ShardIdentity( + ArtifactIdentity( + "example/model", + "0123456789abcdef", + _digest("c"), + "dense-llama", + _digest("b"), + 8, + DerivativeBinding(_digest("a"), 4, 8), + ), + source.recipe, + 4, + 8, + ) + assert source.fingerprint.matches(split.fingerprint) + with pytest.raises(RecipeIdentityError): + ShardIdentity(split.artifact, split.recipe, 3, 8) + + +def test_declared_digest_mismatch_is_recomputed_and_rejected_not_authenticated(): + report = _report(_identity()) + report["identity"]["fingerprint"]["runtime_recipe_digest"] = _digest("f") + + assert _evaluate(report).state == STATE_FINGERPRINT_MISMATCH + + +def test_exact_recipe_registers_dark_until_tracker_certification(): + identity = _identity() + report = _report(identity) + ledger = CertificationLedger() + + dark = _evaluate(report, ledger=ledger) + assert dark.state == STATE_UNCERTIFIED + # Unit fixtures cannot promote a recipe: this is the explicit trust boundary. + with pytest.raises(ValueError, match="synthetic"): + ledger.certify( + parse_identity(identity.to_dict()), + DistributedForwardEvidence( + "session", + 1, + ("a", "b"), + ((0, 4), (4, 8)), + 1, + 8, + identity.fingerprint.key, + synthetic=True, + ), + ) + + with pytest.raises(ValueError, match="fingerprint does not match"): + ledger.certify( + parse_identity(identity.to_dict()), + DistributedForwardEvidence( + "session", + 1, + ("a", "b"), + ((0, 4), (4, 8)), + 1, + 8, + (_digest("e"), _digest("f")), + ), + ) + + +def test_capability_identity_must_match_the_proof_labels(): + identity = _identity() + + wrong_model = _report(identity) + wrong_model["identity"]["artifact"]["artifact_id"] = "other/model" + wrong_model["identity"]["fingerprint"] = None + assert _evaluate(wrong_model).state == STATE_MODEL_MISMATCH + + wrong_recipe = _report(identity) + wrong_recipe["recipe"]["recipe_id"] = "other-recipe" + assert _evaluate(wrong_recipe).state == STATE_RECIPE_MISMATCH + + wrong_backend = _report(identity) + wrong_backend["backend"]["backend_id"] = "other-backend" + assert _evaluate(wrong_backend).state == STATE_RECIPE_MISMATCH + + wrong_quantization = _report(identity) + wrong_quantization["backend"]["quantization"] = "Q5_K_M" + assert _evaluate(wrong_quantization).state == STATE_RECIPE_MISMATCH + + +def test_tracker_server_owns_the_ledger_used_by_registration(): + identity = _identity() + report = _report(identity) + report["validated_at"] = time.time() + payload = {"capability_report": report} + tracker = TrackerServer() + + def evaluate(): + return _capability_from_registration( + payload, + model="example/model", + hf_repo=None, + shard_start=0, + shard_end=3, + recipe_certifications=tracker._recipe_certifications, + ) + + assert evaluate().state == STATE_UNCERTIFIED + tracker._recipe_certifications.certify( + parse_identity(identity.to_dict()), + DistributedForwardEvidence( + "session", + 1, + ("physical-a", "physical-b"), + ((0, 4), (4, 8)), + 1, + 8, + identity.fingerprint.key, + ), + ) + assert evaluate().state == STATE_ADMITTED + + +def test_grpc_handshake_uses_the_dgr_002_fingerprint_and_fails_closed(): + local = _identity() + remote = replace(local.fingerprint, runtime_recipe_digest=_digest("f")).to_proto() + + mismatches = check_handshake(local, remote) + assert mismatches + assert handshake_error(mismatches).code != 0