popov/neuron-tai
1
0
Fork 0
Code Issues 42 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge DGR-025 (certified-artifact-byte recipe identity) from ralph-fable-loop lane

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-21 13:23:17 +03:00
parent 732ee9f91a 03e97ca31a
commit 159284b3b5
10 changed files with 1277 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

422
packages/node/meshnet_node/native_backend.py
View File

@@ -9,10 +9,15 @@ authoritative immutable GGUF artifact pin and must remain identity-free.
from __future__ import annotations
from dataclasses import dataclass
import ctypes
import hashlib
import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from .native_protocol import BUNDLE_VERSION, SCHEMA_VERSION, pb
from .runtime_pin import load_runtime_pin
from .runtime_pin import DEFAULT_LOCK_DIR, RuntimePin, load_runtime_pin
from .runtime_recipe import (
ArtifactIdentity,
DerivativeBinding,
@@ -23,6 +28,323 @@ from .runtime_recipe import (
handshake_error,
)
_HEX40 = re.compile(r"^[0-9a-f]{40}$")
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
# The executing-runtime attestation contract.
#
# The repository lock is world-readable, so a Python object holding
# lock-shaped values proves nothing about the runtime that will execute:
# copying `load_runtime_pin()` into a self-report is exactly the forgery
# DGR-025 forbids. Attestation values are therefore accepted only when
# *extracted from the native artifact itself*, through two channels that must
# agree:
#
# 1. static — the artifact's bytes embed exactly one
# ``MESHNET-RUNTIME-ATTESTATION.v1:<canonical json>`` marker (NUL
# terminated). The DGR-027 CMake ABI-marker lane is where the native
# build bakes it in from the lock at configure time.
# 2. dynamic — the artifact must actually dlopen, and its exported
# ``llama_meshnet_runtime_attestation`` symbol must return that same
# marker. A marker pasted into a plain file is not an executing runtime.
#
# What this cannot prove: a cross-compiler bit-reproducible binary SHA, or
# that an adversary did not *build* a native artifact that embeds lock-true
# values while lying about its source. Manufacturing a lying native build is
# a categorically higher bar than authoring a Python dict, and real
# distributed certification (DGR-025's registered-but-dark ledger) remains
# the final backstop behind this boundary.
ATTESTATION_MARKER_PREFIX = b"MESHNET-RUNTIME-ATTESTATION.v1:"
ATTESTATION_SYMBOL = "llama_meshnet_runtime_attestation"
_ATTESTATION_STR_FIELDS = (
"runtime_name",
"upstream_commit",
"patched_tree",
"patch_stack_digest",
"build_recipe_digest",
)
_ATTESTATION_INT_FIELDS = ("boundary_schema_version", "protocol_schema_version")
# Module-private capability: evidence can only be minted where an artifact
# was actually read, scanned, loaded, and queried.
_EVIDENCE_TOKEN = object()
def attestation_payload(
*,
runtime_name: str,
upstream_commit: str,
patched_tree: str,
patch_stack_digest: str,
build_recipe_digest: str,
boundary_schema_version: int,
protocol_schema_version: int,
) -> bytes:
"""The canonical marker payload for one exact runtime.
This single encoding is shared by the build lane that embeds the marker,
the extractor that parses it, and the binding check that ties attestation
fields to the extracted evidence — so there is exactly one byte string a
given runtime identity can legitimately embed.
"""
return json.dumps(
{
"runtime_name": runtime_name,
"upstream_commit": upstream_commit,
"patched_tree": patched_tree,
"patch_stack_digest": patch_stack_digest,
"build_recipe_digest": build_recipe_digest,
"boundary_schema_version": boundary_schema_version,
"protocol_schema_version": protocol_schema_version,
},
sort_keys=True,
separators=(",", ":"),
ensure_ascii=False,
).encode("utf-8")
def expected_attestation_payload(
pin: RuntimePin,
*,
boundary_schema_version: int = BUNDLE_VERSION,
protocol_schema_version: int = int(SCHEMA_VERSION),
) -> bytes:
"""The marker payload a native build of this lock workspace must embed."""
return attestation_payload(
runtime_name=pin.runtime_name,
upstream_commit=pin.upstream_commit,
patched_tree=pin.patched_tree,
patch_stack_digest=pin.patch_stack_digest,
build_recipe_digest=pin.build_recipe_digest,
boundary_schema_version=boundary_schema_version,
protocol_schema_version=protocol_schema_version,
)
@dataclass(frozen=True)
class NativeArtifactEvidence:
"""Proof that attestation values came out of a loadable native artifact.
``binary_digest`` pins *which* artifact bytes were attested;
``payload_digest`` pins *what* those bytes attested, and is re-derived
from the attestation's own fields on construction so the values cannot be
edited after extraction (``dataclasses.replace`` laundering fails).
Only :func:`attest_loaded_runtime` can mint this object.
"""
artifact_path: str
binary_digest: str
payload_digest: str
_token: object = field(default=None, repr=False, compare=False)
def __post_init__(self) -> None:
if self._token is not _EVIDENCE_TOKEN:
raise RecipeIdentityError(
"native artifact evidence can only be minted by "
"attest_loaded_runtime() from an actually loaded native "
"artifact; it cannot be authored from repository lock values"
)
if not isinstance(self.artifact_path, str) or not self.artifact_path:
raise RecipeIdentityError("native artifact evidence must name the artifact")
for field_name in ("binary_digest", "payload_digest"):
value = getattr(self, field_name)
if not isinstance(value, str) or not _HEX64.fullmatch(value):
raise RecipeIdentityError(
f"native artifact evidence {field_name!r} must be a 64-hex sha256"
)
@dataclass(frozen=True)
class NativeRuntimeAttestation:
"""What the *executing* runtime reports about itself, at load time.
The repository lock says what the runtime is supposed to be; this says
what the loaded runtime *is* — the source tree it was built from, the
patch stack compiled into it, the numerically relevant build recipe, and
the boundary/protocol schema (ABI) it speaks. The values are never
accepted from a caller: they must arrive bound to
:class:`NativeArtifactEvidence`, which only
:func:`attest_loaded_runtime` can produce by reading, loading, and
querying the native artifact itself. Identity construction then compares
them to the lock/build-derived expectation and refuses on any difference,
so a worker cannot serve a lock it is not actually running — and cannot
fake one by copying the lock into a Python self-report.
Deliberately *not* attested: a compiler-specific binary SHA. Binding the
recorded build recipe is honest about what the manifest can prove;
bit-reproducible binary attestation is not claimed.
"""
runtime_name: str
upstream_commit: str
patched_tree: str
patch_stack_digest: str
build_recipe_digest: str
boundary_schema_version: int
protocol_schema_version: int
evidence: NativeArtifactEvidence
def __post_init__(self) -> None:
if not isinstance(self.evidence, NativeArtifactEvidence):
raise RecipeIdentityError(
"runtime attestation values must be extracted from the loaded "
"native artifact via attest_loaded_runtime(); a Python "
"self-report carrying copied lock values is not an attestation"
)
if not isinstance(self.runtime_name, str) or not self.runtime_name.strip():
raise RecipeIdentityError(
"runtime attestation must name the executing runtime"
)
for field_name, pattern, what in (
("upstream_commit", _HEX40, "40-hex upstream commit"),
("patched_tree", _HEX40, "40-hex patched source tree id"),
("patch_stack_digest", _HEX64, "64-hex patch-stack digest"),
("build_recipe_digest", _HEX64, "64-hex build-recipe digest"),
):
value = getattr(self, field_name)
if not isinstance(value, str) or not pattern.fullmatch(value):
raise RecipeIdentityError(
f"runtime attestation {field_name!r} must be an exact {what}"
)
for field_name in _ATTESTATION_INT_FIELDS:
value = getattr(self, field_name)
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
raise RecipeIdentityError(
f"runtime attestation {field_name!r} must be a positive integer"
)
payload = attestation_payload(
**{name: getattr(self, name) for name in _ATTESTATION_STR_FIELDS},
**{name: getattr(self, name) for name in _ATTESTATION_INT_FIELDS},
)
if hashlib.sha256(payload).hexdigest() != self.evidence.payload_digest:
raise RecipeIdentityError(
"runtime attestation fields do not match the attestation "
"extracted from the native artifact; refusing values edited "
"after extraction"
)
def _parse_attestation_payload(payload: bytes) -> dict[str, object]:
"""Strictly parse one embedded marker payload, or refuse."""
try:
doc = json.loads(payload.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RecipeIdentityError(
f"embedded runtime attestation marker is not valid JSON: {exc}"
) from exc
expected_keys = set(_ATTESTATION_STR_FIELDS) | set(_ATTESTATION_INT_FIELDS)
if not isinstance(doc, dict) or set(doc) != expected_keys:
raise RecipeIdentityError(
"embedded runtime attestation marker must record exactly the "
"attestation fields"
)
for name in _ATTESTATION_STR_FIELDS:
if not isinstance(doc[name], str):
raise RecipeIdentityError(
f"embedded runtime attestation field {name!r} must be a string"
)
for name in _ATTESTATION_INT_FIELDS:
if isinstance(doc[name], bool) or not isinstance(doc[name], int):
raise RecipeIdentityError(
f"embedded runtime attestation field {name!r} must be an integer"
)
if attestation_payload(**doc) != payload: # type: ignore[arg-type]
raise RecipeIdentityError(
"embedded runtime attestation marker is not in canonical form"
)
return doc
def attest_loaded_runtime(artifact_path: Path | str) -> NativeRuntimeAttestation:
"""Extract the executing runtime's attestation from its native artifact.
Fails closed when the artifact is missing or empty, embeds no attestation
marker (a runtime built without the attestation lane cannot prove what it
is), embeds conflicting markers, is not a loadable shared object, does
not export :data:`ATTESTATION_SYMBOL`, or reports through that symbol
anything other than the embedded marker.
The returned attestation is bound to the artifact by its byte digest and
to the extracted values by the payload digest. Loading the artifact does
execute its initializers — this is the same artifact the worker is about
to run inference with, so that adds no new execution. An OS-level swap
of the file between the byte read and the dlopen is a documented
residual race; distributed certification remains the final backstop.
"""
path = Path(artifact_path)
try:
data = path.read_bytes()
except FileNotFoundError:
raise RecipeIdentityError(
f"native runtime artifact not found at {path}; without the built "
"native runtime there is no executing identity to attest"
) from None
except OSError as exc:
raise RecipeIdentityError(
f"native runtime artifact at {path} is unreadable: {exc}"
) from exc
if not data:
raise RecipeIdentityError(f"native runtime artifact at {path} is empty")
payloads: list[bytes] = []
cursor = 0
while (start := data.find(ATTESTATION_MARKER_PREFIX, cursor)) >= 0:
end = data.find(b"\x00", start)
if end < 0:
raise RecipeIdentityError(
"embedded runtime attestation marker is not NUL-terminated"
)
payloads.append(data[start + len(ATTESTATION_MARKER_PREFIX) : end])
cursor = end
if not payloads:
raise RecipeIdentityError(
f"native runtime artifact at {path} embeds no runtime attestation "
"marker; a runtime built without the attestation lane cannot "
"prove what it is"
)
if len(set(payloads)) != 1:
raise RecipeIdentityError(
f"native runtime artifact at {path} embeds conflicting runtime "
"attestation markers"
)
payload = payloads[0]
doc = _parse_attestation_payload(payload)
try:
library = ctypes.CDLL(str(path), mode=ctypes.RTLD_LOCAL)
except OSError as exc:
raise RecipeIdentityError(
f"native runtime artifact at {path} is not a loadable native "
"artifact; an attestation marker copied into a plain file is not "
"an executing runtime"
) from exc
try:
symbol = getattr(library, ATTESTATION_SYMBOL)
except AttributeError:
raise RecipeIdentityError(
f"native runtime artifact at {path} does not export "
f"{ATTESTATION_SYMBOL}; the loaded runtime itself must report "
"its attestation"
) from None
symbol.restype = ctypes.c_char_p
symbol.argtypes = []
reported = symbol()
if reported != ATTESTATION_MARKER_PREFIX + payload:
raise RecipeIdentityError(
f"the runtime loaded from {path} reports a different attestation "
"than its artifact embeds; refusing an artifact that disagrees "
"with itself"
)
evidence = NativeArtifactEvidence(
artifact_path=str(path),
binary_digest=hashlib.sha256(data).hexdigest(),
payload_digest=hashlib.sha256(payload).hexdigest(),
_token=_EVIDENCE_TOKEN,
)
return NativeRuntimeAttestation(evidence=evidence, **doc) # type: ignore[arg-type]
@dataclass(frozen=True)
class NativeLoadedArtifactReport:
@@ -32,6 +354,10 @@ class NativeLoadedArtifactReport:
parsed GGUF metadata while the model is live. Byte counts are operational
evidence rather than compatibility axes, but keeping them beside the range
prevents a caller from substituting an unverified range declaration.
``runtime_attestation`` must be the evidence-bound attestation extracted
from the loaded native artifact by :func:`attest_loaded_runtime`; a
report without one cannot be turned into an identity at all, and one
cannot exist without an actual native artifact to extract it from.
"""
owned_start_layer: int
@@ -42,6 +368,7 @@ class NativeLoadedArtifactReport:
architecture: str
architecture_digest: str
layer_count: int
runtime_attestation: NativeRuntimeAttestation
def __post_init__(self) -> None:
if self.owned_start_layer < 0 or self.owned_end_layer <= self.owned_start_layer:
@@ -50,6 +377,10 @@ class NativeLoadedArtifactReport:
raise RecipeIdentityError("native report range is outside GGUF layer metadata")
if min(self.mapped_bytes, self.resident_bytes, self.registered_bytes) < 0:
raise RecipeIdentityError("native report byte counts must be non-negative")
if not isinstance(self.runtime_attestation, NativeRuntimeAttestation):
raise RecipeIdentityError(
"native report must carry the executing runtime's attestation"
)
@dataclass(frozen=True)
@@ -82,7 +413,12 @@ class NativeNumericalRecipe:
@dataclass(frozen=True)
class NativeIdentityInputs:
"""Everything a native backend needs to emit one exact identity."""
"""Everything a native backend needs to emit one exact identity.
``tokenizer_revision`` must be the content-addressed identity computed by
:func:`meshnet_node.runtime_recipe.tokenizer_identity` over the loaded
tokenizer/config bytes; identity construction rejects anything else.
"""
loaded_artifact: NativeLoadedArtifactReport
artifact_pin: ImmutableArtifactPin
@@ -90,8 +426,72 @@ class NativeIdentityInputs:
numerical_recipe: NativeNumericalRecipe
def shard_identity_from_native_report(inputs: NativeIdentityInputs) -> ShardIdentity:
"""Derive identity only from the native report and immutable pinned inputs."""
def _require_attested_runtime(
attested: NativeRuntimeAttestation,
expected: RuntimePin,
recipe: NativeNumericalRecipe,
) -> None:
"""Fail closed unless the executing runtime is the locked, built runtime.
Every comparison is exact and every difference is separately fatal: an
attestation that agrees on the commit but not the patch stack (or the
patched tree, or the build recipe, or the ABI) is a different runtime
wearing the lock's name, and letting it emit the lock's identity is
exactly the substitution DGR-025 exists to prevent.
"""
for what, got, want in (
("runtime name", attested.runtime_name, expected.runtime_name),
("upstream commit", attested.upstream_commit, expected.upstream_commit),
("patched source tree", attested.patched_tree, expected.patched_tree),
(
"ordered patch stack",
attested.patch_stack_digest,
expected.patch_stack_digest,
),
(
"build recipe",
attested.build_recipe_digest,
expected.build_recipe_digest,
),
):
if got != want:
raise RecipeIdentityError(
f"the executing runtime's attested {what} does not match the "
"lock/build-derived expectation; refusing to emit an identity "
"for a runtime this node is not provably running"
)
for what, got, want in (
(
"boundary schema",
attested.boundary_schema_version,
recipe.boundary_schema_version,
),
(
"protocol schema",
attested.protocol_schema_version,
recipe.protocol_schema_version,
),
):
if got != want:
raise RecipeIdentityError(
f"the executing runtime's attested {what} version ({got}) does "
f"not match the recipe's ({want}); an ABI the runtime does not "
"actually speak cannot be part of its identity"
)
def shard_identity_from_native_report(
inputs: NativeIdentityInputs,
*,
lock_dir: Path = DEFAULT_LOCK_DIR,
) -> ShardIdentity:
"""Derive identity only from the native report and immutable pinned inputs.
The ``runtime_version`` axis is never accepted from a caller: it is derived
from the committed lock workspace, and the loaded runtime's attestation
must match that lock/build-derived expectation exactly — otherwise this
raises and no identity exists to register, admit, or certify.
"""
report = inputs.loaded_artifact
pin = inputs.artifact_pin
recipe = inputs.numerical_recipe
@@ -99,7 +499,17 @@ def shard_identity_from_native_report(inputs: NativeIdentityInputs) -> ShardIden
raise RecipeIdentityError(
"native llama.cpp identity requires backend_id 'llama.cpp' or 'llama-cpp'"
)
runtime_version = load_runtime_pin().runtime_version
runtime_pin = load_runtime_pin(lock_dir)
_require_attested_runtime(report.runtime_attestation, runtime_pin, recipe)
# The lock-derived prefix identifies the intended source/patch/build
# recipe. The executing artifact digest identifies the bytes that actually
# supplied the attestation. Without this suffix, any independently built
# shared object could copy the public lock values into its marker and claim
# the exact same compatibility identity as the certified artifact.
runtime_version = (
f"{runtime_pin.runtime_version}"
f"+artifact.{report.runtime_attestation.evidence.binary_digest}"
)
artifact = ArtifactIdentity(
artifact_id=pin.artifact_id,
revision=pin.revision,

63
packages/node/meshnet_node/runtime_pin.py
View File

@@ -8,13 +8,16 @@ still hash to the same recipe. The DGR-027 lock manifest
and the ordered patch stack the native runtime is built from, so the axis
value is *derived* from that manifest, never asserted.
The derived value has three load-bearing parts, and each is separately fatal
The derived value has four load-bearing parts, and each is separately fatal
to compatibility: the runtime name (from the upstream URL), the exact
40-character upstream commit, and a digest over the ordered patch-stack bytes.
A different upstream pin, a reordered stack, or a single changed patch byte
each produce a different axis value, which produces a different recipe digest,
40-character upstream commit, a digest over the ordered patch-stack bytes, and
a digest over the numerically relevant build recipe. A different upstream pin,
a reordered stack, a single changed patch byte, or a changed build flag each
produce a different axis value, which produces a different recipe digest,
which partitions the route — exactly the fail-closed behavior DGR-025 asks
for.
for. The pin also records the patched source tree's git tree id, which the
executing runtime's attestation is compared against
(:mod:`meshnet_node.native_backend`).
Every consistency check here fails closed. The manifest keeps three records of
the stack — ``UPSTREAM_LOCK.json``'s ``patch_series``, ``patches/series``, and
@@ -36,6 +39,7 @@ from pathlib import Path
# Domain separation, matching the runtime_recipe digest convention: a patch
# stack digest can never be confused with an artifact or recipe digest.
PATCH_STACK_DIGEST_DOMAIN = "meshnet.runtime-patch-stack.v1"
BUILD_RECIPE_DIGEST_DOMAIN = "meshnet.runtime-build-recipe.v1"
# The UPSTREAM_LOCK.json layout this reader understands (DGR-027 schema).
RUNTIME_PIN_SCHEMA_VERSION = 1
@@ -58,14 +62,42 @@ def _canonical_sha256(value: object) -> str:
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
def build_recipe_digest(build: object) -> str:
"""A digest over the numerically relevant build recipe.
The lock's ``build`` object records what the runtime is compiled *as* —
the configure flags, standards, and targets that select kernels and
numeric behavior. Two binaries built from one patched tree with different
build recipes can disagree numerically, so the recipe is part of runtime
identity. This is deliberately a digest over the *recorded recipe*, not a
compiler-specific binary SHA: reproducible-binary attestation is not
claimed here.
"""
if not isinstance(build, dict) or not build:
raise RuntimePinError(
"the lock's 'build' section must be a non-empty JSON object; a "
"runtime with an unstated build recipe has an unknowable identity"
)
return _canonical_sha256(
{"domain": BUILD_RECIPE_DIGEST_DOMAIN, "body": build}
)
@dataclass(frozen=True)
class RuntimePin:
"""One exact runtime: a name, an upstream commit, and an ordered patch stack."""
"""One exact runtime: name, upstream commit, patched tree, patch stack, build.
``patched_tree`` is the git tree object id of the source tree *after* the
ordered patch stack is applied — what the runtime was actually compiled
from, as distinct from the upstream commit it started from.
"""
runtime_name: str
upstream_commit: str
patched_tree: str
patch_series: tuple[str, ...]
patch_digests: tuple[str, ...]
build_recipe_digest: str
@property
def patch_stack_digest(self) -> str:
@@ -89,10 +121,16 @@ class RuntimePin:
@property
def runtime_version(self) -> str:
"""The exact ``runtime_version`` recipe axis value for this pin."""
"""The exact ``runtime_version`` recipe axis value for this pin.
Commits to the runtime name, the exact upstream commit, the ordered
patch stack, and the numerically relevant build recipe — each
separately fatal to compatibility.
"""
return (
f"{self.runtime_name}@{self.upstream_commit}"
f"+patchstack.{self.patch_stack_digest}"
f"+build.{self.build_recipe_digest}"
)
@@ -166,6 +204,15 @@ def load_runtime_pin(lock_dir: Path = DEFAULT_LOCK_DIR) -> RuntimePin:
"hexadecimal object id; a moving reference is not a pin"
)
patched_tree = lock.get("patched_tree")
if not isinstance(patched_tree, str) or not _HEX40.match(patched_tree):
raise RuntimePinError(
"UPSTREAM_LOCK.json patched_tree must be the exact 40-character git "
"tree object id of the source tree after the patch stack is applied"
)
build_digest = build_recipe_digest(lock.get("build"))
commit_file = _read_text(lock_dir / "UPSTREAM_COMMIT", "UPSTREAM_COMMIT")
recorded = commit_file.strip().splitlines()[0].strip() if commit_file.strip() else ""
if recorded != commit:
@@ -216,6 +263,8 @@ def load_runtime_pin(lock_dir: Path = DEFAULT_LOCK_DIR) -> RuntimePin:
return RuntimePin(
runtime_name=runtime_name,
upstream_commit=commit,
patched_tree=patched_tree,
patch_series=tuple(lock_series),
patch_digests=tuple(digests),
build_recipe_digest=build_digest,
)

75
packages/node/meshnet_node/runtime_recipe.py
View File

@@ -73,6 +73,7 @@ RECIPE_IDENTITY_SCHEMA_VERSION = 1
ARTIFACT_DIGEST_DOMAIN = "meshnet.model-artifact.v1"
RECIPE_DIGEST_DOMAIN = "meshnet.runtime-recipe.v1"
SHARD_BINDING_DIGEST_DOMAIN = "meshnet.shard-binding.v1"
TOKENIZER_DIGEST_DOMAIN = "meshnet.tokenizer-identity.v1"
# The axes of a runtime recipe. Every one of these changes the numbers a Shard
# produces, so every one of them is part of identity and none of them may be
@@ -126,12 +127,22 @@ _AXIS_MISMATCH: Mapping[str, str] = {
_HEX64 = re.compile(r"^[0-9a-f]{64}$")
_LLAMA_CPP_RUNTIME_PIN = re.compile(
r"^llama\.cpp@[0-9a-f]{40}\+patchstack\.[0-9a-f]{64}$"
r"^llama\.cpp@[0-9a-f]{40}\+patchstack\.[0-9a-f]{64}"
r"\+build\.[0-9a-f]{64}\+artifact\.[0-9a-f]{64}$"
)
_LLAMA_CPP_BACKEND_IDS = frozenset({"llama.cpp", "llama-cpp"})
# A revision that can move is not a pin. DGR-017 learned this on the artifact;
# it is just as true of a tokenizer.
# The one shape a tokenizer identity may take: a digest over the tokenizer's
# actual bytes (see `tokenizer_identity`). Any *label* — `origin/main`,
# `stable`, `release`, a tag, a symbolic ref — names a mutable pointer, and a
# denylist of known-mutable names can never enumerate them all. So the check is
# inverted: instead of rejecting labels we recognize as moving, accept only a
# value that could not be a label in the first place.
_TOKENIZER_IDENTITY = re.compile(r"^tokenizer\.v1:[0-9a-f]{64}$")
# A revision that can move is not a pin. DGR-017 learned this on the artifact.
# Used for diagnosis-only fields (`artifact.revision`); the digested tokenizer
# axis requires the strictly stronger `_TOKENIZER_IDENTITY` form.
_MOVING_REFS = frozenset({"main", "master", "head", "latest", "dev", "trunk"})
@@ -204,7 +215,50 @@ def _require_runtime_pin(value: Any, backend_id: Any) -> str:
if backend in _LLAMA_CPP_BACKEND_IDS and not _LLAMA_CPP_RUNTIME_PIN.fullmatch(text):
raise RecipeIdentityError(
"'recipe.runtime_version' for llama.cpp must be "
"'llama.cpp@<40-hex commit>+patchstack.<64-hex digest>'"
"'llama.cpp@<40-hex commit>+patchstack.<64-hex digest>"
"+build.<64-hex digest>+artifact.<64-hex digest>'"
)
return text
def tokenizer_identity(files: Mapping[str, bytes]) -> str:
"""The content-addressed identity of a tokenizer: a digest over its bytes.
`files` maps each numerically relevant tokenizer/config file name — for a
GGUF, the embedded tokenizer metadata blob; for a safetensors deployment,
`tokenizer.json`, `tokenizer_config.json`, `special_tokens_map.json` — to
that file's exact bytes. The identity commits to each name and each byte
set, so two tokenizers published under one label differ, and a one-byte
edit is a different tokenizer.
"""
if not isinstance(files, Mapping) or not files:
raise RecipeIdentityError(
"tokenizer identity requires at least one named tokenizer/config "
"byte set; an identity over nothing pins nothing"
)
digests: dict[str, str] = {}
for name, body in files.items():
if not isinstance(name, str) or not name.strip():
raise RecipeIdentityError(
"tokenizer identity file names must be non-empty strings"
)
if not isinstance(body, (bytes, bytearray)):
raise RecipeIdentityError(
f"tokenizer identity for {name!r} requires the file's bytes, "
"not a path or label"
)
digests[name] = hashlib.sha256(bytes(body)).hexdigest()
return "tokenizer.v1:" + _digest(TOKENIZER_DIGEST_DOMAIN, {"files": digests})
def _require_tokenizer_identity(value: Any, what: str) -> str:
text = _require_text(value, what)
if not _TOKENIZER_IDENTITY.fullmatch(text):
raise RecipeIdentityError(
f"{what!r} must be a content-addressed tokenizer identity "
"'tokenizer.v1:<64-hex digest>' derived from the tokenizer's bytes "
"(tokenizer_identity); a repository label, tag, branch, or symbolic "
"ref names a mutable pointer, not the bytes it currently resolves to"
)
return text
@@ -389,10 +443,13 @@ class RuntimeRecipe:
one in fp16, produce different logits from the same bytes. Keeping the axes
apart is the entire safety property; see :data:`RECIPE_AXES`.
`tokenizer_revision` and `runtime_version` must be exact pins, never moving
references. For the native runtime the canonical `runtime_version` value —
committing to the exact upstream commit *and* the ordered patch stack — is
derived from the DGR-027 lock manifest by :mod:`meshnet_node.runtime_pin`.
`tokenizer_revision` must be a content-addressed tokenizer identity
(:func:`tokenizer_identity`) — a digest over the tokenizer's actual bytes,
never a repository label that merely points at bytes. `runtime_version`
must be an exact pin; for the native runtime the canonical value —
committing to the exact upstream commit, the ordered patch stack, *and*
the numerically relevant build recipe — is derived from the DGR-027 lock
manifest by :mod:`meshnet_node.runtime_pin`.
The three label fields are diagnosis only and are not digested.
"""
@@ -419,7 +476,7 @@ class RuntimeRecipe:
_require_int(value, f"recipe.{axis}", 1)
else:
_require_text(value, f"recipe.{axis}")
_require_pin(self.tokenizer_revision, "recipe.tokenizer_revision")
_require_tokenizer_identity(self.tokenizer_revision, "recipe.tokenizer_revision")
_require_runtime_pin(self.runtime_version, self.backend_id)
_require_text(self.recipe_id, "recipe.recipe_id")
_require_text(self.recipe_version, "recipe.recipe_version")