Merge branch 'archived_ralph/dgr-001-performance-contract' into merge/all-branches-into-master
# Conflicts: # .claude/memory/MEMORY.md # .scratch/distributed-gguf-runtime/PRD.md # .scratch/distributed-gguf-runtime/RALPH-CONTEXT.md # .scratch/distributed-gguf-runtime/README.md # .scratch/distributed-gguf-runtime/architecture.md # .scratch/distributed-gguf-runtime/evidence/DGR-017/README.md # .scratch/distributed-gguf-runtime/implementation-strategy.md # .scratch/distributed-gguf-runtime/issues/07-add-isolated-concurrent-local-hot-kv-state.md # .scratch/distributed-gguf-runtime/issues/13-harden-failure-cancellation-and-restart-semantics.md # .scratch/distributed-gguf-runtime/milestones.md # .scratch/distributed-gguf-runtime/prd.json # docs/issues/distributed-gguf-runtime/01-lock-the-safetensors-versus-gguf-performance-contract.md # docs/issues/distributed-gguf-runtime/02-adopt-the-versioned-grpc-shard-protocol.md # docs/issues/distributed-gguf-runtime/03-define-exact-artifact-and-runtime-recipe-identity.md # docs/issues/distributed-gguf-runtime/05-implement-dense-llama-range-aware-gguf-ownership.md # docs/issues/distributed-gguf-runtime/06-implement-architecture-defined-boundary-input-output.md
This commit is contained in:
Dobromir Popov
@@ -99,7 +99,12 @@ def compress_activation(body: bytes, policy: CompressionPolicy) -> CompressionRe
|
||||
return CompressionResult(candidate, "zstd", len(body), len(candidate), time.monotonic() - started, "compressed")
|
||||
|
||||
|
||||
def decompress_activation(body: bytes, encoding: str | None) -> CompressionResult:
|
||||
def decompress_activation(
|
||||
body: bytes,
|
||||
encoding: str | None,
|
||||
*,
|
||||
max_output_bytes: int | None = None,
|
||||
) -> CompressionResult:
|
||||
"""Decode a modern zstd body or preserve a legacy raw body with metrics."""
|
||||
started = time.monotonic()
|
||||
if not encoding:
|
||||
@@ -110,8 +115,23 @@ def decompress_activation(body: bytes, encoding: str | None) -> CompressionResul
|
||||
import zstandard as zstd
|
||||
except ImportError as exc:
|
||||
raise ValueError("zstd support is unavailable") from exc
|
||||
if max_output_bytes is not None and max_output_bytes < 0:
|
||||
raise ValueError("max_output_bytes must be non-negative")
|
||||
try:
|
||||
raw = zstd.ZstdDecompressor().decompress(body)
|
||||
if max_output_bytes is None:
|
||||
raw = zstd.ZstdDecompressor().decompress(body)
|
||||
else:
|
||||
# Cap both decoder window allocation and bytes read. zstandard's
|
||||
# max_window_size unit is KiB.
|
||||
max_window_kib = max(1024, (max_output_bytes + 1023) // 1024)
|
||||
decompressor = zstd.ZstdDecompressor(max_window_size=max_window_kib)
|
||||
# `decompress(max_output_size=...)` may trust a frame's advertised
|
||||
# content size. A bounded stream read enforces the limit regardless
|
||||
# of frame metadata and detects trailing expansion with one byte.
|
||||
with decompressor.stream_reader(body) as reader:
|
||||
raw = reader.read(max_output_bytes + 1)
|
||||
if len(raw) > max_output_bytes:
|
||||
raise ValueError("zstd activation body exceeds its output limit")
|
||||
except zstd.ZstdError as exc:
|
||||
raise ValueError("invalid zstd activation body") from exc
|
||||
return CompressionResult(raw, "zstd", len(body), len(raw), time.monotonic() - started, "decompressed")
|
||||
|
||||
186
packages/node/meshnet_node/architecture_boundary.py
Normal file
186
packages/node/meshnet_node/architecture_boundary.py
Normal file
@@ -0,0 +1,186 @@
|
||||
"""Certified architecture adapters for the public TensorBundle boundary.
|
||||
|
||||
The adapter is intentionally small: it owns boundary names and endpoint rules,
|
||||
not transformer execution. llama.cpp owns local graphs; callers select a
|
||||
certified adapter before accepting an activation from another Shard.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
import struct
|
||||
from typing import Callable, Mapping, Sequence
|
||||
|
||||
from .native_protocol import (
|
||||
HIDDEN_STATES,
|
||||
ProtocolError,
|
||||
encode_bundle,
|
||||
encode_tensor,
|
||||
pb,
|
||||
validate_tail_result,
|
||||
)
|
||||
|
||||
|
||||
class Architecture(str, Enum):
|
||||
DENSE = "dense"
|
||||
MOE = "moe"
|
||||
MLA = "mla"
|
||||
|
||||
|
||||
class BoundaryStage(str, Enum):
|
||||
HEAD = "head"
|
||||
MIDDLE = "middle"
|
||||
TAIL = "tail"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProtocolIdentity:
|
||||
request_id: str
|
||||
runtime_recipe_digest: str
|
||||
chat_template_id: str
|
||||
chat_template_version: str
|
||||
reasoning_mode: str
|
||||
architecture: Architecture
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SamplingParameters:
|
||||
temperature: float
|
||||
top_p: float
|
||||
top_k: int
|
||||
seed: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TailOutput:
|
||||
kind: str
|
||||
value: int | object
|
||||
|
||||
@classmethod
|
||||
def sampled_token(cls, token_id: int) -> "TailOutput":
|
||||
if token_id < 0:
|
||||
raise ProtocolError("sampled token id must be non-negative")
|
||||
return cls("sampled_token", token_id)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TypedTailResult:
|
||||
identity: ProtocolIdentity
|
||||
sampling: SamplingParameters
|
||||
output_kind: str
|
||||
message: pb.TailResult
|
||||
|
||||
@property
|
||||
def sampled_token_id(self) -> int | None:
|
||||
return self.message.sampled_token_id if self.output_kind == "sampled_token_id" else None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchitectureBoundaryAdapter:
|
||||
architecture: Architecture
|
||||
required_names: frozenset[str]
|
||||
|
||||
@property
|
||||
def protocol_architecture(self) -> int:
|
||||
return {
|
||||
Architecture.DENSE: pb.ARCHITECTURE_TYPE_DENSE,
|
||||
Architecture.MOE: pb.ARCHITECTURE_TYPE_MOE,
|
||||
Architecture.MLA: pb.ARCHITECTURE_TYPE_MLA,
|
||||
}[self.architecture]
|
||||
|
||||
def bundle_from_token_ids(
|
||||
self,
|
||||
token_ids: Sequence[int],
|
||||
token_embedding: Callable[[int], Sequence[float]],
|
||||
):
|
||||
"""Head-only embedding entry point; middle/tail never receive IDs."""
|
||||
if self.architecture is not Architecture.DENSE:
|
||||
raise ProtocolError("head token embedding is not certified for this architecture")
|
||||
if not token_ids:
|
||||
raise ProtocolError("head requires at least one token id")
|
||||
rows = [tuple(token_embedding(token)) for token in token_ids]
|
||||
if not rows or not rows[0] or any(len(row) != len(rows[0]) for row in rows):
|
||||
raise ProtocolError("token embedding returned inconsistent hidden widths")
|
||||
payload = struct.pack("<" + "f" * (len(rows) * len(rows[0])), *(x for row in rows for x in row))
|
||||
return self.bundle_from_named_payloads({HIDDEN_STATES: payload}, shape=[1, len(rows), len(rows[0])])
|
||||
|
||||
def bundle_from_named_payloads(
|
||||
self, payloads: Mapping[str, bytes], *, shape: Sequence[int] | None = None
|
||||
):
|
||||
names = set(payloads)
|
||||
if not self.required_names <= names:
|
||||
missing = sorted(self.required_names - names)
|
||||
raise ProtocolError(f"{self.architecture.value} boundary requires {missing}")
|
||||
tensors = []
|
||||
for name, payload in payloads.items():
|
||||
tensor_shape = list(shape) if name == HIDDEN_STATES and shape else [len(payload) // 4]
|
||||
if len(payload) % 4:
|
||||
raise ProtocolError(f"{name!r} F32 fixture payload is not word aligned")
|
||||
tensors.append(encode_tensor(name, payload, tensor_shape, pb.DTYPE_FLOAT32))
|
||||
return encode_bundle(
|
||||
tensors,
|
||||
architecture=self.protocol_architecture,
|
||||
boundary_point="pre_tail_residual",
|
||||
)
|
||||
|
||||
def input_for(self, stage: BoundaryStage, bundle):
|
||||
"""Accept architecture state only after the head embedding boundary."""
|
||||
if stage is BoundaryStage.HEAD:
|
||||
raise ProtocolError("head accepts token ids and owns token embedding")
|
||||
if bundle is None:
|
||||
raise ProtocolError(f"{stage.value} requires a TensorBundle")
|
||||
from .native_protocol import decode_bundle
|
||||
|
||||
payloads = decode_bundle(bundle)
|
||||
if bundle.architecture != self.protocol_architecture:
|
||||
raise ProtocolError("boundary architecture does not match certified adapter")
|
||||
if bundle.boundary_point != "pre_tail_residual":
|
||||
raise ProtocolError("unsupported architecture boundary point")
|
||||
if not self.required_names <= set(payloads):
|
||||
raise ProtocolError(f"{self.architecture.value} boundary requires {sorted(self.required_names)}")
|
||||
return bundle
|
||||
|
||||
def tail_result(
|
||||
self, *, identity: ProtocolIdentity, sampling: SamplingParameters, output: TailOutput
|
||||
) -> TypedTailResult:
|
||||
if identity.architecture is not self.architecture:
|
||||
raise ProtocolError("tail result architecture does not match certified adapter")
|
||||
if not identity.request_id or not identity.runtime_recipe_digest:
|
||||
raise ProtocolError("tail result requires exact request and recipe identity")
|
||||
if output.kind != "sampled_token":
|
||||
raise ProtocolError("uncertified tail output kind")
|
||||
message = pb.TailResult(
|
||||
identity=pb.RequestRecipeIdentity(
|
||||
request_id=identity.request_id,
|
||||
runtime_recipe_digest=identity.runtime_recipe_digest,
|
||||
chat_template_id=identity.chat_template_id,
|
||||
chat_template_version=identity.chat_template_version,
|
||||
reasoning_mode=identity.reasoning_mode,
|
||||
architecture=self.protocol_architecture,
|
||||
),
|
||||
sampling=pb.SamplingParameters(
|
||||
temperature=sampling.temperature,
|
||||
top_p=sampling.top_p,
|
||||
top_k=sampling.top_k,
|
||||
seed=sampling.seed,
|
||||
greedy=sampling.temperature == 0.0,
|
||||
),
|
||||
sampled_token_id=int(output.value),
|
||||
)
|
||||
validate_tail_result(message)
|
||||
return TypedTailResult(identity, sampling, "sampled_token_id", message)
|
||||
|
||||
|
||||
_ADAPTERS = {
|
||||
Architecture.DENSE: ArchitectureBoundaryAdapter(Architecture.DENSE, frozenset({HIDDEN_STATES})),
|
||||
Architecture.MOE: ArchitectureBoundaryAdapter(Architecture.MOE, frozenset({HIDDEN_STATES, "router_logits"})),
|
||||
Architecture.MLA: ArchitectureBoundaryAdapter(Architecture.MLA, frozenset({HIDDEN_STATES, "mla_position_state"})),
|
||||
}
|
||||
|
||||
|
||||
def adapter_for(architecture: Architecture | str) -> ArchitectureBoundaryAdapter:
|
||||
try:
|
||||
return _ADAPTERS[Architecture(architecture)]
|
||||
except (KeyError, ValueError):
|
||||
raise ProtocolError(f"unsupported architecture {architecture!r}") from None
|
||||
@@ -20,6 +20,8 @@ import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .runtime_recipe import CompatibilityFingerprint, ShardIdentity
|
||||
|
||||
# Layout of the serialized report. Bump when the JSON shape changes.
|
||||
CAPABILITY_SCHEMA_VERSION = 1
|
||||
|
||||
@@ -330,7 +332,16 @@ def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CapabilityReport:
|
||||
"""One node's validated (or failed) model/shard/recipe/backend combination."""
|
||||
"""One node's validated (or failed) model/shard/recipe/backend combination.
|
||||
|
||||
`identity` is the exact DGR-003 artifact/runtime-recipe block: the separated
|
||||
numerical axes and the compatibility fingerprint derived from them. It is
|
||||
optional and additive — a node that predates DGR-003 presents none, and the
|
||||
tracker falls back to the coarse label comparison it has always done
|
||||
(ADR-0023's compat rollout). A node that *does* present one is held to it:
|
||||
the tracker re-derives the fingerprint and refuses a report whose claim does
|
||||
not match its own derivation.
|
||||
"""
|
||||
|
||||
model: ModelIdentity
|
||||
shard: ShardRange
|
||||
@@ -341,6 +352,7 @@ class CapabilityReport:
|
||||
duration_ms: int
|
||||
diagnostics: tuple[str, ...] = ()
|
||||
schema_version: int = CAPABILITY_SCHEMA_VERSION
|
||||
identity: ShardIdentity | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.status not in VALID_STATUSES:
|
||||
@@ -360,6 +372,11 @@ class CapabilityReport:
|
||||
def passed(self) -> bool:
|
||||
return self.status == STATUS_PASSED
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> CompatibilityFingerprint | None:
|
||||
"""The exact compatibility fingerprint, when this node declares one."""
|
||||
return None if self.identity is None else self.identity.fingerprint
|
||||
|
||||
def identity_key(self) -> tuple[str, int, int, str, str, str, str]:
|
||||
"""The tuple a consumer must match to reuse this proof.
|
||||
|
||||
@@ -380,7 +397,7 @@ class CapabilityReport:
|
||||
return max(0.0, (time.time() if now is None else now) - self.validated_at)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
doc = {
|
||||
"schema_version": self.schema_version,
|
||||
"model": self.model.to_dict(),
|
||||
"shard": self.shard.to_dict(),
|
||||
@@ -391,6 +408,9 @@ class CapabilityReport:
|
||||
"duration_ms": self.duration_ms,
|
||||
"diagnostics": list(self.diagnostics),
|
||||
}
|
||||
if self.identity is not None:
|
||||
doc["identity"] = self.identity.to_dict()
|
||||
return doc
|
||||
|
||||
def to_json(self, indent: int | None = None) -> str:
|
||||
return json.dumps(self.to_dict(), indent=indent, sort_keys=True)
|
||||
@@ -417,6 +437,7 @@ class CapabilityReport:
|
||||
):
|
||||
raise CapabilityReportError("'validated_at' must be a Unix timestamp")
|
||||
|
||||
raw_identity = doc.get("identity")
|
||||
return cls(
|
||||
schema_version=schema_version,
|
||||
model=ModelIdentity.from_dict(doc.get("model")),
|
||||
@@ -427,6 +448,9 @@ class CapabilityReport:
|
||||
validated_at=float(validated_at),
|
||||
duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0),
|
||||
diagnostics=sanitize_diagnostics(doc.get("diagnostics")),
|
||||
identity=(
|
||||
None if raw_identity is None else ShardIdentity.from_dict(raw_identity)
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -461,12 +485,14 @@ def build_capability_report(
|
||||
diagnostics: Any = None,
|
||||
validated_at: float | None = None,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
identity: ShardIdentity | None = None,
|
||||
) -> CapabilityReport:
|
||||
"""Assemble a report from flat validation results.
|
||||
|
||||
`model_config` may be the loaded config mapping (hashed into a fingerprint)
|
||||
or an already-computed ``sha256:…`` string. `validated_at` defaults to now,
|
||||
so callers that need determinism pass it explicitly.
|
||||
so callers that need determinism pass it explicitly. `identity` is the exact
|
||||
DGR-003 artifact/recipe block, when the backend can state one.
|
||||
"""
|
||||
return CapabilityReport(
|
||||
model=ModelIdentity(
|
||||
@@ -491,4 +517,5 @@ def build_capability_report(
|
||||
validated_at=time.time() if validated_at is None else validated_at,
|
||||
duration_ms=duration_ms,
|
||||
diagnostics=sanitize_diagnostics(diagnostics, environ),
|
||||
identity=identity,
|
||||
)
|
||||
|
||||
@@ -150,7 +150,7 @@ def _cmd_default(args) -> int:
|
||||
print("\nSetup cancelled.")
|
||||
return 1
|
||||
save_config(cfg)
|
||||
print(f"\nConfig saved to ~/.config/meshnet/config.json\n")
|
||||
print("\nConfig saved to ~/.config/meshnet/config.json\n")
|
||||
|
||||
# Apply CLI overrides on top of saved config
|
||||
overrides: dict = {}
|
||||
@@ -206,7 +206,7 @@ def _cmd_default(args) -> int:
|
||||
|
||||
def _cmd_models(args) -> int:
|
||||
"""List curated models (with optional HF Hub browse)."""
|
||||
from .wizard import print_models_table, _browse_hf_interactive
|
||||
from .wizard import print_models_table
|
||||
|
||||
if args.browse:
|
||||
from .model_catalog import browse_hf_hub
|
||||
|
||||
@@ -5,7 +5,6 @@ from __future__ import annotations
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import deque
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -114,7 +113,7 @@ def run_dashboard(node, config: dict, start_time: float) -> None:
|
||||
return
|
||||
|
||||
try:
|
||||
from rich.live import Live # type: ignore[import]
|
||||
from rich.live import Live # type: ignore[import] # noqa: F401
|
||||
|
||||
_run_rich_dashboard(node, config, start_time)
|
||||
except ImportError:
|
||||
@@ -126,7 +125,6 @@ def _build_rich_renderable(
|
||||
):
|
||||
from rich.table import Table # type: ignore[import]
|
||||
from rich.panel import Panel # type: ignore[import]
|
||||
from rich.columns import Columns # type: ignore[import]
|
||||
from rich.text import Text # type: ignore[import]
|
||||
|
||||
uptime = time.monotonic() - start_time
|
||||
@@ -178,8 +176,8 @@ def _build_rich_renderable(
|
||||
f"Tokens/sec {tps_bar} {tps:.1f} t/s (EMA)",
|
||||
f"Requests {req_count:,} served",
|
||||
f"Success {stats['success_rate']:.1f}% failed {stats['failed_requests']:,} queue {stats['queue_depth']}",
|
||||
f"Peers 0 connected (gossip: US-017)",
|
||||
f"TAI earned 0.00 TAI (payments: US-006)",
|
||||
"Peers 0 connected (gossip: US-017)",
|
||||
"TAI earned 0.00 TAI (payments: US-006)",
|
||||
f"Uptime {_format_uptime(uptime)}",
|
||||
"",
|
||||
"[q] quit [c] compact view",
|
||||
|
||||
@@ -36,6 +36,7 @@ from .capability import (
|
||||
CapabilityReport,
|
||||
build_capability_report,
|
||||
)
|
||||
from .native_backend import NativeWorkerBackendAdapter
|
||||
from .recipe_manifest import (
|
||||
DEFAULT_RECIPE_ID,
|
||||
Recipe,
|
||||
@@ -449,11 +450,9 @@ def _validate_recipe(
|
||||
category: str | None = None
|
||||
error: BaseException | None = None
|
||||
diagnostics: list[str] = []
|
||||
detail: dict = {}
|
||||
|
||||
try:
|
||||
backend = load_backend(selection, recipe)
|
||||
detail = probe_forward(backend)
|
||||
probe_forward(backend)
|
||||
except DoctorError as exc:
|
||||
category, error = exc.category, exc
|
||||
diagnostics = [str(exc), exc.hint]
|
||||
@@ -464,23 +463,48 @@ def _validate_recipe(
|
||||
duration_ms = int((time.monotonic() - started) * 1000)
|
||||
|
||||
device = _backend_device(backend, selection)
|
||||
# Only the native adapter has an authoritative immutable GGUF report and
|
||||
# deployment pin. The Transformers path deliberately remains dark: a
|
||||
# model/config fingerprint is not an exact ArtifactIdentity.
|
||||
identity = backend.identity if isinstance(backend, NativeWorkerBackendAdapter) else None
|
||||
model_id = selection.model_id if identity is None else identity.artifact.artifact_id
|
||||
shard_start = selection.shard_start if identity is None else identity.shard_start
|
||||
shard_end = selection.shard_end if identity is None else identity.shard_end - 1
|
||||
recipe_id = recipe.id if identity is None else identity.recipe.recipe_id
|
||||
recipe_version = recipe.version if identity is None else identity.recipe.recipe_version
|
||||
catalogue_version = (
|
||||
manifest.catalogue_version if identity is None else identity.recipe.catalogue_version
|
||||
)
|
||||
backend_id = recipe.backend_id if identity is None else identity.recipe.backend_id
|
||||
quantization = (
|
||||
selection.quantization if identity is None else identity.recipe.weight_quantization
|
||||
)
|
||||
runtime = _runtime_versions()
|
||||
model_config = _model_config(backend)
|
||||
revision = None
|
||||
if identity is not None:
|
||||
revision = identity.artifact.revision
|
||||
model_config = "sha256:" + identity.artifact.architecture_digest
|
||||
runtime = {**runtime, "native_runtime": identity.recipe.runtime_version}
|
||||
report = build_capability_report(
|
||||
model_id=selection.model_id,
|
||||
shard_start=selection.shard_start,
|
||||
shard_end=selection.shard_end,
|
||||
recipe_id=recipe.id,
|
||||
recipe_version=recipe.version,
|
||||
catalogue_version=manifest.catalogue_version,
|
||||
backend_id=recipe.backend_id,
|
||||
model_id=model_id,
|
||||
shard_start=shard_start,
|
||||
shard_end=shard_end,
|
||||
recipe_id=recipe_id,
|
||||
recipe_version=recipe_version,
|
||||
catalogue_version=catalogue_version,
|
||||
backend_id=backend_id,
|
||||
device=device,
|
||||
device_name=_backend_device_name(device),
|
||||
quantization=selection.quantization,
|
||||
runtime=_runtime_versions(),
|
||||
model_config=_model_config(backend),
|
||||
quantization=quantization,
|
||||
runtime=runtime,
|
||||
revision=revision,
|
||||
model_config=model_config,
|
||||
status=STATUS_FAILED if category else STATUS_PASSED,
|
||||
duration_ms=duration_ms,
|
||||
diagnostics=[d for d in diagnostics if d] or None,
|
||||
validated_at=clock(),
|
||||
identity=identity,
|
||||
)
|
||||
if category:
|
||||
return RecipeResult(
|
||||
|
||||
123
packages/node/meshnet_node/glm_alpha/__init__.py
Normal file
123
packages/node/meshnet_node/glm_alpha/__init__.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""The locked GLM-5.2 Max alpha target: identity, resource plan, and acceptance contract.
|
||||
|
||||
Three files, three jobs:
|
||||
|
||||
- :mod:`~meshnet_node.glm_alpha.manifest` — *is this the exact artifact?* Pinned
|
||||
repository revisions, six shard digests, and the architecture-critical config
|
||||
snapshot they must agree with.
|
||||
- :mod:`~meshnet_node.glm_alpha.planner` — *can this route hold it?* Deterministic
|
||||
memory, KV, and seam arithmetic over the exact artifact bytes, counting unified
|
||||
memory once.
|
||||
- :mod:`~meshnet_node.glm_alpha.contract` — *what would have counted as success?*
|
||||
The acceptance thresholds, locked before the target runs and digest-bound so a
|
||||
later change cannot be silent.
|
||||
|
||||
Nothing here downloads, loads, or executes a model. This package is the contract
|
||||
DGR-018, DGR-019, and DGR-020 are judged against.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .contract import (
|
||||
ALPHA_CONTRACT_ID,
|
||||
ALPHA_CONTRACT_SCHEMA_VERSION,
|
||||
VERDICT_ALPHA,
|
||||
VERDICT_STOP,
|
||||
AlphaContract,
|
||||
AlphaContractError,
|
||||
compute_contract_digest,
|
||||
load_alpha_contract,
|
||||
parse_alpha_contract,
|
||||
require_contract_target,
|
||||
seal_contract,
|
||||
)
|
||||
from .manifest import (
|
||||
ALPHA_QUANTIZATION,
|
||||
ALPHA_SHARD_COUNT,
|
||||
ArchitectureSnapshot,
|
||||
GlmTargetError,
|
||||
Shard,
|
||||
TargetManifest,
|
||||
canonical_sha256,
|
||||
load_architecture_snapshot,
|
||||
load_target_manifest,
|
||||
parse_architecture_snapshot,
|
||||
parse_target_manifest,
|
||||
require_pinned_target,
|
||||
)
|
||||
from .planner import (
|
||||
AGGREGATE_HARD_FIT_FLOOR_GIB,
|
||||
ALPHA_CONTEXT_TOKENS,
|
||||
ALPHA_KV_DTYPE,
|
||||
MIN_LINK_RATE_GBPS,
|
||||
PLACEMENT_IMBALANCE_FACTOR,
|
||||
RECOMMENDED_LINK_RATE_GBPS,
|
||||
RESERVE_FLOOR_GIB,
|
||||
RESERVE_FRACTION,
|
||||
NodeMemory,
|
||||
ResourcePlanError,
|
||||
RouteFit,
|
||||
SeamPlan,
|
||||
TopologyPlan,
|
||||
kv_bytes,
|
||||
plan_all_tiers,
|
||||
plan_route,
|
||||
plan_seams,
|
||||
plan_topology,
|
||||
)
|
||||
|
||||
|
||||
def load_locked_target() -> tuple[AlphaContract, TargetManifest, ArchitectureSnapshot]:
|
||||
"""Load and cross-bind the packaged alpha contract, manifest, and snapshot."""
|
||||
|
||||
contract = load_alpha_contract()
|
||||
manifest = load_target_manifest()
|
||||
snapshot = load_architecture_snapshot()
|
||||
require_contract_target(contract, manifest, snapshot)
|
||||
return contract, manifest, snapshot
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AGGREGATE_HARD_FIT_FLOOR_GIB",
|
||||
"ALPHA_CONTEXT_TOKENS",
|
||||
"ALPHA_CONTRACT_ID",
|
||||
"ALPHA_CONTRACT_SCHEMA_VERSION",
|
||||
"ALPHA_KV_DTYPE",
|
||||
"ALPHA_QUANTIZATION",
|
||||
"ALPHA_SHARD_COUNT",
|
||||
"MIN_LINK_RATE_GBPS",
|
||||
"PLACEMENT_IMBALANCE_FACTOR",
|
||||
"RECOMMENDED_LINK_RATE_GBPS",
|
||||
"RESERVE_FLOOR_GIB",
|
||||
"RESERVE_FRACTION",
|
||||
"VERDICT_ALPHA",
|
||||
"VERDICT_STOP",
|
||||
"AlphaContract",
|
||||
"AlphaContractError",
|
||||
"ArchitectureSnapshot",
|
||||
"GlmTargetError",
|
||||
"NodeMemory",
|
||||
"ResourcePlanError",
|
||||
"RouteFit",
|
||||
"SeamPlan",
|
||||
"Shard",
|
||||
"TargetManifest",
|
||||
"TopologyPlan",
|
||||
"canonical_sha256",
|
||||
"compute_contract_digest",
|
||||
"kv_bytes",
|
||||
"load_alpha_contract",
|
||||
"load_architecture_snapshot",
|
||||
"load_locked_target",
|
||||
"load_target_manifest",
|
||||
"parse_alpha_contract",
|
||||
"parse_architecture_snapshot",
|
||||
"parse_target_manifest",
|
||||
"plan_all_tiers",
|
||||
"plan_route",
|
||||
"plan_seams",
|
||||
"plan_topology",
|
||||
"require_contract_target",
|
||||
"require_pinned_target",
|
||||
"seal_contract",
|
||||
]
|
||||
359
packages/node/meshnet_node/glm_alpha/contract.py
Normal file
359
packages/node/meshnet_node/glm_alpha/contract.py
Normal file
@@ -0,0 +1,359 @@
|
||||
"""The immutable GLM-5.2 Max alpha acceptance contract.
|
||||
|
||||
The contract exists to answer one question that cannot be answered honestly after
|
||||
the fact: *what would have counted as success?*
|
||||
|
||||
Its thresholds are locked before the target ever runs (DGR-017), and DGR-020 reads
|
||||
them back to publish an ``alpha`` or ``stop`` verdict. The whole point is that the
|
||||
gap between those two moments is where a threshold quietly becomes "0.1 tokens/sec
|
||||
was always the goal". So the document carries ``contract_sha256`` over its canonical content, while the
|
||||
approved v1 digest is pinned independently in code. :func:`load_alpha_contract`
|
||||
recomputes the self-digest and then requires that trusted pre-execution digest.
|
||||
Changing a threshold and re-sealing under the same identity is rejected; an
|
||||
amendment requires a new supported contract identity under human review.
|
||||
|
||||
The parsed contract recursively freezes nested mappings and sequences. Thresholds
|
||||
therefore cannot change between verification and use, and :meth:`AlphaContract.to_dict`
|
||||
returns an isolated mutable copy for diagnostics and tests.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Mapping
|
||||
|
||||
from .manifest import (
|
||||
ALPHA_QUANTIZATION,
|
||||
ALPHA_SHARD_COUNT,
|
||||
ArchitectureSnapshot,
|
||||
GlmTargetError,
|
||||
TargetManifest,
|
||||
canonical_sha256,
|
||||
)
|
||||
|
||||
ALPHA_CONTRACT_SCHEMA_VERSION = 1
|
||||
ALPHA_CONTRACT_VERSION = 1
|
||||
ALPHA_CONTRACT_ID = "glm-5.2-max-alpha/v1"
|
||||
ALPHA_CONTRACT_V1_SHA256 = "aab23220280c053a3c14ff559df3cb5c9e1bf7f0f7188c6519e2e9d9ad036ed9"
|
||||
|
||||
_CONTRACT_RESOURCE = "alpha-contract.json"
|
||||
|
||||
DIGEST_FIELD = "contract_sha256"
|
||||
|
||||
VERDICT_ALPHA = "alpha"
|
||||
VERDICT_STOP = "stop"
|
||||
|
||||
# Every section the roadmap's acceptance matrix (section 5) locks. A contract that
|
||||
# omits one is not a weaker contract, it is an unreviewable one.
|
||||
REQUIRED_SECTIONS: tuple[str, ...] = (
|
||||
"identity_and_fit",
|
||||
"semantic_correctness",
|
||||
"target_run",
|
||||
"performance",
|
||||
"reliability",
|
||||
"storage",
|
||||
)
|
||||
|
||||
|
||||
class AlphaContractError(GlmTargetError):
|
||||
"""Raised when the alpha contract is missing, malformed, or has been mutated."""
|
||||
|
||||
|
||||
def contract_signing_payload(document: Mapping[str, Any]) -> dict:
|
||||
"""The contract content the digest covers: everything except the digest itself."""
|
||||
unsigned = dict(document)
|
||||
unsigned.pop(DIGEST_FIELD, None)
|
||||
return unsigned
|
||||
|
||||
|
||||
def compute_contract_digest(document: Mapping[str, Any]) -> str:
|
||||
"""SHA-256 over the canonical contract content."""
|
||||
return canonical_sha256(contract_signing_payload(_thaw_json(document)))
|
||||
|
||||
|
||||
def _freeze_json(value: Any) -> Any:
|
||||
if isinstance(value, Mapping):
|
||||
return MappingProxyType({str(key): _freeze_json(item) for key, item in value.items()})
|
||||
if isinstance(value, list):
|
||||
return tuple(_freeze_json(item) for item in value)
|
||||
return value
|
||||
|
||||
|
||||
def _thaw_json(value: Any) -> Any:
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _thaw_json(item) for key, item in value.items()}
|
||||
if isinstance(value, tuple):
|
||||
return [_thaw_json(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AlphaContract:
|
||||
"""A locked, digest-bound alpha acceptance contract."""
|
||||
|
||||
schema_version: int
|
||||
contract_version: int
|
||||
contract_id: str
|
||||
locked_at: str
|
||||
locked_by: str
|
||||
target: Mapping[str, Any]
|
||||
sections: Mapping[str, Mapping[str, Any]]
|
||||
verdicts: tuple[str, ...]
|
||||
amendment_policy: str
|
||||
digest: str
|
||||
raw: Mapping[str, Any]
|
||||
source: str = "<memory>"
|
||||
|
||||
def section(self, name: str) -> Mapping[str, Any]:
|
||||
if name not in self.sections:
|
||||
raise AlphaContractError(f"contract section {name!r} is missing from {self.source}")
|
||||
return self.sections[name]
|
||||
|
||||
def threshold(self, section: str, key: str) -> Any:
|
||||
block = self.section(section)
|
||||
if key not in block:
|
||||
raise AlphaContractError(
|
||||
f"threshold {section}.{key} is not locked in {self.source}; an unlocked "
|
||||
"threshold cannot be used to judge a result"
|
||||
)
|
||||
return block[key]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return _thaw_json(self.raw)
|
||||
|
||||
|
||||
def parse_alpha_contract(data: Any, source: str = "<memory>") -> AlphaContract:
|
||||
"""Validate a contract document and verify it has not been mutated since locking."""
|
||||
if not isinstance(data, Mapping):
|
||||
raise AlphaContractError(f"contract root in {source} must be a JSON object")
|
||||
|
||||
schema_version = data.get("schema_version")
|
||||
if (
|
||||
not isinstance(schema_version, int)
|
||||
or isinstance(schema_version, bool)
|
||||
or schema_version != ALPHA_CONTRACT_SCHEMA_VERSION
|
||||
):
|
||||
raise AlphaContractError(
|
||||
f"{source} declares alpha-contract schema version {schema_version!r}, but this "
|
||||
f"node reads version {ALPHA_CONTRACT_SCHEMA_VERSION}"
|
||||
)
|
||||
|
||||
contract_version = data.get("contract_version")
|
||||
if (
|
||||
not isinstance(contract_version, int)
|
||||
or isinstance(contract_version, bool)
|
||||
or contract_version != ALPHA_CONTRACT_VERSION
|
||||
):
|
||||
raise AlphaContractError(
|
||||
f"{source} declares contract version {contract_version!r}, but this node "
|
||||
f"reads version {ALPHA_CONTRACT_VERSION}"
|
||||
)
|
||||
|
||||
contract_id = data.get("contract_id")
|
||||
if contract_id != ALPHA_CONTRACT_ID:
|
||||
raise AlphaContractError(
|
||||
f"{source} declares contract_id {contract_id!r}, but this node is locked "
|
||||
f"to {ALPHA_CONTRACT_ID!r}"
|
||||
)
|
||||
|
||||
for field in ("locked_at", "locked_by"):
|
||||
value = data.get(field)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise AlphaContractError(f"{source} must carry a non-empty {field}")
|
||||
|
||||
declared = data.get(DIGEST_FIELD)
|
||||
if not isinstance(declared, str) or not declared:
|
||||
raise AlphaContractError(
|
||||
f"{source} carries no {DIGEST_FIELD}; an unsealed contract cannot prove it "
|
||||
"predates the results it judges"
|
||||
)
|
||||
|
||||
computed = compute_contract_digest(data)
|
||||
if computed != declared:
|
||||
raise AlphaContractError(
|
||||
f"{source} has been modified since it was locked: its content hashes to "
|
||||
f"{computed}, but it declares {declared}. Alpha thresholds are locked before "
|
||||
"target execution and may not be weakened afterwards. To change them, open a "
|
||||
"new contract_id under human review; do not edit this one."
|
||||
)
|
||||
|
||||
if not data.get("locked_before_target_execution"):
|
||||
raise AlphaContractError(
|
||||
f"{source} does not assert locked_before_target_execution; a contract written "
|
||||
"after the results are known is not a contract"
|
||||
)
|
||||
|
||||
missing = [name for name in REQUIRED_SECTIONS if not isinstance(data.get(name), Mapping)]
|
||||
if missing:
|
||||
raise AlphaContractError(
|
||||
f"{source} is missing locked acceptance section(s) {missing}"
|
||||
)
|
||||
|
||||
verdicts = data.get("verdicts")
|
||||
if not isinstance(verdicts, list) or sorted(verdicts) != sorted([VERDICT_ALPHA, VERDICT_STOP]):
|
||||
raise AlphaContractError(
|
||||
f"{source} must offer exactly the verdicts {[VERDICT_ALPHA, VERDICT_STOP]}; a "
|
||||
"third outcome is how 'it loaded' becomes a pass"
|
||||
)
|
||||
|
||||
target = data.get("target")
|
||||
if not isinstance(target, Mapping):
|
||||
raise AlphaContractError(f"{source} is missing its 'target' block")
|
||||
for field in (
|
||||
"source_repo_id",
|
||||
"source_revision",
|
||||
"gguf_repo_id",
|
||||
"gguf_revision",
|
||||
"quantization",
|
||||
"target_manifest_sha256",
|
||||
"architecture_snapshot_sha256",
|
||||
"reasoning_effort",
|
||||
):
|
||||
if not target.get(field):
|
||||
raise AlphaContractError(f"{source} target block is missing {field!r}")
|
||||
for field in ("source_revision", "gguf_revision"):
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", str(target[field])):
|
||||
raise AlphaContractError(f"{source} target.{field} is not a full commit revision")
|
||||
for field in ("target_manifest_sha256", "architecture_snapshot_sha256"):
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", str(target[field])):
|
||||
raise AlphaContractError(f"{source} target.{field} is not a SHA-256 digest")
|
||||
if target["quantization"] != ALPHA_QUANTIZATION:
|
||||
raise AlphaContractError(
|
||||
f"{source} targets {target['quantization']!r}, not locked alpha quantization "
|
||||
f"{ALPHA_QUANTIZATION!r}"
|
||||
)
|
||||
if target["reasoning_effort"] != "max":
|
||||
raise AlphaContractError(f"{source} must lock reasoning_effort='max'")
|
||||
shard_count = target.get("shard_count")
|
||||
if (
|
||||
not isinstance(shard_count, int)
|
||||
or isinstance(shard_count, bool)
|
||||
or shard_count != ALPHA_SHARD_COUNT
|
||||
):
|
||||
raise AlphaContractError(
|
||||
f"{source} target.shard_count must be exactly {ALPHA_SHARD_COUNT}"
|
||||
)
|
||||
total_bytes = target.get("total_bytes")
|
||||
if not isinstance(total_bytes, int) or isinstance(total_bytes, bool) or total_bytes <= 0:
|
||||
raise AlphaContractError(f"{source} target.total_bytes must be a positive integer")
|
||||
|
||||
amendment_policy = data.get("amendment_policy")
|
||||
if not isinstance(amendment_policy, str) or not amendment_policy.strip():
|
||||
raise AlphaContractError(f"{source} must state its amendment policy")
|
||||
|
||||
if declared != ALPHA_CONTRACT_V1_SHA256:
|
||||
raise AlphaContractError(
|
||||
f"{source} is a re-sealed mutation of {ALPHA_CONTRACT_ID}: digest "
|
||||
f"{declared} does not match the trusted pre-execution digest "
|
||||
f"{ALPHA_CONTRACT_V1_SHA256}. An amendment requires a new supported "
|
||||
"contract identity under human review."
|
||||
)
|
||||
|
||||
frozen = _freeze_json(data)
|
||||
|
||||
return AlphaContract(
|
||||
schema_version=schema_version,
|
||||
contract_version=contract_version,
|
||||
contract_id=contract_id,
|
||||
locked_at=str(data["locked_at"]),
|
||||
locked_by=str(data["locked_by"]),
|
||||
target=frozen["target"],
|
||||
sections=MappingProxyType({name: frozen[name] for name in REQUIRED_SECTIONS}),
|
||||
verdicts=tuple(verdicts),
|
||||
amendment_policy=amendment_policy,
|
||||
digest=declared,
|
||||
raw=frozen,
|
||||
source=source,
|
||||
)
|
||||
|
||||
|
||||
def require_contract_target(
|
||||
contract: AlphaContract,
|
||||
manifest: TargetManifest,
|
||||
snapshot: ArchitectureSnapshot,
|
||||
) -> None:
|
||||
"""Bind the sealed contract to the exact manifest and architecture snapshot.
|
||||
|
||||
Repository revisions alone do not bind shard LFS objects or derived architecture
|
||||
semantics. Call this before planning, admission, download, or execution.
|
||||
"""
|
||||
|
||||
expected = contract.target
|
||||
actual = {
|
||||
"source_repo_id": manifest.source_repo_id,
|
||||
"source_revision": manifest.source_revision,
|
||||
"gguf_repo_id": manifest.gguf_repo_id,
|
||||
"gguf_revision": manifest.gguf_revision,
|
||||
"quantization": manifest.quantization,
|
||||
"shard_count": len(manifest.shards),
|
||||
"total_bytes": manifest.total_bytes,
|
||||
"target_manifest_sha256": manifest.digest,
|
||||
"architecture_snapshot_sha256": snapshot.digest,
|
||||
}
|
||||
if snapshot.source_repo_id != manifest.source_repo_id:
|
||||
raise AlphaContractError(
|
||||
"architecture snapshot repository does not match the target manifest repository"
|
||||
)
|
||||
if snapshot.source_revision != manifest.source_revision:
|
||||
raise AlphaContractError(
|
||||
"architecture snapshot revision does not match the target manifest revision"
|
||||
)
|
||||
mismatches = {
|
||||
key: (expected.get(key), value)
|
||||
for key, value in actual.items()
|
||||
if expected.get(key) != value
|
||||
}
|
||||
if mismatches:
|
||||
details = ", ".join(
|
||||
f"{key}: locked={locked!r}, actual={value!r}"
|
||||
for key, (locked, value) in sorted(mismatches.items())
|
||||
)
|
||||
raise AlphaContractError(f"target documents do not match the sealed contract: {details}")
|
||||
|
||||
|
||||
def load_alpha_contract(path: Path | None = None) -> AlphaContract:
|
||||
"""Load the packaged alpha contract, or one at ``path``."""
|
||||
if path is not None:
|
||||
source = str(path)
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise AlphaContractError(f"cannot read {source}: {exc.strerror or exc}") from exc
|
||||
else:
|
||||
source = f"packaged {_CONTRACT_RESOURCE}"
|
||||
try:
|
||||
raw = (
|
||||
files("meshnet_node.glm_alpha")
|
||||
.joinpath("data", _CONTRACT_RESOURCE)
|
||||
.read_text(encoding="utf-8")
|
||||
)
|
||||
except (OSError, FileNotFoundError, ModuleNotFoundError) as exc:
|
||||
raise AlphaContractError(
|
||||
f"{source} is missing from this node installation ({type(exc).__name__})"
|
||||
) from exc
|
||||
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AlphaContractError(
|
||||
f"{source} is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}"
|
||||
) from exc
|
||||
|
||||
return parse_alpha_contract(data, source=source)
|
||||
|
||||
|
||||
def seal_contract(document: Mapping[str, Any]) -> dict:
|
||||
"""Return the document with a freshly computed digest.
|
||||
|
||||
This is the only supported way to produce a contract file. It is deliberately
|
||||
*not* called at load time: sealing on load would turn every mutation into a
|
||||
valid contract, which is precisely the property the digest exists to deny.
|
||||
"""
|
||||
sealed = dict(document)
|
||||
sealed[DIGEST_FIELD] = compute_contract_digest(document)
|
||||
return sealed
|
||||
122
packages/node/meshnet_node/glm_alpha/data/alpha-contract.json
Normal file
122
packages/node/meshnet_node/glm_alpha/data/alpha-contract.json
Normal file
@@ -0,0 +1,122 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"contract_version": 1,
|
||||
"contract_id": "glm-5.2-max-alpha/v1",
|
||||
"locked_at": "2026-07-13",
|
||||
"locked_by": "DGR-017",
|
||||
"locked_before_target_execution": true,
|
||||
"target": {
|
||||
"source_repo_id": "zai-org/GLM-5.2",
|
||||
"source_revision": "b4734de4facf877f85769a911abafc5283eab3d9",
|
||||
"gguf_repo_id": "unsloth/GLM-5.2-GGUF",
|
||||
"gguf_revision": "abc55e72527792c6e77069c99b4cb7de16fa9f23",
|
||||
"quantization": "UD-IQ1_S",
|
||||
"shard_count": 6,
|
||||
"total_bytes": 216715360960,
|
||||
"target_manifest_sha256": "0b6aed04479d204902bb64c0203f1a46cab26a47b378ecccf85237b63f6c1962",
|
||||
"architecture_snapshot_sha256": "253fbd94b06b42acc4724ec2c7f33914e2d4cc43f54a36dff6af19a80ae6ceb1",
|
||||
"reasoning_effort": "max"
|
||||
},
|
||||
"identity_and_fit": {
|
||||
"require_exact_revisions": true,
|
||||
"require_all_shard_sha256": true,
|
||||
"require_per_node_owned_tensor_report": true,
|
||||
"require_owned_tensor_union_equals_inventory": true,
|
||||
"max_unintended_tensor_overlap": 0,
|
||||
"require_no_single_node_can_admit_complete_recipe": true,
|
||||
"min_node_reserve_fraction": 0.2,
|
||||
"min_node_reserve_gib": 8.0,
|
||||
"require_measured_peak_scratch_inside_reserve": true,
|
||||
"forbid_swap": true,
|
||||
"forbid_overcommit": true,
|
||||
"forbid_mmap_only_fit_claim": true,
|
||||
"forbid_double_counted_unified_memory": true,
|
||||
"aggregate_hard_fit_floor_gib": 224.0,
|
||||
"aggregate_floor_class": "experimental_hard_fit_floor",
|
||||
"recommended_topologies": [
|
||||
"5x64GiB",
|
||||
"3x96GiB",
|
||||
"3x128GiB"
|
||||
],
|
||||
"arithmetic_minimum_requires_measured_placement_evidence": true
|
||||
},
|
||||
"semantic_correctness": {
|
||||
"require_active_moe_routing": true,
|
||||
"require_active_shared_expert": true,
|
||||
"require_active_dsa_lightning_indexer": true,
|
||||
"require_active_sparse_attention": true,
|
||||
"require_active_indexshare_full_and_shared": true,
|
||||
"dense_attention_fallback_satisfies_alpha": false,
|
||||
"require_rendered_reasoning_effort_marker": "<|system|>Reasoning Effort: Max",
|
||||
"f32_seam_fixture_exact_match_tokens": 32,
|
||||
"f32_seam_fixture_requires_exact_match": true,
|
||||
"min_greedy_token_agreement": 0.9,
|
||||
"min_mean_state_cosine_similarity": 0.999,
|
||||
"forbid_nonfinite_tensors": true,
|
||||
"require_fail_closed_on_fingerprint_mismatch": true
|
||||
},
|
||||
"target_run": {
|
||||
"context_tokens": 16384,
|
||||
"kv_dtype": "Q8_0",
|
||||
"concurrency": 1,
|
||||
"prompt_lane_tokens": 4096,
|
||||
"min_output_tokens": 512,
|
||||
"min_output_tokens_with_natural_eos": 128,
|
||||
"require_same_switch_wired_network": true,
|
||||
"min_link_rate_gbps": 2.5,
|
||||
"recommended_link_rate_gbps": 10.0,
|
||||
"require_sentinels": [
|
||||
"coding",
|
||||
"structured_tool_call_json",
|
||||
"multi_step_reasoning"
|
||||
],
|
||||
"require_openai_compatible_response_fields": [
|
||||
"model",
|
||||
"finish_reason",
|
||||
"usage"
|
||||
]
|
||||
},
|
||||
"performance": {
|
||||
"min_median_decode_tokens_per_second": 0.5,
|
||||
"max_ttft_seconds_at_4096_prompt": 600,
|
||||
"max_unexplained_stall_seconds": 60,
|
||||
"warmups": 1,
|
||||
"require_per_stage_telemetry": [
|
||||
"compute",
|
||||
"queue",
|
||||
"kv",
|
||||
"seam_bytes",
|
||||
"seam_latency",
|
||||
"rss",
|
||||
"vram",
|
||||
"backend_timing"
|
||||
],
|
||||
"quality_pass_with_speed_fail_verdict": "stop",
|
||||
"forbid_generalising_results_to_other_hardware": true
|
||||
},
|
||||
"reliability": {
|
||||
"consecutive_clean_cold_starts": 2,
|
||||
"require_cancellation_releases_buffers_and_kv": true,
|
||||
"require_worker_loss_aborts_route": true,
|
||||
"retry_policy": "from_token_zero_on_new_compatible_route",
|
||||
"forbid_silent_kv_migration": true,
|
||||
"require_reject_stale_epoch": true,
|
||||
"require_reject_duplicate_step_id": true,
|
||||
"synthetic_workers_satisfy_alpha": false,
|
||||
"layer_reduced_fixtures_satisfy_alpha": false
|
||||
},
|
||||
"storage": {
|
||||
"mounted_storage_only": true,
|
||||
"forbidden_path_prefixes": [
|
||||
"/home"
|
||||
],
|
||||
"forbid_secrets_in_logs": true,
|
||||
"forbid_unrestricted_prompt_payloads_in_logs": true
|
||||
},
|
||||
"verdicts": [
|
||||
"alpha",
|
||||
"stop"
|
||||
],
|
||||
"amendment_policy": "Thresholds are locked before target execution. They may not be weakened, moved, or reinterpreted after results are known. A change requires a new contract_id and contract_version under human review, and the superseded contract is retained.",
|
||||
"contract_sha256": "aab23220280c053a3c14ff559df3cb5c9e1bf7f0f7188c6519e2e9d9ad036ed9"
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"observed_at": "2026-07-13",
|
||||
"source_repo_id": "zai-org/GLM-5.2",
|
||||
"source_revision": "b4734de4facf877f85769a911abafc5283eab3d9",
|
||||
"source_files": [
|
||||
{
|
||||
"path": "config.json",
|
||||
"size_bytes": 3732,
|
||||
"sha256": "185f93ee6d12548e16a847e279dc0c3c90b1524c970b0866b42fb545747d859a",
|
||||
"url": "https://huggingface.co/zai-org/GLM-5.2/resolve/b4734de4facf877f85769a911abafc5283eab3d9/config.json"
|
||||
},
|
||||
{
|
||||
"path": "chat_template.jinja",
|
||||
"size_bytes": 5076,
|
||||
"sha256": "172dc74a35e1752df75ecfb2b2cf9326d2852bb1379868ebeec9571654489679",
|
||||
"url": "https://huggingface.co/zai-org/GLM-5.2/resolve/b4734de4facf877f85769a911abafc5283eab3d9/chat_template.jinja"
|
||||
},
|
||||
{
|
||||
"path": "generation_config.json",
|
||||
"size_bytes": 194,
|
||||
"sha256": "ac76b43d8683d3b930126870fc8be73d8679308fe752fa1f381096d8354f6a55",
|
||||
"url": "https://huggingface.co/zai-org/GLM-5.2/resolve/b4734de4facf877f85769a911abafc5283eab3d9/generation_config.json"
|
||||
},
|
||||
{
|
||||
"path": "tokenizer_config.json",
|
||||
"size_bytes": 761,
|
||||
"sha256": "98b1271574f41abf89427ae2dda030d94dc9478f0edc5a8bd240db213c6fd5fc",
|
||||
"url": "https://huggingface.co/zai-org/GLM-5.2/resolve/b4734de4facf877f85769a911abafc5283eab3d9/tokenizer_config.json"
|
||||
}
|
||||
],
|
||||
"architecture": {
|
||||
"architectures": [
|
||||
"GlmMoeDsaForCausalLM"
|
||||
],
|
||||
"model_type": "glm_moe_dsa",
|
||||
"num_hidden_layers": 78,
|
||||
"num_nextn_predict_layers": 1,
|
||||
"total_artifact_layers": 79,
|
||||
"first_k_dense_replace": 3,
|
||||
"dense_layers": 3,
|
||||
"sparse_moe_layers": 75,
|
||||
"hidden_size": 6144,
|
||||
"intermediate_size": 12288,
|
||||
"moe_intermediate_size": 2048,
|
||||
"n_routed_experts": 256,
|
||||
"num_experts_per_tok": 8,
|
||||
"n_shared_experts": 1,
|
||||
"scoring_func": "sigmoid",
|
||||
"topk_method": "noaux_tc",
|
||||
"norm_topk_prob": true,
|
||||
"routed_scaling_factor": 2.5,
|
||||
"num_attention_heads": 64,
|
||||
"head_dim": 192,
|
||||
"qk_nope_head_dim": 192,
|
||||
"qk_rope_head_dim": 64,
|
||||
"qk_head_dim": 256,
|
||||
"v_head_dim": 256,
|
||||
"kv_lora_rank": 512,
|
||||
"q_lora_rank": 2048,
|
||||
"mla_cached_values_per_token_per_layer": 576,
|
||||
"index_topk": 2048,
|
||||
"index_head_dim": 128,
|
||||
"index_n_heads": 32,
|
||||
"index_topk_freq": 4,
|
||||
"index_skip_topk_offset": 3,
|
||||
"index_share_for_mtp_iteration": true,
|
||||
"indexer_full_layers": 21,
|
||||
"indexer_shared_layers": 57,
|
||||
"indexer_types_sha256": "ec3b4927af83cf02baf37fb10454c40176ec8bf501ae89334b27a9df5fa17025",
|
||||
"max_position_embeddings": 1048576,
|
||||
"vocab_size": 154880,
|
||||
"rope_theta": 8000000,
|
||||
"dtype": "bfloat16",
|
||||
"tie_word_embeddings": false
|
||||
},
|
||||
"reasoning_effort": {
|
||||
"alpha_mode": "max",
|
||||
"rendered_marker": "<|system|>Reasoning Effort: Max",
|
||||
"template_rule": "effective_reasoning_effort = 'high' if reasoning_effort == 'high' else 'max'",
|
||||
"default_is_max": true,
|
||||
"suppressed_when": "enable_thinking is defined and false",
|
||||
"note": "The template recognises exactly one non-max level ('high'); every other value, including an absent reasoning_effort, renders Max. Alpha therefore asserts the rendered 'Reasoning Effort: Max' marker, not merely the presence of a request field."
|
||||
},
|
||||
"notes": [
|
||||
"indexer_types has 78 entries: layers 0-2 are 'full', then a repeating [shared, shared, shared, full] pattern, giving 21 Full producer layers and 57 Shared consumer layers.",
|
||||
"MLA caches kv_lora_rank (512) + qk_rope_head_dim (64) = 576 values per token per backbone layer.",
|
||||
"num_nextn_predict_layers=1 is the NextN/MTP layer present in the artifact. Alpha does not run MTP; the tensors are loaded or explicitly excluded by a certified recipe and must never be silently reinterpreted as a 79th backbone layer.",
|
||||
"Values are derived from the pinned config.json. The runtime must re-derive them from the artifact and fail closed on contradictory metadata; marketing names are not compatibility identity."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"manifest_version": 1,
|
||||
"observed_at": "2026-07-13",
|
||||
"observed_by": "DGR-017",
|
||||
"alpha_quantization": "UD-IQ1_S",
|
||||
"source_model": {
|
||||
"repo_id": "zai-org/GLM-5.2",
|
||||
"revision": "b4734de4facf877f85769a911abafc5283eab3d9",
|
||||
"last_modified": "2026-07-02T08:08:14.000Z",
|
||||
"weight_license": "mit",
|
||||
"code_documentation_license": "apache-2.0",
|
||||
"url": "https://huggingface.co/zai-org/GLM-5.2",
|
||||
"revision_url": "https://huggingface.co/zai-org/GLM-5.2/tree/b4734de4facf877f85769a911abafc5283eab3d9",
|
||||
"api_url": "https://huggingface.co/api/models/zai-org/GLM-5.2"
|
||||
},
|
||||
"gguf_artifact": {
|
||||
"repo_id": "unsloth/GLM-5.2-GGUF",
|
||||
"revision": "abc55e72527792c6e77069c99b4cb7de16fa9f23",
|
||||
"last_modified": "2026-06-23T15:18:23.000Z",
|
||||
"license": "mit",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF",
|
||||
"revision_url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/tree/abc55e72527792c6e77069c99b4cb7de16fa9f23",
|
||||
"api_url": "https://huggingface.co/api/models/unsloth/GLM-5.2-GGUF",
|
||||
"quantization": "UD-IQ1_S",
|
||||
"shard_count": 6,
|
||||
"total_bytes": 216715360960,
|
||||
"total_gib": 201.832,
|
||||
"total_gb": 216.715,
|
||||
"shards": [
|
||||
{
|
||||
"index": 1,
|
||||
"path": "UD-IQ1_S/GLM-5.2-UD-IQ1_S-00001-of-00006.gguf",
|
||||
"size_bytes": 9423744,
|
||||
"sha256": "46b6148389219ae45167cb8124fbb18ef7d432daf619b4faf9e06ea80d3f4777",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/resolve/abc55e72527792c6e77069c99b4cb7de16fa9f23/UD-IQ1_S/GLM-5.2-UD-IQ1_S-00001-of-00006.gguf"
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"path": "UD-IQ1_S/GLM-5.2-UD-IQ1_S-00002-of-00006.gguf",
|
||||
"size_bytes": 49208128256,
|
||||
"sha256": "f2180207285e04fcaa5b8c53ba6e77ad5cc58666b6e7c6b04a5eded3fe8bef09",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/resolve/abc55e72527792c6e77069c99b4cb7de16fa9f23/UD-IQ1_S/GLM-5.2-UD-IQ1_S-00002-of-00006.gguf"
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"path": "UD-IQ1_S/GLM-5.2-UD-IQ1_S-00003-of-00006.gguf",
|
||||
"size_bytes": 49684417024,
|
||||
"sha256": "b1c0c5a302cc8d5d9ea0bcd4467c01db72c26839f820f7e882079582ea0a8d2b",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/resolve/abc55e72527792c6e77069c99b4cb7de16fa9f23/UD-IQ1_S/GLM-5.2-UD-IQ1_S-00003-of-00006.gguf"
|
||||
},
|
||||
{
|
||||
"index": 4,
|
||||
"path": "UD-IQ1_S/GLM-5.2-UD-IQ1_S-00004-of-00006.gguf",
|
||||
"size_bytes": 49396052864,
|
||||
"sha256": "a6a42da6975e29f89866dcde2956e9e50e6ea26635fb5063b74f3973f4f863b6",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/resolve/abc55e72527792c6e77069c99b4cb7de16fa9f23/UD-IQ1_S/GLM-5.2-UD-IQ1_S-00004-of-00006.gguf"
|
||||
},
|
||||
{
|
||||
"index": 5,
|
||||
"path": "UD-IQ1_S/GLM-5.2-UD-IQ1_S-00005-of-00006.gguf",
|
||||
"size_bytes": 49246275936,
|
||||
"sha256": "a4a9851a50db533f21ef824e5d8038f04e6782e7d602d18e5fdd6643f68ccccb",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/resolve/abc55e72527792c6e77069c99b4cb7de16fa9f23/UD-IQ1_S/GLM-5.2-UD-IQ1_S-00005-of-00006.gguf"
|
||||
},
|
||||
{
|
||||
"index": 6,
|
||||
"path": "UD-IQ1_S/GLM-5.2-UD-IQ1_S-00006-of-00006.gguf",
|
||||
"size_bytes": 19171063136,
|
||||
"sha256": "3b767f55df64e0432d52fcf1a14eb47a1ef3bbc91339e2ae220f38602237d7d7",
|
||||
"url": "https://huggingface.co/unsloth/GLM-5.2-GGUF/resolve/abc55e72527792c6e77069c99b4cb7de16fa9f23/UD-IQ1_S/GLM-5.2-UD-IQ1_S-00006-of-00006.gguf"
|
||||
}
|
||||
]
|
||||
},
|
||||
"diagnostic_fallback": {
|
||||
"quantization": "UD-IQ1_M",
|
||||
"shard_count": 6,
|
||||
"total_bytes": 228492966624,
|
||||
"total_gib": 212.801,
|
||||
"total_gb": 228.493,
|
||||
"policy": "First diagnostic fallback only if UD-IQ1_S exposes a runtime or quality defect. It does not satisfy the alpha 'lowest published quantization' target unless human review changes the target contract."
|
||||
},
|
||||
"storage": {
|
||||
"mounted_storage_only": true,
|
||||
"forbidden_path_prefixes": [
|
||||
"/home"
|
||||
],
|
||||
"note": "Model artifacts resolve through the machine-specific .env.<hostname> mounted-drive configuration. A path under /home fails admission closed."
|
||||
}
|
||||
}
|
||||
490
packages/node/meshnet_node/glm_alpha/manifest.py
Normal file
490
packages/node/meshnet_node/glm_alpha/manifest.py
Normal file
@@ -0,0 +1,490 @@
|
||||
"""The pinned GLM-5.2 Max target manifest and architecture snapshot.
|
||||
|
||||
This module is the *identity* half of the alpha target contract. It answers one
|
||||
question: is the artifact on this disk the exact artifact alpha was locked
|
||||
against?
|
||||
|
||||
Identity is pinned by repository revision **and** by every shard's LFS SHA-256.
|
||||
A revision alone is not enough — a repository can be force-pushed, and a tag can
|
||||
be moved — and a size alone is not enough, because two different quantizations of
|
||||
the same model land within a rounding error of each other. The manifest therefore
|
||||
carries both, plus the aggregate byte total, and cross-checks the aggregate
|
||||
against the sum of the shards. A manifest whose declared total disagrees with its
|
||||
own shards is rejected rather than trusted, because that is the exact shape a
|
||||
hand-edited "it fits now" manifest takes.
|
||||
|
||||
Nothing here downloads a weight payload. Sizes and hashes come from the Hugging
|
||||
Face metadata API (see ``scripts/refresh_glm_target_manifest.py``); verification
|
||||
against a local file is DGR-018's job, using the digests locked here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from importlib.resources import files
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
TARGET_MANIFEST_SCHEMA_VERSION = 1
|
||||
ARCHITECTURE_SNAPSHOT_SCHEMA_VERSION = 1
|
||||
|
||||
ALPHA_QUANTIZATION = "UD-IQ1_S"
|
||||
ALPHA_SHARD_COUNT = 6
|
||||
|
||||
_SHA256_RE = re.compile(r"\A[0-9a-f]{64}\Z")
|
||||
|
||||
_MANIFEST_RESOURCE = "target-manifest.json"
|
||||
_ARCHITECTURE_RESOURCE = "architecture-snapshot.json"
|
||||
|
||||
GIB = 1024**3
|
||||
GB = 1000**3
|
||||
|
||||
|
||||
class GlmTargetError(ValueError):
|
||||
"""Raised when the target manifest or architecture snapshot is not the pinned target."""
|
||||
|
||||
|
||||
def canonical_sha256(value: Any) -> str:
|
||||
"""SHA-256 over canonical JSON — the repository's digest convention."""
|
||||
payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _require_mapping(value: Any, what: str) -> Mapping[str, Any]:
|
||||
if not isinstance(value, Mapping):
|
||||
raise GlmTargetError(f"{what} must be a JSON object, got {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
def _require_text(value: Any, what: str) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise GlmTargetError(f"{what} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _require_int(value: Any, what: str) -> int:
|
||||
if not isinstance(value, int) or isinstance(value, bool):
|
||||
raise GlmTargetError(f"{what} must be an integer, got {type(value).__name__}")
|
||||
return value
|
||||
|
||||
|
||||
def _require_sha256(value: Any, what: str) -> str:
|
||||
text = _require_text(value, what)
|
||||
if not _SHA256_RE.match(text):
|
||||
raise GlmTargetError(
|
||||
f"{what} must be a lowercase 64-character hex SHA-256, got {text!r}"
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def _require_revision(value: Any, what: str) -> str:
|
||||
text = _require_text(value, what)
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", text):
|
||||
raise GlmTargetError(
|
||||
f"{what} must be a full 40-character commit revision, got {text!r}; "
|
||||
"a branch name or short SHA is not an immutable pin"
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Shard:
|
||||
"""One GGUF shard of the alpha artifact."""
|
||||
|
||||
index: int
|
||||
path: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
url: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"index": self.index,
|
||||
"path": self.path,
|
||||
"size_bytes": self.size_bytes,
|
||||
"sha256": self.sha256,
|
||||
"url": self.url,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TargetManifest:
|
||||
"""The pinned, self-consistent GLM-5.2 ``UD-IQ1_S`` target."""
|
||||
|
||||
schema_version: int
|
||||
manifest_version: int
|
||||
observed_at: str
|
||||
quantization: str
|
||||
source_repo_id: str
|
||||
source_revision: str
|
||||
source_license: str
|
||||
gguf_repo_id: str
|
||||
gguf_revision: str
|
||||
gguf_license: str
|
||||
total_bytes: int
|
||||
shards: tuple[Shard, ...]
|
||||
raw: Mapping[str, Any]
|
||||
source: str = "<memory>"
|
||||
|
||||
@property
|
||||
def total_gib(self) -> float:
|
||||
return self.total_bytes / GIB
|
||||
|
||||
@property
|
||||
def total_gb(self) -> float:
|
||||
return self.total_bytes / GB
|
||||
|
||||
def shard(self, index: int) -> Shard:
|
||||
for shard in self.shards:
|
||||
if shard.index == index:
|
||||
return shard
|
||||
raise GlmTargetError(f"shard {index} is not in {self.source}")
|
||||
|
||||
@property
|
||||
def digest(self) -> str:
|
||||
"""Stable identity of this manifest, for the DGR-003 runtime recipe."""
|
||||
return canonical_sha256(self.raw)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return dict(self.raw)
|
||||
|
||||
|
||||
def _parse_shards(raw: Any, expected_count: int, expected_total: int) -> tuple[Shard, ...]:
|
||||
if not isinstance(raw, list):
|
||||
raise GlmTargetError("gguf_artifact.shards must be a JSON array")
|
||||
if len(raw) != expected_count:
|
||||
raise GlmTargetError(
|
||||
f"the alpha artifact has exactly {expected_count} shards, "
|
||||
f"but the manifest lists {len(raw)}"
|
||||
)
|
||||
|
||||
shards: list[Shard] = []
|
||||
seen_index: set[int] = set()
|
||||
seen_sha: set[str] = set()
|
||||
for position, entry in enumerate(raw):
|
||||
item = _require_mapping(entry, f"shards[{position}]")
|
||||
index = _require_int(item.get("index"), f"shards[{position}].index")
|
||||
if index in seen_index:
|
||||
raise GlmTargetError(f"duplicate shard index {index} in the manifest")
|
||||
seen_index.add(index)
|
||||
size_bytes = _require_int(item.get("size_bytes"), f"shards[{index}].size_bytes")
|
||||
if size_bytes <= 0:
|
||||
raise GlmTargetError(f"shards[{index}].size_bytes must be positive")
|
||||
sha256 = _require_sha256(item.get("sha256"), f"shards[{index}].sha256")
|
||||
if sha256 in seen_sha:
|
||||
raise GlmTargetError(
|
||||
f"shard {index} repeats SHA-256 {sha256}; two distinct shards cannot "
|
||||
"have the same content digest"
|
||||
)
|
||||
seen_sha.add(sha256)
|
||||
shards.append(
|
||||
Shard(
|
||||
index=index,
|
||||
path=_require_text(item.get("path"), f"shards[{index}].path"),
|
||||
size_bytes=size_bytes,
|
||||
sha256=sha256,
|
||||
url=_require_text(item.get("url"), f"shards[{index}].url"),
|
||||
)
|
||||
)
|
||||
|
||||
expected_indices = set(range(1, expected_count + 1))
|
||||
if seen_index != expected_indices:
|
||||
missing = sorted(expected_indices - seen_index)
|
||||
raise GlmTargetError(
|
||||
f"the manifest is missing shard(s) {missing}; all {expected_count} "
|
||||
"shards of the alpha artifact must be pinned"
|
||||
)
|
||||
|
||||
summed = sum(shard.size_bytes for shard in shards)
|
||||
if summed != expected_total:
|
||||
raise GlmTargetError(
|
||||
f"declared total_bytes {expected_total} does not equal the sum of the "
|
||||
f"shard sizes {summed}; the manifest is not self-consistent"
|
||||
)
|
||||
|
||||
return tuple(sorted(shards, key=lambda shard: shard.index))
|
||||
|
||||
|
||||
def parse_target_manifest(data: Any, source: str = "<memory>") -> TargetManifest:
|
||||
"""Validate an already-decoded target manifest, failing closed."""
|
||||
doc = _require_mapping(data, f"manifest root in {source}")
|
||||
|
||||
schema_version = _require_int(doc.get("schema_version"), f"'schema_version' in {source}")
|
||||
if schema_version != TARGET_MANIFEST_SCHEMA_VERSION:
|
||||
raise GlmTargetError(
|
||||
f"{source} declares target-manifest schema version {schema_version}, "
|
||||
f"but this node reads version {TARGET_MANIFEST_SCHEMA_VERSION}"
|
||||
)
|
||||
|
||||
quantization = _require_text(doc.get("alpha_quantization"), f"'alpha_quantization' in {source}")
|
||||
if quantization != ALPHA_QUANTIZATION:
|
||||
raise GlmTargetError(
|
||||
f"{source} pins quantization {quantization!r}, but the locked alpha "
|
||||
f"quantization is {ALPHA_QUANTIZATION!r}; a different quantization is a "
|
||||
"different target and requires a human contract change"
|
||||
)
|
||||
|
||||
source_model = _require_mapping(doc.get("source_model"), f"'source_model' in {source}")
|
||||
gguf = _require_mapping(doc.get("gguf_artifact"), f"'gguf_artifact' in {source}")
|
||||
|
||||
gguf_quant = _require_text(gguf.get("quantization"), f"gguf_artifact.quantization in {source}")
|
||||
if gguf_quant != quantization:
|
||||
raise GlmTargetError(
|
||||
f"{source} declares alpha_quantization {quantization!r} but the GGUF "
|
||||
f"artifact block says {gguf_quant!r}"
|
||||
)
|
||||
|
||||
shard_count = _require_int(gguf.get("shard_count"), f"gguf_artifact.shard_count in {source}")
|
||||
if shard_count != ALPHA_SHARD_COUNT:
|
||||
raise GlmTargetError(
|
||||
f"{source} declares {shard_count} shards; the pinned alpha artifact has "
|
||||
f"exactly {ALPHA_SHARD_COUNT}"
|
||||
)
|
||||
|
||||
total_bytes = _require_int(gguf.get("total_bytes"), f"gguf_artifact.total_bytes in {source}")
|
||||
shards = _parse_shards(gguf.get("shards"), shard_count, total_bytes)
|
||||
|
||||
return TargetManifest(
|
||||
schema_version=schema_version,
|
||||
manifest_version=_require_int(doc.get("manifest_version"), f"'manifest_version' in {source}"),
|
||||
observed_at=_require_text(doc.get("observed_at"), f"'observed_at' in {source}"),
|
||||
quantization=quantization,
|
||||
source_repo_id=_require_text(source_model.get("repo_id"), "source_model.repo_id"),
|
||||
source_revision=_require_revision(source_model.get("revision"), "source_model.revision"),
|
||||
source_license=_require_text(source_model.get("weight_license"), "source_model.weight_license"),
|
||||
gguf_repo_id=_require_text(gguf.get("repo_id"), "gguf_artifact.repo_id"),
|
||||
gguf_revision=_require_revision(gguf.get("revision"), "gguf_artifact.revision"),
|
||||
gguf_license=_require_text(gguf.get("license"), "gguf_artifact.license"),
|
||||
total_bytes=total_bytes,
|
||||
shards=shards,
|
||||
raw=doc,
|
||||
source=source,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchitectureSnapshot:
|
||||
"""Architecture-critical metadata derived from the pinned ``config.json``."""
|
||||
|
||||
schema_version: int
|
||||
source_repo_id: str
|
||||
source_revision: str
|
||||
architecture: Mapping[str, Any]
|
||||
reasoning_effort: Mapping[str, Any]
|
||||
source_files: Mapping[str, Mapping[str, Any]]
|
||||
raw: Mapping[str, Any]
|
||||
source: str = "<memory>"
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
if key not in self.architecture:
|
||||
raise GlmTargetError(f"architecture field {key!r} is missing from {self.source}")
|
||||
return self.architecture[key]
|
||||
|
||||
@property
|
||||
def digest(self) -> str:
|
||||
return canonical_sha256(self.raw)
|
||||
|
||||
def file_sha256(self, path: str) -> str:
|
||||
entry = self.source_files.get(path)
|
||||
if entry is None:
|
||||
raise GlmTargetError(f"{path!r} is not pinned in {self.source}")
|
||||
return str(entry["sha256"])
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return dict(self.raw)
|
||||
|
||||
|
||||
# Fields the distributed runtime cannot plan or shard without. Absent or
|
||||
# contradictory values fail closed rather than defaulting.
|
||||
REQUIRED_ARCHITECTURE_FIELDS: tuple[str, ...] = (
|
||||
"model_type",
|
||||
"num_hidden_layers",
|
||||
"num_nextn_predict_layers",
|
||||
"total_artifact_layers",
|
||||
"first_k_dense_replace",
|
||||
"dense_layers",
|
||||
"sparse_moe_layers",
|
||||
"hidden_size",
|
||||
"n_routed_experts",
|
||||
"num_experts_per_tok",
|
||||
"n_shared_experts",
|
||||
"kv_lora_rank",
|
||||
"qk_rope_head_dim",
|
||||
"mla_cached_values_per_token_per_layer",
|
||||
"index_topk",
|
||||
"index_head_dim",
|
||||
"indexer_full_layers",
|
||||
"indexer_shared_layers",
|
||||
"max_position_embeddings",
|
||||
"vocab_size",
|
||||
)
|
||||
|
||||
|
||||
def parse_architecture_snapshot(data: Any, source: str = "<memory>") -> ArchitectureSnapshot:
|
||||
"""Validate an architecture snapshot and its internal arithmetic."""
|
||||
doc = _require_mapping(data, f"snapshot root in {source}")
|
||||
|
||||
schema_version = _require_int(doc.get("schema_version"), f"'schema_version' in {source}")
|
||||
if schema_version != ARCHITECTURE_SNAPSHOT_SCHEMA_VERSION:
|
||||
raise GlmTargetError(
|
||||
f"{source} declares architecture-snapshot schema version {schema_version}, "
|
||||
f"but this node reads version {ARCHITECTURE_SNAPSHOT_SCHEMA_VERSION}"
|
||||
)
|
||||
|
||||
arch = _require_mapping(doc.get("architecture"), f"'architecture' in {source}")
|
||||
missing = [field for field in REQUIRED_ARCHITECTURE_FIELDS if field not in arch]
|
||||
if missing:
|
||||
raise GlmTargetError(
|
||||
f"{source} is missing architecture-critical field(s) {missing}; the "
|
||||
"runtime cannot shard or plan an architecture it cannot fully describe"
|
||||
)
|
||||
|
||||
layers = _require_int(arch["num_hidden_layers"], "num_hidden_layers")
|
||||
nextn = _require_int(arch["num_nextn_predict_layers"], "num_nextn_predict_layers")
|
||||
total_layers = _require_int(arch["total_artifact_layers"], "total_artifact_layers")
|
||||
if total_layers != layers + nextn:
|
||||
raise GlmTargetError(
|
||||
f"total_artifact_layers {total_layers} != num_hidden_layers {layers} + "
|
||||
f"num_nextn_predict_layers {nextn}; the NextN layer must be counted "
|
||||
"explicitly, never folded into the backbone"
|
||||
)
|
||||
|
||||
dense = _require_int(arch["dense_layers"], "dense_layers")
|
||||
sparse = _require_int(arch["sparse_moe_layers"], "sparse_moe_layers")
|
||||
if dense != _require_int(arch["first_k_dense_replace"], "first_k_dense_replace"):
|
||||
raise GlmTargetError("dense_layers must equal first_k_dense_replace")
|
||||
if dense + sparse != layers:
|
||||
raise GlmTargetError(
|
||||
f"dense_layers {dense} + sparse_moe_layers {sparse} != num_hidden_layers {layers}"
|
||||
)
|
||||
|
||||
full = _require_int(arch["indexer_full_layers"], "indexer_full_layers")
|
||||
shared = _require_int(arch["indexer_shared_layers"], "indexer_shared_layers")
|
||||
if full + shared != layers:
|
||||
raise GlmTargetError(
|
||||
f"indexer_full_layers {full} + indexer_shared_layers {shared} != "
|
||||
f"num_hidden_layers {layers}; every layer holds exactly one IndexShare role"
|
||||
)
|
||||
if full <= 0:
|
||||
raise GlmTargetError(
|
||||
"indexer_full_layers must be positive; a route with no Full producer layer "
|
||||
"has no index for its Shared consumers to reuse"
|
||||
)
|
||||
|
||||
mla = _require_int(
|
||||
arch["mla_cached_values_per_token_per_layer"], "mla_cached_values_per_token_per_layer"
|
||||
)
|
||||
expected_mla = _require_int(arch["kv_lora_rank"], "kv_lora_rank") + _require_int(
|
||||
arch["qk_rope_head_dim"], "qk_rope_head_dim"
|
||||
)
|
||||
if mla != expected_mla:
|
||||
raise GlmTargetError(
|
||||
f"mla_cached_values_per_token_per_layer {mla} != kv_lora_rank + "
|
||||
f"qk_rope_head_dim ({expected_mla})"
|
||||
)
|
||||
|
||||
reasoning = _require_mapping(doc.get("reasoning_effort"), f"'reasoning_effort' in {source}")
|
||||
if reasoning.get("alpha_mode") != "max":
|
||||
raise GlmTargetError(
|
||||
f"{source} does not lock reasoning_effort=max; alpha is defined as the "
|
||||
"Max reasoning mode of this exact checkpoint"
|
||||
)
|
||||
_require_text(reasoning.get("rendered_marker"), "reasoning_effort.rendered_marker")
|
||||
|
||||
files_raw = doc.get("source_files")
|
||||
if not isinstance(files_raw, list) or not files_raw:
|
||||
raise GlmTargetError(f"'source_files' in {source} must be a non-empty JSON array")
|
||||
source_files: dict[str, Mapping[str, Any]] = {}
|
||||
for position, entry in enumerate(files_raw):
|
||||
item = _require_mapping(entry, f"source_files[{position}]")
|
||||
path = _require_text(item.get("path"), f"source_files[{position}].path")
|
||||
_require_sha256(item.get("sha256"), f"source_files[{path}].sha256")
|
||||
source_files[path] = item
|
||||
for required in ("config.json", "chat_template.jinja"):
|
||||
if required not in source_files:
|
||||
raise GlmTargetError(
|
||||
f"{source} does not pin {required!r}; config and chat-template drift "
|
||||
"silently changes runtime semantics"
|
||||
)
|
||||
|
||||
return ArchitectureSnapshot(
|
||||
schema_version=schema_version,
|
||||
source_repo_id=_require_text(doc.get("source_repo_id"), "source_repo_id"),
|
||||
source_revision=_require_revision(doc.get("source_revision"), "source_revision"),
|
||||
architecture=arch,
|
||||
reasoning_effort=reasoning,
|
||||
source_files=source_files,
|
||||
raw=doc,
|
||||
source=source,
|
||||
)
|
||||
|
||||
|
||||
def _read_resource(resource: str, path: Path | None) -> tuple[str, str]:
|
||||
if path is not None:
|
||||
try:
|
||||
return str(path), path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise GlmTargetError(f"cannot read {path}: {exc.strerror or exc}") from exc
|
||||
source = f"packaged {resource}"
|
||||
try:
|
||||
raw = files("meshnet_node.glm_alpha").joinpath("data", resource).read_text(encoding="utf-8")
|
||||
except (OSError, FileNotFoundError, ModuleNotFoundError) as exc:
|
||||
raise GlmTargetError(
|
||||
f"{source} is missing from this node installation ({type(exc).__name__})"
|
||||
) from exc
|
||||
return source, raw
|
||||
|
||||
|
||||
def _load_json(resource: str, path: Path | None) -> tuple[str, Any]:
|
||||
source, raw = _read_resource(resource, path)
|
||||
try:
|
||||
return source, json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise GlmTargetError(
|
||||
f"{source} is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}"
|
||||
) from exc
|
||||
|
||||
|
||||
def load_target_manifest(path: Path | None = None) -> TargetManifest:
|
||||
"""Load the packaged target manifest, or one at ``path``."""
|
||||
source, data = _load_json(_MANIFEST_RESOURCE, path)
|
||||
return parse_target_manifest(data, source=source)
|
||||
|
||||
|
||||
def load_architecture_snapshot(path: Path | None = None) -> ArchitectureSnapshot:
|
||||
"""Load the packaged architecture snapshot, or one at ``path``."""
|
||||
source, data = _load_json(_ARCHITECTURE_RESOURCE, path)
|
||||
return parse_architecture_snapshot(data, source=source)
|
||||
|
||||
|
||||
def require_pinned_target(
|
||||
manifest: TargetManifest,
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
expected_source_revision: str,
|
||||
expected_gguf_revision: str,
|
||||
) -> None:
|
||||
"""Reject any target whose revisions are not the ones alpha was locked against.
|
||||
|
||||
Callers pass the revisions from the locked alpha contract, so a swapped
|
||||
manifest cannot quietly re-point the target at a different upstream commit.
|
||||
"""
|
||||
if manifest.source_revision != expected_source_revision:
|
||||
raise GlmTargetError(
|
||||
f"source revision {manifest.source_revision} does not match the locked "
|
||||
f"alpha revision {expected_source_revision}"
|
||||
)
|
||||
if manifest.gguf_revision != expected_gguf_revision:
|
||||
raise GlmTargetError(
|
||||
f"GGUF revision {manifest.gguf_revision} does not match the locked alpha "
|
||||
f"revision {expected_gguf_revision}"
|
||||
)
|
||||
if snapshot.source_revision != manifest.source_revision:
|
||||
raise GlmTargetError(
|
||||
f"the architecture snapshot was taken at {snapshot.source_revision} but the "
|
||||
f"manifest pins {manifest.source_revision}; config metadata and weights must "
|
||||
"come from one revision"
|
||||
)
|
||||
522
packages/node/meshnet_node/glm_alpha/planner.py
Normal file
522
packages/node/meshnet_node/glm_alpha/planner.py
Normal file
@@ -0,0 +1,522 @@
|
||||
"""Deterministic memory, KV, and network planner for the GLM-5.2 Max alpha route.
|
||||
|
||||
Everything here is arithmetic over the exact pinned artifact bytes and the exact
|
||||
pinned architecture. There is no measurement, no probing, and no heuristic tuned
|
||||
to a result — the planner is written *before* the target runs so that a later
|
||||
story cannot discover a topology that "works" and then rationalise it.
|
||||
|
||||
Three ideas do the real work.
|
||||
|
||||
**Unified memory is one pool.** On an integrated-GPU machine the "VRAM" the driver
|
||||
reports is carved out of the same physical DRAM the OS is already counting. Adding
|
||||
them produces a node that appears to hold twice what it holds, and the failure mode
|
||||
is not a clean admission rejection — it is an OOM or a swap-thrash halfway through
|
||||
a 200 GiB load. :class:`NodeMemory` therefore refuses to be constructed from an
|
||||
additive claim about one shared pool.
|
||||
|
||||
**The reserve is not optional headroom.** Weights plus KV are not the whole
|
||||
resident cost: backend workspaces, quantization scratch, the graph plan, the
|
||||
process, and the OS all live outside them, and the largest of those scale with the
|
||||
backend rather than with the shard. Alpha reserves ``max(20% of physically usable
|
||||
memory, 8 GiB)`` per node, and the *remainder* is the placement budget.
|
||||
|
||||
**Equal layer counts are not equal bytes.** Embeddings and the output head are
|
||||
endpoint-only; three layers are dense and 75 are MoE; shared experts, indexer
|
||||
tensors, and quant block alignment all skew the per-node share. Until DGR-018/019
|
||||
report measured per-tensor placement, the planner carries an explicit
|
||||
:data:`PLACEMENT_IMBALANCE_FACTOR` and reports the arithmetic minimum and the
|
||||
recommended count as two separate numbers. The arithmetic minimum is a fit probe;
|
||||
it is admissible only with exact measured placement evidence behind it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
from .manifest import GIB, ArchitectureSnapshot, TargetManifest
|
||||
|
||||
# Q8_0 stores 32 int8 quants plus one fp16 scale per block: 34 bytes / 32 values.
|
||||
Q8_0_BYTES_PER_VALUE = 34 / 32
|
||||
F16_BYTES_PER_VALUE = 2.0
|
||||
|
||||
KV_DTYPES: dict[str, float] = {
|
||||
"Q8_0": Q8_0_BYTES_PER_VALUE,
|
||||
"F16": F16_BYTES_PER_VALUE,
|
||||
}
|
||||
|
||||
# The alpha KV configuration, locked by the roadmap.
|
||||
ALPHA_KV_DTYPE = "Q8_0"
|
||||
ALPHA_CONTEXT_TOKENS = 16384
|
||||
ALPHA_CONCURRENCY = 1
|
||||
|
||||
# The reserve every node holds outside its weight-plus-KV placement budget.
|
||||
RESERVE_FRACTION = 0.20
|
||||
RESERVE_FLOOR_GIB = 8.0
|
||||
|
||||
# The aggregate runtime-accessible memory at which the artifact *just* fits.
|
||||
# This is an experimental hard-fit floor, not an operational envelope: it has no
|
||||
# room for a backend that allocates more scratch than another, and none for the
|
||||
# imbalance below.
|
||||
AGGREGATE_HARD_FIT_FLOOR_GIB = 224.0
|
||||
|
||||
# How much more than an equal share the worst-placed node is expected to hold.
|
||||
# 1.10 is the roadmap's recommended-topology column expressed as arithmetic: it
|
||||
# reproduces 10 / 6 / 5 / 3 / 3 nodes for the 32 / 48 / 64 / 96 / 128 GiB tiers.
|
||||
# DGR-019 must replace it with measured per-tensor placement.
|
||||
PLACEMENT_IMBALANCE_FACTOR = 1.10
|
||||
|
||||
# Alpha network floor. A link rate is a bandwidth claim, never a speed claim.
|
||||
MIN_LINK_RATE_GBPS = 2.5
|
||||
RECOMMENDED_LINK_RATE_GBPS = 10.0
|
||||
|
||||
BF16_BYTES = 2
|
||||
DSA_SIDEBAND_INT32_BYTES = 4
|
||||
|
||||
IndexerLayout = Literal["optimized", "conservative"]
|
||||
|
||||
|
||||
class ResourcePlanError(ValueError):
|
||||
"""Raised when a node or route cannot be accounted for honestly."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NodeMemory:
|
||||
"""One node's physically usable memory, counted once.
|
||||
|
||||
``physical_usable_gib`` is what the node can actually place bytes into after
|
||||
firmware and fixed carve-outs — not the marketing capacity, and not a sum of
|
||||
two views of the same DRAM.
|
||||
"""
|
||||
|
||||
name: str
|
||||
physical_usable_gib: float
|
||||
unified: bool
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not isinstance(self.name, str) or not self.name.strip():
|
||||
raise ResourcePlanError("node name must be a non-empty physical-host identity")
|
||||
if (
|
||||
isinstance(self.physical_usable_gib, bool)
|
||||
or not isinstance(self.physical_usable_gib, (int, float))
|
||||
or not math.isfinite(self.physical_usable_gib)
|
||||
or self.physical_usable_gib <= 0
|
||||
):
|
||||
raise ResourcePlanError(
|
||||
f"node {self.name!r} must declare finite positive usable memory"
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_host(
|
||||
cls,
|
||||
name: str,
|
||||
*,
|
||||
system_ram_gib: float,
|
||||
gpu_memory_gib: float = 0.0,
|
||||
unified: bool,
|
||||
) -> "NodeMemory":
|
||||
"""Build a node from a host's reported RAM and GPU memory.
|
||||
|
||||
On a unified machine the GPU memory *is* system RAM, so it is counted once
|
||||
and never added. Passing a non-zero ``gpu_memory_gib`` alongside
|
||||
``unified=True`` is the double-count this project has already decided is a
|
||||
bug (RALPH-CONTEXT runtime decision 16), so it is rejected rather than
|
||||
silently discarded: a caller who believes an integrated GPU adds memory has
|
||||
a wrong model of the machine, and quietly ignoring the argument would let
|
||||
that belief survive.
|
||||
"""
|
||||
if not isinstance(unified, bool):
|
||||
raise ResourcePlanError(f"node {name!r} unified flag must be boolean")
|
||||
for value, label, allow_zero in (
|
||||
(system_ram_gib, "system RAM", False),
|
||||
(gpu_memory_gib, "GPU memory", True),
|
||||
):
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, (int, float))
|
||||
or not math.isfinite(value)
|
||||
or value < 0
|
||||
or (not allow_zero and value == 0)
|
||||
):
|
||||
qualifier = "finite non-negative" if allow_zero else "finite positive"
|
||||
raise ResourcePlanError(f"node {name!r} must declare {qualifier} {label}")
|
||||
if unified:
|
||||
if gpu_memory_gib:
|
||||
raise ResourcePlanError(
|
||||
f"node {name!r} declares unified memory and {gpu_memory_gib} GiB of "
|
||||
"separate GPU memory. Integrated-GPU memory is carved out of the same "
|
||||
"physical DRAM as system RAM; adding them double-counts one pool. "
|
||||
"Pass unified=True with system_ram_gib only."
|
||||
)
|
||||
usable = system_ram_gib
|
||||
else:
|
||||
usable = system_ram_gib + gpu_memory_gib
|
||||
return cls(name=name, physical_usable_gib=usable, unified=unified)
|
||||
|
||||
@property
|
||||
def reserve_gib(self) -> float:
|
||||
"""``max(20% of physically usable memory, 8 GiB)``."""
|
||||
return max(RESERVE_FRACTION * self.physical_usable_gib, RESERVE_FLOOR_GIB)
|
||||
|
||||
@property
|
||||
def placement_budget_gib(self) -> float:
|
||||
"""What remains for weights plus KV after the reserve."""
|
||||
return self.physical_usable_gib - self.reserve_gib
|
||||
|
||||
|
||||
def kv_bytes(
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
concurrency: int = ALPHA_CONCURRENCY,
|
||||
dtype: str = ALPHA_KV_DTYPE,
|
||||
indexer_layout: IndexerLayout = "conservative",
|
||||
include_indexer: bool = True,
|
||||
) -> int:
|
||||
"""Bytes of MLA (and DSA indexer) KV cache for the whole model.
|
||||
|
||||
``indexer_layout`` is the honest part. Correct DSA only needs indexer keys for
|
||||
the Full producer layers, but the current experimental implementation may
|
||||
allocate them across every backbone layer. Alpha budgets ``conservative``
|
||||
(all 78) so that a route admitted by this planner cannot be surprised by the
|
||||
implementation it actually gets.
|
||||
"""
|
||||
if (
|
||||
not isinstance(context_tokens, int)
|
||||
or isinstance(context_tokens, bool)
|
||||
or context_tokens <= 0
|
||||
or not isinstance(concurrency, int)
|
||||
or isinstance(concurrency, bool)
|
||||
or concurrency <= 0
|
||||
):
|
||||
raise ResourcePlanError("context_tokens and concurrency must be positive integers")
|
||||
if dtype not in KV_DTYPES:
|
||||
raise ResourcePlanError(
|
||||
f"unsupported KV dtype {dtype!r}; alpha locks {ALPHA_KV_DTYPE} "
|
||||
f"(known: {', '.join(sorted(KV_DTYPES))})"
|
||||
)
|
||||
bytes_per_value = KV_DTYPES[dtype]
|
||||
|
||||
layers = int(snapshot["num_hidden_layers"])
|
||||
mla_values = int(snapshot["mla_cached_values_per_token_per_layer"])
|
||||
total_values = mla_values * layers
|
||||
|
||||
if include_indexer:
|
||||
if indexer_layout == "optimized":
|
||||
indexer_layers = int(snapshot["indexer_full_layers"])
|
||||
elif indexer_layout == "conservative":
|
||||
indexer_layers = layers
|
||||
else: # pragma: no cover - Literal keeps this unreachable from typed callers
|
||||
raise ResourcePlanError(f"unknown indexer_layout {indexer_layout!r}")
|
||||
total_values += int(snapshot["index_head_dim"]) * indexer_layers
|
||||
|
||||
return int(total_values * context_tokens * concurrency * bytes_per_value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TopologyPlan:
|
||||
"""The node count a homogeneous tier needs, and how it was reached."""
|
||||
|
||||
physical_usable_gib: float
|
||||
reserve_gib: float
|
||||
placement_budget_gib: float
|
||||
weight_gib: float
|
||||
kv_gib: float
|
||||
total_placement_gib: float
|
||||
arithmetic_minimum_nodes: int
|
||||
recommended_nodes: int
|
||||
imbalance_factor: float
|
||||
|
||||
@property
|
||||
def is_arithmetic_minimum_topology(self) -> bool:
|
||||
"""True when the recommendation offers no imbalance headroom at all."""
|
||||
return self.recommended_nodes == self.arithmetic_minimum_nodes
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"physical_usable_gib": round(self.physical_usable_gib, 3),
|
||||
"reserve_gib": round(self.reserve_gib, 3),
|
||||
"placement_budget_gib": round(self.placement_budget_gib, 3),
|
||||
"weight_gib": round(self.weight_gib, 3),
|
||||
"kv_gib": round(self.kv_gib, 3),
|
||||
"total_placement_gib": round(self.total_placement_gib, 3),
|
||||
"arithmetic_minimum_nodes": self.arithmetic_minimum_nodes,
|
||||
"recommended_nodes": self.recommended_nodes,
|
||||
"imbalance_factor": self.imbalance_factor,
|
||||
}
|
||||
|
||||
|
||||
def plan_topology(
|
||||
manifest: TargetManifest,
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
physical_usable_gib: float,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
concurrency: int = ALPHA_CONCURRENCY,
|
||||
kv_dtype: str = ALPHA_KV_DTYPE,
|
||||
indexer_layout: IndexerLayout = "conservative",
|
||||
imbalance_factor: float = PLACEMENT_IMBALANCE_FACTOR,
|
||||
) -> TopologyPlan:
|
||||
"""Minimum and recommended node count for a homogeneous tier of this size."""
|
||||
if (
|
||||
isinstance(imbalance_factor, bool)
|
||||
or not isinstance(imbalance_factor, (int, float))
|
||||
or not math.isfinite(imbalance_factor)
|
||||
or imbalance_factor < 1.0
|
||||
):
|
||||
raise ResourcePlanError(
|
||||
"imbalance_factor must be finite and at least 1.0; a lower value would "
|
||||
"assume the worst-placed node holds less than an equal share"
|
||||
)
|
||||
|
||||
node = NodeMemory(
|
||||
name=f"{physical_usable_gib:g}GiB-tier",
|
||||
physical_usable_gib=physical_usable_gib,
|
||||
unified=False,
|
||||
)
|
||||
budget = node.placement_budget_gib
|
||||
if budget <= 0:
|
||||
raise ResourcePlanError(
|
||||
f"a {physical_usable_gib:g} GiB node has no placement budget after its "
|
||||
f"{node.reserve_gib:.1f} GiB reserve"
|
||||
)
|
||||
|
||||
weight_gib = manifest.total_bytes / GIB
|
||||
kv_gib = (
|
||||
kv_bytes(
|
||||
snapshot,
|
||||
context_tokens=context_tokens,
|
||||
concurrency=concurrency,
|
||||
dtype=kv_dtype,
|
||||
indexer_layout=indexer_layout,
|
||||
)
|
||||
/ GIB
|
||||
)
|
||||
total = weight_gib + kv_gib
|
||||
|
||||
return TopologyPlan(
|
||||
physical_usable_gib=physical_usable_gib,
|
||||
reserve_gib=node.reserve_gib,
|
||||
placement_budget_gib=budget,
|
||||
weight_gib=weight_gib,
|
||||
kv_gib=kv_gib,
|
||||
total_placement_gib=total,
|
||||
arithmetic_minimum_nodes=math.ceil(total / budget),
|
||||
recommended_nodes=math.ceil(total * imbalance_factor / budget),
|
||||
imbalance_factor=imbalance_factor,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RouteFit:
|
||||
"""Whether a concrete, possibly heterogeneous set of nodes can hold the target."""
|
||||
|
||||
node_count: int
|
||||
aggregate_usable_gib: float
|
||||
aggregate_placement_budget_gib: float
|
||||
required_placement_gib: float
|
||||
fits: bool
|
||||
meets_hard_fit_floor: bool
|
||||
no_single_node_can_admit_target: bool
|
||||
headroom_gib: float
|
||||
reasons: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"node_count": self.node_count,
|
||||
"aggregate_usable_gib": round(self.aggregate_usable_gib, 3),
|
||||
"aggregate_placement_budget_gib": round(self.aggregate_placement_budget_gib, 3),
|
||||
"required_placement_gib": round(self.required_placement_gib, 3),
|
||||
"fits": self.fits,
|
||||
"meets_hard_fit_floor": self.meets_hard_fit_floor,
|
||||
"no_single_node_can_admit_target": self.no_single_node_can_admit_target,
|
||||
"headroom_gib": round(self.headroom_gib, 3),
|
||||
"reasons": list(self.reasons),
|
||||
}
|
||||
|
||||
|
||||
def plan_route(
|
||||
manifest: TargetManifest,
|
||||
snapshot: ArchitectureSnapshot,
|
||||
nodes: list[NodeMemory],
|
||||
*,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
concurrency: int = ALPHA_CONCURRENCY,
|
||||
kv_dtype: str = ALPHA_KV_DTYPE,
|
||||
indexer_layout: IndexerLayout = "conservative",
|
||||
) -> RouteFit:
|
||||
"""Evaluate a concrete route. Every node's memory is already counted once."""
|
||||
if len(nodes) < 2:
|
||||
raise ResourcePlanError(
|
||||
"the alpha target is distributed by definition; a route needs at least two "
|
||||
"physical nodes"
|
||||
)
|
||||
names = [node.name for node in nodes]
|
||||
if len(set(names)) != len(names):
|
||||
raise ResourcePlanError(
|
||||
"duplicate node names in the route; one physical machine counted twice is "
|
||||
"the same double-count as adding integrated-GPU memory to system RAM"
|
||||
)
|
||||
|
||||
weight_gib = manifest.total_bytes / GIB
|
||||
kv_gib = (
|
||||
kv_bytes(
|
||||
snapshot,
|
||||
context_tokens=context_tokens,
|
||||
concurrency=concurrency,
|
||||
dtype=kv_dtype,
|
||||
indexer_layout=indexer_layout,
|
||||
)
|
||||
/ GIB
|
||||
)
|
||||
required = weight_gib + kv_gib
|
||||
|
||||
aggregate_usable = sum(node.physical_usable_gib for node in nodes)
|
||||
aggregate_budget = sum(node.placement_budget_gib for node in nodes)
|
||||
fits = aggregate_budget >= required
|
||||
largest_budget = max(node.placement_budget_gib for node in nodes)
|
||||
no_single_node = largest_budget < required
|
||||
|
||||
reasons: list[str] = []
|
||||
if not fits:
|
||||
reasons.append(
|
||||
f"aggregate placement budget {aggregate_budget:.1f} GiB is below the "
|
||||
f"{required:.1f} GiB the target needs after each node's reserve"
|
||||
)
|
||||
if not no_single_node:
|
||||
reasons.append(
|
||||
"at least one node could admit the complete target alone; that is a "
|
||||
"single-host run, not distributed alpha"
|
||||
)
|
||||
if aggregate_usable < AGGREGATE_HARD_FIT_FLOOR_GIB:
|
||||
reasons.append(
|
||||
f"aggregate usable memory {aggregate_usable:.1f} GiB is below the "
|
||||
f"{AGGREGATE_HARD_FIT_FLOOR_GIB:g} GiB experimental hard-fit floor"
|
||||
)
|
||||
|
||||
return RouteFit(
|
||||
node_count=len(nodes),
|
||||
aggregate_usable_gib=aggregate_usable,
|
||||
aggregate_placement_budget_gib=aggregate_budget,
|
||||
required_placement_gib=required,
|
||||
fits=fits,
|
||||
meets_hard_fit_floor=aggregate_usable >= AGGREGATE_HARD_FIT_FLOOR_GIB,
|
||||
no_single_node_can_admit_target=no_single_node,
|
||||
headroom_gib=aggregate_budget - required,
|
||||
reasons=tuple(reasons),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SeamPlan:
|
||||
"""Bytes and latency across the activation seams of a route.
|
||||
|
||||
Bandwidth and latency are reported separately on purpose. Decode moves almost
|
||||
nothing — 12 KiB per token per seam — so a faster link barely helps it. What
|
||||
decode pays is *serial*: every generated token crosses every seam in order, so
|
||||
the cost that matters is ``seams x per-hop latency``. A route that claims to be
|
||||
fast because it is on 10 GbE has confused the two.
|
||||
"""
|
||||
|
||||
node_count: int
|
||||
seam_count: int
|
||||
hidden_size: int
|
||||
bytes_per_token_per_seam: int
|
||||
prefill_bytes_per_seam: int
|
||||
decode_bytes_per_seam_per_token: int
|
||||
dsa_sideband_bytes_per_query: int
|
||||
link_rate_gbps: float
|
||||
meets_alpha_minimum: bool
|
||||
is_recommended_link: bool
|
||||
decode_serialization_ms_per_token: float
|
||||
decode_latency_ms_per_token: float
|
||||
decode_bandwidth_share_ms_per_token: float
|
||||
prefill_serialization_ms: float
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"node_count": self.node_count,
|
||||
"seam_count": self.seam_count,
|
||||
"hidden_size": self.hidden_size,
|
||||
"bytes_per_token_per_seam": self.bytes_per_token_per_seam,
|
||||
"prefill_bytes_per_seam": self.prefill_bytes_per_seam,
|
||||
"decode_bytes_per_seam_per_token": self.decode_bytes_per_seam_per_token,
|
||||
"dsa_sideband_bytes_per_query": self.dsa_sideband_bytes_per_query,
|
||||
"link_rate_gbps": self.link_rate_gbps,
|
||||
"meets_alpha_minimum": self.meets_alpha_minimum,
|
||||
"is_recommended_link": self.is_recommended_link,
|
||||
"decode_serialization_ms_per_token": round(self.decode_serialization_ms_per_token, 4),
|
||||
"decode_latency_ms_per_token": round(self.decode_latency_ms_per_token, 4),
|
||||
"decode_bandwidth_share_ms_per_token": round(
|
||||
self.decode_bandwidth_share_ms_per_token, 4
|
||||
),
|
||||
"prefill_serialization_ms": round(self.prefill_serialization_ms, 3),
|
||||
}
|
||||
|
||||
|
||||
def plan_seams(
|
||||
snapshot: ArchitectureSnapshot,
|
||||
*,
|
||||
node_count: int,
|
||||
context_tokens: int = ALPHA_CONTEXT_TOKENS,
|
||||
link_rate_gbps: float = MIN_LINK_RATE_GBPS,
|
||||
per_hop_latency_ms: float = 0.5,
|
||||
) -> SeamPlan:
|
||||
"""Model seam bytes, wire serialization, and serial per-hop latency separately."""
|
||||
if not isinstance(node_count, int) or isinstance(node_count, bool) or node_count < 2:
|
||||
raise ResourcePlanError("a seam exists only between two nodes")
|
||||
if not isinstance(context_tokens, int) or isinstance(context_tokens, bool) or context_tokens <= 0:
|
||||
raise ResourcePlanError("context_tokens must be a positive integer")
|
||||
if (
|
||||
isinstance(link_rate_gbps, bool)
|
||||
or not isinstance(link_rate_gbps, (int, float))
|
||||
or not math.isfinite(link_rate_gbps)
|
||||
or link_rate_gbps <= 0
|
||||
):
|
||||
raise ResourcePlanError("link_rate_gbps must be finite and positive")
|
||||
if (
|
||||
isinstance(per_hop_latency_ms, bool)
|
||||
or not isinstance(per_hop_latency_ms, (int, float))
|
||||
or not math.isfinite(per_hop_latency_ms)
|
||||
or per_hop_latency_ms < 0
|
||||
):
|
||||
raise ResourcePlanError("per_hop_latency_ms must be finite and non-negative")
|
||||
|
||||
hidden = int(snapshot["hidden_size"])
|
||||
bytes_per_token = hidden * BF16_BYTES
|
||||
seams = node_count - 1
|
||||
|
||||
bits_per_ms = link_rate_gbps * 1e9 / 1e3
|
||||
decode_serialization_ms = (bytes_per_token * 8) / bits_per_ms
|
||||
prefill_serialization_ms = (bytes_per_token * context_tokens * 8) / bits_per_ms
|
||||
|
||||
return SeamPlan(
|
||||
node_count=node_count,
|
||||
seam_count=seams,
|
||||
hidden_size=hidden,
|
||||
bytes_per_token_per_seam=bytes_per_token,
|
||||
prefill_bytes_per_seam=bytes_per_token * context_tokens,
|
||||
decode_bytes_per_seam_per_token=bytes_per_token,
|
||||
dsa_sideband_bytes_per_query=int(snapshot["index_topk"]) * DSA_SIDEBAND_INT32_BYTES,
|
||||
link_rate_gbps=link_rate_gbps,
|
||||
meets_alpha_minimum=link_rate_gbps >= MIN_LINK_RATE_GBPS,
|
||||
is_recommended_link=link_rate_gbps >= RECOMMENDED_LINK_RATE_GBPS,
|
||||
decode_serialization_ms_per_token=decode_serialization_ms * seams,
|
||||
decode_latency_ms_per_token=per_hop_latency_ms * seams,
|
||||
decode_bandwidth_share_ms_per_token=decode_serialization_ms * seams,
|
||||
prefill_serialization_ms=prefill_serialization_ms * seams,
|
||||
)
|
||||
|
||||
|
||||
ALPHA_TIERS_GIB: tuple[float, ...] = (32.0, 48.0, 64.0, 96.0, 128.0)
|
||||
|
||||
|
||||
def plan_all_tiers(
|
||||
manifest: TargetManifest, snapshot: ArchitectureSnapshot
|
||||
) -> dict[str, TopologyPlan]:
|
||||
"""The alpha tier table, recomputed from the pinned artifact and architecture."""
|
||||
return {
|
||||
f"{tier:g}": plan_topology(manifest, snapshot, physical_usable_gib=tier)
|
||||
for tier in ALPHA_TIERS_GIB
|
||||
}
|
||||
185
packages/node/meshnet_node/native_backend.py
Normal file
185
packages/node/meshnet_node/native_backend.py
Normal file
@@ -0,0 +1,185 @@
|
||||
"""Authoritative identity boundary for a native GGUF Shard backend.
|
||||
|
||||
The native loader owns the facts about the mapped artifact. This module turns
|
||||
that immutable report and separately pinned deployment inputs into the one
|
||||
``ShardIdentity`` DGR-003 permits a native worker to emit. It is intentionally
|
||||
not an adapter for the legacy Transformers backend: that backend has no
|
||||
authoritative immutable GGUF artifact pin and must remain identity-free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .native_protocol import BUNDLE_VERSION, SCHEMA_VERSION, pb
|
||||
from .runtime_recipe import (
|
||||
ArtifactIdentity,
|
||||
DerivativeBinding,
|
||||
RecipeIdentityError,
|
||||
RuntimeRecipe,
|
||||
ShardIdentity,
|
||||
check_session_open,
|
||||
handshake_error,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NativeLoadedArtifactReport:
|
||||
"""Immutable GGUF facts returned by the loaded native model.
|
||||
|
||||
The report is copied from ``llama_model_meshnet_range_report`` plus the
|
||||
parsed GGUF metadata while the model is live. Byte counts are operational
|
||||
evidence rather than compatibility axes, but keeping them beside the range
|
||||
prevents a caller from substituting an unverified range declaration.
|
||||
"""
|
||||
|
||||
owned_start_layer: int
|
||||
owned_end_layer: int
|
||||
mapped_bytes: int
|
||||
resident_bytes: int
|
||||
registered_bytes: int
|
||||
architecture: str
|
||||
architecture_digest: str
|
||||
layer_count: int
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.owned_start_layer < 0 or self.owned_end_layer <= self.owned_start_layer:
|
||||
raise RecipeIdentityError("native report has an invalid owned layer range")
|
||||
if self.layer_count < 1 or self.owned_end_layer > self.layer_count:
|
||||
raise RecipeIdentityError("native report range is outside GGUF layer metadata")
|
||||
if min(self.mapped_bytes, self.resident_bytes, self.registered_bytes) < 0:
|
||||
raise RecipeIdentityError("native report byte counts must be non-negative")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImmutableArtifactPin:
|
||||
"""Deployment-supplied immutable bytes pin, never inferred from a model name."""
|
||||
|
||||
artifact_id: str
|
||||
revision: str
|
||||
content_digest: str
|
||||
derived_from: DerivativeBinding | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NativeNumericalRecipe:
|
||||
"""Immutable numerical inputs selected for this native worker instance."""
|
||||
|
||||
weight_quantization: str
|
||||
activation_dtype: str
|
||||
compute_dtype: str
|
||||
kv_dtype: str
|
||||
kv_layout: str
|
||||
architecture_adapter: str
|
||||
backend_id: str
|
||||
runtime_version: str
|
||||
recipe_id: str
|
||||
recipe_version: str
|
||||
catalogue_version: str
|
||||
boundary_schema_version: int = BUNDLE_VERSION
|
||||
protocol_schema_version: int = int(SCHEMA_VERSION)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NativeIdentityInputs:
|
||||
"""Everything a native backend needs to emit one exact identity."""
|
||||
|
||||
loaded_artifact: NativeLoadedArtifactReport
|
||||
artifact_pin: ImmutableArtifactPin
|
||||
tokenizer_revision: str
|
||||
numerical_recipe: NativeNumericalRecipe
|
||||
|
||||
|
||||
def shard_identity_from_native_report(inputs: NativeIdentityInputs) -> ShardIdentity:
|
||||
"""Derive identity only from the native report and immutable pinned inputs."""
|
||||
report = inputs.loaded_artifact
|
||||
pin = inputs.artifact_pin
|
||||
recipe = inputs.numerical_recipe
|
||||
artifact = ArtifactIdentity(
|
||||
artifact_id=pin.artifact_id,
|
||||
revision=pin.revision,
|
||||
content_digest=pin.content_digest,
|
||||
architecture=report.architecture,
|
||||
architecture_digest=report.architecture_digest,
|
||||
layer_count=report.layer_count,
|
||||
derived_from=pin.derived_from,
|
||||
)
|
||||
return ShardIdentity(
|
||||
artifact=artifact,
|
||||
recipe=RuntimeRecipe(
|
||||
weight_quantization=recipe.weight_quantization,
|
||||
activation_dtype=recipe.activation_dtype,
|
||||
compute_dtype=recipe.compute_dtype,
|
||||
kv_dtype=recipe.kv_dtype,
|
||||
kv_layout=recipe.kv_layout,
|
||||
tokenizer_revision=inputs.tokenizer_revision,
|
||||
architecture_adapter=recipe.architecture_adapter,
|
||||
backend_id=recipe.backend_id,
|
||||
runtime_version=recipe.runtime_version,
|
||||
boundary_schema_version=recipe.boundary_schema_version,
|
||||
protocol_schema_version=recipe.protocol_schema_version,
|
||||
recipe_id=recipe.recipe_id,
|
||||
recipe_version=recipe.recipe_version,
|
||||
catalogue_version=recipe.catalogue_version,
|
||||
),
|
||||
shard_start=report.owned_start_layer,
|
||||
shard_end=report.owned_end_layer,
|
||||
)
|
||||
|
||||
|
||||
class NativeSessionRejected(RecipeIdentityError):
|
||||
"""A native worker refused a ``SessionOpen`` before allocating session state."""
|
||||
|
||||
def __init__(self, error: "pb.ShardError") -> None:
|
||||
super().__init__(f"native SessionOpen rejected: {error.code}")
|
||||
self.error = error
|
||||
|
||||
|
||||
class NativeWorkerBackendAdapter:
|
||||
"""Small backend-facing adapter around the native loaded-artifact seam."""
|
||||
|
||||
def __init__(self, identity_inputs: NativeIdentityInputs) -> None:
|
||||
self.identity_inputs = identity_inputs
|
||||
self.identity = shard_identity_from_native_report(identity_inputs)
|
||||
|
||||
@property
|
||||
def loaded_artifact_report(self) -> NativeLoadedArtifactReport:
|
||||
return self.identity_inputs.loaded_artifact
|
||||
|
||||
def check_session_open(
|
||||
self,
|
||||
opened: "pb.SessionOpen",
|
||||
*,
|
||||
expected_route_session_id: str | None = None,
|
||||
expected_route_epoch: int | None = None,
|
||||
) -> None:
|
||||
"""Reject incompatible/stale opens at the native worker boundary."""
|
||||
mismatches = check_session_open(
|
||||
self.identity,
|
||||
opened,
|
||||
expected_route_session_id=expected_route_session_id,
|
||||
expected_route_epoch=expected_route_epoch,
|
||||
)
|
||||
error = handshake_error(mismatches)
|
||||
if error is not None:
|
||||
raise NativeSessionRejected(error)
|
||||
|
||||
def on_session_open(
|
||||
self,
|
||||
opened: "pb.SessionOpen",
|
||||
*,
|
||||
expected_route_session_id: str | None = None,
|
||||
expected_route_epoch: int | None = None,
|
||||
) -> "pb.SessionAccepted":
|
||||
"""The native worker's SessionOpen boundary, before session allocation."""
|
||||
self.check_session_open(
|
||||
opened,
|
||||
expected_route_session_id=expected_route_session_id,
|
||||
expected_route_epoch=expected_route_epoch,
|
||||
)
|
||||
return pb.SessionAccepted(
|
||||
schema_version=SCHEMA_VERSION,
|
||||
route_session_id=opened.route_session_id,
|
||||
route_epoch=opened.route_epoch,
|
||||
fingerprint=self.identity.fingerprint.to_proto(),
|
||||
)
|
||||
79
packages/node/meshnet_node/native_protocol/__init__.py
Normal file
79
packages/node/meshnet_node/native_protocol/__init__.py
Normal file
@@ -0,0 +1,79 @@
|
||||
"""The native Shard protocol: Protobuf over gRPC/HTTP2 (ADR-0020).
|
||||
|
||||
`packages/node/native/proto/shard_runtime.proto` is the contract. This package
|
||||
is how Python speaks it: the generated stubs plus the validation, framing and
|
||||
chunking rules that the stubs cannot express.
|
||||
|
||||
Import the message types from here rather than reaching into `.generated`, so
|
||||
the location of build output stays an implementation detail::
|
||||
|
||||
from meshnet_node.native_protocol import pb, encode_tensor, decode_bundle
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .generated import shard_runtime_pb2 as pb
|
||||
from .codec import (
|
||||
BUNDLE_VERSION,
|
||||
DEFAULT_MAX_CHUNK_BYTES,
|
||||
DEFAULT_MAX_FRAGMENT_BYTES,
|
||||
DEFAULT_MAX_FRAGMENTS_PER_TENSOR,
|
||||
DEFAULT_MAX_INFLIGHT_CHUNKS,
|
||||
DEFAULT_MAX_PREFILL_CHUNK_TOKENS,
|
||||
DEFAULT_MAX_TENSORS_PER_BUNDLE,
|
||||
DEFAULT_MAX_TENSOR_DIMENSION,
|
||||
DEFAULT_MAX_TENSOR_RANK,
|
||||
HIDDEN_STATES,
|
||||
SCHEMA_VERSION,
|
||||
PayloadCorrupt,
|
||||
PrefillChunk,
|
||||
ProtocolError,
|
||||
checksum_of,
|
||||
crc32c,
|
||||
decode_bundle,
|
||||
decode_step_bundle,
|
||||
encode_decode_step,
|
||||
decode_tensor,
|
||||
default_flow_control,
|
||||
encode_bundle,
|
||||
encode_tensor,
|
||||
expected_bytes,
|
||||
itemsize,
|
||||
negotiate_flow_control,
|
||||
plan_prefill_chunks,
|
||||
validate_session_message_size,
|
||||
validate_tail_result,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"BUNDLE_VERSION",
|
||||
"DEFAULT_MAX_CHUNK_BYTES",
|
||||
"DEFAULT_MAX_FRAGMENT_BYTES",
|
||||
"DEFAULT_MAX_FRAGMENTS_PER_TENSOR",
|
||||
"DEFAULT_MAX_INFLIGHT_CHUNKS",
|
||||
"DEFAULT_MAX_PREFILL_CHUNK_TOKENS",
|
||||
"DEFAULT_MAX_TENSORS_PER_BUNDLE",
|
||||
"DEFAULT_MAX_TENSOR_DIMENSION",
|
||||
"DEFAULT_MAX_TENSOR_RANK",
|
||||
"HIDDEN_STATES",
|
||||
"SCHEMA_VERSION",
|
||||
"PayloadCorrupt",
|
||||
"PrefillChunk",
|
||||
"ProtocolError",
|
||||
"checksum_of",
|
||||
"crc32c",
|
||||
"decode_bundle",
|
||||
"decode_step_bundle",
|
||||
"encode_decode_step",
|
||||
"decode_tensor",
|
||||
"default_flow_control",
|
||||
"encode_bundle",
|
||||
"encode_tensor",
|
||||
"expected_bytes",
|
||||
"itemsize",
|
||||
"negotiate_flow_control",
|
||||
"pb",
|
||||
"plan_prefill_chunks",
|
||||
"validate_session_message_size",
|
||||
"validate_tail_result",
|
||||
]
|
||||
608
packages/node/meshnet_node/native_protocol/codec.py
Normal file
608
packages/node/meshnet_node/native_protocol/codec.py
Normal file
@@ -0,0 +1,608 @@
|
||||
"""Encode and decode the native Shard protocol's named-tensor bundles.
|
||||
|
||||
The generated stubs give us message *structure*; they cannot enforce the
|
||||
invariants that keep a distributed forward correct. A bundle whose declared
|
||||
shape disagrees with its byte count, whose fragments leave a hole, or whose
|
||||
checksum does not match is not a slightly-wrong activation — it is silently
|
||||
wrong tokens for the rest of the generation. So decoding is validating: every
|
||||
path into a tensor's bytes goes through :func:`decode_tensor`, which refuses a
|
||||
payload it cannot fully account for.
|
||||
|
||||
Compression is a transport optimisation and is decided by the same policy layer
|
||||
the existing HTTP seam already uses (``activation_compression``), so a node's
|
||||
tuned thresholds apply to both transports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import struct
|
||||
from typing import Iterable, Sequence
|
||||
|
||||
from ..activation_compression import (
|
||||
CompressionPolicy,
|
||||
compress_activation,
|
||||
decompress_activation,
|
||||
)
|
||||
from .generated import shard_runtime_pb2 as pb
|
||||
|
||||
# The schema generation this build speaks. A peer offering something else is
|
||||
# rejected at the handshake rather than being half-understood.
|
||||
SCHEMA_VERSION = pb.SCHEMA_VERSION_1
|
||||
|
||||
# Generation of the tensor-bundle layout, versioned independently of the
|
||||
# protocol so a boundary payload can evolve without a protocol bump.
|
||||
BUNDLE_VERSION = 1
|
||||
|
||||
# Token-aligned prefill chunk bound. 128 tokens is the size ADR-0008 already
|
||||
# uses on the HTTP seam; keeping it identical means seam bytes stay comparable
|
||||
# across transports.
|
||||
DEFAULT_MAX_PREFILL_CHUNK_TOKENS = 128
|
||||
|
||||
# gRPC's default maximum receive size. Fragmenting below it keeps us inside the
|
||||
# default limits of any conformant peer instead of requiring every client to
|
||||
# raise its window.
|
||||
DEFAULT_MAX_CHUNK_BYTES = 4 * 1024 * 1024
|
||||
|
||||
# Leave room for envelope and framing overhead inside one chunk message.
|
||||
DEFAULT_MAX_FRAGMENT_BYTES = 1024 * 1024
|
||||
|
||||
DEFAULT_MAX_INFLIGHT_CHUNKS = 8
|
||||
DEFAULT_MAX_FRAGMENTS_PER_TENSOR = 64
|
||||
DEFAULT_MAX_TENSORS_PER_BUNDLE = 64
|
||||
DEFAULT_MAX_TENSOR_RANK = 8
|
||||
DEFAULT_MAX_TENSOR_DIMENSION = (1 << 31) - 1
|
||||
|
||||
# Canonical boundary tensor name for a dense transformer hidden state.
|
||||
HIDDEN_STATES = "hidden_states"
|
||||
|
||||
_DTYPE_ITEMSIZE: dict[int, int] = {
|
||||
pb.DTYPE_BFLOAT16: 2,
|
||||
pb.DTYPE_FLOAT16: 2,
|
||||
pb.DTYPE_FLOAT32: 4,
|
||||
pb.DTYPE_INT32: 4,
|
||||
pb.DTYPE_INT64: 8,
|
||||
pb.DTYPE_UINT8: 1,
|
||||
pb.DTYPE_INT8: 1,
|
||||
pb.DTYPE_BOOL: 1,
|
||||
}
|
||||
|
||||
|
||||
class ProtocolError(Exception):
|
||||
"""A peer sent something this build cannot safely interpret."""
|
||||
|
||||
|
||||
class PayloadCorrupt(ProtocolError):
|
||||
"""A tensor payload failed validation: size, coverage, or checksum."""
|
||||
|
||||
|
||||
def itemsize(dtype: int) -> int:
|
||||
try:
|
||||
return _DTYPE_ITEMSIZE[dtype]
|
||||
except KeyError:
|
||||
raise ProtocolError(f"unsupported dtype {dtype}") from None
|
||||
|
||||
|
||||
def expected_bytes(
|
||||
shape: Sequence[int],
|
||||
dtype: int,
|
||||
*,
|
||||
max_rank: int = DEFAULT_MAX_TENSOR_RANK,
|
||||
max_dimension: int = DEFAULT_MAX_TENSOR_DIMENSION,
|
||||
max_bytes: int | None = None,
|
||||
) -> int:
|
||||
"""Byte count a tensor of `shape` and `dtype` must occupy."""
|
||||
if len(shape) > max_rank:
|
||||
raise ProtocolError(f"tensor rank {len(shape)} exceeds limit {max_rank}")
|
||||
if any(dim < 0 or dim > max_dimension for dim in shape):
|
||||
raise ProtocolError(
|
||||
f"dimension outside 0..{max_dimension} in shape {list(shape)}"
|
||||
)
|
||||
size = itemsize(dtype)
|
||||
count = 1
|
||||
for dim in shape:
|
||||
count *= dim
|
||||
if max_bytes is not None and count * size > max_bytes:
|
||||
raise ProtocolError(f"tensor shape {list(shape)} exceeds byte limit {max_bytes}")
|
||||
return count * size
|
||||
|
||||
|
||||
# --- CRC32C ----------------------------------------------------------------
|
||||
#
|
||||
# CRC32C (Castagnoli), not zlib's CRC32: it is the checksum gRPC, and the
|
||||
# storage systems these payloads pass through, already use, and hardware
|
||||
# implements it. `google_crc32c` is used when present; the table fallback keeps
|
||||
# the default test suite dependency-free.
|
||||
|
||||
_CRC32C_POLY = 0x82F63B78
|
||||
_CRC32C_TABLE: list[int] = []
|
||||
for _i in range(256):
|
||||
_c = _i
|
||||
for _ in range(8):
|
||||
_c = (_c >> 1) ^ (_CRC32C_POLY if _c & 1 else 0)
|
||||
_CRC32C_TABLE.append(_c)
|
||||
|
||||
try: # pragma: no cover - depends on an optional native package
|
||||
from google_crc32c import value as _fast_crc32c
|
||||
except ImportError: # pragma: no cover
|
||||
_fast_crc32c = None
|
||||
|
||||
|
||||
def crc32c(data: bytes) -> int:
|
||||
if _fast_crc32c is not None: # pragma: no cover - optional fast path
|
||||
return _fast_crc32c(data)
|
||||
crc = 0xFFFFFFFF
|
||||
for byte in data:
|
||||
crc = (crc >> 8) ^ _CRC32C_TABLE[(crc ^ byte) & 0xFF]
|
||||
return crc ^ 0xFFFFFFFF
|
||||
|
||||
|
||||
def checksum_of(data: bytes) -> pb.Checksum:
|
||||
return pb.Checksum(
|
||||
algorithm=pb.CHECKSUM_ALGORITHM_CRC32C,
|
||||
value=struct.pack(">I", crc32c(data)),
|
||||
)
|
||||
|
||||
|
||||
# --- Tensors ---------------------------------------------------------------
|
||||
|
||||
|
||||
def encode_tensor(
|
||||
name: str,
|
||||
data: bytes,
|
||||
shape: Sequence[int],
|
||||
dtype: int = pb.DTYPE_BFLOAT16,
|
||||
*,
|
||||
policy: CompressionPolicy | None = None,
|
||||
max_chunk_bytes: int = DEFAULT_MAX_CHUNK_BYTES,
|
||||
max_fragment_bytes: int = DEFAULT_MAX_FRAGMENT_BYTES,
|
||||
max_fragments: int = DEFAULT_MAX_FRAGMENTS_PER_TENSOR,
|
||||
) -> pb.NamedTensor:
|
||||
"""Build a NamedTensor, compressing and fragmenting as needed.
|
||||
|
||||
`data` is the uncompressed little-endian payload. The checksum is taken over
|
||||
it *before* compression so it stays valid whichever framing a hop chooses.
|
||||
"""
|
||||
if max_chunk_bytes <= 0 or max_fragment_bytes <= 0 or max_fragments <= 0:
|
||||
raise ProtocolError("tensor byte/count bounds must be positive")
|
||||
declared = expected_bytes(shape, dtype, max_bytes=max_chunk_bytes)
|
||||
if len(data) != declared:
|
||||
raise ProtocolError(
|
||||
f"tensor {name!r} declares shape {list(shape)} ({declared} bytes) "
|
||||
f"but carries {len(data)} bytes"
|
||||
)
|
||||
|
||||
body = data
|
||||
compression = pb.COMPRESSION_NONE
|
||||
if policy is not None:
|
||||
result = compress_activation(data, policy)
|
||||
if result.compressed:
|
||||
body = result.body
|
||||
compression = pb.COMPRESSION_ZSTD
|
||||
|
||||
tensor = pb.NamedTensor(
|
||||
name=name,
|
||||
shape=list(shape),
|
||||
dtype=dtype,
|
||||
byte_order=pb.BYTE_ORDER_LITTLE_ENDIAN,
|
||||
total_bytes=len(data),
|
||||
compression=compression,
|
||||
checksum=checksum_of(data),
|
||||
)
|
||||
|
||||
# Fragment the wire body (compressed if we compressed). Offsets walk the
|
||||
# wire body so a receiver can verify coverage without assuming arrival
|
||||
# order; a zstd frame is not decodable per fragment, so reassembly comes
|
||||
# first and decompression happens once, in decode_tensor.
|
||||
slices = [body[i : i + max_fragment_bytes] for i in range(0, len(body), max_fragment_bytes)]
|
||||
if not slices:
|
||||
# A zero-element tensor is legal (e.g. an empty mask) and still needs a
|
||||
# fragment, so coverage checks have something to verify.
|
||||
slices = [b""]
|
||||
if len(slices) > max_fragments:
|
||||
raise ProtocolError(
|
||||
f"tensor {name!r} needs {len(slices)} fragments, exceeding limit {max_fragments}"
|
||||
)
|
||||
|
||||
offset = 0
|
||||
for index, piece in enumerate(slices):
|
||||
tensor.fragments.append(
|
||||
pb.TensorFragment(
|
||||
fragment_index=index,
|
||||
fragment_count=len(slices),
|
||||
byte_offset=offset,
|
||||
payload=piece,
|
||||
)
|
||||
)
|
||||
offset += len(piece)
|
||||
return tensor
|
||||
|
||||
|
||||
def decode_tensor(
|
||||
tensor: pb.NamedTensor,
|
||||
*,
|
||||
max_chunk_bytes: int = DEFAULT_MAX_CHUNK_BYTES,
|
||||
max_fragment_bytes: int = DEFAULT_MAX_FRAGMENT_BYTES,
|
||||
max_fragments: int = DEFAULT_MAX_FRAGMENTS_PER_TENSOR,
|
||||
) -> bytes:
|
||||
"""Reassemble, decompress and validate a NamedTensor's payload.
|
||||
|
||||
Raises PayloadCorrupt rather than returning a payload it cannot fully
|
||||
account for: a hole in the fragments or a bad checksum means the activation
|
||||
is not what the sender computed, and continuing would corrupt the route.
|
||||
"""
|
||||
if max_chunk_bytes <= 0 or max_fragment_bytes <= 0 or max_fragments <= 0:
|
||||
raise ProtocolError("negotiated byte/count bounds must be positive")
|
||||
if tensor.total_bytes > max_chunk_bytes:
|
||||
raise ProtocolError(
|
||||
f"tensor {tensor.name!r} declares {tensor.total_bytes} bytes, exceeding "
|
||||
f"the {max_chunk_bytes}-byte negotiated chunk bound"
|
||||
)
|
||||
if tensor.byte_order == pb.BYTE_ORDER_BIG_ENDIAN:
|
||||
raise ProtocolError(f"tensor {tensor.name!r} is big-endian; wire order is little-endian")
|
||||
if tensor.byte_order != pb.BYTE_ORDER_LITTLE_ENDIAN:
|
||||
raise ProtocolError(f"tensor {tensor.name!r} declares no byte order")
|
||||
declared = expected_bytes(
|
||||
tensor.shape, tensor.dtype, max_bytes=max_chunk_bytes
|
||||
)
|
||||
if declared != tensor.total_bytes:
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} shape {list(tensor.shape)} implies {declared} bytes "
|
||||
f"but declares {tensor.total_bytes}"
|
||||
)
|
||||
|
||||
if not tensor.fragments:
|
||||
raise PayloadCorrupt(f"tensor {tensor.name!r} carries no fragments")
|
||||
if len(tensor.fragments) > max_fragments:
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} carries {len(tensor.fragments)} fragments, "
|
||||
f"exceeding limit {max_fragments}"
|
||||
)
|
||||
if any(len(fragment.payload) > max_fragment_bytes for fragment in tensor.fragments):
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} carries a fragment larger than "
|
||||
f"{max_fragment_bytes} bytes"
|
||||
)
|
||||
wire_bytes = sum(len(fragment.payload) for fragment in tensor.fragments)
|
||||
if wire_bytes > max_chunk_bytes:
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} wire body exceeds the "
|
||||
f"{max_chunk_bytes}-byte negotiated chunk bound"
|
||||
)
|
||||
|
||||
fragments = sorted(tensor.fragments, key=lambda f: f.byte_offset)
|
||||
count = fragments[0].fragment_count
|
||||
if any(f.fragment_count != count for f in fragments):
|
||||
raise PayloadCorrupt(f"tensor {tensor.name!r} has inconsistent fragment_count")
|
||||
if len(fragments) != count:
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} expects {count} fragments but carries {len(fragments)}"
|
||||
)
|
||||
if {f.fragment_index for f in fragments} != set(range(count)):
|
||||
raise PayloadCorrupt(f"tensor {tensor.name!r} has duplicate or missing fragment indices")
|
||||
|
||||
# Contiguity: offsets must tile the body exactly, with no hole and no overlap.
|
||||
body = bytearray()
|
||||
for fragment in fragments:
|
||||
if fragment.byte_offset != len(body):
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} fragment {fragment.fragment_index} starts at "
|
||||
f"{fragment.byte_offset}, expected {len(body)}"
|
||||
)
|
||||
body.extend(fragment.payload)
|
||||
|
||||
if tensor.compression == pb.COMPRESSION_ZSTD:
|
||||
try:
|
||||
data = decompress_activation(
|
||||
bytes(body), "zstd", max_output_bytes=tensor.total_bytes
|
||||
).body
|
||||
except ValueError as exc:
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} has invalid bounded zstd payload"
|
||||
) from exc
|
||||
elif tensor.compression == pb.COMPRESSION_NONE:
|
||||
data = bytes(body)
|
||||
else:
|
||||
raise ProtocolError(
|
||||
f"tensor {tensor.name!r} uses unspecified or unsupported compression"
|
||||
)
|
||||
|
||||
if len(data) != tensor.total_bytes:
|
||||
raise PayloadCorrupt(
|
||||
f"tensor {tensor.name!r} declares {tensor.total_bytes} bytes but "
|
||||
f"reassembled {len(data)}"
|
||||
)
|
||||
|
||||
algorithm = tensor.checksum.algorithm
|
||||
if algorithm == pb.CHECKSUM_ALGORITHM_CRC32C:
|
||||
if tensor.checksum.value != struct.pack(">I", crc32c(data)):
|
||||
raise PayloadCorrupt(f"tensor {tensor.name!r} failed its CRC32C check")
|
||||
elif algorithm != pb.CHECKSUM_ALGORITHM_NONE:
|
||||
raise ProtocolError(
|
||||
f"tensor {tensor.name!r} uses unspecified or unsupported checksum algorithm"
|
||||
)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def encode_bundle(
|
||||
tensors: Iterable[pb.NamedTensor],
|
||||
*,
|
||||
architecture: int = pb.ARCHITECTURE_TYPE_UNSPECIFIED,
|
||||
boundary_point: str = "",
|
||||
max_chunk_bytes: int = DEFAULT_MAX_CHUNK_BYTES,
|
||||
max_tensors: int = DEFAULT_MAX_TENSORS_PER_BUNDLE,
|
||||
) -> pb.TensorBundle:
|
||||
if max_chunk_bytes <= 0 or max_tensors <= 0:
|
||||
raise ProtocolError("bundle byte/count bounds must be positive")
|
||||
tensor_list = list(tensors)
|
||||
if len(tensor_list) > max_tensors:
|
||||
raise ProtocolError(
|
||||
f"bundle carries {len(tensor_list)} tensors, exceeding limit {max_tensors}"
|
||||
)
|
||||
bundle = pb.TensorBundle(
|
||||
bundle_version=BUNDLE_VERSION,
|
||||
tensors=tensor_list,
|
||||
architecture=architecture,
|
||||
boundary_point=boundary_point,
|
||||
)
|
||||
if bundle.ByteSize() > max_chunk_bytes:
|
||||
raise ProtocolError(
|
||||
f"serialized tensor bundle exceeds the {max_chunk_bytes}-byte "
|
||||
"negotiated chunk bound"
|
||||
)
|
||||
return bundle
|
||||
|
||||
|
||||
def decode_bundle(
|
||||
bundle: pb.TensorBundle,
|
||||
*,
|
||||
max_chunk_bytes: int = DEFAULT_MAX_CHUNK_BYTES,
|
||||
max_fragment_bytes: int = DEFAULT_MAX_FRAGMENT_BYTES,
|
||||
max_fragments: int = DEFAULT_MAX_FRAGMENTS_PER_TENSOR,
|
||||
max_tensors: int = DEFAULT_MAX_TENSORS_PER_BUNDLE,
|
||||
) -> dict[str, bytes]:
|
||||
"""Validate every tensor in a bundle and return name -> payload."""
|
||||
if bundle.bundle_version != BUNDLE_VERSION:
|
||||
raise ProtocolError(
|
||||
f"bundle version {bundle.bundle_version} is not supported by this build "
|
||||
f"({BUNDLE_VERSION})"
|
||||
)
|
||||
if (
|
||||
max_chunk_bytes <= 0
|
||||
or max_fragment_bytes <= 0
|
||||
or max_fragments <= 0
|
||||
or max_tensors <= 0
|
||||
):
|
||||
raise ProtocolError("negotiated byte/count bounds must be positive")
|
||||
if len(bundle.tensors) > max_tensors:
|
||||
raise ProtocolError(
|
||||
f"bundle carries {len(bundle.tensors)} tensors, exceeding limit {max_tensors}"
|
||||
)
|
||||
if bundle.ByteSize() > max_chunk_bytes:
|
||||
raise ProtocolError(
|
||||
f"serialized tensor bundle exceeds the {max_chunk_bytes}-byte "
|
||||
"negotiated chunk bound"
|
||||
)
|
||||
payloads: dict[str, bytes] = {}
|
||||
for tensor in bundle.tensors:
|
||||
if not tensor.name:
|
||||
raise ProtocolError("bundle carries an unnamed tensor")
|
||||
if tensor.name in payloads:
|
||||
raise ProtocolError(f"bundle carries duplicate tensor {tensor.name!r}")
|
||||
payloads[tensor.name] = decode_tensor(
|
||||
tensor,
|
||||
max_chunk_bytes=max_chunk_bytes,
|
||||
max_fragment_bytes=max_fragment_bytes,
|
||||
max_fragments=max_fragments,
|
||||
)
|
||||
return payloads
|
||||
|
||||
|
||||
def encode_decode_step(
|
||||
bundle: pb.TensorBundle,
|
||||
*,
|
||||
idempotency_step: int,
|
||||
position: int,
|
||||
expected_past_len: int,
|
||||
work_id: str,
|
||||
deadline_unix_nanos: int = 0,
|
||||
prefer_compact_one_tensor: bool = True,
|
||||
) -> pb.DecodeStep:
|
||||
"""Encode a decode boundary, retaining the deliberate compact fallback."""
|
||||
step = pb.DecodeStep(
|
||||
idempotency_step=idempotency_step,
|
||||
position=position,
|
||||
expected_past_len=expected_past_len,
|
||||
work_id=work_id,
|
||||
deadline_unix_nanos=deadline_unix_nanos,
|
||||
)
|
||||
if prefer_compact_one_tensor and len(bundle.tensors) == 1:
|
||||
step.tensor.CopyFrom(bundle.tensors[0])
|
||||
else:
|
||||
step.bundle.CopyFrom(bundle)
|
||||
return step
|
||||
|
||||
|
||||
def validate_tail_result(result: pb.TailResult) -> None:
|
||||
"""Fail closed unless a tail completion is bound to its exact recipe."""
|
||||
identity = result.identity
|
||||
required = (
|
||||
identity.request_id,
|
||||
identity.runtime_recipe_digest,
|
||||
identity.chat_template_id,
|
||||
identity.chat_template_version,
|
||||
identity.reasoning_mode,
|
||||
)
|
||||
if not all(required) or identity.architecture == pb.ARCHITECTURE_TYPE_UNSPECIFIED:
|
||||
raise ProtocolError("tail result lacks exact request/recipe/template identity")
|
||||
if result.WhichOneof("output") not in {"logits", "sampled_token_id"}:
|
||||
raise ProtocolError("tail result lacks logits or sampled token output")
|
||||
|
||||
|
||||
def decode_step_bundle(
|
||||
step: pb.DecodeStep,
|
||||
*,
|
||||
max_chunk_bytes: int = DEFAULT_MAX_CHUNK_BYTES,
|
||||
max_fragment_bytes: int = DEFAULT_MAX_FRAGMENT_BYTES,
|
||||
max_fragments: int = DEFAULT_MAX_FRAGMENTS_PER_TENSOR,
|
||||
max_tensors: int = DEFAULT_MAX_TENSORS_PER_BUNDLE,
|
||||
) -> dict[str, bytes]:
|
||||
"""Decode a fast-path boundary with the DGR-006 compatibility rule.
|
||||
|
||||
`bundle` is authoritative because it can carry architecture sidebands. The
|
||||
old `tensor` field remains the compact representation for a certified
|
||||
one-tensor boundary and is accepted by new readers during rollout.
|
||||
"""
|
||||
if step.HasField("bundle"):
|
||||
return decode_bundle(
|
||||
step.bundle,
|
||||
max_chunk_bytes=max_chunk_bytes,
|
||||
max_fragment_bytes=max_fragment_bytes,
|
||||
max_fragments=max_fragments,
|
||||
max_tensors=max_tensors,
|
||||
)
|
||||
if step.HasField("tensor"):
|
||||
return decode_bundle(
|
||||
encode_bundle([step.tensor], max_chunk_bytes=max_chunk_bytes),
|
||||
max_chunk_bytes=max_chunk_bytes,
|
||||
max_fragment_bytes=max_fragment_bytes,
|
||||
max_fragments=max_fragments,
|
||||
max_tensors=max_tensors,
|
||||
)
|
||||
raise ProtocolError("decode step carries neither TensorBundle nor legacy tensor")
|
||||
|
||||
|
||||
def validate_session_message_size(
|
||||
message: pb.SessionRequest | pb.SessionResponse,
|
||||
*,
|
||||
max_chunk_bytes: int = DEFAULT_MAX_CHUNK_BYTES,
|
||||
) -> int:
|
||||
"""Reject an oversized complete stream frame, including protobuf overhead.
|
||||
|
||||
Bundle validation alone is insufficient because the envelope and oneof
|
||||
framing are part of the same gRPC message. Senders call this immediately
|
||||
before writing; receivers configure gRPC's receive limit to the same value
|
||||
and call it again before semantic decoding.
|
||||
"""
|
||||
if max_chunk_bytes <= 0:
|
||||
raise ProtocolError("max_chunk_bytes must be positive")
|
||||
if not isinstance(message, (pb.SessionRequest, pb.SessionResponse)):
|
||||
raise ProtocolError("size validation requires a session request or response")
|
||||
size = message.ByteSize()
|
||||
if size > max_chunk_bytes:
|
||||
raise ProtocolError(
|
||||
f"serialized session message is {size} bytes, exceeding the "
|
||||
f"{max_chunk_bytes}-byte negotiated chunk bound"
|
||||
)
|
||||
return size
|
||||
|
||||
|
||||
# --- Bounded prefill chunking ----------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PrefillChunk:
|
||||
"""One token-aligned slice of a prefill."""
|
||||
|
||||
chunk_index: int
|
||||
chunk_count: int
|
||||
first_position: int
|
||||
token_count: int
|
||||
|
||||
@property
|
||||
def final_chunk(self) -> bool:
|
||||
return self.chunk_index == self.chunk_count - 1
|
||||
|
||||
def chunk_info(self) -> pb.ChunkInfo:
|
||||
return pb.ChunkInfo(
|
||||
chunk_index=self.chunk_index,
|
||||
chunk_count=self.chunk_count,
|
||||
final_chunk=self.final_chunk,
|
||||
)
|
||||
|
||||
def position(self) -> pb.PositionSpan:
|
||||
return pb.PositionSpan(
|
||||
first_position=self.first_position, token_count=self.token_count
|
||||
)
|
||||
|
||||
|
||||
def plan_prefill_chunks(
|
||||
total_tokens: int,
|
||||
*,
|
||||
first_position: int = 0,
|
||||
max_tokens: int = DEFAULT_MAX_PREFILL_CHUNK_TOKENS,
|
||||
) -> list[PrefillChunk]:
|
||||
"""Split a prefill into bounded, token-aligned chunks.
|
||||
|
||||
Splits fall on token boundaries only (ADR-0008): a fragment of a token's
|
||||
hidden state is not a thing a receiver can execute.
|
||||
"""
|
||||
if total_tokens <= 0:
|
||||
raise ProtocolError("a prefill must carry at least one token")
|
||||
if max_tokens <= 0:
|
||||
raise ProtocolError("max_tokens must be positive")
|
||||
|
||||
count = (total_tokens + max_tokens - 1) // max_tokens
|
||||
chunks = []
|
||||
for index in range(count):
|
||||
offset = index * max_tokens
|
||||
chunks.append(
|
||||
PrefillChunk(
|
||||
chunk_index=index,
|
||||
chunk_count=count,
|
||||
first_position=first_position + offset,
|
||||
token_count=min(max_tokens, total_tokens - offset),
|
||||
)
|
||||
)
|
||||
return chunks
|
||||
|
||||
|
||||
def default_flow_control() -> pb.FlowControl:
|
||||
return pb.FlowControl(
|
||||
credits_granted=DEFAULT_MAX_INFLIGHT_CHUNKS,
|
||||
max_inflight_chunks=DEFAULT_MAX_INFLIGHT_CHUNKS,
|
||||
max_chunk_bytes=DEFAULT_MAX_CHUNK_BYTES,
|
||||
max_prefill_chunk_tokens=DEFAULT_MAX_PREFILL_CHUNK_TOKENS,
|
||||
)
|
||||
|
||||
|
||||
def negotiate_flow_control(
|
||||
proposed: pb.FlowControl, limits: pb.FlowControl
|
||||
) -> pb.FlowControl:
|
||||
"""Settle a stream's limits: the strictest bound of either peer wins.
|
||||
|
||||
Taking the minimum means neither peer can raise the other's ceiling, so a
|
||||
misconfigured — or hostile — sender cannot talk a worker into unbounded
|
||||
queues by proposing a large window.
|
||||
"""
|
||||
|
||||
def _min(a: int, b: int, fallback: int) -> int:
|
||||
candidates = [v for v in (a, b) if v > 0]
|
||||
return min(candidates) if candidates else fallback
|
||||
|
||||
max_inflight_chunks = _min(
|
||||
proposed.max_inflight_chunks,
|
||||
limits.max_inflight_chunks,
|
||||
DEFAULT_MAX_INFLIGHT_CHUNKS,
|
||||
)
|
||||
credits_granted = min(
|
||||
_min(
|
||||
proposed.credits_granted,
|
||||
limits.credits_granted,
|
||||
DEFAULT_MAX_INFLIGHT_CHUNKS,
|
||||
),
|
||||
max_inflight_chunks,
|
||||
)
|
||||
return pb.FlowControl(
|
||||
credits_granted=credits_granted,
|
||||
max_inflight_chunks=max_inflight_chunks,
|
||||
max_chunk_bytes=_min(
|
||||
proposed.max_chunk_bytes, limits.max_chunk_bytes, DEFAULT_MAX_CHUNK_BYTES
|
||||
),
|
||||
max_prefill_chunk_tokens=_min(
|
||||
proposed.max_prefill_chunk_tokens,
|
||||
limits.max_prefill_chunk_tokens,
|
||||
DEFAULT_MAX_PREFILL_CHUNK_TOKENS,
|
||||
),
|
||||
)
|
||||
166
packages/node/meshnet_node/native_protocol/conformance.py
Normal file
166
packages/node/meshnet_node/native_protocol/conformance.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""Canonical conformance vectors for the native Shard protocol.
|
||||
|
||||
Two independently-written codecs that each round-trip their own output prove
|
||||
nothing about each other. These vectors are the shared reference: Python builds
|
||||
the canonical message, the bytes are committed under
|
||||
`packages/node/native/testdata/`, and the C++ test parses those exact bytes and
|
||||
asserts the same field values. A change that alters the wire meaning of a field
|
||||
breaks the vector in both languages instead of drifting silently in one.
|
||||
|
||||
The vector deliberately exercises every field group the protocol promises to
|
||||
carry — identity, epoch, fingerprint, range, phase, position, idempotency,
|
||||
cache expectation, deadline, chunking, compression, checksum and a multi-
|
||||
fragment named tensor — so it doubles as an executable inventory of the
|
||||
contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
|
||||
from . import codec
|
||||
from .generated import shard_runtime_pb2 as pb
|
||||
|
||||
# Committed vectors live beside the schema, under `packages/node/native/`.
|
||||
# parents[2] is `packages/node`: native_protocol -> meshnet_node -> node.
|
||||
TESTDATA_DIR = pathlib.Path(__file__).resolve().parents[2] / "native/testdata"
|
||||
|
||||
GOLDEN_SESSION_REQUEST = "session_request_golden.binpb"
|
||||
GOLDEN_CAPABILITY_REPORT = "capability_report_golden.binpb"
|
||||
GOLDEN_DECODE_STEP = "decode_step_golden.binpb"
|
||||
|
||||
# Written by the C++ conformance test into its build tree; the Python test picks
|
||||
# it up when present to prove the two languages agree byte-for-byte.
|
||||
CPP_ROUNDTRIP = "cpp_roundtrip.binpb"
|
||||
|
||||
# Fixed, non-default values. Every one is chosen to be distinguishable from a
|
||||
# proto3 default so an unset field can never masquerade as a correct one.
|
||||
WORK_ID = "work-7f3a"
|
||||
ROUTE_SESSION_ID = "rs-2b91"
|
||||
ROUTE_EPOCH = 7
|
||||
IDEMPOTENCY_STEP = 42
|
||||
FIRST_POSITION = 256
|
||||
TOKEN_COUNT = 128
|
||||
EXPECTED_PAST_LEN = 256
|
||||
DEADLINE_UNIX_NANOS = 1_800_000_000_000_000_000
|
||||
MODEL_ARTIFACT_DIGEST = "sha256:1f0c9d2e"
|
||||
RUNTIME_RECIPE_DIGEST = "sha256:ab77e410"
|
||||
RECIPE_ID = "llama-gguf-q4km-rocm"
|
||||
RECIPE_VERSION = "3"
|
||||
CATALOGUE_VERSION = "2026.07.1"
|
||||
START_LAYER = 12
|
||||
END_LAYER = 24
|
||||
EFFECTIVE_START_LAYER = 16
|
||||
HIDDEN_SIZE = 8
|
||||
|
||||
# A payload big enough to force more than one fragment at the bound below, so
|
||||
# the vector actually exercises reassembly rather than the one-fragment path.
|
||||
FRAGMENT_BYTES = 64
|
||||
TENSOR_SHAPE = [1, TOKEN_COUNT, HIDDEN_SIZE]
|
||||
|
||||
|
||||
def canonical_payload() -> bytes:
|
||||
"""Deterministic bfloat16-sized payload for the canonical tensor."""
|
||||
total = codec.expected_bytes(TENSOR_SHAPE, pb.DTYPE_BFLOAT16)
|
||||
return bytes((i * 7 + 11) % 256 for i in range(total))
|
||||
|
||||
|
||||
def canonical_session_request() -> pb.SessionRequest:
|
||||
"""The canonical prefill chunk carried on a session stream."""
|
||||
tensor = codec.encode_tensor(
|
||||
codec.HIDDEN_STATES,
|
||||
canonical_payload(),
|
||||
TENSOR_SHAPE,
|
||||
pb.DTYPE_BFLOAT16,
|
||||
max_fragment_bytes=FRAGMENT_BYTES,
|
||||
)
|
||||
envelope = pb.Envelope(
|
||||
schema_version=pb.SCHEMA_VERSION_1,
|
||||
work_id=WORK_ID,
|
||||
route_session_id=ROUTE_SESSION_ID,
|
||||
route_epoch=ROUTE_EPOCH,
|
||||
fingerprint=pb.Fingerprint(
|
||||
model_artifact_digest=MODEL_ARTIFACT_DIGEST,
|
||||
runtime_recipe_digest=RUNTIME_RECIPE_DIGEST,
|
||||
recipe_id=RECIPE_ID,
|
||||
recipe_version=RECIPE_VERSION,
|
||||
catalogue_version=CATALOGUE_VERSION,
|
||||
),
|
||||
shard_range=pb.ShardRange(
|
||||
start_layer=START_LAYER,
|
||||
end_layer=END_LAYER,
|
||||
effective_start_layer=EFFECTIVE_START_LAYER,
|
||||
),
|
||||
phase=pb.PHASE_PREFILL,
|
||||
position=pb.PositionSpan(
|
||||
first_position=FIRST_POSITION, token_count=TOKEN_COUNT
|
||||
),
|
||||
idempotency_step=IDEMPOTENCY_STEP,
|
||||
cache_expectation=pb.CacheExpectation(
|
||||
mode=pb.CACHE_MODE_PREFILL, expected_past_len=EXPECTED_PAST_LEN
|
||||
),
|
||||
deadline_unix_nanos=DEADLINE_UNIX_NANOS,
|
||||
chunk=pb.ChunkInfo(chunk_index=1, chunk_count=3, final_chunk=False),
|
||||
)
|
||||
return pb.SessionRequest(
|
||||
chunk=pb.ActivationChunk(
|
||||
envelope=envelope, bundle=codec.encode_bundle([tensor])
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def canonical_capability_report() -> pb.CapabilityReport:
|
||||
"""The canonical capability report a worker answers admission with."""
|
||||
return pb.CapabilityReport(
|
||||
schema_version=pb.SCHEMA_VERSION_1,
|
||||
fingerprint=pb.Fingerprint(
|
||||
model_artifact_digest=MODEL_ARTIFACT_DIGEST,
|
||||
runtime_recipe_digest=RUNTIME_RECIPE_DIGEST,
|
||||
recipe_id=RECIPE_ID,
|
||||
recipe_version=RECIPE_VERSION,
|
||||
catalogue_version=CATALOGUE_VERSION,
|
||||
),
|
||||
shard_range=pb.ShardRange(
|
||||
start_layer=START_LAYER,
|
||||
end_layer=END_LAYER,
|
||||
effective_start_layer=EFFECTIVE_START_LAYER,
|
||||
),
|
||||
backend="rocm",
|
||||
device="gfx1151",
|
||||
validated=True,
|
||||
max_concurrent_sessions=4,
|
||||
max_context_tokens=8192,
|
||||
flow_control=codec.default_flow_control(),
|
||||
accepted_compression=[pb.COMPRESSION_NONE, pb.COMPRESSION_ZSTD],
|
||||
supported_schema_versions=[pb.SCHEMA_VERSION_1],
|
||||
validated_at_unix_nanos=DEADLINE_UNIX_NANOS,
|
||||
)
|
||||
|
||||
|
||||
def canonical_decode_step() -> pb.SessionRequest:
|
||||
"""The DGR-006 multi-tensor decode boundary vector."""
|
||||
hidden = codec.encode_tensor(
|
||||
codec.HIDDEN_STATES, bytes(range(16)), [1, 1, 4], pb.DTYPE_FLOAT32
|
||||
)
|
||||
index_topk = codec.encode_tensor(
|
||||
"index_topk", (3).to_bytes(4, "little"), [1], pb.DTYPE_INT32
|
||||
)
|
||||
return pb.SessionRequest(
|
||||
decode=pb.DecodeStep(
|
||||
idempotency_step=43,
|
||||
position=384,
|
||||
expected_past_len=384,
|
||||
work_id="decode-7f3a",
|
||||
deadline_unix_nanos=DEADLINE_UNIX_NANOS,
|
||||
bundle=codec.encode_bundle(
|
||||
[hidden, index_topk],
|
||||
architecture=pb.ARCHITECTURE_TYPE_MLA,
|
||||
boundary_point="pre_tail_residual",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def serialize(message) -> bytes:
|
||||
"""Serialize deterministically, so committed golden bytes are stable."""
|
||||
return message.SerializeToString(deterministic=True)
|
||||
@@ -0,0 +1,2 @@
|
||||
# Generated by scripts/generate_native_protocol.py. Do not edit.
|
||||
"""Generated protobuf/gRPC stubs for the native Shard protocol."""
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,586 @@
|
||||
from google.protobuf.internal import containers as _containers
|
||||
from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from collections.abc import Iterable as _Iterable, Mapping as _Mapping
|
||||
from typing import ClassVar as _ClassVar, Optional as _Optional, Union as _Union
|
||||
|
||||
DESCRIPTOR: _descriptor.FileDescriptor
|
||||
|
||||
class SchemaVersion(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
SCHEMA_VERSION_UNSPECIFIED: _ClassVar[SchemaVersion]
|
||||
SCHEMA_VERSION_1: _ClassVar[SchemaVersion]
|
||||
|
||||
class ArchitectureType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
ARCHITECTURE_TYPE_UNSPECIFIED: _ClassVar[ArchitectureType]
|
||||
ARCHITECTURE_TYPE_DENSE: _ClassVar[ArchitectureType]
|
||||
ARCHITECTURE_TYPE_MOE: _ClassVar[ArchitectureType]
|
||||
ARCHITECTURE_TYPE_MLA: _ClassVar[ArchitectureType]
|
||||
|
||||
class DType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
DTYPE_UNSPECIFIED: _ClassVar[DType]
|
||||
DTYPE_BFLOAT16: _ClassVar[DType]
|
||||
DTYPE_FLOAT16: _ClassVar[DType]
|
||||
DTYPE_FLOAT32: _ClassVar[DType]
|
||||
DTYPE_INT32: _ClassVar[DType]
|
||||
DTYPE_INT64: _ClassVar[DType]
|
||||
DTYPE_UINT8: _ClassVar[DType]
|
||||
DTYPE_INT8: _ClassVar[DType]
|
||||
DTYPE_BOOL: _ClassVar[DType]
|
||||
|
||||
class ByteOrder(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
BYTE_ORDER_UNSPECIFIED: _ClassVar[ByteOrder]
|
||||
BYTE_ORDER_LITTLE_ENDIAN: _ClassVar[ByteOrder]
|
||||
BYTE_ORDER_BIG_ENDIAN: _ClassVar[ByteOrder]
|
||||
|
||||
class Compression(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
COMPRESSION_UNSPECIFIED: _ClassVar[Compression]
|
||||
COMPRESSION_NONE: _ClassVar[Compression]
|
||||
COMPRESSION_ZSTD: _ClassVar[Compression]
|
||||
|
||||
class ChecksumAlgorithm(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
CHECKSUM_ALGORITHM_UNSPECIFIED: _ClassVar[ChecksumAlgorithm]
|
||||
CHECKSUM_ALGORITHM_NONE: _ClassVar[ChecksumAlgorithm]
|
||||
CHECKSUM_ALGORITHM_CRC32C: _ClassVar[ChecksumAlgorithm]
|
||||
|
||||
class Phase(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
PHASE_UNSPECIFIED: _ClassVar[Phase]
|
||||
PHASE_PREFILL: _ClassVar[Phase]
|
||||
PHASE_DECODE: _ClassVar[Phase]
|
||||
PHASE_RELEASE: _ClassVar[Phase]
|
||||
PHASE_CANCEL: _ClassVar[Phase]
|
||||
|
||||
class CacheMode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
CACHE_MODE_UNSPECIFIED: _ClassVar[CacheMode]
|
||||
CACHE_MODE_STATELESS: _ClassVar[CacheMode]
|
||||
CACHE_MODE_PREFILL: _ClassVar[CacheMode]
|
||||
CACHE_MODE_DECODE: _ClassVar[CacheMode]
|
||||
|
||||
class ErrorCode(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
ERROR_CODE_UNSPECIFIED: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_SCHEMA_UNSUPPORTED: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_FINGERPRINT_MISMATCH: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_EPOCH_STALE: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_SHARD_RANGE_MISMATCH: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_CACHE_MISS: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_RESOURCE_EXHAUSTED: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_PAYLOAD_CORRUPT: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_CANCELLED: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_DEADLINE_EXCEEDED: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_FLOW_CONTROL_VIOLATION: _ClassVar[ErrorCode]
|
||||
ERROR_CODE_INTERNAL: _ClassVar[ErrorCode]
|
||||
|
||||
class ServingState(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
|
||||
__slots__ = ()
|
||||
SERVING_STATE_UNSPECIFIED: _ClassVar[ServingState]
|
||||
SERVING_STATE_SERVING: _ClassVar[ServingState]
|
||||
SERVING_STATE_DRAINING: _ClassVar[ServingState]
|
||||
SERVING_STATE_NOT_SERVING: _ClassVar[ServingState]
|
||||
SCHEMA_VERSION_UNSPECIFIED: SchemaVersion
|
||||
SCHEMA_VERSION_1: SchemaVersion
|
||||
ARCHITECTURE_TYPE_UNSPECIFIED: ArchitectureType
|
||||
ARCHITECTURE_TYPE_DENSE: ArchitectureType
|
||||
ARCHITECTURE_TYPE_MOE: ArchitectureType
|
||||
ARCHITECTURE_TYPE_MLA: ArchitectureType
|
||||
DTYPE_UNSPECIFIED: DType
|
||||
DTYPE_BFLOAT16: DType
|
||||
DTYPE_FLOAT16: DType
|
||||
DTYPE_FLOAT32: DType
|
||||
DTYPE_INT32: DType
|
||||
DTYPE_INT64: DType
|
||||
DTYPE_UINT8: DType
|
||||
DTYPE_INT8: DType
|
||||
DTYPE_BOOL: DType
|
||||
BYTE_ORDER_UNSPECIFIED: ByteOrder
|
||||
BYTE_ORDER_LITTLE_ENDIAN: ByteOrder
|
||||
BYTE_ORDER_BIG_ENDIAN: ByteOrder
|
||||
COMPRESSION_UNSPECIFIED: Compression
|
||||
COMPRESSION_NONE: Compression
|
||||
COMPRESSION_ZSTD: Compression
|
||||
CHECKSUM_ALGORITHM_UNSPECIFIED: ChecksumAlgorithm
|
||||
CHECKSUM_ALGORITHM_NONE: ChecksumAlgorithm
|
||||
CHECKSUM_ALGORITHM_CRC32C: ChecksumAlgorithm
|
||||
PHASE_UNSPECIFIED: Phase
|
||||
PHASE_PREFILL: Phase
|
||||
PHASE_DECODE: Phase
|
||||
PHASE_RELEASE: Phase
|
||||
PHASE_CANCEL: Phase
|
||||
CACHE_MODE_UNSPECIFIED: CacheMode
|
||||
CACHE_MODE_STATELESS: CacheMode
|
||||
CACHE_MODE_PREFILL: CacheMode
|
||||
CACHE_MODE_DECODE: CacheMode
|
||||
ERROR_CODE_UNSPECIFIED: ErrorCode
|
||||
ERROR_CODE_SCHEMA_UNSUPPORTED: ErrorCode
|
||||
ERROR_CODE_FINGERPRINT_MISMATCH: ErrorCode
|
||||
ERROR_CODE_EPOCH_STALE: ErrorCode
|
||||
ERROR_CODE_SHARD_RANGE_MISMATCH: ErrorCode
|
||||
ERROR_CODE_CACHE_MISS: ErrorCode
|
||||
ERROR_CODE_RESOURCE_EXHAUSTED: ErrorCode
|
||||
ERROR_CODE_PAYLOAD_CORRUPT: ErrorCode
|
||||
ERROR_CODE_CANCELLED: ErrorCode
|
||||
ERROR_CODE_DEADLINE_EXCEEDED: ErrorCode
|
||||
ERROR_CODE_FLOW_CONTROL_VIOLATION: ErrorCode
|
||||
ERROR_CODE_INTERNAL: ErrorCode
|
||||
SERVING_STATE_UNSPECIFIED: ServingState
|
||||
SERVING_STATE_SERVING: ServingState
|
||||
SERVING_STATE_DRAINING: ServingState
|
||||
SERVING_STATE_NOT_SERVING: ServingState
|
||||
|
||||
class Checksum(_message.Message):
|
||||
__slots__ = ("algorithm", "value")
|
||||
ALGORITHM_FIELD_NUMBER: _ClassVar[int]
|
||||
VALUE_FIELD_NUMBER: _ClassVar[int]
|
||||
algorithm: ChecksumAlgorithm
|
||||
value: bytes
|
||||
def __init__(self, algorithm: _Optional[_Union[ChecksumAlgorithm, str]] = ..., value: _Optional[bytes] = ...) -> None: ...
|
||||
|
||||
class TensorFragment(_message.Message):
|
||||
__slots__ = ("fragment_index", "fragment_count", "byte_offset", "payload")
|
||||
FRAGMENT_INDEX_FIELD_NUMBER: _ClassVar[int]
|
||||
FRAGMENT_COUNT_FIELD_NUMBER: _ClassVar[int]
|
||||
BYTE_OFFSET_FIELD_NUMBER: _ClassVar[int]
|
||||
PAYLOAD_FIELD_NUMBER: _ClassVar[int]
|
||||
fragment_index: int
|
||||
fragment_count: int
|
||||
byte_offset: int
|
||||
payload: bytes
|
||||
def __init__(self, fragment_index: _Optional[int] = ..., fragment_count: _Optional[int] = ..., byte_offset: _Optional[int] = ..., payload: _Optional[bytes] = ...) -> None: ...
|
||||
|
||||
class NamedTensor(_message.Message):
|
||||
__slots__ = ("name", "shape", "dtype", "byte_order", "total_bytes", "compression", "checksum", "fragments")
|
||||
NAME_FIELD_NUMBER: _ClassVar[int]
|
||||
SHAPE_FIELD_NUMBER: _ClassVar[int]
|
||||
DTYPE_FIELD_NUMBER: _ClassVar[int]
|
||||
BYTE_ORDER_FIELD_NUMBER: _ClassVar[int]
|
||||
TOTAL_BYTES_FIELD_NUMBER: _ClassVar[int]
|
||||
COMPRESSION_FIELD_NUMBER: _ClassVar[int]
|
||||
CHECKSUM_FIELD_NUMBER: _ClassVar[int]
|
||||
FRAGMENTS_FIELD_NUMBER: _ClassVar[int]
|
||||
name: str
|
||||
shape: _containers.RepeatedScalarFieldContainer[int]
|
||||
dtype: DType
|
||||
byte_order: ByteOrder
|
||||
total_bytes: int
|
||||
compression: Compression
|
||||
checksum: Checksum
|
||||
fragments: _containers.RepeatedCompositeFieldContainer[TensorFragment]
|
||||
def __init__(self, name: _Optional[str] = ..., shape: _Optional[_Iterable[int]] = ..., dtype: _Optional[_Union[DType, str]] = ..., byte_order: _Optional[_Union[ByteOrder, str]] = ..., total_bytes: _Optional[int] = ..., compression: _Optional[_Union[Compression, str]] = ..., checksum: _Optional[_Union[Checksum, _Mapping]] = ..., fragments: _Optional[_Iterable[_Union[TensorFragment, _Mapping]]] = ...) -> None: ...
|
||||
|
||||
class TensorBundle(_message.Message):
|
||||
__slots__ = ("bundle_version", "tensors", "architecture", "boundary_point")
|
||||
BUNDLE_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
TENSORS_FIELD_NUMBER: _ClassVar[int]
|
||||
ARCHITECTURE_FIELD_NUMBER: _ClassVar[int]
|
||||
BOUNDARY_POINT_FIELD_NUMBER: _ClassVar[int]
|
||||
bundle_version: int
|
||||
tensors: _containers.RepeatedCompositeFieldContainer[NamedTensor]
|
||||
architecture: ArchitectureType
|
||||
boundary_point: str
|
||||
def __init__(self, bundle_version: _Optional[int] = ..., tensors: _Optional[_Iterable[_Union[NamedTensor, _Mapping]]] = ..., architecture: _Optional[_Union[ArchitectureType, str]] = ..., boundary_point: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class Fingerprint(_message.Message):
|
||||
__slots__ = ("model_artifact_digest", "runtime_recipe_digest", "recipe_id", "recipe_version", "catalogue_version")
|
||||
MODEL_ARTIFACT_DIGEST_FIELD_NUMBER: _ClassVar[int]
|
||||
RUNTIME_RECIPE_DIGEST_FIELD_NUMBER: _ClassVar[int]
|
||||
RECIPE_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
RECIPE_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
CATALOGUE_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
model_artifact_digest: str
|
||||
runtime_recipe_digest: str
|
||||
recipe_id: str
|
||||
recipe_version: str
|
||||
catalogue_version: str
|
||||
def __init__(self, model_artifact_digest: _Optional[str] = ..., runtime_recipe_digest: _Optional[str] = ..., recipe_id: _Optional[str] = ..., recipe_version: _Optional[str] = ..., catalogue_version: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ShardRange(_message.Message):
|
||||
__slots__ = ("start_layer", "end_layer", "effective_start_layer")
|
||||
START_LAYER_FIELD_NUMBER: _ClassVar[int]
|
||||
END_LAYER_FIELD_NUMBER: _ClassVar[int]
|
||||
EFFECTIVE_START_LAYER_FIELD_NUMBER: _ClassVar[int]
|
||||
start_layer: int
|
||||
end_layer: int
|
||||
effective_start_layer: int
|
||||
def __init__(self, start_layer: _Optional[int] = ..., end_layer: _Optional[int] = ..., effective_start_layer: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class PositionSpan(_message.Message):
|
||||
__slots__ = ("first_position", "token_count")
|
||||
FIRST_POSITION_FIELD_NUMBER: _ClassVar[int]
|
||||
TOKEN_COUNT_FIELD_NUMBER: _ClassVar[int]
|
||||
first_position: int
|
||||
token_count: int
|
||||
def __init__(self, first_position: _Optional[int] = ..., token_count: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class ChunkInfo(_message.Message):
|
||||
__slots__ = ("chunk_index", "chunk_count", "final_chunk")
|
||||
CHUNK_INDEX_FIELD_NUMBER: _ClassVar[int]
|
||||
CHUNK_COUNT_FIELD_NUMBER: _ClassVar[int]
|
||||
FINAL_CHUNK_FIELD_NUMBER: _ClassVar[int]
|
||||
chunk_index: int
|
||||
chunk_count: int
|
||||
final_chunk: bool
|
||||
def __init__(self, chunk_index: _Optional[int] = ..., chunk_count: _Optional[int] = ..., final_chunk: _Optional[bool] = ...) -> None: ...
|
||||
|
||||
class CacheExpectation(_message.Message):
|
||||
__slots__ = ("mode", "expected_past_len")
|
||||
MODE_FIELD_NUMBER: _ClassVar[int]
|
||||
EXPECTED_PAST_LEN_FIELD_NUMBER: _ClassVar[int]
|
||||
mode: CacheMode
|
||||
expected_past_len: int
|
||||
def __init__(self, mode: _Optional[_Union[CacheMode, str]] = ..., expected_past_len: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class CacheResult(_message.Message):
|
||||
__slots__ = ("mode", "past_len", "cache_hit")
|
||||
MODE_FIELD_NUMBER: _ClassVar[int]
|
||||
PAST_LEN_FIELD_NUMBER: _ClassVar[int]
|
||||
CACHE_HIT_FIELD_NUMBER: _ClassVar[int]
|
||||
mode: CacheMode
|
||||
past_len: int
|
||||
cache_hit: bool
|
||||
def __init__(self, mode: _Optional[_Union[CacheMode, str]] = ..., past_len: _Optional[int] = ..., cache_hit: _Optional[bool] = ...) -> None: ...
|
||||
|
||||
class Envelope(_message.Message):
|
||||
__slots__ = ("schema_version", "work_id", "route_session_id", "route_epoch", "fingerprint", "shard_range", "phase", "position", "idempotency_step", "cache_expectation", "deadline_unix_nanos", "chunk")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
FINGERPRINT_FIELD_NUMBER: _ClassVar[int]
|
||||
SHARD_RANGE_FIELD_NUMBER: _ClassVar[int]
|
||||
PHASE_FIELD_NUMBER: _ClassVar[int]
|
||||
POSITION_FIELD_NUMBER: _ClassVar[int]
|
||||
IDEMPOTENCY_STEP_FIELD_NUMBER: _ClassVar[int]
|
||||
CACHE_EXPECTATION_FIELD_NUMBER: _ClassVar[int]
|
||||
DEADLINE_UNIX_NANOS_FIELD_NUMBER: _ClassVar[int]
|
||||
CHUNK_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
work_id: str
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
fingerprint: Fingerprint
|
||||
shard_range: ShardRange
|
||||
phase: Phase
|
||||
position: PositionSpan
|
||||
idempotency_step: int
|
||||
cache_expectation: CacheExpectation
|
||||
deadline_unix_nanos: int
|
||||
chunk: ChunkInfo
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., work_id: _Optional[str] = ..., route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ..., fingerprint: _Optional[_Union[Fingerprint, _Mapping]] = ..., shard_range: _Optional[_Union[ShardRange, _Mapping]] = ..., phase: _Optional[_Union[Phase, str]] = ..., position: _Optional[_Union[PositionSpan, _Mapping]] = ..., idempotency_step: _Optional[int] = ..., cache_expectation: _Optional[_Union[CacheExpectation, _Mapping]] = ..., deadline_unix_nanos: _Optional[int] = ..., chunk: _Optional[_Union[ChunkInfo, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class ShardError(_message.Message):
|
||||
__slots__ = ("code", "detail", "retryable", "actual_past_len")
|
||||
CODE_FIELD_NUMBER: _ClassVar[int]
|
||||
DETAIL_FIELD_NUMBER: _ClassVar[int]
|
||||
RETRYABLE_FIELD_NUMBER: _ClassVar[int]
|
||||
ACTUAL_PAST_LEN_FIELD_NUMBER: _ClassVar[int]
|
||||
code: ErrorCode
|
||||
detail: str
|
||||
retryable: bool
|
||||
actual_past_len: int
|
||||
def __init__(self, code: _Optional[_Union[ErrorCode, str]] = ..., detail: _Optional[str] = ..., retryable: _Optional[bool] = ..., actual_past_len: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class FlowControl(_message.Message):
|
||||
__slots__ = ("credits_granted", "max_inflight_chunks", "max_chunk_bytes", "max_prefill_chunk_tokens")
|
||||
CREDITS_GRANTED_FIELD_NUMBER: _ClassVar[int]
|
||||
MAX_INFLIGHT_CHUNKS_FIELD_NUMBER: _ClassVar[int]
|
||||
MAX_CHUNK_BYTES_FIELD_NUMBER: _ClassVar[int]
|
||||
MAX_PREFILL_CHUNK_TOKENS_FIELD_NUMBER: _ClassVar[int]
|
||||
credits_granted: int
|
||||
max_inflight_chunks: int
|
||||
max_chunk_bytes: int
|
||||
max_prefill_chunk_tokens: int
|
||||
def __init__(self, credits_granted: _Optional[int] = ..., max_inflight_chunks: _Optional[int] = ..., max_chunk_bytes: _Optional[int] = ..., max_prefill_chunk_tokens: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class SessionOpen(_message.Message):
|
||||
__slots__ = ("schema_version", "route_session_id", "route_epoch", "fingerprint", "shard_range", "proposed_flow_control", "accepted_compression")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
FINGERPRINT_FIELD_NUMBER: _ClassVar[int]
|
||||
SHARD_RANGE_FIELD_NUMBER: _ClassVar[int]
|
||||
PROPOSED_FLOW_CONTROL_FIELD_NUMBER: _ClassVar[int]
|
||||
ACCEPTED_COMPRESSION_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
fingerprint: Fingerprint
|
||||
shard_range: ShardRange
|
||||
proposed_flow_control: FlowControl
|
||||
accepted_compression: _containers.RepeatedScalarFieldContainer[Compression]
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ..., fingerprint: _Optional[_Union[Fingerprint, _Mapping]] = ..., shard_range: _Optional[_Union[ShardRange, _Mapping]] = ..., proposed_flow_control: _Optional[_Union[FlowControl, _Mapping]] = ..., accepted_compression: _Optional[_Iterable[_Union[Compression, str]]] = ...) -> None: ...
|
||||
|
||||
class SessionAccepted(_message.Message):
|
||||
__slots__ = ("schema_version", "route_session_id", "route_epoch", "flow_control", "accepted_compression", "fingerprint")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
FLOW_CONTROL_FIELD_NUMBER: _ClassVar[int]
|
||||
ACCEPTED_COMPRESSION_FIELD_NUMBER: _ClassVar[int]
|
||||
FINGERPRINT_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
flow_control: FlowControl
|
||||
accepted_compression: _containers.RepeatedScalarFieldContainer[Compression]
|
||||
fingerprint: Fingerprint
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ..., flow_control: _Optional[_Union[FlowControl, _Mapping]] = ..., accepted_compression: _Optional[_Iterable[_Union[Compression, str]]] = ..., fingerprint: _Optional[_Union[Fingerprint, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class ActivationChunk(_message.Message):
|
||||
__slots__ = ("envelope", "bundle")
|
||||
ENVELOPE_FIELD_NUMBER: _ClassVar[int]
|
||||
BUNDLE_FIELD_NUMBER: _ClassVar[int]
|
||||
envelope: Envelope
|
||||
bundle: TensorBundle
|
||||
def __init__(self, envelope: _Optional[_Union[Envelope, _Mapping]] = ..., bundle: _Optional[_Union[TensorBundle, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class DecodeStep(_message.Message):
|
||||
__slots__ = ("idempotency_step", "position", "expected_past_len", "tensor", "work_id", "deadline_unix_nanos", "bundle")
|
||||
IDEMPOTENCY_STEP_FIELD_NUMBER: _ClassVar[int]
|
||||
POSITION_FIELD_NUMBER: _ClassVar[int]
|
||||
EXPECTED_PAST_LEN_FIELD_NUMBER: _ClassVar[int]
|
||||
TENSOR_FIELD_NUMBER: _ClassVar[int]
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
DEADLINE_UNIX_NANOS_FIELD_NUMBER: _ClassVar[int]
|
||||
BUNDLE_FIELD_NUMBER: _ClassVar[int]
|
||||
idempotency_step: int
|
||||
position: int
|
||||
expected_past_len: int
|
||||
tensor: NamedTensor
|
||||
work_id: str
|
||||
deadline_unix_nanos: int
|
||||
bundle: TensorBundle
|
||||
def __init__(self, idempotency_step: _Optional[int] = ..., position: _Optional[int] = ..., expected_past_len: _Optional[int] = ..., tensor: _Optional[_Union[NamedTensor, _Mapping]] = ..., work_id: _Optional[str] = ..., deadline_unix_nanos: _Optional[int] = ..., bundle: _Optional[_Union[TensorBundle, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class RequestRecipeIdentity(_message.Message):
|
||||
__slots__ = ("request_id", "runtime_recipe_digest", "chat_template_id", "chat_template_version", "reasoning_mode", "architecture")
|
||||
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
RUNTIME_RECIPE_DIGEST_FIELD_NUMBER: _ClassVar[int]
|
||||
CHAT_TEMPLATE_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
CHAT_TEMPLATE_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
REASONING_MODE_FIELD_NUMBER: _ClassVar[int]
|
||||
ARCHITECTURE_FIELD_NUMBER: _ClassVar[int]
|
||||
request_id: str
|
||||
runtime_recipe_digest: str
|
||||
chat_template_id: str
|
||||
chat_template_version: str
|
||||
reasoning_mode: str
|
||||
architecture: ArchitectureType
|
||||
def __init__(self, request_id: _Optional[str] = ..., runtime_recipe_digest: _Optional[str] = ..., chat_template_id: _Optional[str] = ..., chat_template_version: _Optional[str] = ..., reasoning_mode: _Optional[str] = ..., architecture: _Optional[_Union[ArchitectureType, str]] = ...) -> None: ...
|
||||
|
||||
class SamplingParameters(_message.Message):
|
||||
__slots__ = ("temperature", "top_p", "top_k", "seed", "greedy")
|
||||
TEMPERATURE_FIELD_NUMBER: _ClassVar[int]
|
||||
TOP_P_FIELD_NUMBER: _ClassVar[int]
|
||||
TOP_K_FIELD_NUMBER: _ClassVar[int]
|
||||
SEED_FIELD_NUMBER: _ClassVar[int]
|
||||
GREEDY_FIELD_NUMBER: _ClassVar[int]
|
||||
temperature: float
|
||||
top_p: float
|
||||
top_k: int
|
||||
seed: int
|
||||
greedy: bool
|
||||
def __init__(self, temperature: _Optional[float] = ..., top_p: _Optional[float] = ..., top_k: _Optional[int] = ..., seed: _Optional[int] = ..., greedy: _Optional[bool] = ...) -> None: ...
|
||||
|
||||
class TailResult(_message.Message):
|
||||
__slots__ = ("identity", "sampling", "logits", "sampled_token_id")
|
||||
IDENTITY_FIELD_NUMBER: _ClassVar[int]
|
||||
SAMPLING_FIELD_NUMBER: _ClassVar[int]
|
||||
LOGITS_FIELD_NUMBER: _ClassVar[int]
|
||||
SAMPLED_TOKEN_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
identity: RequestRecipeIdentity
|
||||
sampling: SamplingParameters
|
||||
logits: TensorBundle
|
||||
sampled_token_id: int
|
||||
def __init__(self, identity: _Optional[_Union[RequestRecipeIdentity, _Mapping]] = ..., sampling: _Optional[_Union[SamplingParameters, _Mapping]] = ..., logits: _Optional[_Union[TensorBundle, _Mapping]] = ..., sampled_token_id: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class ReleaseSignal(_message.Message):
|
||||
__slots__ = ("route_session_id", "route_epoch", "work_id")
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
work_id: str
|
||||
def __init__(self, route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ..., work_id: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class CancelSignal(_message.Message):
|
||||
__slots__ = ("route_session_id", "route_epoch", "work_id", "reason")
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
REASON_FIELD_NUMBER: _ClassVar[int]
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
work_id: str
|
||||
reason: str
|
||||
def __init__(self, route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ..., work_id: _Optional[str] = ..., reason: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class Ack(_message.Message):
|
||||
__slots__ = ("work_id", "idempotency_step", "cache_result", "duplicate", "execution_nanos")
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
IDEMPOTENCY_STEP_FIELD_NUMBER: _ClassVar[int]
|
||||
CACHE_RESULT_FIELD_NUMBER: _ClassVar[int]
|
||||
DUPLICATE_FIELD_NUMBER: _ClassVar[int]
|
||||
EXECUTION_NANOS_FIELD_NUMBER: _ClassVar[int]
|
||||
work_id: str
|
||||
idempotency_step: int
|
||||
cache_result: CacheResult
|
||||
duplicate: bool
|
||||
execution_nanos: int
|
||||
def __init__(self, work_id: _Optional[str] = ..., idempotency_step: _Optional[int] = ..., cache_result: _Optional[_Union[CacheResult, _Mapping]] = ..., duplicate: _Optional[bool] = ..., execution_nanos: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class ShardStatus(_message.Message):
|
||||
__slots__ = ("work_id", "route_session_id", "idempotency_step", "error", "terminal")
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
IDEMPOTENCY_STEP_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
TERMINAL_FIELD_NUMBER: _ClassVar[int]
|
||||
work_id: str
|
||||
route_session_id: str
|
||||
idempotency_step: int
|
||||
error: ShardError
|
||||
terminal: bool
|
||||
def __init__(self, work_id: _Optional[str] = ..., route_session_id: _Optional[str] = ..., idempotency_step: _Optional[int] = ..., error: _Optional[_Union[ShardError, _Mapping]] = ..., terminal: _Optional[bool] = ...) -> None: ...
|
||||
|
||||
class SessionRequest(_message.Message):
|
||||
__slots__ = ("open", "chunk", "decode", "flow_control", "release", "cancel")
|
||||
OPEN_FIELD_NUMBER: _ClassVar[int]
|
||||
CHUNK_FIELD_NUMBER: _ClassVar[int]
|
||||
DECODE_FIELD_NUMBER: _ClassVar[int]
|
||||
FLOW_CONTROL_FIELD_NUMBER: _ClassVar[int]
|
||||
RELEASE_FIELD_NUMBER: _ClassVar[int]
|
||||
CANCEL_FIELD_NUMBER: _ClassVar[int]
|
||||
open: SessionOpen
|
||||
chunk: ActivationChunk
|
||||
decode: DecodeStep
|
||||
flow_control: FlowControl
|
||||
release: ReleaseSignal
|
||||
cancel: CancelSignal
|
||||
def __init__(self, open: _Optional[_Union[SessionOpen, _Mapping]] = ..., chunk: _Optional[_Union[ActivationChunk, _Mapping]] = ..., decode: _Optional[_Union[DecodeStep, _Mapping]] = ..., flow_control: _Optional[_Union[FlowControl, _Mapping]] = ..., release: _Optional[_Union[ReleaseSignal, _Mapping]] = ..., cancel: _Optional[_Union[CancelSignal, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class SessionResponse(_message.Message):
|
||||
__slots__ = ("accepted", "chunk", "ack", "flow_control", "status", "tail_result")
|
||||
ACCEPTED_FIELD_NUMBER: _ClassVar[int]
|
||||
CHUNK_FIELD_NUMBER: _ClassVar[int]
|
||||
ACK_FIELD_NUMBER: _ClassVar[int]
|
||||
FLOW_CONTROL_FIELD_NUMBER: _ClassVar[int]
|
||||
STATUS_FIELD_NUMBER: _ClassVar[int]
|
||||
TAIL_RESULT_FIELD_NUMBER: _ClassVar[int]
|
||||
accepted: SessionAccepted
|
||||
chunk: ActivationChunk
|
||||
ack: Ack
|
||||
flow_control: FlowControl
|
||||
status: ShardStatus
|
||||
tail_result: TailResult
|
||||
def __init__(self, accepted: _Optional[_Union[SessionAccepted, _Mapping]] = ..., chunk: _Optional[_Union[ActivationChunk, _Mapping]] = ..., ack: _Optional[_Union[Ack, _Mapping]] = ..., flow_control: _Optional[_Union[FlowControl, _Mapping]] = ..., status: _Optional[_Union[ShardStatus, _Mapping]] = ..., tail_result: _Optional[_Union[TailResult, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class CapabilityRequest(_message.Message):
|
||||
__slots__ = ("schema_version",)
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ...) -> None: ...
|
||||
|
||||
class CapabilityReport(_message.Message):
|
||||
__slots__ = ("schema_version", "fingerprint", "shard_range", "backend", "device", "validated", "detail", "max_concurrent_sessions", "max_context_tokens", "flow_control", "accepted_compression", "supported_schema_versions", "validated_at_unix_nanos")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
FINGERPRINT_FIELD_NUMBER: _ClassVar[int]
|
||||
SHARD_RANGE_FIELD_NUMBER: _ClassVar[int]
|
||||
BACKEND_FIELD_NUMBER: _ClassVar[int]
|
||||
DEVICE_FIELD_NUMBER: _ClassVar[int]
|
||||
VALIDATED_FIELD_NUMBER: _ClassVar[int]
|
||||
DETAIL_FIELD_NUMBER: _ClassVar[int]
|
||||
MAX_CONCURRENT_SESSIONS_FIELD_NUMBER: _ClassVar[int]
|
||||
MAX_CONTEXT_TOKENS_FIELD_NUMBER: _ClassVar[int]
|
||||
FLOW_CONTROL_FIELD_NUMBER: _ClassVar[int]
|
||||
ACCEPTED_COMPRESSION_FIELD_NUMBER: _ClassVar[int]
|
||||
SUPPORTED_SCHEMA_VERSIONS_FIELD_NUMBER: _ClassVar[int]
|
||||
VALIDATED_AT_UNIX_NANOS_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
fingerprint: Fingerprint
|
||||
shard_range: ShardRange
|
||||
backend: str
|
||||
device: str
|
||||
validated: bool
|
||||
detail: str
|
||||
max_concurrent_sessions: int
|
||||
max_context_tokens: int
|
||||
flow_control: FlowControl
|
||||
accepted_compression: _containers.RepeatedScalarFieldContainer[Compression]
|
||||
supported_schema_versions: _containers.RepeatedScalarFieldContainer[SchemaVersion]
|
||||
validated_at_unix_nanos: int
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., fingerprint: _Optional[_Union[Fingerprint, _Mapping]] = ..., shard_range: _Optional[_Union[ShardRange, _Mapping]] = ..., backend: _Optional[str] = ..., device: _Optional[str] = ..., validated: _Optional[bool] = ..., detail: _Optional[str] = ..., max_concurrent_sessions: _Optional[int] = ..., max_context_tokens: _Optional[int] = ..., flow_control: _Optional[_Union[FlowControl, _Mapping]] = ..., accepted_compression: _Optional[_Iterable[_Union[Compression, str]]] = ..., supported_schema_versions: _Optional[_Iterable[_Union[SchemaVersion, str]]] = ..., validated_at_unix_nanos: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class HealthRequest(_message.Message):
|
||||
__slots__ = ("schema_version",)
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ...) -> None: ...
|
||||
|
||||
class HealthReport(_message.Message):
|
||||
__slots__ = ("schema_version", "state", "active_sessions", "queued_chunks", "batch_occupancy", "kv_pressure", "resident_bytes", "detail")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
STATE_FIELD_NUMBER: _ClassVar[int]
|
||||
ACTIVE_SESSIONS_FIELD_NUMBER: _ClassVar[int]
|
||||
QUEUED_CHUNKS_FIELD_NUMBER: _ClassVar[int]
|
||||
BATCH_OCCUPANCY_FIELD_NUMBER: _ClassVar[int]
|
||||
KV_PRESSURE_FIELD_NUMBER: _ClassVar[int]
|
||||
RESIDENT_BYTES_FIELD_NUMBER: _ClassVar[int]
|
||||
DETAIL_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
state: ServingState
|
||||
active_sessions: int
|
||||
queued_chunks: int
|
||||
batch_occupancy: int
|
||||
kv_pressure: float
|
||||
resident_bytes: int
|
||||
detail: str
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., state: _Optional[_Union[ServingState, str]] = ..., active_sessions: _Optional[int] = ..., queued_chunks: _Optional[int] = ..., batch_occupancy: _Optional[int] = ..., kv_pressure: _Optional[float] = ..., resident_bytes: _Optional[int] = ..., detail: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class ReleaseRequest(_message.Message):
|
||||
__slots__ = ("schema_version", "route_session_id", "route_epoch")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ...) -> None: ...
|
||||
|
||||
class ReleaseResponse(_message.Message):
|
||||
__slots__ = ("released", "error")
|
||||
RELEASED_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
released: bool
|
||||
error: ShardError
|
||||
def __init__(self, released: _Optional[bool] = ..., error: _Optional[_Union[ShardError, _Mapping]] = ...) -> None: ...
|
||||
|
||||
class CancelRequest(_message.Message):
|
||||
__slots__ = ("schema_version", "route_session_id", "route_epoch", "work_id", "reason")
|
||||
SCHEMA_VERSION_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_SESSION_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
ROUTE_EPOCH_FIELD_NUMBER: _ClassVar[int]
|
||||
WORK_ID_FIELD_NUMBER: _ClassVar[int]
|
||||
REASON_FIELD_NUMBER: _ClassVar[int]
|
||||
schema_version: SchemaVersion
|
||||
route_session_id: str
|
||||
route_epoch: int
|
||||
work_id: str
|
||||
reason: str
|
||||
def __init__(self, schema_version: _Optional[_Union[SchemaVersion, str]] = ..., route_session_id: _Optional[str] = ..., route_epoch: _Optional[int] = ..., work_id: _Optional[str] = ..., reason: _Optional[str] = ...) -> None: ...
|
||||
|
||||
class CancelResponse(_message.Message):
|
||||
__slots__ = ("cancelled_work_items", "error")
|
||||
CANCELLED_WORK_ITEMS_FIELD_NUMBER: _ClassVar[int]
|
||||
ERROR_FIELD_NUMBER: _ClassVar[int]
|
||||
cancelled_work_items: int
|
||||
error: ShardError
|
||||
def __init__(self, cancelled_work_items: _Optional[int] = ..., error: _Optional[_Union[ShardError, _Mapping]] = ...) -> None: ...
|
||||
@@ -0,0 +1,295 @@
|
||||
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
||||
"""Client and server classes corresponding to protobuf-defined services."""
|
||||
import grpc
|
||||
import warnings
|
||||
|
||||
from . import shard_runtime_pb2 as shard__runtime__pb2
|
||||
|
||||
GRPC_GENERATED_VERSION = '1.82.1'
|
||||
GRPC_VERSION = grpc.__version__
|
||||
_version_not_supported = False
|
||||
|
||||
try:
|
||||
from grpc._utilities import first_version_is_lower
|
||||
_version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION)
|
||||
except ImportError:
|
||||
_version_not_supported = True
|
||||
|
||||
if _version_not_supported:
|
||||
raise RuntimeError(
|
||||
f'The grpc package installed is at version {GRPC_VERSION},'
|
||||
+ ' but the generated code in shard_runtime_pb2_grpc.py depends on'
|
||||
+ f' grpcio>={GRPC_GENERATED_VERSION}.'
|
||||
+ f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}'
|
||||
+ f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.'
|
||||
)
|
||||
|
||||
|
||||
class ShardRuntimeStub:
|
||||
"""---------------------------------------------------------------------------
|
||||
Service
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, channel):
|
||||
"""Constructor.
|
||||
|
||||
Args:
|
||||
channel: A grpc.Channel.
|
||||
"""
|
||||
self.GetCapability = channel.unary_unary(
|
||||
'/meshnet.shard.v1.ShardRuntime/GetCapability',
|
||||
request_serializer=shard__runtime__pb2.CapabilityRequest.SerializeToString,
|
||||
response_deserializer=shard__runtime__pb2.CapabilityReport.FromString,
|
||||
_registered_method=True)
|
||||
self.Health = channel.unary_unary(
|
||||
'/meshnet.shard.v1.ShardRuntime/Health',
|
||||
request_serializer=shard__runtime__pb2.HealthRequest.SerializeToString,
|
||||
response_deserializer=shard__runtime__pb2.HealthReport.FromString,
|
||||
_registered_method=True)
|
||||
self.Session = channel.stream_stream(
|
||||
'/meshnet.shard.v1.ShardRuntime/Session',
|
||||
request_serializer=shard__runtime__pb2.SessionRequest.SerializeToString,
|
||||
response_deserializer=shard__runtime__pb2.SessionResponse.FromString,
|
||||
_registered_method=True)
|
||||
self.Release = channel.unary_unary(
|
||||
'/meshnet.shard.v1.ShardRuntime/Release',
|
||||
request_serializer=shard__runtime__pb2.ReleaseRequest.SerializeToString,
|
||||
response_deserializer=shard__runtime__pb2.ReleaseResponse.FromString,
|
||||
_registered_method=True)
|
||||
self.Cancel = channel.unary_unary(
|
||||
'/meshnet.shard.v1.ShardRuntime/Cancel',
|
||||
request_serializer=shard__runtime__pb2.CancelRequest.SerializeToString,
|
||||
response_deserializer=shard__runtime__pb2.CancelResponse.FromString,
|
||||
_registered_method=True)
|
||||
|
||||
|
||||
class ShardRuntimeServicer:
|
||||
"""---------------------------------------------------------------------------
|
||||
Service
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
|
||||
def GetCapability(self, request, context):
|
||||
"""What this worker can execute. Read before a route is built.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Health(self, request, context):
|
||||
"""Live load and serving state.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Session(self, request_iterator, context):
|
||||
"""One long-lived bidirectional stream per Route Session Activation Seam.
|
||||
|
||||
The stream opens with SessionOpen/SessionAccepted, then carries bounded
|
||||
prefill chunks and decode steps in both directions for the life of the
|
||||
session. Per-token channel creation is a non-goal: the handshake cost is
|
||||
paid once and the hot path carries only what changes.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Release(self, request, context):
|
||||
"""Drop session state out of band. Idempotent.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
def Cancel(self, request, context):
|
||||
"""Cancel out of band, on a fresh call.
|
||||
|
||||
In-band CancelSignal is preferred, but a sender that is blocked on flow
|
||||
control cannot write one — a cancel that can only travel down a wedged
|
||||
stream is not a cancel. This RPC always has a path to the worker.
|
||||
"""
|
||||
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
||||
context.set_details('Method not implemented!')
|
||||
raise NotImplementedError('Method not implemented!')
|
||||
|
||||
|
||||
def add_ShardRuntimeServicer_to_server(servicer, server):
|
||||
rpc_method_handlers = {
|
||||
'GetCapability': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.GetCapability,
|
||||
request_deserializer=shard__runtime__pb2.CapabilityRequest.FromString,
|
||||
response_serializer=shard__runtime__pb2.CapabilityReport.SerializeToString,
|
||||
),
|
||||
'Health': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Health,
|
||||
request_deserializer=shard__runtime__pb2.HealthRequest.FromString,
|
||||
response_serializer=shard__runtime__pb2.HealthReport.SerializeToString,
|
||||
),
|
||||
'Session': grpc.stream_stream_rpc_method_handler(
|
||||
servicer.Session,
|
||||
request_deserializer=shard__runtime__pb2.SessionRequest.FromString,
|
||||
response_serializer=shard__runtime__pb2.SessionResponse.SerializeToString,
|
||||
),
|
||||
'Release': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Release,
|
||||
request_deserializer=shard__runtime__pb2.ReleaseRequest.FromString,
|
||||
response_serializer=shard__runtime__pb2.ReleaseResponse.SerializeToString,
|
||||
),
|
||||
'Cancel': grpc.unary_unary_rpc_method_handler(
|
||||
servicer.Cancel,
|
||||
request_deserializer=shard__runtime__pb2.CancelRequest.FromString,
|
||||
response_serializer=shard__runtime__pb2.CancelResponse.SerializeToString,
|
||||
),
|
||||
}
|
||||
generic_handler = grpc.method_handlers_generic_handler(
|
||||
'meshnet.shard.v1.ShardRuntime', rpc_method_handlers)
|
||||
server.add_generic_rpc_handlers((generic_handler,))
|
||||
server.add_registered_method_handlers('meshnet.shard.v1.ShardRuntime', rpc_method_handlers)
|
||||
|
||||
|
||||
# This class is part of an EXPERIMENTAL API.
|
||||
class ShardRuntime:
|
||||
"""---------------------------------------------------------------------------
|
||||
Service
|
||||
---------------------------------------------------------------------------
|
||||
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def GetCapability(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/meshnet.shard.v1.ShardRuntime/GetCapability',
|
||||
shard__runtime__pb2.CapabilityRequest.SerializeToString,
|
||||
shard__runtime__pb2.CapabilityReport.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def Health(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/meshnet.shard.v1.ShardRuntime/Health',
|
||||
shard__runtime__pb2.HealthRequest.SerializeToString,
|
||||
shard__runtime__pb2.HealthReport.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def Session(request_iterator,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.stream_stream(
|
||||
request_iterator,
|
||||
target,
|
||||
'/meshnet.shard.v1.ShardRuntime/Session',
|
||||
shard__runtime__pb2.SessionRequest.SerializeToString,
|
||||
shard__runtime__pb2.SessionResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def Release(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/meshnet.shard.v1.ShardRuntime/Release',
|
||||
shard__runtime__pb2.ReleaseRequest.SerializeToString,
|
||||
shard__runtime__pb2.ReleaseResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
|
||||
@staticmethod
|
||||
def Cancel(request,
|
||||
target,
|
||||
options=(),
|
||||
channel_credentials=None,
|
||||
call_credentials=None,
|
||||
insecure=False,
|
||||
compression=None,
|
||||
wait_for_ready=None,
|
||||
timeout=None,
|
||||
metadata=None):
|
||||
return grpc.experimental.unary_unary(
|
||||
request,
|
||||
target,
|
||||
'/meshnet.shard.v1.ShardRuntime/Cancel',
|
||||
shard__runtime__pb2.CancelRequest.SerializeToString,
|
||||
shard__runtime__pb2.CancelResponse.FromString,
|
||||
options,
|
||||
channel_credentials,
|
||||
insecure,
|
||||
call_credentials,
|
||||
compression,
|
||||
wait_for_ready,
|
||||
timeout,
|
||||
metadata,
|
||||
_registered_method=True)
|
||||
902
packages/node/meshnet_node/performance_contract.py
Normal file
902
packages/node/meshnet_node/performance_contract.py
Normal file
@@ -0,0 +1,902 @@
|
||||
"""The versioned safetensors-versus-GGUF performance contract.
|
||||
|
||||
The contract is the decision rule the native GGUF track is judged by, written
|
||||
down *before* the numbers arrive and consumed later by the release gate
|
||||
(DGR-014). Its thresholds are ratios against the Transformers/safetensors
|
||||
reference recipe rather than absolute tokens/sec, because the absolute figure is
|
||||
a property of whichever machine ran the benchmark and would have to be re-argued
|
||||
on every host; a ratio is a claim about the runtime.
|
||||
|
||||
Three rules give the contract its teeth:
|
||||
|
||||
* **Thresholds are locked.** ``CONTRACT_SCHEMA_VERSION`` and ``locked_at``
|
||||
travel with the document. Moving a threshold after seeing results is a new
|
||||
contract version and a human decision, not a tweak.
|
||||
* **Only like-for-like comparisons count.** A recipe measured on a different
|
||||
device than the reference is marked non-comparable and is granted no benefit,
|
||||
so a GPU-versus-CPU mismatch can never be laundered into a speed win.
|
||||
* **Quantized recipes never claim numerical equivalence.** Quality is gated on
|
||||
the near-lossless quality lane; the performance-fit lane is judged on speed,
|
||||
memory and fit alone.
|
||||
|
||||
The verdict is one of ``promote``, ``optimize`` or ``stop`` — the three outcomes
|
||||
the release gate is allowed to reach.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
|
||||
|
||||
from .recipe_benchmark import Lane, REPORT_SCHEMA_VERSION
|
||||
|
||||
# Layout of the contract document understood by this reader.
|
||||
CONTRACT_SCHEMA_VERSION = 1
|
||||
PROVENANCE_SCHEMA_VERSION = 1
|
||||
REAL_REPORT_PRODUCER = "meshnet_node.recipe_drivers.run_configured_benchmark/v1"
|
||||
|
||||
VERDICT_PROMOTE = "promote"
|
||||
VERDICT_OPTIMIZE = "optimize"
|
||||
VERDICT_STOP = "stop"
|
||||
|
||||
|
||||
class PerformanceContractError(ValueError):
|
||||
"""Raised when a contract is missing, malformed, or of an unsupported version."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContractThresholds:
|
||||
"""The locked decision thresholds.
|
||||
|
||||
Every value is a ratio of a GGUF recipe's metric to the reference recipe's
|
||||
metric on the same machine, same device, same plan.
|
||||
|
||||
A *meaningful speed benefit* means the GGUF recipe decodes at least 25%
|
||||
faster for a single request without making time-to-first-token materially
|
||||
worse, or sustains at least 25% more aggregate throughput under concurrency.
|
||||
Either route is a real win for the product: one helps a single user, the
|
||||
other helps a loaded node.
|
||||
|
||||
A *meaningful fit benefit* means peak resident memory (RSS plus VRAM) drops
|
||||
by at least 25%. Fit is the product thesis — models larger than one
|
||||
consumer node — so it is measured in resident bytes, not in how small the
|
||||
file on disk is. Artifact size has its own reported threshold because a
|
||||
smaller download is a real but secondary good.
|
||||
|
||||
25% is chosen to sit well clear of ordinary run-to-run variance on a busy
|
||||
developer machine while still being a benefit a user would notice. A 5%
|
||||
edge would not justify owning a native runtime and a patch stack.
|
||||
"""
|
||||
|
||||
min_decode_speedup: float = 1.25
|
||||
max_ttft_ratio: float = 1.25
|
||||
min_aggregate_throughput_speedup: float = 1.25
|
||||
max_resident_memory_ratio: float = 0.75
|
||||
max_artifact_size_ratio: float = 0.60
|
||||
min_quality_exact_match_rate: float = 0.90
|
||||
min_quality_mean_similarity: float = 0.97
|
||||
max_failure_rate: float = 0.0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PerformanceContract:
|
||||
"""A locked, versioned contract plus the baseline it was locked against."""
|
||||
|
||||
contract_version: int
|
||||
locked_at: str
|
||||
locked_by: str
|
||||
plan_id: str
|
||||
thresholds: ContractThresholds
|
||||
baseline: Mapping[str, Any]
|
||||
stop_condition: str
|
||||
notes: str = ""
|
||||
schema_version: int = CONTRACT_SCHEMA_VERSION
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"schema_version": self.schema_version,
|
||||
"contract_version": self.contract_version,
|
||||
"locked_at": self.locked_at,
|
||||
"locked_by": self.locked_by,
|
||||
"plan_id": self.plan_id,
|
||||
"thresholds": self.thresholds.to_dict(),
|
||||
"baseline": dict(self.baseline),
|
||||
"stop_condition": self.stop_condition,
|
||||
"notes": self.notes,
|
||||
}
|
||||
|
||||
|
||||
STOP_CONDITION = (
|
||||
"Stop the native llama.cpp/GGUF track when, on the same machine and device "
|
||||
"as the Transformers/safetensors reference and under this plan, no "
|
||||
"performance-fit GGUF recipe delivers either a meaningful speed benefit "
|
||||
"(>=25% higher single-request decode tokens/sec without a >25% worse TTFT, "
|
||||
"or >=25% higher aggregate throughput under concurrency) or a meaningful fit "
|
||||
"benefit (>=25% lower peak resident memory), or when the near-lossless "
|
||||
"quality lane fails, which indicates a broken runtime rather than a "
|
||||
"quantization trade-off."
|
||||
)
|
||||
|
||||
|
||||
def _recipe_entries(report: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
|
||||
return {entry["recipe"]["id"]: entry for entry in report["recipes"]}
|
||||
|
||||
|
||||
def _cell(entry: Mapping[str, Any], concurrency: int) -> Mapping[str, Any] | None:
|
||||
return entry["concurrency"].get(str(concurrency))
|
||||
|
||||
|
||||
def _resident_bytes(cell: Mapping[str, Any]) -> int:
|
||||
return int(cell["peak_rss_bytes"]) + int(cell["peak_vram_bytes"])
|
||||
|
||||
|
||||
def _ratio(value: float, reference: float) -> float:
|
||||
"""Ratio guarded against a zero reference, which means "not measured"."""
|
||||
if reference <= 0:
|
||||
return 0.0
|
||||
return round(value / reference, 4)
|
||||
|
||||
|
||||
def _canonical_sha256(value: Any) -> str:
|
||||
payload = json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def report_signing_payload(report: Mapping[str, Any]) -> bytes:
|
||||
"""Canonical report bytes covered by the Ed25519 signature."""
|
||||
provenance = report.get("provenance")
|
||||
if not isinstance(provenance, Mapping):
|
||||
raise PerformanceContractError("real benchmark report lacks signed provenance")
|
||||
unsigned = dict(report)
|
||||
unsigned_provenance = dict(provenance)
|
||||
unsigned_provenance.pop("signature", None)
|
||||
unsigned["provenance"] = unsigned_provenance
|
||||
return json.dumps(
|
||||
unsigned, sort_keys=True, separators=(",", ":"), ensure_ascii=False
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _decode_base64(value: Any, label: str) -> bytes:
|
||||
if not isinstance(value, str):
|
||||
raise PerformanceContractError(f"report lacks {label}")
|
||||
try:
|
||||
return base64.b64decode(value, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise PerformanceContractError(f"report carries invalid {label}") from exc
|
||||
|
||||
|
||||
def _verify_real_provenance(
|
||||
contract: PerformanceContract, report: Mapping[str, Any]
|
||||
) -> None:
|
||||
provenance = report.get("provenance")
|
||||
if not isinstance(provenance, Mapping):
|
||||
raise PerformanceContractError("real benchmark report lacks signed provenance")
|
||||
if provenance.get("schema_version") != PROVENANCE_SCHEMA_VERSION:
|
||||
raise PerformanceContractError("report provenance schema is unsupported")
|
||||
if provenance.get("producer") != REAL_REPORT_PRODUCER:
|
||||
raise PerformanceContractError("report was not emitted by the canonical real runner")
|
||||
if provenance.get("signature_algorithm") != "ed25519":
|
||||
raise PerformanceContractError("report provenance is not Ed25519 signed")
|
||||
for field in ("run_id", "started_at", "completed_at"):
|
||||
if not provenance.get(field):
|
||||
raise PerformanceContractError(f"report provenance lacks {field}")
|
||||
|
||||
required_config = contract.baseline.get("required_config_sha256")
|
||||
if not required_config or provenance.get("config_sha256") != required_config:
|
||||
raise PerformanceContractError("report config digest does not match the locked config")
|
||||
|
||||
encoded_public_key = contract.baseline.get("required_signer_public_key")
|
||||
public_key_bytes = _decode_base64(encoded_public_key, "locked signer public key")
|
||||
if len(public_key_bytes) != 32:
|
||||
raise PerformanceContractError("locked Ed25519 public key must be 32 bytes")
|
||||
expected_fingerprint = hashlib.sha256(public_key_bytes).hexdigest()
|
||||
if provenance.get("signer_public_key_sha256") != expected_fingerprint:
|
||||
raise PerformanceContractError("report signer fingerprint does not match the contract")
|
||||
|
||||
signature = _decode_base64(provenance.get("signature"), "Ed25519 signature")
|
||||
try:
|
||||
Ed25519PublicKey.from_public_bytes(public_key_bytes).verify(
|
||||
signature, report_signing_payload(report)
|
||||
)
|
||||
except (InvalidSignature, ValueError) as exc:
|
||||
raise PerformanceContractError("report Ed25519 signature verification failed") from exc
|
||||
|
||||
|
||||
def _validate_report(contract: PerformanceContract, report: Mapping[str, Any]) -> None:
|
||||
"""Fail closed when a report is not the experiment the contract locked."""
|
||||
try:
|
||||
schema_version = report["schema_version"]
|
||||
evidence_class = report["evidence_class"]
|
||||
plan = report["plan"]
|
||||
recipes = report["recipes"]
|
||||
reference_id = report["reference_recipe_id"]
|
||||
host = report["host"]
|
||||
except (KeyError, TypeError) as exc:
|
||||
raise PerformanceContractError("benchmark report is missing required structure") from exc
|
||||
|
||||
if schema_version != REPORT_SCHEMA_VERSION:
|
||||
raise PerformanceContractError(
|
||||
f"report schema {schema_version!r} is not supported schema {REPORT_SCHEMA_VERSION}"
|
||||
)
|
||||
if plan.get("plan_id") != contract.plan_id:
|
||||
raise PerformanceContractError(
|
||||
f"report plan {plan.get('plan_id')!r} does not match locked plan {contract.plan_id!r}"
|
||||
)
|
||||
required_plan_sha256 = contract.baseline.get("required_plan_sha256")
|
||||
measured_plan_sha256 = _canonical_sha256(plan)
|
||||
if required_plan_sha256 and measured_plan_sha256 != required_plan_sha256:
|
||||
raise PerformanceContractError(
|
||||
f"report plan digest {measured_plan_sha256} does not match locked digest "
|
||||
f"{required_plan_sha256}"
|
||||
)
|
||||
minimum_repeats = int(contract.baseline.get("minimum_repeats", 0))
|
||||
minimum_prompts = int(contract.baseline.get("minimum_prompt_count", 0))
|
||||
if int(plan.get("repeats", 0)) < minimum_repeats:
|
||||
raise PerformanceContractError("report has too few repeats for the locked contract")
|
||||
if len(plan.get("prompts", ())) < minimum_prompts:
|
||||
raise PerformanceContractError("report has too few prompts for the locked contract")
|
||||
minimum_output_tokens = int(contract.baseline.get("minimum_output_tokens", 0))
|
||||
if int(plan.get("sampling", {}).get("max_output_tokens", 0)) < minimum_output_tokens:
|
||||
raise PerformanceContractError("report output length is below the locked contract")
|
||||
required_evidence = contract.baseline.get("required_evidence_class")
|
||||
if required_evidence and evidence_class != required_evidence:
|
||||
raise PerformanceContractError(
|
||||
f"report evidence class {evidence_class!r} does not satisfy {required_evidence!r}"
|
||||
)
|
||||
if evidence_class in {"local-real", "multi-machine-real"}:
|
||||
_verify_real_provenance(contract, report)
|
||||
if required_evidence and (
|
||||
not isinstance(host, Mapping)
|
||||
or any(key not in host for key in ("hostname", "platform", "python", "cpu_count"))
|
||||
):
|
||||
raise PerformanceContractError("report lacks measured host provenance")
|
||||
if not isinstance(recipes, list) or not recipes:
|
||||
raise PerformanceContractError("report contains no recipes")
|
||||
|
||||
recipe_ids = [entry.get("recipe", {}).get("id") for entry in recipes]
|
||||
if len(set(recipe_ids)) != len(recipe_ids) or None in recipe_ids:
|
||||
raise PerformanceContractError("report recipe IDs must be present and unique")
|
||||
required_recipes = set(contract.baseline.get("required_recipes", ()))
|
||||
missing_recipes = required_recipes - set(recipe_ids)
|
||||
if missing_recipes:
|
||||
raise PerformanceContractError(
|
||||
f"report is missing required recipes {sorted(missing_recipes)}"
|
||||
)
|
||||
if reference_id not in recipe_ids:
|
||||
raise PerformanceContractError("report reference recipe is absent")
|
||||
|
||||
levels = {int(level) for level in plan.get("concurrency_levels", ())}
|
||||
required_levels = {
|
||||
int(level) for level in contract.baseline.get("required_concurrency_levels", ())
|
||||
}
|
||||
if not required_levels.issubset(levels):
|
||||
raise PerformanceContractError(
|
||||
f"report concurrency {sorted(levels)} lacks required levels {sorted(required_levels)}"
|
||||
)
|
||||
if not plan.get("prompts"):
|
||||
raise PerformanceContractError("report plan contains no prompts")
|
||||
|
||||
model_id = plan.get("model_id")
|
||||
model_revision = plan.get("model_revision")
|
||||
required_device = contract.baseline.get("required_device")
|
||||
required_artifacts = dict(contract.baseline.get("required_artifact_sha256") or {})
|
||||
required_runtimes = dict(contract.baseline.get("required_recipe_runtime") or {})
|
||||
required_backends = dict(contract.baseline.get("required_backend_detail") or {})
|
||||
required_host = dict(contract.baseline.get("required_host_identity") or {})
|
||||
for field, expected in required_host.items():
|
||||
if host.get(field) != expected:
|
||||
raise PerformanceContractError(
|
||||
f"report host/runtime field {field!r} does not match the locked identity"
|
||||
)
|
||||
for entry in recipes:
|
||||
recipe = entry.get("recipe", {})
|
||||
recipe_id = recipe.get("id")
|
||||
if required_device and recipe.get("device") != required_device:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe.get('id')!r} did not run on locked device {required_device!r}"
|
||||
)
|
||||
if required_evidence:
|
||||
if recipe.get("source_model_id") != model_id:
|
||||
raise PerformanceContractError("report mixes source model IDs")
|
||||
if recipe.get("source_model_revision") != model_revision:
|
||||
raise PerformanceContractError("report mixes source model revisions")
|
||||
digest = recipe.get("artifact_sha256", "")
|
||||
if not isinstance(digest, str) or len(digest) != 64:
|
||||
raise PerformanceContractError("report lacks an artifact SHA-256 digest")
|
||||
expected_digest = required_artifacts.get(recipe_id)
|
||||
if not expected_digest or digest != expected_digest:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} artifact digest does not match the contract"
|
||||
)
|
||||
expected_runtime = required_runtimes.get(recipe_id)
|
||||
if not isinstance(expected_runtime, Mapping):
|
||||
raise PerformanceContractError(
|
||||
f"contract lacks runtime identity for recipe {recipe_id!r}"
|
||||
)
|
||||
actual_runtime = {
|
||||
field: recipe.get(field) for field in expected_runtime
|
||||
}
|
||||
if actual_runtime != dict(expected_runtime):
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} runtime identity does not match the contract"
|
||||
)
|
||||
if entry.get("available"):
|
||||
expected_backend = required_backends.get(recipe_id)
|
||||
actual_backend = entry.get("load", {}).get("backend_detail")
|
||||
if not expected_backend or actual_backend != expected_backend:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} backend identity does not match the contract"
|
||||
)
|
||||
if entry.get("available"):
|
||||
cells = entry.get("concurrency", {})
|
||||
missing_cells = required_levels - {int(level) for level in cells}
|
||||
if missing_cells:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe.get('id')!r} lacks concurrency cells {sorted(missing_cells)}"
|
||||
)
|
||||
|
||||
reference = next(entry for entry in recipes if entry["recipe"]["id"] == reference_id)
|
||||
if not reference.get("available"):
|
||||
raise PerformanceContractError("reference recipe is unavailable")
|
||||
reference_failures = sum(
|
||||
int(cell.get("failures", 0)) for cell in reference.get("concurrency", {}).values()
|
||||
)
|
||||
if reference_failures:
|
||||
raise PerformanceContractError("reference recipe contains failed requests")
|
||||
|
||||
def token_counts(entry: Mapping[str, Any]) -> dict[tuple[str, int, int], list[tuple[int, int]]]:
|
||||
counts: dict[tuple[str, int, int], list[tuple[int, int]]] = {}
|
||||
for outcome in entry.get("outcomes", ()):
|
||||
if not outcome.get("ok"):
|
||||
continue
|
||||
key = (
|
||||
str(outcome.get("prompt_id")),
|
||||
int(outcome.get("concurrency", 0)),
|
||||
int(outcome.get("repeat", -1)),
|
||||
)
|
||||
counts.setdefault(key, []).append(
|
||||
(int(outcome.get("prompt_tokens", 0)), int(outcome.get("decode_tokens", 0)))
|
||||
)
|
||||
return {key: sorted(values) for key, values in counts.items()}
|
||||
|
||||
def outcome_counts(entry: Mapping[str, Any]) -> Counter[tuple[str, int, int]]:
|
||||
return Counter(
|
||||
(
|
||||
str(outcome.get("prompt_id")),
|
||||
int(outcome.get("concurrency", 0)),
|
||||
int(outcome.get("repeat", -1)),
|
||||
)
|
||||
for outcome in entry.get("outcomes", ())
|
||||
)
|
||||
|
||||
prompt_ids = {str(prompt["id"]) for prompt in plan["prompts"]}
|
||||
repeats = int(plan["repeats"])
|
||||
for entry in recipes:
|
||||
if not entry.get("available"):
|
||||
continue
|
||||
recipe_id = str(entry["recipe"]["id"])
|
||||
cells = entry.get("concurrency", {})
|
||||
actual_levels = {int(level) for level in cells}
|
||||
if actual_levels != levels:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} has unexpected concurrency cells"
|
||||
)
|
||||
for outcome in entry.get("outcomes", ()):
|
||||
try:
|
||||
outcome_recipe_id = str(outcome["recipe_id"])
|
||||
prompt_id = str(outcome["prompt_id"])
|
||||
level = int(outcome["concurrency"])
|
||||
repeat = int(outcome["repeat"])
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} contains a malformed raw outcome"
|
||||
) from exc
|
||||
if outcome_recipe_id != recipe_id:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} contains an outcome for {outcome_recipe_id!r}"
|
||||
)
|
||||
if prompt_id not in prompt_ids or level not in levels or not 0 <= repeat < repeats:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} contains an out-of-domain raw outcome"
|
||||
)
|
||||
if not isinstance(outcome.get("ok"), bool):
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} contains an outcome without boolean ok status"
|
||||
)
|
||||
|
||||
coverage = outcome_counts(entry)
|
||||
for prompt_id in prompt_ids:
|
||||
for level in levels:
|
||||
for repeat in range(repeats):
|
||||
if coverage[(prompt_id, level, repeat)] != level:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} lacks complete request coverage"
|
||||
)
|
||||
for level in levels:
|
||||
cell = cells.get(str(level), cells.get(level))
|
||||
raw = [
|
||||
outcome
|
||||
for outcome in entry.get("outcomes", ())
|
||||
if int(outcome["concurrency"]) == level
|
||||
]
|
||||
if int(cell.get("concurrency", -1)) != level:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} cell identity does not match concurrency {level}"
|
||||
)
|
||||
if int(cell.get("requests", -1)) != len(raw):
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} aggregate requests do not match raw outcomes"
|
||||
)
|
||||
raw_failures = sum(not outcome["ok"] for outcome in raw)
|
||||
if int(cell.get("failures", -1)) != raw_failures:
|
||||
raise PerformanceContractError(
|
||||
f"recipe {recipe_id!r} aggregate failures do not match raw outcomes"
|
||||
)
|
||||
|
||||
reference_counts = token_counts(reference)
|
||||
for prompt_id in prompt_ids:
|
||||
for level in levels:
|
||||
for repeat in range(repeats):
|
||||
values = reference_counts.get((prompt_id, level, repeat), ())
|
||||
if len(values) != level:
|
||||
raise PerformanceContractError(
|
||||
"reference recipe lacks complete prompt/repeat/concurrency coverage"
|
||||
)
|
||||
if contract.thresholds.max_failure_rate != 0:
|
||||
raise PerformanceContractError(
|
||||
"nonzero failure tolerance requires an explicit failed-request token policy"
|
||||
)
|
||||
for entry in recipes:
|
||||
if not entry.get("available"):
|
||||
continue
|
||||
for key, values in token_counts(entry).items():
|
||||
if Counter(values) - Counter(reference_counts.get(key, ())):
|
||||
raise PerformanceContractError(
|
||||
f"recipe {entry['recipe']['id']!r} used different prompt/decode token counts"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecipeEvaluation:
|
||||
"""How one GGUF recipe fared against the reference under the contract."""
|
||||
|
||||
recipe_id: str
|
||||
lane: str
|
||||
comparable: bool
|
||||
incomparable_reason: str
|
||||
speed_benefit: bool
|
||||
fit_benefit: bool
|
||||
quality_pass: bool | None
|
||||
failures: int
|
||||
measurements: Mapping[str, Any]
|
||||
reasons: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
data = asdict(self)
|
||||
data["measurements"] = dict(self.measurements)
|
||||
data["reasons"] = list(self.reasons)
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ContractEvaluation:
|
||||
"""The release-gate answer: a verdict plus every reason behind it."""
|
||||
|
||||
contract_version: int
|
||||
plan_id: str
|
||||
verdict: str
|
||||
quality_lane_pass: bool
|
||||
speed_benefit: bool
|
||||
fit_benefit: bool
|
||||
stop_condition_met: bool
|
||||
recipes: tuple[RecipeEvaluation, ...]
|
||||
rationale: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"contract_version": self.contract_version,
|
||||
"plan_id": self.plan_id,
|
||||
"verdict": self.verdict,
|
||||
"quality_lane_pass": self.quality_lane_pass,
|
||||
"speed_benefit": self.speed_benefit,
|
||||
"fit_benefit": self.fit_benefit,
|
||||
"stop_condition_met": self.stop_condition_met,
|
||||
"recipes": [recipe.to_dict() for recipe in self.recipes],
|
||||
"rationale": list(self.rationale),
|
||||
}
|
||||
|
||||
|
||||
def _evaluate_recipe(
|
||||
entry: Mapping[str, Any],
|
||||
reference: Mapping[str, Any],
|
||||
drift_by_recipe: Mapping[str, Mapping[str, Any]],
|
||||
thresholds: ContractThresholds,
|
||||
concurrency_levels: list[int],
|
||||
expected_prompt_count: int,
|
||||
) -> RecipeEvaluation:
|
||||
recipe = entry["recipe"]
|
||||
lane = recipe["lane"]
|
||||
reasons: list[str] = []
|
||||
|
||||
if not entry["available"]:
|
||||
return RecipeEvaluation(
|
||||
recipe_id=recipe["id"], lane=lane, comparable=False,
|
||||
incomparable_reason=entry["unavailable_reason"] or "recipe was not measured",
|
||||
speed_benefit=False, fit_benefit=False, quality_pass=None, failures=0,
|
||||
measurements={}, reasons=("recipe unavailable; no benefit granted",),
|
||||
)
|
||||
|
||||
if recipe["device"] != reference["recipe"]["device"]:
|
||||
return RecipeEvaluation(
|
||||
recipe_id=recipe["id"], lane=lane, comparable=False,
|
||||
incomparable_reason=(
|
||||
f"recipe ran on device {recipe['device']!r} but the reference ran on "
|
||||
f"{reference['recipe']['device']!r}; a cross-device ratio is not a runtime result"
|
||||
),
|
||||
speed_benefit=False, fit_benefit=False, quality_pass=None, failures=0,
|
||||
measurements={}, reasons=("cross-device comparison; no benefit granted",),
|
||||
)
|
||||
|
||||
single = _cell(entry, 1)
|
||||
reference_single = _cell(reference, 1)
|
||||
measurements: dict[str, Any] = {}
|
||||
speed_benefit = False
|
||||
|
||||
if single and reference_single:
|
||||
decode_speedup = _ratio(
|
||||
single["decode_tokens_per_sec"], reference_single["decode_tokens_per_sec"]
|
||||
)
|
||||
ttft_ratio = _ratio(single["ttft_p50_ms"], reference_single["ttft_p50_ms"])
|
||||
measurements["decode_speedup"] = decode_speedup
|
||||
measurements["ttft_ratio"] = ttft_ratio
|
||||
single_request_win = (
|
||||
decode_speedup >= thresholds.min_decode_speedup
|
||||
and 0 < ttft_ratio <= thresholds.max_ttft_ratio
|
||||
)
|
||||
if single_request_win:
|
||||
speed_benefit = lane == Lane.PERFORMANCE_FIT.value
|
||||
reasons.append(
|
||||
f"single-request decode {decode_speedup:.2f}x reference "
|
||||
f"(>= {thresholds.min_decode_speedup:.2f}x) at TTFT ratio {ttft_ratio:.2f}"
|
||||
)
|
||||
else:
|
||||
reasons.append(
|
||||
f"no single-request speed win: decode {decode_speedup:.2f}x, TTFT {ttft_ratio:.2f}x"
|
||||
)
|
||||
|
||||
concurrent = [level for level in concurrency_levels if level > 1]
|
||||
if concurrent:
|
||||
top = max(concurrent)
|
||||
cell, reference_cell = _cell(entry, top), _cell(reference, top)
|
||||
if cell and reference_cell:
|
||||
aggregate_speedup = _ratio(
|
||||
cell["aggregate_decode_tokens_per_sec"],
|
||||
reference_cell["aggregate_decode_tokens_per_sec"],
|
||||
)
|
||||
measurements["aggregate_throughput_speedup"] = aggregate_speedup
|
||||
measurements["aggregate_concurrency"] = top
|
||||
if aggregate_speedup >= thresholds.min_aggregate_throughput_speedup:
|
||||
speed_benefit = lane == Lane.PERFORMANCE_FIT.value
|
||||
reasons.append(
|
||||
f"aggregate throughput at concurrency {top} is {aggregate_speedup:.2f}x reference "
|
||||
f"(>= {thresholds.min_aggregate_throughput_speedup:.2f}x)"
|
||||
)
|
||||
else:
|
||||
reasons.append(
|
||||
f"no concurrency speed win: aggregate throughput at {top} is "
|
||||
f"{aggregate_speedup:.2f}x reference"
|
||||
)
|
||||
|
||||
fit_benefit = False
|
||||
if single and reference_single:
|
||||
resident_ratio = _ratio(_resident_bytes(single), _resident_bytes(reference_single))
|
||||
artifact_ratio = _ratio(
|
||||
entry["load"]["artifact_bytes"], reference["load"]["artifact_bytes"]
|
||||
)
|
||||
measurements["resident_memory_ratio"] = resident_ratio
|
||||
measurements["artifact_size_ratio"] = artifact_ratio
|
||||
if 0 < resident_ratio <= thresholds.max_resident_memory_ratio:
|
||||
fit_benefit = lane == Lane.PERFORMANCE_FIT.value
|
||||
reasons.append(
|
||||
f"peak resident memory is {resident_ratio:.2f}x reference "
|
||||
f"(<= {thresholds.max_resident_memory_ratio:.2f}x)"
|
||||
)
|
||||
else:
|
||||
reasons.append(f"no fit win: peak resident memory is {resident_ratio:.2f}x reference")
|
||||
measurements["artifact_size_win"] = (
|
||||
0 < artifact_ratio <= thresholds.max_artifact_size_ratio
|
||||
)
|
||||
|
||||
failures = sum(cell["failures"] for cell in entry["concurrency"].values())
|
||||
requests = sum(cell["requests"] for cell in entry["concurrency"].values())
|
||||
failure_rate = _ratio(failures, requests) if requests else 0.0
|
||||
measurements["failure_rate"] = failure_rate
|
||||
failure_limit_exceeded = requests == 0 or (
|
||||
failures > 0
|
||||
if thresholds.max_failure_rate == 0
|
||||
else failures / requests > thresholds.max_failure_rate
|
||||
)
|
||||
if failure_limit_exceeded:
|
||||
reasons.append(f"failure rate {failure_rate:.2%} exceeds the contract limit")
|
||||
speed_benefit = False
|
||||
fit_benefit = False
|
||||
|
||||
# Quality is a claim only the near-lossless lane is allowed to make. A
|
||||
# quantized recipe's drift is recorded elsewhere and deliberately not read
|
||||
# here: Q4 disagreeing with bf16 is the trade-off, not a failure.
|
||||
quality_pass: bool | None = None
|
||||
if lane == Lane.QUALITY.value:
|
||||
drift = drift_by_recipe.get(recipe["id"])
|
||||
if drift is None:
|
||||
quality_pass = False
|
||||
reasons.append("quality-lane recipe has no drift measurement against the reference")
|
||||
else:
|
||||
complete_coverage = drift.get("compared_prompts") == expected_prompt_count
|
||||
quality_pass = (
|
||||
complete_coverage
|
||||
and failures == 0
|
||||
and drift["exact_match_rate"] >= thresholds.min_quality_exact_match_rate
|
||||
and drift["mean_similarity"] >= thresholds.min_quality_mean_similarity
|
||||
)
|
||||
measurements["compared_prompts"] = drift.get("compared_prompts", 0)
|
||||
measurements["expected_prompts"] = expected_prompt_count
|
||||
measurements["exact_match_rate"] = drift["exact_match_rate"]
|
||||
measurements["mean_similarity"] = drift["mean_similarity"]
|
||||
if not complete_coverage:
|
||||
reasons.append(
|
||||
f"quality lane compared {drift.get('compared_prompts', 0)} of "
|
||||
f"{expected_prompt_count} required prompts"
|
||||
)
|
||||
if failures:
|
||||
reasons.append("quality lane contains failed requests")
|
||||
reasons.append(
|
||||
f"quality lane exact-match {drift['exact_match_rate']:.2f} / similarity "
|
||||
f"{drift['mean_similarity']:.3f} versus the reference "
|
||||
f"({'pass' if quality_pass else 'fail'})"
|
||||
)
|
||||
|
||||
return RecipeEvaluation(
|
||||
recipe_id=recipe["id"], lane=lane, comparable=True, incomparable_reason="",
|
||||
speed_benefit=speed_benefit, fit_benefit=fit_benefit, quality_pass=quality_pass,
|
||||
failures=failures, measurements=measurements, reasons=tuple(reasons),
|
||||
)
|
||||
|
||||
|
||||
def evaluate_contract(
|
||||
contract: PerformanceContract,
|
||||
report: Mapping[str, Any],
|
||||
) -> ContractEvaluation:
|
||||
"""Judge a benchmark report against the locked contract.
|
||||
|
||||
Only performance-fit recipes can earn a speed or fit benefit; the quality
|
||||
lane decides only whether the GGUF runtime is numerically sane. A GGUF
|
||||
runtime that fails the quality lane is broken, and no amount of speed
|
||||
redeems it, so the verdict is ``stop`` regardless of the other numbers.
|
||||
"""
|
||||
_validate_report(contract, report)
|
||||
entries = _recipe_entries(report)
|
||||
reference_id = report["reference_recipe_id"]
|
||||
reference = entries.get(reference_id)
|
||||
if reference is None:
|
||||
raise PerformanceContractError(
|
||||
f"report names reference recipe {reference_id!r}, which it does not contain"
|
||||
)
|
||||
|
||||
drift_by_recipe = {entry["recipe_id"]: entry for entry in report["drift"]}
|
||||
concurrency_levels = list(report["plan"]["concurrency_levels"])
|
||||
|
||||
evaluations = tuple(
|
||||
_evaluate_recipe(
|
||||
entry,
|
||||
reference,
|
||||
drift_by_recipe,
|
||||
contract.thresholds,
|
||||
concurrency_levels,
|
||||
len(report["plan"]["prompts"]),
|
||||
)
|
||||
for recipe_id, entry in entries.items()
|
||||
if recipe_id != reference_id
|
||||
)
|
||||
|
||||
quality_lane = [
|
||||
evaluation for evaluation in evaluations
|
||||
if evaluation.lane == Lane.QUALITY.value and evaluation.comparable
|
||||
]
|
||||
quality_lane_pass = bool(quality_lane) and all(
|
||||
evaluation.quality_pass for evaluation in quality_lane
|
||||
)
|
||||
performance_lane = [
|
||||
evaluation for evaluation in evaluations
|
||||
if evaluation.lane == Lane.PERFORMANCE_FIT.value
|
||||
]
|
||||
speed_benefit = any(evaluation.speed_benefit for evaluation in performance_lane)
|
||||
fit_benefit = any(evaluation.fit_benefit for evaluation in performance_lane)
|
||||
|
||||
rationale: list[str] = []
|
||||
if not quality_lane:
|
||||
rationale.append(
|
||||
"no comparable near-lossless GGUF recipe was measured, so the runtime's "
|
||||
"numerical correctness is unproven"
|
||||
)
|
||||
elif not quality_lane_pass:
|
||||
rationale.append(
|
||||
"the near-lossless quality lane failed: the GGUF runtime disagrees with the "
|
||||
"safetensors reference beyond what near-lossless weights can explain"
|
||||
)
|
||||
else:
|
||||
rationale.append("the near-lossless quality lane passed against the safetensors reference")
|
||||
|
||||
rationale.append(
|
||||
"a meaningful speed benefit was measured" if speed_benefit
|
||||
else "no performance-fit recipe delivered a meaningful speed benefit"
|
||||
)
|
||||
rationale.append(
|
||||
"a meaningful fit benefit was measured" if fit_benefit
|
||||
else "no performance-fit recipe delivered a meaningful fit benefit"
|
||||
)
|
||||
|
||||
stop_condition_met = not quality_lane_pass or not (speed_benefit or fit_benefit)
|
||||
if stop_condition_met:
|
||||
verdict = VERDICT_STOP
|
||||
elif speed_benefit and fit_benefit:
|
||||
verdict = VERDICT_PROMOTE
|
||||
else:
|
||||
verdict = VERDICT_OPTIMIZE
|
||||
rationale.append(
|
||||
"exactly one of speed or fit cleared the contract: the benefit is real but partial, "
|
||||
"so the measured bottleneck needs a bounded optimization task before promotion"
|
||||
)
|
||||
|
||||
return ContractEvaluation(
|
||||
contract_version=contract.contract_version,
|
||||
plan_id=contract.plan_id,
|
||||
verdict=verdict,
|
||||
quality_lane_pass=quality_lane_pass,
|
||||
speed_benefit=speed_benefit,
|
||||
fit_benefit=fit_benefit,
|
||||
stop_condition_met=stop_condition_met,
|
||||
recipes=evaluations,
|
||||
rationale=tuple(rationale),
|
||||
)
|
||||
|
||||
|
||||
def parse_contract(data: Any, source: str = "<memory>") -> PerformanceContract:
|
||||
"""Validate an already-decoded contract document."""
|
||||
if not isinstance(data, Mapping):
|
||||
raise PerformanceContractError(f"contract root in {source} must be a JSON object")
|
||||
|
||||
schema_version = data.get("schema_version")
|
||||
if not isinstance(schema_version, int) or isinstance(schema_version, bool):
|
||||
raise PerformanceContractError(f"'schema_version' in {source} must be an integer")
|
||||
if schema_version != CONTRACT_SCHEMA_VERSION:
|
||||
raise PerformanceContractError(
|
||||
f"{source} declares contract schema version {schema_version}, but this node reads "
|
||||
f"version {CONTRACT_SCHEMA_VERSION}; upgrade the node or use a supported contract"
|
||||
)
|
||||
|
||||
for required in ("contract_version", "locked_at", "locked_by", "plan_id", "stop_condition"):
|
||||
if not data.get(required):
|
||||
raise PerformanceContractError(f"{source} is missing {required!r}")
|
||||
|
||||
raw_thresholds = data.get("thresholds")
|
||||
if not isinstance(raw_thresholds, Mapping):
|
||||
raise PerformanceContractError(f"'thresholds' in {source} must be a JSON object")
|
||||
known = set(ContractThresholds().to_dict())
|
||||
unknown = set(raw_thresholds) - known
|
||||
missing = known - set(raw_thresholds)
|
||||
if unknown or missing:
|
||||
raise PerformanceContractError(
|
||||
f"{source} threshold keys differ from v1; unknown={sorted(unknown)}, "
|
||||
f"missing={sorted(missing)}"
|
||||
)
|
||||
thresholds = ContractThresholds(**{
|
||||
key: float(value) for key, value in raw_thresholds.items()
|
||||
})
|
||||
contract_version = int(data["contract_version"])
|
||||
if contract_version != 1 or thresholds != ContractThresholds():
|
||||
raise PerformanceContractError(
|
||||
f"{source} changes immutable v1 thresholds without a supported contract version"
|
||||
)
|
||||
if str(data["stop_condition"]) != STOP_CONDITION:
|
||||
raise PerformanceContractError(
|
||||
f"{source} stop condition differs from executable v1 semantics"
|
||||
)
|
||||
|
||||
return PerformanceContract(
|
||||
contract_version=contract_version,
|
||||
locked_at=str(data["locked_at"]),
|
||||
locked_by=str(data["locked_by"]),
|
||||
plan_id=str(data["plan_id"]),
|
||||
thresholds=thresholds,
|
||||
baseline=dict(data.get("baseline") or {}),
|
||||
stop_condition=str(data["stop_condition"]),
|
||||
notes=str(data.get("notes", "")),
|
||||
schema_version=schema_version,
|
||||
)
|
||||
|
||||
|
||||
def load_contract(path: Path) -> PerformanceContract:
|
||||
"""Load and validate the contract at ``path``."""
|
||||
try:
|
||||
raw = path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
raise PerformanceContractError(
|
||||
f"cannot read performance contract {path}: {exc.strerror or exc}"
|
||||
) from exc
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise PerformanceContractError(
|
||||
f"{path} is not valid JSON: {exc.msg} at line {exc.lineno} column {exc.colno}"
|
||||
) from exc
|
||||
return parse_contract(data, source=str(path))
|
||||
|
||||
|
||||
def baseline_from_report(report: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Distil the reference numbers a later gate needs to compare against."""
|
||||
entries = _recipe_entries(report)
|
||||
baseline: dict[str, Any] = {
|
||||
"evidence_class": report["evidence_class"],
|
||||
"model_id": report["plan"]["model_id"],
|
||||
"model_revision": report["plan"]["model_revision"],
|
||||
"plan_sha256": _canonical_sha256(report["plan"]),
|
||||
"reference_recipe_id": report["reference_recipe_id"],
|
||||
"host": report["host"],
|
||||
"provenance": dict(report.get("provenance") or {}),
|
||||
"artifact_sha256": {
|
||||
recipe_id: entry["recipe"]["artifact_sha256"]
|
||||
for recipe_id, entry in entries.items()
|
||||
},
|
||||
"recipe_runtime": {
|
||||
recipe_id: {
|
||||
field: entry["recipe"].get(field)
|
||||
for field in ("runtime", "weight_format", "weight_quantization", "device")
|
||||
}
|
||||
for recipe_id, entry in entries.items()
|
||||
},
|
||||
"backend_detail": {
|
||||
recipe_id: entry.get("load", {}).get("backend_detail")
|
||||
for recipe_id, entry in entries.items()
|
||||
if entry.get("available")
|
||||
},
|
||||
"recipes": {},
|
||||
}
|
||||
for recipe_id, entry in entries.items():
|
||||
if not entry["available"]:
|
||||
baseline["recipes"][recipe_id] = {"available": False,
|
||||
"reason": entry["unavailable_reason"]}
|
||||
continue
|
||||
baseline["recipes"][recipe_id] = {
|
||||
"available": True,
|
||||
"lane": entry["recipe"]["lane"],
|
||||
"device": entry["recipe"]["device"],
|
||||
"artifact_bytes": entry["load"]["artifact_bytes"],
|
||||
"concurrency": {
|
||||
level: {
|
||||
"ttft_p50_ms": cell["ttft_p50_ms"],
|
||||
"ttft_p95_ms": cell["ttft_p95_ms"],
|
||||
"latency_p50_ms": cell["latency_p50_ms"],
|
||||
"latency_p95_ms": cell["latency_p95_ms"],
|
||||
"prefill_tokens_per_sec": cell["prefill_tokens_per_sec"],
|
||||
"decode_tokens_per_sec": cell["decode_tokens_per_sec"],
|
||||
"aggregate_decode_tokens_per_sec": cell["aggregate_decode_tokens_per_sec"],
|
||||
"peak_rss_bytes": cell["peak_rss_bytes"],
|
||||
"peak_vram_bytes": cell["peak_vram_bytes"],
|
||||
"failures": cell["failures"],
|
||||
}
|
||||
for level, cell in entry["concurrency"].items()
|
||||
},
|
||||
}
|
||||
return baseline
|
||||
694
packages/node/meshnet_node/recipe_benchmark.py
Normal file
694
packages/node/meshnet_node/recipe_benchmark.py
Normal file
@@ -0,0 +1,694 @@
|
||||
"""Controlled safetensors-versus-GGUF recipe benchmark.
|
||||
|
||||
This is a *model recipe* benchmark, unlike
|
||||
:mod:`meshnet_node.route_session_benchmark`, which is a transport harness. It
|
||||
answers one question: on one machine, with one model revision and one fixed
|
||||
workload, what do the Transformers/safetensors recipe and the whole-model
|
||||
llama.cpp/GGUF recipes actually cost in speed, memory, fit, and output drift?
|
||||
|
||||
Two ideas keep the answer honest.
|
||||
|
||||
**Lanes.** A recipe belongs to exactly one :class:`Lane`. The quality lane
|
||||
holds near-lossless recipes (bf16/f16 weights) whose outputs may legitimately be
|
||||
compared for numerical agreement. The performance-fit lane holds quantized
|
||||
recipes (Q8_0, Q4_K_M, ...). Quantized recipes are judged on speed, memory, and
|
||||
artifact size only; their drift is *reported* but never read as evidence that
|
||||
Q4 and bf16 are numerically equivalent, because they are not. The lane is a
|
||||
property of the recipe, so nothing downstream can quietly cross the boundary.
|
||||
|
||||
**Drivers.** The measurement core here is pure and runtime-free: it drives a
|
||||
:class:`RecipeDriver` and computes metrics. Real runtimes live in
|
||||
:mod:`meshnet_node.recipe_drivers` and are imported only on demand, which keeps
|
||||
the default test suite deterministic, GPU-free and model-download-free while the
|
||||
same code path produces the real evidence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from difflib import SequenceMatcher
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping, Protocol, Sequence
|
||||
|
||||
# Layout of the report document produced by :func:`build_report`.
|
||||
REPORT_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
class Lane(str, Enum):
|
||||
"""Why a recipe is being measured at all.
|
||||
|
||||
``QUALITY`` recipes carry near-lossless weights, so comparing their output
|
||||
with the reference recipe is a meaningful correctness signal.
|
||||
``PERFORMANCE_FIT`` recipes carry quantized weights: they exist to be faster
|
||||
or to fit, and their drift is descriptive, never a pass/fail equivalence
|
||||
claim.
|
||||
"""
|
||||
|
||||
QUALITY = "quality"
|
||||
PERFORMANCE_FIT = "performance-fit"
|
||||
|
||||
|
||||
class BenchmarkError(RuntimeError):
|
||||
"""Raised when a benchmark cannot be run as specified."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SamplingPolicy:
|
||||
"""The sampling policy every recipe must be given, identically.
|
||||
|
||||
Greedy by default: sampling noise would otherwise be indistinguishable from
|
||||
quantization drift, and the whole point of the quality lane is to tell those
|
||||
two apart.
|
||||
"""
|
||||
|
||||
temperature: float = 0.0
|
||||
top_p: float = 1.0
|
||||
top_k: int = 1
|
||||
seed: int = 1234
|
||||
max_output_tokens: int = 64
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PromptSpec:
|
||||
"""One fixed prompt, tagged with the context length it is meant to exercise."""
|
||||
|
||||
id: str
|
||||
text: str
|
||||
context_class: str = "short"
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BenchmarkPlan:
|
||||
"""The controlled variables: identical for every recipe in a report.
|
||||
|
||||
A plan is the experiment. If two recipes were measured under different
|
||||
plans, their numbers are not comparable and the report must not pretend they
|
||||
are, so the plan is recorded once at the top of the document rather than
|
||||
per-recipe.
|
||||
"""
|
||||
|
||||
plan_id: str
|
||||
model_id: str
|
||||
model_revision: str
|
||||
prompts: tuple[PromptSpec, ...]
|
||||
sampling: SamplingPolicy = SamplingPolicy()
|
||||
concurrency_levels: tuple[int, ...] = (1, 4)
|
||||
repeats: int = 1
|
||||
warmup_requests: int = 1
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if not self.prompts:
|
||||
raise BenchmarkError("a benchmark plan needs at least one prompt")
|
||||
if not self.concurrency_levels or any(level < 1 for level in self.concurrency_levels):
|
||||
raise BenchmarkError("concurrency levels must all be >= 1")
|
||||
if self.repeats < 1:
|
||||
raise BenchmarkError("repeats must be >= 1")
|
||||
if 1 not in self.concurrency_levels or 4 not in self.concurrency_levels:
|
||||
raise BenchmarkError("a controlled baseline must include concurrency levels 1 and 4")
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"plan_id": self.plan_id,
|
||||
"model_id": self.model_id,
|
||||
"model_revision": self.model_revision,
|
||||
"prompts": [prompt.to_dict() for prompt in self.prompts],
|
||||
"sampling": self.sampling.to_dict(),
|
||||
"concurrency_levels": list(self.concurrency_levels),
|
||||
"repeats": self.repeats,
|
||||
"warmup_requests": self.warmup_requests,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RecipeSpec:
|
||||
"""One runtime recipe under test.
|
||||
|
||||
``is_reference`` marks the single recipe every other recipe's output drift is
|
||||
measured against — the current Transformers/safetensors route, which
|
||||
decision Gate 8 keeps as the correctness backend.
|
||||
"""
|
||||
|
||||
id: str
|
||||
runtime: str
|
||||
weight_format: str
|
||||
weight_quantization: str
|
||||
lane: Lane
|
||||
device: str
|
||||
artifact_path: str = ""
|
||||
source_model_id: str = ""
|
||||
source_model_revision: str = ""
|
||||
artifact_sha256: str = ""
|
||||
is_reference: bool = False
|
||||
notes: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
data = asdict(self)
|
||||
data["lane"] = self.lane.value
|
||||
return data
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LoadStats:
|
||||
"""What loading the recipe cost, before any token is generated."""
|
||||
|
||||
artifact_bytes: int
|
||||
load_ms: float
|
||||
rss_bytes: int = 0
|
||||
vram_bytes: int = 0
|
||||
backend_detail: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GenerationSample:
|
||||
"""One completed generation as reported by a driver.
|
||||
|
||||
``prefill_ms``/``decode_ms`` are the runtime's own split where it exposes one
|
||||
(llama.cpp does); drivers that cannot split honestly report ``prefill_ms`` as
|
||||
the time to the first token. ``queue_wait_ms`` separates time spent waiting
|
||||
for a runtime slot from time spent computing, so a concurrency-4 TTFT is not
|
||||
silently read as a slower prefill.
|
||||
"""
|
||||
|
||||
text: str
|
||||
prompt_tokens: int
|
||||
decode_tokens: int
|
||||
ttft_ms: float
|
||||
prefill_ms: float
|
||||
decode_ms: float
|
||||
total_ms: float
|
||||
queue_wait_ms: float = 0.0
|
||||
|
||||
|
||||
class RecipeDriver(Protocol):
|
||||
"""The seam every runtime implements; the measurement core knows nothing else."""
|
||||
|
||||
def load(self) -> LoadStats:
|
||||
"""Load the artifact and return its cost."""
|
||||
|
||||
def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
|
||||
"""Run one complete generation under the given sampling policy."""
|
||||
|
||||
def memory_probe(self) -> tuple[int, int]:
|
||||
"""Return ``(rss_bytes, vram_bytes)`` observed right now."""
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release the runtime."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RequestOutcome:
|
||||
"""One request attempt, successful or not.
|
||||
|
||||
A failure is a first-class result, not an exception that aborts the run: a
|
||||
recipe that cannot sustain concurrency 4 has told us something, and the
|
||||
report must carry it rather than lose it.
|
||||
"""
|
||||
|
||||
recipe_id: str
|
||||
concurrency: int
|
||||
prompt_id: str
|
||||
repeat: int
|
||||
ok: bool
|
||||
latency_ms: float
|
||||
ttft_ms: float = 0.0
|
||||
prefill_ms: float = 0.0
|
||||
decode_ms: float = 0.0
|
||||
queue_wait_ms: float = 0.0
|
||||
prompt_tokens: int = 0
|
||||
decode_tokens: int = 0
|
||||
text: str = ""
|
||||
error: str = ""
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def _percentile(values: Sequence[float], percentile: float) -> float:
|
||||
"""Nearest-rank percentile; 0.0 for an empty sample."""
|
||||
ordered = sorted(values)
|
||||
if not ordered:
|
||||
return 0.0
|
||||
rank = max(1, -(-len(ordered) * percentile // 100))
|
||||
return round(ordered[int(rank) - 1], 4)
|
||||
|
||||
|
||||
def _mean(values: Sequence[float]) -> float:
|
||||
return round(statistics.fmean(values), 4) if values else 0.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConcurrencyMetrics:
|
||||
"""Aggregate metrics for one recipe at one concurrency level."""
|
||||
|
||||
concurrency: int
|
||||
requests: int
|
||||
failures: int
|
||||
wall_ms: float
|
||||
ttft_p50_ms: float
|
||||
ttft_p95_ms: float
|
||||
latency_p50_ms: float
|
||||
latency_p95_ms: float
|
||||
prefill_tokens_per_sec: float
|
||||
decode_tokens_per_sec: float
|
||||
aggregate_decode_tokens_per_sec: float
|
||||
peak_rss_bytes: int
|
||||
peak_vram_bytes: int
|
||||
failure_reasons: tuple[str, ...] = ()
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
data = asdict(self)
|
||||
data["failure_reasons"] = list(self.failure_reasons)
|
||||
return data
|
||||
|
||||
|
||||
def summarize_concurrency(
|
||||
outcomes: Sequence[RequestOutcome],
|
||||
*,
|
||||
concurrency: int,
|
||||
wall_ms: float,
|
||||
peak_rss_bytes: int,
|
||||
peak_vram_bytes: int,
|
||||
) -> ConcurrencyMetrics:
|
||||
"""Aggregate one recipe/concurrency cell.
|
||||
|
||||
Per-request rates are averaged over successful requests; aggregate
|
||||
throughput is total decoded tokens over the wall clock of the whole cell,
|
||||
which is the only figure that credits a runtime for overlapping work.
|
||||
"""
|
||||
ok = [outcome for outcome in outcomes if outcome.ok]
|
||||
failures = [outcome for outcome in outcomes if not outcome.ok]
|
||||
decode_tokens = sum(outcome.decode_tokens for outcome in ok)
|
||||
|
||||
prefill_rates = [
|
||||
outcome.prompt_tokens / (outcome.prefill_ms / 1000)
|
||||
for outcome in ok
|
||||
if outcome.prefill_ms > 0 and outcome.prompt_tokens
|
||||
]
|
||||
decode_rates = [
|
||||
outcome.decode_tokens / (outcome.decode_ms / 1000)
|
||||
for outcome in ok
|
||||
if outcome.decode_ms > 0 and outcome.decode_tokens
|
||||
]
|
||||
return ConcurrencyMetrics(
|
||||
concurrency=concurrency,
|
||||
requests=len(outcomes),
|
||||
failures=len(failures),
|
||||
wall_ms=round(wall_ms, 4),
|
||||
ttft_p50_ms=_percentile([outcome.ttft_ms for outcome in ok], 50),
|
||||
ttft_p95_ms=_percentile([outcome.ttft_ms for outcome in ok], 95),
|
||||
latency_p50_ms=_percentile([outcome.latency_ms for outcome in ok], 50),
|
||||
latency_p95_ms=_percentile([outcome.latency_ms for outcome in ok], 95),
|
||||
prefill_tokens_per_sec=_mean(prefill_rates),
|
||||
decode_tokens_per_sec=_mean(decode_rates),
|
||||
aggregate_decode_tokens_per_sec=round(decode_tokens / max(1e-6, wall_ms / 1000), 4),
|
||||
peak_rss_bytes=peak_rss_bytes,
|
||||
peak_vram_bytes=peak_vram_bytes,
|
||||
failure_reasons=tuple(sorted({outcome.error for outcome in failures if outcome.error})),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecipeMeasurement:
|
||||
"""Everything measured for one recipe across every concurrency level."""
|
||||
|
||||
recipe: RecipeSpec
|
||||
load: LoadStats
|
||||
metrics: dict[int, ConcurrencyMetrics] = field(default_factory=dict)
|
||||
outcomes: list[RequestOutcome] = field(default_factory=list)
|
||||
unavailable_reason: str = ""
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return not self.unavailable_reason
|
||||
|
||||
def outputs_by_prompt(self) -> dict[str, str]:
|
||||
"""First successful output per prompt, at the lowest concurrency measured.
|
||||
|
||||
Drift is a property of the recipe, not of load: concurrency must not
|
||||
change greedy output, so the least-contended sample is the fair one.
|
||||
"""
|
||||
best: dict[str, tuple[int, str]] = {}
|
||||
for outcome in self.outcomes:
|
||||
if not outcome.ok:
|
||||
continue
|
||||
seen = best.get(outcome.prompt_id)
|
||||
if seen is None or outcome.concurrency < seen[0]:
|
||||
best[outcome.prompt_id] = (outcome.concurrency, outcome.text)
|
||||
return {prompt_id: text for prompt_id, (_, text) in best.items()}
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"recipe": self.recipe.to_dict(),
|
||||
"available": self.available,
|
||||
"unavailable_reason": self.unavailable_reason,
|
||||
"load": self.load.to_dict(),
|
||||
"concurrency": {
|
||||
str(level): metrics.to_dict() for level, metrics in sorted(self.metrics.items())
|
||||
},
|
||||
"outcomes": [outcome.to_dict() for outcome in self.outcomes],
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DriftReport:
|
||||
"""Output drift of one recipe against the reference recipe.
|
||||
|
||||
``advisory`` is true for every performance-fit recipe: the number is
|
||||
published, but a Q4 recipe disagreeing with bf16 is expected behaviour, not a
|
||||
defect, and no gate may read it as one.
|
||||
"""
|
||||
|
||||
recipe_id: str
|
||||
lane: Lane
|
||||
reference_id: str
|
||||
compared_prompts: int
|
||||
exact_match_rate: float
|
||||
mean_similarity: float
|
||||
advisory: bool
|
||||
per_prompt: tuple[dict[str, Any], ...] = ()
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"recipe_id": self.recipe_id,
|
||||
"lane": self.lane.value,
|
||||
"reference_id": self.reference_id,
|
||||
"compared_prompts": self.compared_prompts,
|
||||
"exact_match_rate": self.exact_match_rate,
|
||||
"mean_similarity": self.mean_similarity,
|
||||
"advisory": self.advisory,
|
||||
"per_prompt": list(self.per_prompt),
|
||||
}
|
||||
|
||||
|
||||
def _first_divergence(left: str, right: str) -> int:
|
||||
"""Index of the first differing character, or -1 when the strings agree."""
|
||||
for index, (a, b) in enumerate(zip(left, right)):
|
||||
if a != b:
|
||||
return index
|
||||
return -1 if len(left) == len(right) else min(len(left), len(right))
|
||||
|
||||
|
||||
def compute_drift(
|
||||
measurement: RecipeMeasurement,
|
||||
reference: RecipeMeasurement,
|
||||
) -> DriftReport:
|
||||
"""Compare one recipe's greedy outputs with the reference recipe's."""
|
||||
reference_outputs = reference.outputs_by_prompt()
|
||||
outputs = measurement.outputs_by_prompt()
|
||||
shared = sorted(set(outputs) & set(reference_outputs))
|
||||
|
||||
per_prompt: list[dict[str, Any]] = []
|
||||
exact = 0
|
||||
similarities: list[float] = []
|
||||
for prompt_id in shared:
|
||||
got, want = outputs[prompt_id], reference_outputs[prompt_id]
|
||||
matches = got == want
|
||||
exact += matches
|
||||
similarity = round(SequenceMatcher(None, want, got).ratio(), 4)
|
||||
similarities.append(similarity)
|
||||
per_prompt.append({
|
||||
"prompt_id": prompt_id,
|
||||
"exact_match": matches,
|
||||
"similarity": similarity,
|
||||
"first_divergence_char": _first_divergence(want, got),
|
||||
"reference_text": want,
|
||||
"recipe_text": got,
|
||||
})
|
||||
|
||||
return DriftReport(
|
||||
recipe_id=measurement.recipe.id,
|
||||
lane=measurement.recipe.lane,
|
||||
reference_id=reference.recipe.id,
|
||||
compared_prompts=len(shared),
|
||||
exact_match_rate=round(exact / len(shared), 4) if shared else 0.0,
|
||||
mean_similarity=_mean(similarities),
|
||||
advisory=measurement.recipe.lane is Lane.PERFORMANCE_FIT,
|
||||
per_prompt=tuple(per_prompt),
|
||||
)
|
||||
|
||||
|
||||
class _PeakMemory:
|
||||
"""Continuously sample a driver's memory while requests are in flight."""
|
||||
|
||||
def __init__(self, driver: RecipeDriver) -> None:
|
||||
self._driver = driver
|
||||
self.peak_rss = 0
|
||||
self.peak_vram = 0
|
||||
|
||||
def sample(self) -> None:
|
||||
try:
|
||||
rss, vram = self._driver.memory_probe()
|
||||
except Exception: # a probe must never fail a benchmark run
|
||||
return
|
||||
self.peak_rss = max(self.peak_rss, rss)
|
||||
self.peak_vram = max(self.peak_vram, vram)
|
||||
|
||||
def sample_until(self, stop: threading.Event, interval_s: float = 0.01) -> None:
|
||||
"""Sample until ``stop`` is set, including one final post-request probe."""
|
||||
while not stop.wait(interval_s):
|
||||
self.sample()
|
||||
self.sample()
|
||||
|
||||
|
||||
def _run_request(
|
||||
driver: RecipeDriver,
|
||||
recipe: RecipeSpec,
|
||||
prompt: PromptSpec,
|
||||
sampling: SamplingPolicy,
|
||||
concurrency: int,
|
||||
repeat: int,
|
||||
memory: _PeakMemory,
|
||||
) -> RequestOutcome:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
sample = driver.generate(prompt.text, sampling)
|
||||
except Exception as exc: # a failed request is data, not a crashed benchmark
|
||||
return RequestOutcome(
|
||||
recipe_id=recipe.id,
|
||||
concurrency=concurrency,
|
||||
prompt_id=prompt.id,
|
||||
repeat=repeat,
|
||||
ok=False,
|
||||
latency_ms=round((time.monotonic() - started) * 1000, 4),
|
||||
error=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
finally:
|
||||
memory.sample()
|
||||
|
||||
return RequestOutcome(
|
||||
recipe_id=recipe.id,
|
||||
concurrency=concurrency,
|
||||
prompt_id=prompt.id,
|
||||
repeat=repeat,
|
||||
ok=True,
|
||||
latency_ms=round(sample.total_ms, 4),
|
||||
ttft_ms=round(sample.ttft_ms, 4),
|
||||
prefill_ms=round(sample.prefill_ms, 4),
|
||||
decode_ms=round(sample.decode_ms, 4),
|
||||
queue_wait_ms=round(sample.queue_wait_ms, 4),
|
||||
prompt_tokens=sample.prompt_tokens,
|
||||
decode_tokens=sample.decode_tokens,
|
||||
text=sample.text,
|
||||
)
|
||||
|
||||
|
||||
def measure_recipe(
|
||||
driver: RecipeDriver,
|
||||
recipe: RecipeSpec,
|
||||
plan: BenchmarkPlan,
|
||||
) -> RecipeMeasurement:
|
||||
"""Load one recipe and run the whole plan against it.
|
||||
|
||||
The driver is closed exactly once, whatever happens, so a recipe that dies at
|
||||
concurrency 4 still releases its weights before the next recipe loads.
|
||||
"""
|
||||
load = driver.load()
|
||||
measurement = RecipeMeasurement(recipe=recipe, load=load)
|
||||
|
||||
try:
|
||||
for _ in range(plan.warmup_requests):
|
||||
try:
|
||||
driver.generate(plan.prompts[0].text, plan.sampling)
|
||||
except Exception: # a failing warmup is reported by the real requests
|
||||
break
|
||||
|
||||
for concurrency in plan.concurrency_levels:
|
||||
# Measure each cell independently and sample while requests are in
|
||||
# flight; post-request probes alone miss transient KV/workspace peaks.
|
||||
memory = _PeakMemory(driver)
|
||||
memory.sample()
|
||||
stop_sampling = threading.Event()
|
||||
sampler = threading.Thread(
|
||||
target=memory.sample_until,
|
||||
args=(stop_sampling,),
|
||||
name=f"recipe-memory-{recipe.id}-c{concurrency}",
|
||||
daemon=True,
|
||||
)
|
||||
requests = [
|
||||
(prompt, repeat)
|
||||
for repeat in range(plan.repeats)
|
||||
for prompt in plan.prompts
|
||||
for _ in range(concurrency)
|
||||
]
|
||||
started = time.monotonic()
|
||||
sampler.start()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
outcomes = list(pool.map(
|
||||
lambda item: _run_request(
|
||||
driver, recipe, item[0], plan.sampling, concurrency, item[1], memory
|
||||
),
|
||||
requests,
|
||||
))
|
||||
finally:
|
||||
stop_sampling.set()
|
||||
sampler.join()
|
||||
wall_ms = (time.monotonic() - started) * 1000
|
||||
|
||||
measurement.outcomes.extend(outcomes)
|
||||
measurement.metrics[concurrency] = summarize_concurrency(
|
||||
outcomes,
|
||||
concurrency=concurrency,
|
||||
wall_ms=wall_ms,
|
||||
peak_rss_bytes=memory.peak_rss,
|
||||
peak_vram_bytes=memory.peak_vram,
|
||||
)
|
||||
finally:
|
||||
driver.close()
|
||||
|
||||
return measurement
|
||||
|
||||
|
||||
def build_report(
|
||||
plan: BenchmarkPlan,
|
||||
measurements: Sequence[RecipeMeasurement],
|
||||
*,
|
||||
host: dict[str, Any],
|
||||
evidence_class: str,
|
||||
provenance: Mapping[str, Any] | None = None,
|
||||
) -> dict:
|
||||
"""Assemble the machine-readable benchmark document.
|
||||
|
||||
``evidence_class`` is one of ``synthetic``, ``local-real`` or
|
||||
``multi-machine-real`` and is never inferred: a report that cannot say how it
|
||||
was produced cannot be trusted by a release gate.
|
||||
"""
|
||||
if evidence_class not in {"synthetic", "local-real", "multi-machine-real"}:
|
||||
raise BenchmarkError(f"unknown evidence class {evidence_class!r}")
|
||||
if evidence_class != "synthetic" and not isinstance(provenance, Mapping):
|
||||
raise BenchmarkError("non-synthetic reports require canonical signed provenance")
|
||||
|
||||
references = [m for m in measurements if m.recipe.is_reference]
|
||||
if len(references) != 1:
|
||||
raise BenchmarkError(
|
||||
f"exactly one reference recipe is required, got {len(references)}"
|
||||
)
|
||||
reference = references[0]
|
||||
if reference.recipe.lane is not Lane.QUALITY:
|
||||
raise BenchmarkError("the reference recipe must sit in the quality lane")
|
||||
|
||||
drift = [
|
||||
compute_drift(measurement, reference).to_dict()
|
||||
for measurement in measurements
|
||||
if measurement is not reference and measurement.available
|
||||
]
|
||||
report = {
|
||||
"schema_version": REPORT_SCHEMA_VERSION,
|
||||
"evidence_class": evidence_class,
|
||||
"plan": plan.to_dict(),
|
||||
"host": host,
|
||||
"reference_recipe_id": reference.recipe.id,
|
||||
"recipes": [measurement.to_dict() for measurement in measurements],
|
||||
"drift": drift,
|
||||
}
|
||||
if provenance is not None:
|
||||
report["provenance"] = dict(provenance)
|
||||
return report
|
||||
|
||||
|
||||
def format_summary(report: dict) -> str:
|
||||
"""Render the human-readable companion to the JSON artifact."""
|
||||
plan = report["plan"]
|
||||
lines = [
|
||||
f"Recipe benchmark {plan['plan_id']} ({report['evidence_class']})",
|
||||
f"model {plan['model_id']}@{plan['model_revision']}",
|
||||
]
|
||||
for entry in report["recipes"]:
|
||||
recipe = entry["recipe"]
|
||||
if not entry["available"]:
|
||||
lines.append(f"{recipe['id']:38} UNAVAILABLE: {entry['unavailable_reason']}")
|
||||
continue
|
||||
artifact_gb = entry["load"]["artifact_bytes"] / 1e9
|
||||
for level, metrics in entry["concurrency"].items():
|
||||
lines.append(
|
||||
f"{recipe['id']:38} [{recipe['lane']:16}] c={level:>2} "
|
||||
f"ttft p50/p95 {metrics['ttft_p50_ms']:8.1f}/{metrics['ttft_p95_ms']:8.1f} ms; "
|
||||
f"prefill {metrics['prefill_tokens_per_sec']:7.1f} tok/s; "
|
||||
f"decode {metrics['decode_tokens_per_sec']:6.1f} tok/s; "
|
||||
f"aggregate {metrics['aggregate_decode_tokens_per_sec']:7.1f} tok/s; "
|
||||
f"rss {metrics['peak_rss_bytes'] / 1e9:5.2f} GB; "
|
||||
f"vram {metrics['peak_vram_bytes'] / 1e9:5.2f} GB; "
|
||||
f"artifact {artifact_gb:5.2f} GB; failures {metrics['failures']}"
|
||||
)
|
||||
for entry in report["drift"]:
|
||||
tag = "advisory" if entry["advisory"] else "gated"
|
||||
lines.append(
|
||||
f"drift {entry['recipe_id']:32} vs {entry['reference_id']:28} "
|
||||
f"exact {entry['exact_match_rate']:.2f}; similarity {entry['mean_similarity']:.3f} ({tag})"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run the controlled safetensors-versus-GGUF recipe benchmark"
|
||||
)
|
||||
parser.add_argument("--config", type=Path, required=True, help="benchmark configuration JSON")
|
||||
parser.add_argument(
|
||||
"--profile",
|
||||
choices=("contract-v1", "gpu-diagnostic"),
|
||||
default="contract-v1",
|
||||
help="validation and provenance profile (GPU diagnostics are not v1-eligible)",
|
||||
)
|
||||
parser.add_argument("--json-out", type=Path, help="write the JSON report to this path")
|
||||
parser.add_argument("--summary-out", type=Path, help="write the text summary to this path")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
from .recipe_drivers import ( # heavy runtimes: import on demand
|
||||
run_configured_benchmark,
|
||||
run_configured_gpu_diagnostic,
|
||||
)
|
||||
|
||||
runner = (
|
||||
run_configured_gpu_diagnostic
|
||||
if args.profile == "gpu-diagnostic"
|
||||
else run_configured_benchmark
|
||||
)
|
||||
report = runner(json.loads(args.config.read_text(encoding="utf-8")))
|
||||
summary = format_summary(report)
|
||||
if args.json_out:
|
||||
args.json_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
if args.summary_out:
|
||||
args.summary_out.write_text(summary + "\n", encoding="utf-8")
|
||||
print(summary)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover - CLI entry point
|
||||
raise SystemExit(main())
|
||||
855
packages/node/meshnet_node/recipe_drivers.py
Normal file
855
packages/node/meshnet_node/recipe_drivers.py
Normal file
@@ -0,0 +1,855 @@
|
||||
"""Real runtime drivers for the recipe benchmark.
|
||||
|
||||
This module is the only place that imports torch, transformers, or spawns a
|
||||
llama.cpp server, and :mod:`meshnet_node.recipe_benchmark` imports it lazily.
|
||||
That keeps the default test suite deterministic, GPU-free and download-free
|
||||
while the real evidence runs through exactly the same measurement core.
|
||||
|
||||
Fairness is the whole point of a baseline, so both drivers are held to the same
|
||||
rules:
|
||||
|
||||
* They are handed a **pre-formatted prompt string**. Neither applies a chat
|
||||
template, because a template applied twice — or differently — by two runtimes
|
||||
would show up as a speed and drift difference that has nothing to do with the
|
||||
runtime.
|
||||
* They are given the **same CPU thread budget**, so the comparison measures
|
||||
kernels rather than how many cores each runtime felt entitled to take.
|
||||
* They report the runtime's **own prefill/decode split** where it has one, and
|
||||
say so honestly where it does not.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import socket
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from cryptography.hazmat.primitives import serialization
|
||||
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
|
||||
|
||||
from .performance_contract import (
|
||||
PROVENANCE_SCHEMA_VERSION,
|
||||
REAL_REPORT_PRODUCER,
|
||||
_canonical_sha256,
|
||||
report_signing_payload,
|
||||
)
|
||||
from .recipe_benchmark import (
|
||||
BenchmarkError,
|
||||
BenchmarkPlan,
|
||||
GenerationSample,
|
||||
Lane,
|
||||
LoadStats,
|
||||
PromptSpec,
|
||||
RecipeSpec,
|
||||
SamplingPolicy,
|
||||
build_report,
|
||||
measure_recipe,
|
||||
)
|
||||
|
||||
REAL_INFERENCE_ENV = "MESHNET_ENABLE_REAL_INFERENCE_TESTS"
|
||||
EVIDENCE_SIGNING_KEY_ENV = "MESHNET_EVIDENCE_SIGNING_KEY"
|
||||
CONTRACT_V1_PROFILE = "contract-v1"
|
||||
GPU_DIAGNOSTIC_PROFILE = "gpu-diagnostic"
|
||||
GPU_DIAGNOSTIC_REPORT_PRODUCER = "meshnet_node.recipe_drivers.run_configured_gpu_diagnostic/v1"
|
||||
|
||||
|
||||
def real_inference_enabled() -> bool:
|
||||
"""Real runtimes stay off unless the operator opts in explicitly."""
|
||||
return os.environ.get(REAL_INFERENCE_ENV) == "1"
|
||||
|
||||
|
||||
def require_real_inference() -> None:
|
||||
if not real_inference_enabled():
|
||||
raise BenchmarkError(
|
||||
f"real model execution is opt-in: set {REAL_INFERENCE_ENV}=1 to run this benchmark"
|
||||
)
|
||||
|
||||
|
||||
def _load_evidence_signing_key() -> Ed25519PrivateKey:
|
||||
raw_path = os.environ.get(EVIDENCE_SIGNING_KEY_ENV)
|
||||
if not raw_path:
|
||||
raise BenchmarkError(
|
||||
f"real evidence requires {EVIDENCE_SIGNING_KEY_ENV} to name an Ed25519 private key"
|
||||
)
|
||||
path = Path(raw_path).expanduser().resolve(strict=True)
|
||||
if os.name != "nt" and stat.S_IMODE(path.stat().st_mode) != 0o600:
|
||||
raise BenchmarkError("evidence signing key must have mode 0600")
|
||||
key = serialization.load_pem_private_key(path.read_bytes(), password=None)
|
||||
if not isinstance(key, Ed25519PrivateKey):
|
||||
raise BenchmarkError("evidence signing key must be Ed25519")
|
||||
return key
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _sign_report(report: dict[str, Any], key: Ed25519PrivateKey) -> None:
|
||||
public_key = key.public_key().public_bytes(
|
||||
serialization.Encoding.Raw, serialization.PublicFormat.Raw
|
||||
)
|
||||
report["provenance"]["signer_public_key_sha256"] = hashlib.sha256(
|
||||
public_key
|
||||
).hexdigest()
|
||||
report["provenance"]["signature"] = base64.b64encode(
|
||||
key.sign(report_signing_payload(report))
|
||||
).decode("ascii")
|
||||
|
||||
|
||||
def _process_rss(pid: int | None = None) -> int:
|
||||
"""Resident bytes for a process and its children, or 0 when unobservable."""
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
return 0
|
||||
try:
|
||||
process = psutil.Process(pid) if pid else psutil.Process()
|
||||
total = process.memory_info().rss
|
||||
for child in process.children(recursive=True):
|
||||
try:
|
||||
total += child.memory_info().rss
|
||||
except psutil.Error:
|
||||
continue
|
||||
return int(total)
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _directory_bytes(path: Path) -> int:
|
||||
if path.is_file():
|
||||
return path.stat().st_size
|
||||
return sum(entry.stat().st_size for entry in path.rglob("*") if entry.is_file())
|
||||
|
||||
|
||||
def _artifact_sha256(path: Path) -> str:
|
||||
"""Hash an artifact file or a deterministic directory content manifest.
|
||||
|
||||
A file uses the ordinary SHA-256 digest. A directory hashes each sorted
|
||||
relative path, resolved file size, and file bytes, so tokenizer/config drift
|
||||
cannot hide behind a weight-only digest.
|
||||
"""
|
||||
digest = hashlib.sha256()
|
||||
if path.is_file():
|
||||
entries = [(None, path)]
|
||||
else:
|
||||
entries = [
|
||||
(entry.relative_to(path).as_posix(), entry)
|
||||
for entry in sorted(path.rglob("*"))
|
||||
if entry.is_file()
|
||||
]
|
||||
if not entries:
|
||||
raise BenchmarkError(f"artifact directory is empty: {path}")
|
||||
|
||||
for relative, entry in entries:
|
||||
if relative is not None:
|
||||
encoded = relative.encode("utf-8")
|
||||
digest.update(len(encoded).to_bytes(8, "big"))
|
||||
digest.update(encoded)
|
||||
digest.update(entry.stat().st_size.to_bytes(8, "big"))
|
||||
with entry.open("rb") as stream:
|
||||
while chunk := stream.read(8 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _host_manifest(config: Mapping[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""Capture non-secret host facts with the report rather than trusting prose."""
|
||||
manifest: dict[str, Any] = {
|
||||
"hostname": socket.gethostname(),
|
||||
"platform": platform.platform(),
|
||||
"python": sys.version.split()[0],
|
||||
"cpu_count": os.cpu_count(),
|
||||
}
|
||||
try:
|
||||
import torch
|
||||
import transformers
|
||||
|
||||
manifest["torch_version"] = torch.__version__
|
||||
manifest["transformers_version"] = transformers.__version__
|
||||
manifest["cuda_available"] = bool(torch.cuda.is_available())
|
||||
if torch.cuda.is_available():
|
||||
manifest["accelerator_name"] = torch.cuda.get_device_name(0)
|
||||
manifest["accelerator_runtime"] = getattr(torch.version, "cuda", None) or getattr(
|
||||
torch.version, "hip", None
|
||||
)
|
||||
except ImportError:
|
||||
manifest["torch_version"] = None
|
||||
|
||||
llama_identities: dict[str, dict[str, str]] = {}
|
||||
for spec in (config or {}).get("recipes", ()):
|
||||
driver = spec.get("driver", {})
|
||||
if driver.get("type") != "llama-cpp-server":
|
||||
continue
|
||||
binary = Path(driver["binary"]).resolve(strict=True)
|
||||
key = str(binary)
|
||||
if key in llama_identities:
|
||||
continue
|
||||
version_result = subprocess.run(
|
||||
[str(binary), "--version"],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
llama_identities[key] = {
|
||||
"sha256": _artifact_sha256(binary),
|
||||
"version": " | ".join(version_result.stdout.strip().splitlines()),
|
||||
}
|
||||
if llama_identities:
|
||||
manifest["llama_server_identities"] = llama_identities
|
||||
return manifest
|
||||
|
||||
|
||||
def _validate_config(
|
||||
config: Mapping[str, Any], *, profile: str = CONTRACT_V1_PROFILE
|
||||
) -> None:
|
||||
"""Reject comparisons that mix models, artifacts, devices, or budgets."""
|
||||
if profile not in {CONTRACT_V1_PROFILE, GPU_DIAGNOSTIC_PROFILE}:
|
||||
raise BenchmarkError(f"unknown benchmark validation profile {profile!r}")
|
||||
try:
|
||||
plan = config["plan"]
|
||||
root = Path(config["artifact_storage_root"]).resolve(strict=True)
|
||||
recipes = config["recipes"]
|
||||
except (KeyError, TypeError, OSError) as exc:
|
||||
raise BenchmarkError(
|
||||
"benchmark config needs an existing artifact_storage_root, plan, and recipes"
|
||||
) from exc
|
||||
if not root.is_absolute() or root == Path("/home") or Path("/home") in root.parents:
|
||||
raise BenchmarkError("model artifacts must use configured mounted-drive storage, never /home")
|
||||
if not isinstance(recipes, list) or not recipes:
|
||||
raise BenchmarkError("benchmark config needs at least one recipe")
|
||||
|
||||
sampling = plan.get("sampling", {})
|
||||
if (
|
||||
float(sampling.get("temperature", 0.0)) != 0.0
|
||||
or int(sampling.get("top_k", 1)) != 1
|
||||
or float(sampling.get("top_p", 1.0)) != 1.0
|
||||
):
|
||||
raise BenchmarkError("the quality comparison requires greedy sampling")
|
||||
|
||||
if len(plan.get("prompts", ())) < 3 or int(plan.get("repeats", 0)) < 3:
|
||||
raise BenchmarkError("contract-grade evidence requires at least 3 prompts and 3 repeats")
|
||||
if int(plan.get("warmup_requests", 0)) < 1:
|
||||
raise BenchmarkError("contract-grade evidence requires at least one warmup")
|
||||
if int(sampling.get("max_output_tokens", 0)) < 32:
|
||||
raise BenchmarkError("contract-grade evidence requires at least 32 output tokens")
|
||||
|
||||
thread_budgets: set[int] = set()
|
||||
max_concurrency = max(int(level) for level in plan.get("concurrency_levels", (1, 4)))
|
||||
for spec in recipes:
|
||||
if spec.get("source_model_id") != plan.get("model_id"):
|
||||
raise BenchmarkError("every recipe must declare the plan's exact source_model_id")
|
||||
if spec.get("source_model_revision") != plan.get("model_revision"):
|
||||
raise BenchmarkError("every recipe must declare the plan's exact source_model_revision")
|
||||
|
||||
digest = spec.get("artifact_sha256", "")
|
||||
if not isinstance(digest, str) or re.fullmatch(r"[0-9a-f]{64}", digest) is None:
|
||||
raise BenchmarkError("every recipe must declare a lowercase SHA-256 artifact digest")
|
||||
artifact = Path(spec.get("artifact_path", "")).resolve(strict=True)
|
||||
if artifact != root and root not in artifact.parents:
|
||||
raise BenchmarkError("every model artifact must be beneath artifact_storage_root")
|
||||
actual_digest = _artifact_sha256(artifact)
|
||||
if not hmac.compare_digest(digest, actual_digest):
|
||||
raise BenchmarkError(
|
||||
f"artifact digest mismatch for {spec.get('id', '<unknown>')}: "
|
||||
f"declared {digest}, measured {actual_digest}"
|
||||
)
|
||||
|
||||
driver = spec.get("driver")
|
||||
if not isinstance(driver, Mapping):
|
||||
raise BenchmarkError("every recipe must declare a driver block")
|
||||
if "artifact_sha256" in driver:
|
||||
raise BenchmarkError(
|
||||
"driver artifact_sha256 is forbidden; the validated recipe digest is authoritative"
|
||||
)
|
||||
kind = driver.get("type")
|
||||
if kind == "transformers":
|
||||
driver_artifact = Path(driver.get("model_path", "")).resolve(strict=True)
|
||||
elif kind == "llama-cpp-server":
|
||||
driver_artifact = Path(driver.get("gguf_path", "")).resolve(strict=True)
|
||||
binary = Path(driver.get("binary", "")).resolve(strict=True)
|
||||
binary_digest = driver.get("binary_sha256", "")
|
||||
if (
|
||||
not isinstance(binary_digest, str)
|
||||
or re.fullmatch(r"[0-9a-f]{64}", binary_digest) is None
|
||||
or not hmac.compare_digest(binary_digest, _artifact_sha256(binary))
|
||||
):
|
||||
raise BenchmarkError("llama.cpp binary SHA-256 mismatch")
|
||||
if int(driver.get("n_parallel", max_concurrency)) < max_concurrency:
|
||||
raise BenchmarkError("llama.cpp parallel slots must cover maximum concurrency")
|
||||
driver_device = driver.get("device", "cpu")
|
||||
gpu_layers = int(driver.get("n_gpu_layers", 0))
|
||||
if profile == CONTRACT_V1_PROFILE and (
|
||||
driver_device != "cpu" or gpu_layers != 0
|
||||
):
|
||||
raise BenchmarkError(
|
||||
"v1 benchmark supports CPU-only llama.cpp until process VRAM is measurable"
|
||||
)
|
||||
if profile == GPU_DIAGNOSTIC_PROFILE and (
|
||||
driver_device != "cuda" or gpu_layers <= 0
|
||||
):
|
||||
raise BenchmarkError(
|
||||
"GPU diagnostic requires CUDA/ROCm llama.cpp with positive n_gpu_layers"
|
||||
)
|
||||
else:
|
||||
raise BenchmarkError(f"unknown driver type {kind!r}")
|
||||
if driver_artifact != artifact:
|
||||
raise BenchmarkError("driver artifact path must match the hashed recipe artifact")
|
||||
if driver.get("device", "cpu") != spec.get("device"):
|
||||
raise BenchmarkError("recipe and driver must declare the same device")
|
||||
if profile == CONTRACT_V1_PROFILE and spec.get("device") != "cpu":
|
||||
raise BenchmarkError("contract-v1 requires every recipe to run on CPU")
|
||||
if profile == GPU_DIAGNOSTIC_PROFILE and spec.get("device") != "cuda":
|
||||
raise BenchmarkError("GPU diagnostic requires every recipe to declare device 'cuda'")
|
||||
thread_budgets.add(int(driver.get("threads", 8)))
|
||||
|
||||
host = config.get("host", {})
|
||||
if not isinstance(host, Mapping):
|
||||
raise BenchmarkError("benchmark host metadata must be an object")
|
||||
if profile == GPU_DIAGNOSTIC_PROFILE and host.get(
|
||||
"benchmark_lane"
|
||||
) != "rocm-gpu-diagnostic":
|
||||
raise BenchmarkError("GPU diagnostic requires the rocm-gpu-diagnostic host marker")
|
||||
|
||||
if len(thread_budgets) != 1:
|
||||
raise BenchmarkError("every recipe must use the same CPU thread budget")
|
||||
|
||||
|
||||
class TransformersDriver:
|
||||
"""The current Transformers/safetensors recipe: the correctness reference.
|
||||
|
||||
Generation is a hand-written prefill-then-decode loop rather than
|
||||
``model.generate`` because the benchmark needs the two phases separated: one
|
||||
forward over the prompt gives an exact prefill time and TTFT, and the cached
|
||||
single-token steps that follow give an exact decode rate. ``generate`` would
|
||||
hand back one blended number.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: str,
|
||||
*,
|
||||
artifact_sha256: str | None = None,
|
||||
device: str = "cpu",
|
||||
dtype: str = "bfloat16",
|
||||
threads: int = 8,
|
||||
) -> None:
|
||||
self.model_path = Path(model_path)
|
||||
self.artifact_sha256 = artifact_sha256
|
||||
self.device = device
|
||||
self.dtype = dtype
|
||||
self.threads = threads
|
||||
self._model: Any = None
|
||||
self._tokenizer: Any = None
|
||||
self._torch: Any = None
|
||||
self._rss_baseline = 0
|
||||
|
||||
def load(self) -> LoadStats:
|
||||
if self.artifact_sha256 is not None:
|
||||
measured_artifact_sha256 = _artifact_sha256(self.model_path)
|
||||
if not hmac.compare_digest(self.artifact_sha256, measured_artifact_sha256):
|
||||
raise BenchmarkError("Transformers artifact changed after config validation")
|
||||
self._rss_baseline = _process_rss()
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
self._torch = torch
|
||||
torch.set_num_threads(self.threads)
|
||||
torch.manual_seed(0)
|
||||
|
||||
started = time.monotonic()
|
||||
self._tokenizer = AutoTokenizer.from_pretrained(
|
||||
str(self.model_path), local_files_only=True
|
||||
)
|
||||
self._model = AutoModelForCausalLM.from_pretrained(
|
||||
str(self.model_path),
|
||||
dtype=getattr(torch, self.dtype),
|
||||
local_files_only=True,
|
||||
)
|
||||
self._model.to(self.device)
|
||||
self._model.eval()
|
||||
load_ms = (time.monotonic() - started) * 1000
|
||||
|
||||
return LoadStats(
|
||||
artifact_bytes=_directory_bytes(self.model_path),
|
||||
load_ms=round(load_ms, 4),
|
||||
rss_bytes=max(0, _process_rss() - self._rss_baseline),
|
||||
vram_bytes=self._vram_bytes(),
|
||||
backend_detail=(
|
||||
f"torch {torch.__version__}; dtype {self.dtype}; "
|
||||
f"device {self.device}; intra-op threads {self.threads}"
|
||||
),
|
||||
)
|
||||
|
||||
def _vram_bytes(self) -> int:
|
||||
torch = self._torch
|
||||
if torch is None or self.device == "cpu":
|
||||
return 0
|
||||
try:
|
||||
if torch.cuda.is_available():
|
||||
return int(torch.cuda.max_memory_allocated())
|
||||
except Exception:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
|
||||
if self._model is None:
|
||||
raise BenchmarkError("TransformersDriver.generate called before load()")
|
||||
torch = self._torch
|
||||
|
||||
# add_special_tokens=False: the plan owns the prompt format, and the
|
||||
# llama.cpp recipe is given the identical string.
|
||||
encoded = self._tokenizer(prompt, return_tensors="pt", add_special_tokens=False)
|
||||
input_ids = encoded["input_ids"].to(self.device)
|
||||
prompt_tokens = int(input_ids.shape[-1])
|
||||
eos_ids = {self._tokenizer.eos_token_id} | set(
|
||||
getattr(self._model.generation_config, "eos_token_id", None) or []
|
||||
if isinstance(getattr(self._model.generation_config, "eos_token_id", None), list)
|
||||
else []
|
||||
)
|
||||
eos_ids.discard(None)
|
||||
|
||||
started = time.monotonic()
|
||||
with torch.inference_mode():
|
||||
outputs = self._model(input_ids=input_ids, use_cache=True)
|
||||
past = outputs.past_key_values
|
||||
next_id = self._select(outputs.logits[:, -1, :], sampling)
|
||||
ttft_ms = (time.monotonic() - started) * 1000
|
||||
|
||||
token_ids = [int(next_id.item())]
|
||||
decode_started = time.monotonic()
|
||||
while len(token_ids) < sampling.max_output_tokens and token_ids[-1] not in eos_ids:
|
||||
outputs = self._model(
|
||||
input_ids=next_id.view(1, 1), past_key_values=past, use_cache=True
|
||||
)
|
||||
past = outputs.past_key_values
|
||||
next_id = self._select(outputs.logits[:, -1, :], sampling)
|
||||
token_ids.append(int(next_id.item()))
|
||||
decode_ms = (time.monotonic() - decode_started) * 1000
|
||||
|
||||
total_ms = (time.monotonic() - started) * 1000
|
||||
emitted = [token for token in token_ids if token not in eos_ids]
|
||||
return GenerationSample(
|
||||
text=self._tokenizer.decode(emitted, skip_special_tokens=True),
|
||||
prompt_tokens=prompt_tokens,
|
||||
# The first token is produced by the prefill forward, so the decode
|
||||
# rate must not be credited with it.
|
||||
decode_tokens=max(0, len(token_ids) - 1),
|
||||
ttft_ms=ttft_ms,
|
||||
prefill_ms=ttft_ms,
|
||||
decode_ms=decode_ms,
|
||||
total_ms=total_ms,
|
||||
)
|
||||
|
||||
def _select(self, logits: Any, sampling: SamplingPolicy) -> Any:
|
||||
if sampling.temperature > 0:
|
||||
raise BenchmarkError(
|
||||
"this benchmark is greedy-only: sampling noise is indistinguishable from "
|
||||
"quantization drift, which is precisely what the quality lane must isolate"
|
||||
)
|
||||
return logits.argmax(dim=-1)
|
||||
|
||||
def memory_probe(self) -> tuple[int, int]:
|
||||
return max(0, _process_rss() - self._rss_baseline), self._vram_bytes()
|
||||
|
||||
def close(self) -> None:
|
||||
self._model = None
|
||||
self._tokenizer = None
|
||||
if self._torch is not None:
|
||||
import gc
|
||||
|
||||
gc.collect()
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
return int(probe.getsockname()[1])
|
||||
|
||||
|
||||
def _gpu_offload_evidence(log_text: str, requested_layers: int) -> str:
|
||||
"""Extract measured ROCm device and layer placement from llama-server logs."""
|
||||
device_matches = re.findall(
|
||||
r"-\s+(ROCm\d+)\s+:\s+(.+?)\s+\(\d+\s+MiB", log_text
|
||||
)
|
||||
offload_matches = re.findall(
|
||||
r"offloaded\s+(\d+)/(\d+)\s+layers\s+to\s+GPU", log_text
|
||||
)
|
||||
if not device_matches:
|
||||
raise BenchmarkError("GPU diagnostic found no measured ROCm device in llama-server logs")
|
||||
if not offload_matches:
|
||||
raise BenchmarkError("GPU diagnostic found no measured layer offload in llama-server logs")
|
||||
|
||||
backend, device_name = device_matches[-1]
|
||||
offloaded, total = (int(value) for value in offload_matches[-1])
|
||||
required = min(requested_layers, total)
|
||||
if requested_layers <= 0 or offloaded < required:
|
||||
raise BenchmarkError(
|
||||
f"GPU diagnostic requested {requested_layers} layers but measured "
|
||||
f"only {offloaded}/{total} offloaded"
|
||||
)
|
||||
return (
|
||||
f"measured accelerator {backend}: {device_name}; "
|
||||
f"measured offload {offloaded}/{total} layers"
|
||||
)
|
||||
|
||||
|
||||
def _gpu_layer_config_detail(device: str, layers: int) -> str:
|
||||
# Immutable v1 pins the historical CPU wording exactly. GPU diagnostics use
|
||||
# the clearer requested/measured distinction without changing v1 identity.
|
||||
return f"gpu layers {layers}" if device == "cpu" else f"requested gpu layers {layers}"
|
||||
|
||||
|
||||
class LlamaCppServerDriver:
|
||||
"""The whole-model llama.cpp/GGUF recipe, driven through ``llama-server``.
|
||||
|
||||
``llama-server`` is used rather than an in-process binding because it is the
|
||||
shape llama.cpp is actually deployed in and the only one that offers
|
||||
continuous batching across parallel slots — which is the runtime property
|
||||
this project cares about most. It also reports its own prefill/decode
|
||||
timings per request, so the decode rate is the runtime's own number and not
|
||||
an inference drawn from a client-side stopwatch.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
binary: str,
|
||||
gguf_path: str,
|
||||
*,
|
||||
binary_sha256: str,
|
||||
artifact_sha256: str | None = None,
|
||||
device: str = "cpu",
|
||||
threads: int = 8,
|
||||
n_parallel: int = 4,
|
||||
context_per_slot: int = 1024,
|
||||
n_gpu_layers: int = 0,
|
||||
startup_timeout_s: float = 120.0,
|
||||
) -> None:
|
||||
self.binary = Path(binary)
|
||||
self.binary_sha256 = binary_sha256
|
||||
self.gguf_path = Path(gguf_path)
|
||||
self.artifact_sha256 = artifact_sha256
|
||||
self.device = device
|
||||
self.threads = threads
|
||||
self.n_parallel = n_parallel
|
||||
self.context_per_slot = context_per_slot
|
||||
self.n_gpu_layers = n_gpu_layers
|
||||
self.startup_timeout_s = startup_timeout_s
|
||||
self._process: subprocess.Popen | None = None
|
||||
self._port = 0
|
||||
self._log: Any = None
|
||||
|
||||
@property
|
||||
def _url(self) -> str:
|
||||
return f"http://127.0.0.1:{self._port}"
|
||||
|
||||
def _log_text(self) -> str:
|
||||
if self._log is None:
|
||||
return ""
|
||||
try:
|
||||
size = min(os.fstat(self._log.fileno()).st_size, 8 * 1024 * 1024)
|
||||
return os.pread(self._log.fileno(), size, 0).decode("utf-8", errors="replace")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
def _log_excerpt(self) -> str:
|
||||
return self._log_text()[-4096:].strip()
|
||||
|
||||
def load(self) -> LoadStats:
|
||||
if not self.binary.exists():
|
||||
raise BenchmarkError(f"llama-server binary not found at {self.binary}")
|
||||
if not self.gguf_path.exists():
|
||||
raise BenchmarkError(f"GGUF artifact not found at {self.gguf_path}")
|
||||
if self.artifact_sha256 is not None:
|
||||
measured_artifact_sha256 = _artifact_sha256(self.gguf_path)
|
||||
if not hmac.compare_digest(self.artifact_sha256, measured_artifact_sha256):
|
||||
raise BenchmarkError("GGUF artifact changed after config validation")
|
||||
measured_binary_sha256 = _artifact_sha256(self.binary)
|
||||
if not hmac.compare_digest(self.binary_sha256, measured_binary_sha256):
|
||||
raise BenchmarkError("llama-server binary changed after config validation")
|
||||
version = " | ".join(
|
||||
subprocess.run(
|
||||
[str(self.binary), "--version"],
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=10,
|
||||
).stdout.strip().splitlines()
|
||||
)
|
||||
|
||||
self._port = _free_port()
|
||||
command = [str(self.binary)]
|
||||
if self.device == "cuda":
|
||||
# Debug verbosity is required to capture measured device placement
|
||||
# and the offloaded/total layer count in signed diagnostics.
|
||||
command.extend(["-lv", "5"])
|
||||
command.extend([
|
||||
"--model", str(self.gguf_path),
|
||||
"--host", "127.0.0.1",
|
||||
"--port", str(self._port),
|
||||
"--threads", str(self.threads),
|
||||
"--parallel", str(self.n_parallel),
|
||||
# Every slot must hold a whole request, so the pool is sized for the
|
||||
# worst case rather than letting llama.cpp silently truncate context.
|
||||
"--ctx-size", str(self.context_per_slot * self.n_parallel),
|
||||
"--n-gpu-layers", str(self.n_gpu_layers),
|
||||
"--no-webui",
|
||||
])
|
||||
started = time.monotonic()
|
||||
self._log = tempfile.TemporaryFile(mode="w+b")
|
||||
self._process = subprocess.Popen(
|
||||
command, stdout=self._log, stderr=subprocess.STDOUT
|
||||
)
|
||||
self._await_health(started)
|
||||
load_ms = (time.monotonic() - started) * 1000
|
||||
gpu_evidence = ""
|
||||
if self.device == "cuda":
|
||||
gpu_evidence = _gpu_offload_evidence(self._log_text(), self.n_gpu_layers)
|
||||
|
||||
return LoadStats(
|
||||
artifact_bytes=self.gguf_path.stat().st_size,
|
||||
load_ms=round(load_ms, 4),
|
||||
rss_bytes=_process_rss(self._process.pid),
|
||||
vram_bytes=0,
|
||||
backend_detail=(
|
||||
f"{version}; binary sha256 {measured_binary_sha256}; "
|
||||
f"threads {self.threads}; parallel slots {self.n_parallel}; "
|
||||
f"ctx/slot {self.context_per_slot}; "
|
||||
f"{_gpu_layer_config_detail(self.device, self.n_gpu_layers)}"
|
||||
+ (f"; {gpu_evidence}" if gpu_evidence else "")
|
||||
),
|
||||
)
|
||||
|
||||
def _await_health(self, started: float) -> None:
|
||||
while time.monotonic() - started < self.startup_timeout_s:
|
||||
if self._process is not None and self._process.poll() is not None:
|
||||
raise BenchmarkError(
|
||||
f"llama-server exited with code {self._process.returncode} during startup; "
|
||||
f"log tail: {self._log_excerpt()}"
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(f"{self._url}/health", timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except (urllib.error.URLError, OSError):
|
||||
time.sleep(0.25)
|
||||
raise BenchmarkError(
|
||||
f"llama-server did not become healthy within {self.startup_timeout_s:.0f}s; "
|
||||
f"log tail: {self._log_excerpt()}"
|
||||
)
|
||||
|
||||
def generate(self, prompt: str, sampling: SamplingPolicy) -> GenerationSample:
|
||||
if self._process is None:
|
||||
raise BenchmarkError("LlamaCppServerDriver.generate called before load()")
|
||||
if sampling.temperature > 0:
|
||||
raise BenchmarkError("this benchmark is greedy-only; see TransformersDriver._select")
|
||||
|
||||
body = json.dumps({
|
||||
"prompt": prompt,
|
||||
"n_predict": sampling.max_output_tokens,
|
||||
"temperature": 0.0,
|
||||
"top_k": 1,
|
||||
"top_p": 1.0,
|
||||
"seed": sampling.seed,
|
||||
# Prompt cache reuse across repeats would measure the cache, not the
|
||||
# prefill, and the safetensors recipe has no equivalent.
|
||||
"cache_prompt": False,
|
||||
"stream": True,
|
||||
}).encode()
|
||||
request = urllib.request.Request(
|
||||
f"{self._url}/completion", data=body,
|
||||
headers={"Content-Type": "application/json"}, method="POST",
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
chunks: list[str] = []
|
||||
timings: Mapping[str, Any] = {}
|
||||
with urllib.request.urlopen(request, timeout=600) as response:
|
||||
for raw in response:
|
||||
line = raw.decode("utf-8").strip()
|
||||
if not line.startswith("data:"):
|
||||
continue
|
||||
payload = json.loads(line[len("data:"):].strip())
|
||||
content = payload.get("content", "")
|
||||
chunks.append(content)
|
||||
if payload.get("stop"):
|
||||
timings = payload.get("timings") or {}
|
||||
total_ms = (time.monotonic() - started) * 1000
|
||||
|
||||
if not timings:
|
||||
raise BenchmarkError("llama-server returned no timings; cannot report an honest split")
|
||||
|
||||
prefill_ms = float(timings.get("prompt_ms", 0.0))
|
||||
decode_ms = float(timings.get("predicted_ms", 0.0))
|
||||
return GenerationSample(
|
||||
text="".join(chunks),
|
||||
prompt_tokens=int(timings.get("prompt_n", 0)),
|
||||
# llama.cpp starts predicted_ms after sampling the first token while
|
||||
# predicted_n includes it. Exclude that token to match the
|
||||
# Transformers inter-token decode metric.
|
||||
decode_tokens=max(0, int(timings.get("predicted_n", 0)) - 1),
|
||||
# Use the runtime's prompt/first-token timing, matching the
|
||||
# in-process Transformers boundary. HTTP/SSE and slot delay remain
|
||||
# represented by total latency and queue_wait_ms.
|
||||
ttft_ms=prefill_ms,
|
||||
prefill_ms=prefill_ms,
|
||||
decode_ms=decode_ms,
|
||||
total_ms=total_ms,
|
||||
# Whatever the wall clock saw but the runtime did not attribute to
|
||||
# compute is time this request spent waiting for a slot.
|
||||
queue_wait_ms=max(0.0, total_ms - prefill_ms - decode_ms),
|
||||
)
|
||||
|
||||
def memory_probe(self) -> tuple[int, int]:
|
||||
if self._process is None:
|
||||
return 0, 0
|
||||
return _process_rss(self._process.pid), 0
|
||||
|
||||
def close(self) -> None:
|
||||
if self._process is not None:
|
||||
if self._process.poll() is None:
|
||||
self._process.terminate()
|
||||
try:
|
||||
self._process.wait(timeout=20)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._process.kill()
|
||||
self._process.wait(timeout=10)
|
||||
self._process = None
|
||||
if self._log is not None:
|
||||
self._log.close()
|
||||
self._log = None
|
||||
|
||||
|
||||
def build_driver(spec: Mapping[str, Any], plan: BenchmarkPlan) -> RecipeDriverBundle:
|
||||
"""Construct the driver named by a recipe's ``driver`` block."""
|
||||
driver_spec = dict(spec["driver"])
|
||||
kind = driver_spec.pop("type")
|
||||
driver_spec["artifact_sha256"] = spec.get("artifact_sha256")
|
||||
if kind == "transformers":
|
||||
return TransformersDriver(**driver_spec)
|
||||
if kind == "llama-cpp-server":
|
||||
driver_spec.setdefault("n_parallel", max(plan.concurrency_levels))
|
||||
return LlamaCppServerDriver(**driver_spec)
|
||||
raise BenchmarkError(f"unknown driver type {kind!r}")
|
||||
|
||||
|
||||
RecipeDriverBundle = Any # a RecipeDriver; named for readability at the call site
|
||||
|
||||
|
||||
def _plan_from_config(config: Mapping[str, Any]) -> BenchmarkPlan:
|
||||
raw = config["plan"]
|
||||
return BenchmarkPlan(
|
||||
plan_id=raw["plan_id"],
|
||||
model_id=raw["model_id"],
|
||||
model_revision=raw["model_revision"],
|
||||
prompts=tuple(PromptSpec(**prompt) for prompt in raw["prompts"]),
|
||||
sampling=SamplingPolicy(**raw.get("sampling", {})),
|
||||
concurrency_levels=tuple(raw.get("concurrency_levels", (1, 4))),
|
||||
repeats=int(raw.get("repeats", 1)),
|
||||
warmup_requests=int(raw.get("warmup_requests", 1)),
|
||||
)
|
||||
|
||||
|
||||
def _recipe_from_config(spec: Mapping[str, Any]) -> RecipeSpec:
|
||||
return RecipeSpec(
|
||||
id=spec["id"],
|
||||
runtime=spec["runtime"],
|
||||
weight_format=spec["weight_format"],
|
||||
weight_quantization=spec["weight_quantization"],
|
||||
lane=Lane(spec["lane"]),
|
||||
device=spec["device"],
|
||||
artifact_path=spec.get("artifact_path", ""),
|
||||
source_model_id=spec.get("source_model_id", ""),
|
||||
source_model_revision=spec.get("source_model_revision", ""),
|
||||
artifact_sha256=spec.get("artifact_sha256", ""),
|
||||
is_reference=bool(spec.get("is_reference", False)),
|
||||
notes=spec.get("notes", ""),
|
||||
)
|
||||
|
||||
|
||||
def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
|
||||
"""Run contract-v1 evidence through the CPU-only, fail-closed profile."""
|
||||
return _run_profiled_benchmark(config, profile=CONTRACT_V1_PROFILE)
|
||||
|
||||
|
||||
def run_configured_gpu_diagnostic(config: Mapping[str, Any]) -> dict:
|
||||
"""Run signed ROCm diagnostics that are intentionally ineligible for v1."""
|
||||
return _run_profiled_benchmark(config, profile=GPU_DIAGNOSTIC_PROFILE)
|
||||
|
||||
|
||||
def _producer_for_profile(profile: str) -> str:
|
||||
if profile == CONTRACT_V1_PROFILE:
|
||||
return REAL_REPORT_PRODUCER
|
||||
if profile == GPU_DIAGNOSTIC_PROFILE:
|
||||
return GPU_DIAGNOSTIC_REPORT_PRODUCER
|
||||
raise BenchmarkError(f"unknown benchmark validation profile {profile!r}")
|
||||
|
||||
|
||||
def _run_profiled_benchmark(config: Mapping[str, Any], *, profile: str) -> dict:
|
||||
"""Validate a closed profile and derive its producer before measurement."""
|
||||
require_real_inference()
|
||||
_validate_config(config, profile=profile)
|
||||
producer = _producer_for_profile(profile)
|
||||
evidence_class = config.get("evidence_class", "local-real")
|
||||
if evidence_class not in {"local-real", "multi-machine-real"}:
|
||||
raise BenchmarkError("canonical real runner cannot emit synthetic evidence")
|
||||
signing_key = _load_evidence_signing_key()
|
||||
started_at = _utc_now()
|
||||
run_id = str(uuid.uuid4())
|
||||
config_sha256 = _canonical_sha256(config)
|
||||
plan = _plan_from_config(config)
|
||||
|
||||
from .recipe_benchmark import RecipeMeasurement # local import keeps the seam obvious
|
||||
|
||||
measurements = []
|
||||
for spec in config["recipes"]:
|
||||
recipe = _recipe_from_config(spec)
|
||||
driver = None
|
||||
try:
|
||||
driver = build_driver(spec, plan)
|
||||
measurements.append(measure_recipe(driver, recipe, plan))
|
||||
except Exception as exc:
|
||||
measurements.append(RecipeMeasurement(
|
||||
recipe=recipe,
|
||||
load=LoadStats(artifact_bytes=0, load_ms=0.0),
|
||||
unavailable_reason=f"{type(exc).__name__}: {exc}",
|
||||
))
|
||||
finally:
|
||||
if driver is not None:
|
||||
driver.close()
|
||||
|
||||
report = build_report(
|
||||
plan,
|
||||
measurements,
|
||||
host={**dict(config.get("host", {})), **_host_manifest(config)},
|
||||
evidence_class=evidence_class,
|
||||
provenance={
|
||||
"schema_version": PROVENANCE_SCHEMA_VERSION,
|
||||
"producer": producer,
|
||||
"run_id": run_id,
|
||||
"started_at": started_at,
|
||||
"completed_at": _utc_now(),
|
||||
"config_sha256": config_sha256,
|
||||
"signature_algorithm": "ed25519",
|
||||
},
|
||||
)
|
||||
_sign_report(report, signing_key)
|
||||
return report
|
||||
1049
packages/node/meshnet_node/runtime_recipe.py
Normal file
1049
packages/node/meshnet_node/runtime_recipe.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -105,7 +105,7 @@ class _StubHTTPServer(http.server.HTTPServer):
|
||||
|
||||
|
||||
class _StubHandler(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
||||
def log_message(self, fmt, *args): # suppress request logs in tests
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
|
||||
@@ -19,7 +19,6 @@ from .model_backend import (
|
||||
InsufficientVRAMError,
|
||||
KVCacheMiss,
|
||||
MissingModelDependencyError,
|
||||
Quantization,
|
||||
TailTokenResult,
|
||||
TorchModelShard,
|
||||
_tensor_from_bfloat16_bytes,
|
||||
@@ -46,7 +45,7 @@ class _DirectRequestUncertainError(ConnectionError):
|
||||
"""A direct request may have reached the downstream node but did not finish."""
|
||||
|
||||
|
||||
from .server import (
|
||||
from .server import ( # noqa: E402
|
||||
_WIRE_VERSION,
|
||||
_parse_shape,
|
||||
_validate_activation_body,
|
||||
@@ -399,7 +398,7 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
|
||||
# Finite responses below provide Content-Length; streams are chunked.
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
||||
def log_message(self, fmt, *args): # suppress request logs in tests
|
||||
pass
|
||||
|
||||
def _request_id(self) -> str:
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
Reference in New Issue
Block a user