feat: checkpoint distributed gguf runtime stories

This commit is contained in:
Dobromir Popov
2026-07-15 23:42:58 +03:00
parent eaf00f6add
commit 1fe31ef38d
60 changed files with 8478 additions and 105 deletions

View File

@@ -58,6 +58,7 @@ STATE_MODEL_MISMATCH = "model-mismatch"
STATE_SHARD_MISMATCH = "shard-mismatch"
STATE_RECIPE_MISMATCH = "recipe-mismatch"
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
STATE_COMPATIBILITY_MISMATCH = "compatibility-mismatch"
ALL_STATES = (
STATE_ADMITTED,
@@ -69,6 +70,7 @@ ALL_STATES = (
STATE_SHARD_MISMATCH,
STATE_RECIPE_MISMATCH,
STATE_CATALOGUE_INCOMPATIBLE,
STATE_COMPATIBILITY_MISMATCH,
)
# --- Compatibility policy for nodes that predate the capability protocol. ---
@@ -155,12 +157,17 @@ class CapabilityState:
model_id: str | None = None
shard_start: int | None = None
shard_end: int | None = None
owns_embedding: bool | None = None
owns_final_head: bool | None = None
recipe_id: str | None = None
recipe_version: str | None = None
catalogue_version: str | None = None
backend_id: str | None = None
device: str | None = None
quantization: str | None = None
artifact_hash: str | None = None
compatibility_fingerprint: str | None = None
runtime_recipe_fingerprint: str | None = None
validated_at: float | None = None
recorded_at: float = 0.0
schema_version: int | None = None
@@ -187,12 +194,17 @@ class CapabilityState:
"model_id": self.model_id,
"shard_start": self.shard_start,
"shard_end": self.shard_end,
"owns_embedding": self.owns_embedding,
"owns_final_head": self.owns_final_head,
"recipe_id": self.recipe_id,
"recipe_version": self.recipe_version,
"catalogue_version": self.catalogue_version,
"backend_id": self.backend_id,
"device": self.device,
"quantization": self.quantization,
"artifact_hash": self.artifact_hash,
"compatibility_fingerprint": self.compatibility_fingerprint,
"runtime_recipe_fingerprint": self.runtime_recipe_fingerprint,
"validated_at": self.validated_at,
"recorded_at": self.recorded_at,
"schema_version": self.schema_version,
@@ -222,6 +234,7 @@ def evaluate_report(
shard_end: int | None,
declared_recipe_id: str | None = None,
declared_recipe_version: str | None = None,
declared_compatibility_fingerprint: str | None = None,
now: float | None = None,
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
) -> CapabilityState:
@@ -308,6 +321,17 @@ def evaluate_report(
f"the node declared v{declared_recipe_version}",
)
if (
declared_compatibility_fingerprint is not None
and base.compatibility_fingerprint != declared_compatibility_fingerprint
):
return base.with_state(
STATE_COMPATIBILITY_MISMATCH,
"proof compatibility fingerprint does not match the node's declared "
"artifact/runtime recipe; the artifact, tokenizer, architecture, "
"boundary schema, activation recipe or cache layout differs",
)
if status != STATUS_PASSED:
return base.with_state(
STATE_FAILED,
@@ -344,6 +368,8 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
shard = _object(doc.get("shard"), "shard")
recipe = _object(doc.get("recipe"), "recipe")
backend = _object(doc.get("backend"), "backend")
artifact = _object_or_none(doc.get("artifact"), "artifact")
runtime_recipe = _object_or_none(doc.get("runtime_recipe"), "runtime_recipe")
validated_at = doc.get("validated_at")
if isinstance(validated_at, bool) or not isinstance(validated_at, (int, float)):
@@ -357,6 +383,8 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
"model_id": _text(model.get("model_id"), "model.model_id"),
"shard_start": _index(shard.get("start"), "shard.start"),
"shard_end": _index(shard.get("end"), "shard.end"),
"owns_embedding": _maybe_bool(shard.get("owns_embedding")),
"owns_final_head": _maybe_bool(shard.get("owns_final_head")),
"recipe_id": _text(recipe.get("recipe_id"), "recipe.recipe_id"),
"recipe_version": _text(recipe.get("recipe_version"), "recipe.recipe_version"),
"catalogue_version": _text(
@@ -367,6 +395,15 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
"quantization": _optional_text(
backend.get("quantization"), "backend.quantization"
),
"artifact_hash": _optional_text(
artifact.get("artifact_hash"), "artifact.artifact_hash"
),
"compatibility_fingerprint": _optional_text(
doc.get("compatibility_fingerprint"), "compatibility_fingerprint"
),
"runtime_recipe_fingerprint": _optional_text(
runtime_recipe.get("fingerprint"), "runtime_recipe.fingerprint"
),
"validated_at": float(validated_at),
"schema_version": schema_version,
"diagnostics": _diagnostics(doc.get("diagnostics")),
@@ -380,6 +417,12 @@ def _object(value: Any, field_name: str) -> Mapping[str, Any]:
return value
def _object_or_none(value: Any, field_name: str) -> Mapping[str, Any]:
if value is None:
return {}
return _object(value, field_name)
def _text(value: Any, field_name: str) -> str:
if not isinstance(value, str) or not value.strip():
raise _ReportError(f"{field_name!r} must be a non-empty string")
@@ -404,6 +447,12 @@ def _maybe_int(value: Any) -> int | None:
return value
def _maybe_bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
return None
def _diagnostics(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()