popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: checkpoint distributed gguf runtime stories

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-15 23:42:58 +03:00
parent eaf00f6add
commit 1fe31ef38d
60 changed files with 8478 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
packages/tracker/meshnet_tracker/capability.py
View File

@@ -58,6 +58,7 @@ STATE_MODEL_MISMATCH = "model-mismatch"
STATE_SHARD_MISMATCH = "shard-mismatch"
STATE_RECIPE_MISMATCH = "recipe-mismatch"
STATE_CATALOGUE_INCOMPATIBLE = "catalogue-incompatible"
STATE_COMPATIBILITY_MISMATCH = "compatibility-mismatch"
ALL_STATES = (
STATE_ADMITTED,
@@ -69,6 +70,7 @@ ALL_STATES = (
STATE_SHARD_MISMATCH,
STATE_RECIPE_MISMATCH,
STATE_CATALOGUE_INCOMPATIBLE,
STATE_COMPATIBILITY_MISMATCH,
)
# --- Compatibility policy for nodes that predate the capability protocol. ---
@@ -155,12 +157,17 @@ class CapabilityState:
model_id: str | None = None
shard_start: int | None = None
shard_end: int | None = None
owns_embedding: bool | None = None
owns_final_head: bool | None = None
recipe_id: str | None = None
recipe_version: str | None = None
catalogue_version: str | None = None
backend_id: str | None = None
device: str | None = None
quantization: str | None = None
artifact_hash: str | None = None
compatibility_fingerprint: str | None = None
runtime_recipe_fingerprint: str | None = None
validated_at: float | None = None
recorded_at: float = 0.0
schema_version: int | None = None
@@ -187,12 +194,17 @@ class CapabilityState:
"model_id": self.model_id,
"shard_start": self.shard_start,
"shard_end": self.shard_end,
"owns_embedding": self.owns_embedding,
"owns_final_head": self.owns_final_head,
"recipe_id": self.recipe_id,
"recipe_version": self.recipe_version,
"catalogue_version": self.catalogue_version,
"backend_id": self.backend_id,
"device": self.device,
"quantization": self.quantization,
"artifact_hash": self.artifact_hash,
"compatibility_fingerprint": self.compatibility_fingerprint,
"runtime_recipe_fingerprint": self.runtime_recipe_fingerprint,
"validated_at": self.validated_at,
"recorded_at": self.recorded_at,
"schema_version": self.schema_version,
@@ -222,6 +234,7 @@ def evaluate_report(
shard_end: int | None,
declared_recipe_id: str | None = None,
declared_recipe_version: str | None = None,
declared_compatibility_fingerprint: str | None = None,
now: float | None = None,
max_age_seconds: float = DEFAULT_MAX_REPORT_AGE_SECONDS,
) -> CapabilityState:
@@ -308,6 +321,17 @@ def evaluate_report(
f"the node declared v{declared_recipe_version}",
)
if (
declared_compatibility_fingerprint is not None
and base.compatibility_fingerprint != declared_compatibility_fingerprint
):
return base.with_state(
STATE_COMPATIBILITY_MISMATCH,
"proof compatibility fingerprint does not match the node's declared "
"artifact/runtime recipe; the artifact, tokenizer, architecture, "
"boundary schema, activation recipe or cache layout differs",
)
if status != STATUS_PASSED:
return base.with_state(
STATE_FAILED,
@@ -344,6 +368,8 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
shard = _object(doc.get("shard"), "shard")
recipe = _object(doc.get("recipe"), "recipe")
backend = _object(doc.get("backend"), "backend")
artifact = _object_or_none(doc.get("artifact"), "artifact")
runtime_recipe = _object_or_none(doc.get("runtime_recipe"), "runtime_recipe")
validated_at = doc.get("validated_at")
if isinstance(validated_at, bool) or not isinstance(validated_at, (int, float)):
@@ -357,6 +383,8 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
"model_id": _text(model.get("model_id"), "model.model_id"),
"shard_start": _index(shard.get("start"), "shard.start"),
"shard_end": _index(shard.get("end"), "shard.end"),
"owns_embedding": _maybe_bool(shard.get("owns_embedding")),
"owns_final_head": _maybe_bool(shard.get("owns_final_head")),
"recipe_id": _text(recipe.get("recipe_id"), "recipe.recipe_id"),
"recipe_version": _text(recipe.get("recipe_version"), "recipe.recipe_version"),
"catalogue_version": _text(
@@ -367,6 +395,15 @@ def _parse_report(doc: Mapping[str, Any]) -> dict:
"quantization": _optional_text(
backend.get("quantization"), "backend.quantization"
),
"artifact_hash": _optional_text(
artifact.get("artifact_hash"), "artifact.artifact_hash"
),
"compatibility_fingerprint": _optional_text(
doc.get("compatibility_fingerprint"), "compatibility_fingerprint"
),
"runtime_recipe_fingerprint": _optional_text(
runtime_recipe.get("fingerprint"), "runtime_recipe.fingerprint"
),
"validated_at": float(validated_at),
"schema_version": schema_version,
"diagnostics": _diagnostics(doc.get("diagnostics")),
@@ -380,6 +417,12 @@ def _object(value: Any, field_name: str) -> Mapping[str, Any]:
return value
def _object_or_none(value: Any, field_name: str) -> Mapping[str, Any]:
if value is None:
return {}
return _object(value, field_name)
def _text(value: Any, field_name: str) -> str:
if not isinstance(value, str) or not value.strip():
raise _ReportError(f"{field_name!r} must be a non-empty string")
@@ -404,6 +447,12 @@ def _maybe_int(value: Any) -> int | None:
return value
def _maybe_bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
return None
def _diagnostics(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()

34
packages/tracker/meshnet_tracker/server.py
View File

@@ -56,6 +56,7 @@ from .capability import (
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
POLICY_COMPAT,
POLICY_ENFORCE,
STATE_COMPATIBILITY_MISMATCH,
STATE_ABSENT,
STATE_ADMITTED,
STATE_MODEL_MISMATCH,
@@ -598,6 +599,7 @@ class _NodeEntry:
"model_tokens_per_sec",
"pending_directives", "last_heartbeat", "tracker_mode",
"relay_addr", "cert_fingerprint", "peer_id", "friendly_name",
"compatibility_fingerprint",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
"current_requests",
@@ -636,6 +638,7 @@ class _NodeEntry:
cert_fingerprint: str | None = None,
peer_id: str | None = None,
friendly_name: str | None = None,
compatibility_fingerprint: str | None = None,
capability: "CapabilityState | None" = None,
) -> None:
self.node_id = node_id
@@ -664,6 +667,7 @@ class _NodeEntry:
self.cert_fingerprint = cert_fingerprint
self.peer_id = peer_id
self.friendly_name = friendly_name
self.compatibility_fingerprint = compatibility_fingerprint
# No proof presented is `absent`, never `admitted` — a node can only earn
# `admitted` by presenting a report that covers what it advertises.
self.capability: CapabilityState = capability or absent_state()
@@ -782,6 +786,16 @@ def _node_admission(node: "_NodeEntry") -> CapabilityState:
f"proof is for layers {state.shard_start}–{state.shard_end}, but the "
f"node now serves layers {node.shard_start}–{node.shard_end}",
)
if (
node.compatibility_fingerprint
and state.compatibility_fingerprint
and state.compatibility_fingerprint != node.compatibility_fingerprint
):
return state.with_state(
STATE_COMPATIBILITY_MISMATCH,
"proof compatibility fingerprint no longer matches the node's "
"declared artifact/runtime recipe",
)
return state
@@ -811,6 +825,12 @@ def _capability_from_registration(
declared_recipe_version=(
recipe_version if isinstance(recipe_version, str) else None
),
declared_compatibility_fingerprint=(
value.strip()
if isinstance((value := payload.get("compatibility_fingerprint")), str)
and value.strip()
else None
),
)
@@ -4588,6 +4608,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
relay_addr = body.get("relay_addr") or None
cert_fingerprint = body.get("cert_fingerprint") or None
peer_id = body.get("peer_id") or None
compatibility_fingerprint = body.get("compatibility_fingerprint")
if compatibility_fingerprint is not None and (
not isinstance(compatibility_fingerprint, str) or not compatibility_fingerprint.strip()
):
self._send_json(400, {"error": "compatibility_fingerprint must be a string"})
return
compatibility_fingerprint = compatibility_fingerprint.strip() if isinstance(compatibility_fingerprint, str) else None
try:
friendly_name = _normalize_friendly_name(body.get("friendly_name"))
except ValueError as exc:
@@ -4647,6 +4674,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
cert_fingerprint=cert_fingerprint,
peer_id=peer_id,
friendly_name=friendly_name,
compatibility_fingerprint=compatibility_fingerprint,
capability=capability,
)
with server.lock:
@@ -7052,6 +7080,12 @@ class TrackerServer:
else None
),
friendly_name=_normalize_friendly_name(payload.get("friendly_name")),
compatibility_fingerprint=(
value.strip()
if isinstance((value := payload.get("compatibility_fingerprint")), str)
and value.strip()
else None
),
# A replicated registration carries its proof: without this, a proven
# node would be routable on the leader and dark on every follower.
capability=_capability_from_registration(