feat: checkpoint distributed gguf runtime stories

This commit is contained in:
Dobromir Popov
2026-07-15 23:42:58 +03:00
parent eaf00f6add
commit 1fe31ef38d
60 changed files with 8478 additions and 105 deletions

View File

@@ -56,6 +56,7 @@ from .capability import (
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
POLICY_COMPAT,
POLICY_ENFORCE,
STATE_COMPATIBILITY_MISMATCH,
STATE_ABSENT,
STATE_ADMITTED,
STATE_MODEL_MISMATCH,
@@ -598,6 +599,7 @@ class _NodeEntry:
"model_tokens_per_sec",
"pending_directives", "last_heartbeat", "tracker_mode",
"relay_addr", "cert_fingerprint", "peer_id", "friendly_name",
"compatibility_fingerprint",
# heartbeat stats (reported by node, cumulative)
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
"current_requests",
@@ -636,6 +638,7 @@ class _NodeEntry:
cert_fingerprint: str | None = None,
peer_id: str | None = None,
friendly_name: str | None = None,
compatibility_fingerprint: str | None = None,
capability: "CapabilityState | None" = None,
) -> None:
self.node_id = node_id
@@ -664,6 +667,7 @@ class _NodeEntry:
self.cert_fingerprint = cert_fingerprint
self.peer_id = peer_id
self.friendly_name = friendly_name
self.compatibility_fingerprint = compatibility_fingerprint
# No proof presented is `absent`, never `admitted` — a node can only earn
# `admitted` by presenting a report that covers what it advertises.
self.capability: CapabilityState = capability or absent_state()
@@ -782,6 +786,16 @@ def _node_admission(node: "_NodeEntry") -> CapabilityState:
f"proof is for layers {state.shard_start}{state.shard_end}, but the "
f"node now serves layers {node.shard_start}{node.shard_end}",
)
if (
node.compatibility_fingerprint
and state.compatibility_fingerprint
and state.compatibility_fingerprint != node.compatibility_fingerprint
):
return state.with_state(
STATE_COMPATIBILITY_MISMATCH,
"proof compatibility fingerprint no longer matches the node's "
"declared artifact/runtime recipe",
)
return state
@@ -811,6 +825,12 @@ def _capability_from_registration(
declared_recipe_version=(
recipe_version if isinstance(recipe_version, str) else None
),
declared_compatibility_fingerprint=(
value.strip()
if isinstance((value := payload.get("compatibility_fingerprint")), str)
and value.strip()
else None
),
)
@@ -4588,6 +4608,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
relay_addr = body.get("relay_addr") or None
cert_fingerprint = body.get("cert_fingerprint") or None
peer_id = body.get("peer_id") or None
compatibility_fingerprint = body.get("compatibility_fingerprint")
if compatibility_fingerprint is not None and (
not isinstance(compatibility_fingerprint, str) or not compatibility_fingerprint.strip()
):
self._send_json(400, {"error": "compatibility_fingerprint must be a string"})
return
compatibility_fingerprint = compatibility_fingerprint.strip() if isinstance(compatibility_fingerprint, str) else None
try:
friendly_name = _normalize_friendly_name(body.get("friendly_name"))
except ValueError as exc:
@@ -4647,6 +4674,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
cert_fingerprint=cert_fingerprint,
peer_id=peer_id,
friendly_name=friendly_name,
compatibility_fingerprint=compatibility_fingerprint,
capability=capability,
)
with server.lock:
@@ -7052,6 +7080,12 @@ class TrackerServer:
else None
),
friendly_name=_normalize_friendly_name(payload.get("friendly_name")),
compatibility_fingerprint=(
value.strip()
if isinstance((value := payload.get("compatibility_fingerprint")), str)
and value.strip()
else None
),
# A replicated registration carries its proof: without this, a proven
# node would be routable on the leader and dark on every follower.
capability=_capability_from_registration(