feat: checkpoint distributed gguf runtime stories
This commit is contained in:
@@ -56,6 +56,7 @@ from .capability import (
|
||||
DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
|
||||
POLICY_COMPAT,
|
||||
POLICY_ENFORCE,
|
||||
STATE_COMPATIBILITY_MISMATCH,
|
||||
STATE_ABSENT,
|
||||
STATE_ADMITTED,
|
||||
STATE_MODEL_MISMATCH,
|
||||
@@ -598,6 +599,7 @@ class _NodeEntry:
|
||||
"model_tokens_per_sec",
|
||||
"pending_directives", "last_heartbeat", "tracker_mode",
|
||||
"relay_addr", "cert_fingerprint", "peer_id", "friendly_name",
|
||||
"compatibility_fingerprint",
|
||||
# heartbeat stats (reported by node, cumulative)
|
||||
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
|
||||
"current_requests",
|
||||
@@ -636,6 +638,7 @@ class _NodeEntry:
|
||||
cert_fingerprint: str | None = None,
|
||||
peer_id: str | None = None,
|
||||
friendly_name: str | None = None,
|
||||
compatibility_fingerprint: str | None = None,
|
||||
capability: "CapabilityState | None" = None,
|
||||
) -> None:
|
||||
self.node_id = node_id
|
||||
@@ -664,6 +667,7 @@ class _NodeEntry:
|
||||
self.cert_fingerprint = cert_fingerprint
|
||||
self.peer_id = peer_id
|
||||
self.friendly_name = friendly_name
|
||||
self.compatibility_fingerprint = compatibility_fingerprint
|
||||
# No proof presented is `absent`, never `admitted` — a node can only earn
|
||||
# `admitted` by presenting a report that covers what it advertises.
|
||||
self.capability: CapabilityState = capability or absent_state()
|
||||
@@ -782,6 +786,16 @@ def _node_admission(node: "_NodeEntry") -> CapabilityState:
|
||||
f"proof is for layers {state.shard_start}–{state.shard_end}, but the "
|
||||
f"node now serves layers {node.shard_start}–{node.shard_end}",
|
||||
)
|
||||
if (
|
||||
node.compatibility_fingerprint
|
||||
and state.compatibility_fingerprint
|
||||
and state.compatibility_fingerprint != node.compatibility_fingerprint
|
||||
):
|
||||
return state.with_state(
|
||||
STATE_COMPATIBILITY_MISMATCH,
|
||||
"proof compatibility fingerprint no longer matches the node's "
|
||||
"declared artifact/runtime recipe",
|
||||
)
|
||||
return state
|
||||
|
||||
|
||||
@@ -811,6 +825,12 @@ def _capability_from_registration(
|
||||
declared_recipe_version=(
|
||||
recipe_version if isinstance(recipe_version, str) else None
|
||||
),
|
||||
declared_compatibility_fingerprint=(
|
||||
value.strip()
|
||||
if isinstance((value := payload.get("compatibility_fingerprint")), str)
|
||||
and value.strip()
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -4588,6 +4608,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
relay_addr = body.get("relay_addr") or None
|
||||
cert_fingerprint = body.get("cert_fingerprint") or None
|
||||
peer_id = body.get("peer_id") or None
|
||||
compatibility_fingerprint = body.get("compatibility_fingerprint")
|
||||
if compatibility_fingerprint is not None and (
|
||||
not isinstance(compatibility_fingerprint, str) or not compatibility_fingerprint.strip()
|
||||
):
|
||||
self._send_json(400, {"error": "compatibility_fingerprint must be a string"})
|
||||
return
|
||||
compatibility_fingerprint = compatibility_fingerprint.strip() if isinstance(compatibility_fingerprint, str) else None
|
||||
try:
|
||||
friendly_name = _normalize_friendly_name(body.get("friendly_name"))
|
||||
except ValueError as exc:
|
||||
@@ -4647,6 +4674,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
cert_fingerprint=cert_fingerprint,
|
||||
peer_id=peer_id,
|
||||
friendly_name=friendly_name,
|
||||
compatibility_fingerprint=compatibility_fingerprint,
|
||||
capability=capability,
|
||||
)
|
||||
with server.lock:
|
||||
@@ -7052,6 +7080,12 @@ class TrackerServer:
|
||||
else None
|
||||
),
|
||||
friendly_name=_normalize_friendly_name(payload.get("friendly_name")),
|
||||
compatibility_fingerprint=(
|
||||
value.strip()
|
||||
if isinstance((value := payload.get("compatibility_fingerprint")), str)
|
||||
and value.strip()
|
||||
else None
|
||||
),
|
||||
# A replicated registration carries its proof: without this, a proven
|
||||
# node would be routable on the leader and dark on every follower.
|
||||
capability=_capability_from_registration(
|
||||
|
||||
Reference in New Issue
Block a user