522 lines
18 KiB
Python
522 lines
18 KiB
Python
"""Model-agnostic node capability report.
|
|
|
|
A capability report is the node's local proof that one concrete combination —
|
|
model artifact, shard range, recipe, backend/device — actually executed. It is
|
|
plain versioned data: arbitrary model ids pass through verbatim, and no model,
|
|
vendor, or kernel name is a default or a code-path discriminator here.
|
|
|
|
Later stories consume this: `doctor` produces a report from a real forward
|
|
(NCA-002), startup refuses to register without a fresh passing one (NCA-003),
|
|
and the tracker routes only to admitted, matching capabilities (NCA-004).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Mapping
|
|
|
|
from .runtime_recipe import CompatibilityFingerprint, ShardIdentity
|
|
|
|
# Layout of the serialized report. Bump when the JSON shape changes.
|
|
CAPABILITY_SCHEMA_VERSION = 1
|
|
|
|
STATUS_PASSED = "passed"
|
|
STATUS_FAILED = "failed"
|
|
STATUS_SKIPPED = "skipped"
|
|
VALID_STATUSES = (STATUS_PASSED, STATUS_FAILED, STATUS_SKIPPED)
|
|
|
|
# Diagnostics are operator-facing, not a log sink: keep them short and few.
|
|
MAX_DIAGNOSTIC_CHARS = 500
|
|
MAX_DIAGNOSTICS = 20
|
|
|
|
REDACTED = "[redacted]"
|
|
|
|
# An env var whose *name* contains one of these holds a secret by convention.
|
|
_SECRET_NAME_HINTS = (
|
|
"TOKEN",
|
|
"SECRET",
|
|
"PASSWORD",
|
|
"PASSWD",
|
|
"CREDENTIAL",
|
|
"APIKEY",
|
|
"API_KEY",
|
|
"PRIVATE_KEY",
|
|
"ACCESS_KEY",
|
|
)
|
|
# Below this length a value is too generic to redact without mangling prose.
|
|
_MIN_SECRET_LEN = 6
|
|
|
|
# Provider-shaped bearer credentials that can appear in a backend error string.
|
|
_CREDENTIAL_PATTERNS = (
|
|
re.compile(r"\b[A-Za-z0-9_]{2,6}_[A-Za-z0-9]{16,}\b"), # hf_…, ghp_…, sk_live_…
|
|
re.compile(r"\bsk-[A-Za-z0-9_-]{16,}\b"),
|
|
re.compile(r"(?i)\bbearer\s+\S+"),
|
|
re.compile(r"(?i)\b(?:token|api[_-]?key|password|secret)\s*[=:]\s*\S+"),
|
|
)
|
|
|
|
|
|
class CapabilityReportError(ValueError):
|
|
"""Raised when report input is malformed.
|
|
|
|
Messages name the offending field and the expected shape, and carry no
|
|
caller-supplied payload beyond the field path itself.
|
|
"""
|
|
|
|
|
|
def _secret_env_values(environ: Mapping[str, str] | None = None) -> list[str]:
|
|
env = os.environ if environ is None else environ
|
|
values: list[str] = []
|
|
for name, value in env.items():
|
|
if not isinstance(value, str) or len(value) < _MIN_SECRET_LEN:
|
|
continue
|
|
upper = name.upper()
|
|
if any(hint in upper for hint in _SECRET_NAME_HINTS):
|
|
values.append(value)
|
|
# Redact longest first so a value that contains another is not partially masked.
|
|
return sorted(values, key=len, reverse=True)
|
|
|
|
|
|
def sanitize_diagnostic(
|
|
text: str,
|
|
environ: Mapping[str, str] | None = None,
|
|
) -> str:
|
|
"""Return `text` with credentials and host identity stripped, clipped to length."""
|
|
cleaned = " ".join(str(text).split())
|
|
|
|
for secret in _secret_env_values(environ):
|
|
cleaned = cleaned.replace(secret, REDACTED)
|
|
|
|
for pattern in _CREDENTIAL_PATTERNS:
|
|
cleaned = pattern.sub(REDACTED, cleaned)
|
|
|
|
home = os.path.expanduser("~")
|
|
if home and home not in ("/", ""):
|
|
cleaned = cleaned.replace(home, "~")
|
|
|
|
if len(cleaned) > MAX_DIAGNOSTIC_CHARS:
|
|
cleaned = cleaned[: MAX_DIAGNOSTIC_CHARS - 1].rstrip() + "…"
|
|
return cleaned
|
|
|
|
|
|
def sanitize_diagnostics(
|
|
diagnostics: Any,
|
|
environ: Mapping[str, str] | None = None,
|
|
) -> tuple[str, ...]:
|
|
"""Sanitize and bound a diagnostics sequence."""
|
|
if diagnostics is None:
|
|
return ()
|
|
if isinstance(diagnostics, str):
|
|
raise CapabilityReportError(
|
|
"'diagnostics' must be a list of strings, got a bare string"
|
|
)
|
|
try:
|
|
items = list(diagnostics)
|
|
except TypeError as exc:
|
|
raise CapabilityReportError(
|
|
f"'diagnostics' must be a list of strings, got {type(diagnostics).__name__}"
|
|
) from exc
|
|
|
|
out: list[str] = []
|
|
for index, item in enumerate(items[:MAX_DIAGNOSTICS]):
|
|
if not isinstance(item, str):
|
|
raise CapabilityReportError(
|
|
f"'diagnostics[{index}]' must be a string, got {type(item).__name__}"
|
|
)
|
|
cleaned = sanitize_diagnostic(item, environ)
|
|
if cleaned:
|
|
out.append(cleaned)
|
|
dropped = len(items) - MAX_DIAGNOSTICS
|
|
if dropped > 0:
|
|
out.append(f"… {dropped} further diagnostic(s) omitted")
|
|
return tuple(out)
|
|
|
|
|
|
def config_fingerprint(config: Any) -> str | None:
|
|
"""Return a stable content hash of a model config mapping.
|
|
|
|
Two nodes that loaded the same artifact revision with the same config
|
|
produce the same fingerprint; anything unserializable degrades to its
|
|
string form rather than failing the report.
|
|
"""
|
|
if config is None:
|
|
return None
|
|
if isinstance(config, str):
|
|
return config if config.startswith("sha256:") else "sha256:" + _sha256(config)
|
|
if not isinstance(config, Mapping):
|
|
raise CapabilityReportError(
|
|
f"model config must be a mapping or a fingerprint string, "
|
|
f"got {type(config).__name__}"
|
|
)
|
|
canonical = json.dumps(
|
|
config, sort_keys=True, separators=(",", ":"), default=str, ensure_ascii=False
|
|
)
|
|
return "sha256:" + _sha256(canonical)
|
|
|
|
|
|
def _sha256(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _require_text(value: Any, field_name: str) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
raise CapabilityReportError(f"{field_name!r} must be a non-empty string")
|
|
return value
|
|
|
|
|
|
def _optional_text(value: Any, field_name: str) -> str | None:
|
|
if value is None:
|
|
return None
|
|
return _require_text(value, field_name)
|
|
|
|
|
|
def _require_int(value: Any, field_name: str, minimum: int) -> int:
|
|
if isinstance(value, bool) or not isinstance(value, int):
|
|
raise CapabilityReportError(f"{field_name!r} must be an integer")
|
|
if value < minimum:
|
|
raise CapabilityReportError(f"{field_name!r} must be >= {minimum}, got {value}")
|
|
return value
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ModelIdentity:
|
|
"""Which artifact was validated. `model_id` is opaque and preserved verbatim."""
|
|
|
|
model_id: str
|
|
revision: str | None = None
|
|
config_fingerprint: str | None = None
|
|
|
|
def __post_init__(self) -> None:
|
|
_require_text(self.model_id, "model.model_id")
|
|
_optional_text(self.revision, "model.revision")
|
|
_optional_text(self.config_fingerprint, "model.config_fingerprint")
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"model_id": self.model_id,
|
|
"revision": self.revision,
|
|
"config_fingerprint": self.config_fingerprint,
|
|
}
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: Any) -> ModelIdentity:
|
|
doc = _as_mapping(data, "model")
|
|
return cls(
|
|
model_id=_require_text(doc.get("model_id"), "model.model_id"),
|
|
revision=_optional_text(doc.get("revision"), "model.revision"),
|
|
config_fingerprint=_optional_text(
|
|
doc.get("config_fingerprint"), "model.config_fingerprint"
|
|
),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ShardRange:
|
|
"""Inclusive layer range, matching the CLI and backend convention."""
|
|
|
|
start: int
|
|
end: int
|
|
|
|
def __post_init__(self) -> None:
|
|
_require_int(self.start, "shard.start", 0)
|
|
_require_int(self.end, "shard.end", 0)
|
|
if self.end < self.start:
|
|
raise CapabilityReportError(
|
|
f"'shard.end' ({self.end}) must be >= 'shard.start' ({self.start})"
|
|
)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {"start": self.start, "end": self.end}
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: Any) -> ShardRange:
|
|
doc = _as_mapping(data, "shard")
|
|
return cls(
|
|
start=_require_int(doc.get("start"), "shard.start", 0),
|
|
end=_require_int(doc.get("end"), "shard.end", 0),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RecipeIdentity:
|
|
"""Which recipe, from which catalogue, was exercised."""
|
|
|
|
recipe_id: str
|
|
recipe_version: str
|
|
catalogue_version: str
|
|
|
|
def __post_init__(self) -> None:
|
|
_require_text(self.recipe_id, "recipe.recipe_id")
|
|
_require_text(self.recipe_version, "recipe.recipe_version")
|
|
_require_text(self.catalogue_version, "recipe.catalogue_version")
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"recipe_id": self.recipe_id,
|
|
"recipe_version": self.recipe_version,
|
|
"catalogue_version": self.catalogue_version,
|
|
}
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: Any) -> RecipeIdentity:
|
|
doc = _as_mapping(data, "recipe")
|
|
return cls(
|
|
recipe_id=_require_text(doc.get("recipe_id"), "recipe.recipe_id"),
|
|
recipe_version=_require_text(
|
|
doc.get("recipe_version"), "recipe.recipe_version"
|
|
),
|
|
catalogue_version=_require_text(
|
|
doc.get("catalogue_version"), "recipe.catalogue_version"
|
|
),
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BackendIdentity:
|
|
"""Which execution stack ran it. All fields are opaque labels, never branches."""
|
|
|
|
backend_id: str
|
|
device: str
|
|
device_name: str | None = None
|
|
quantization: str | None = None
|
|
runtime: Mapping[str, str] = field(default_factory=dict)
|
|
|
|
def __post_init__(self) -> None:
|
|
_require_text(self.backend_id, "backend.backend_id")
|
|
_require_text(self.device, "backend.device")
|
|
_optional_text(self.device_name, "backend.device_name")
|
|
_optional_text(self.quantization, "backend.quantization")
|
|
for key, value in self.runtime.items():
|
|
if not isinstance(key, str) or not isinstance(value, str):
|
|
raise CapabilityReportError(
|
|
"'backend.runtime' must map string names to string versions"
|
|
)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"backend_id": self.backend_id,
|
|
"device": self.device,
|
|
"device_name": self.device_name,
|
|
"quantization": self.quantization,
|
|
"runtime": dict(self.runtime),
|
|
}
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: Any) -> BackendIdentity:
|
|
doc = _as_mapping(data, "backend")
|
|
runtime = doc.get("runtime") or {}
|
|
if not isinstance(runtime, Mapping):
|
|
raise CapabilityReportError("'backend.runtime' must be a JSON object")
|
|
return cls(
|
|
backend_id=_require_text(doc.get("backend_id"), "backend.backend_id"),
|
|
device=_require_text(doc.get("device"), "backend.device"),
|
|
device_name=_optional_text(doc.get("device_name"), "backend.device_name"),
|
|
quantization=_optional_text(
|
|
doc.get("quantization"), "backend.quantization"
|
|
),
|
|
runtime={str(k): str(v) for k, v in runtime.items()},
|
|
)
|
|
|
|
|
|
def _as_mapping(data: Any, field_name: str) -> Mapping[str, Any]:
|
|
if not isinstance(data, Mapping):
|
|
raise CapabilityReportError(
|
|
f"{field_name!r} must be a JSON object, got {type(data).__name__}"
|
|
)
|
|
return data
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CapabilityReport:
|
|
"""One node's validated (or failed) model/shard/recipe/backend combination.
|
|
|
|
`identity` is the exact DGR-003 artifact/runtime-recipe block: the separated
|
|
numerical axes and the compatibility fingerprint derived from them. It is
|
|
optional and additive — a node that predates DGR-003 presents none, and the
|
|
tracker falls back to the coarse label comparison it has always done
|
|
(ADR-0023's compat rollout). A node that *does* present one is held to it:
|
|
the tracker re-derives the fingerprint and refuses a report whose claim does
|
|
not match its own derivation.
|
|
"""
|
|
|
|
model: ModelIdentity
|
|
shard: ShardRange
|
|
recipe: RecipeIdentity
|
|
backend: BackendIdentity
|
|
status: str
|
|
validated_at: float
|
|
duration_ms: int
|
|
diagnostics: tuple[str, ...] = ()
|
|
schema_version: int = CAPABILITY_SCHEMA_VERSION
|
|
identity: ShardIdentity | None = None
|
|
|
|
def __post_init__(self) -> None:
|
|
if self.status not in VALID_STATUSES:
|
|
raise CapabilityReportError(
|
|
f"'status' must be one of {', '.join(VALID_STATUSES)}; got {self.status!r}"
|
|
)
|
|
if isinstance(self.validated_at, bool) or not isinstance(
|
|
self.validated_at, (int, float)
|
|
):
|
|
raise CapabilityReportError("'validated_at' must be a Unix timestamp")
|
|
if self.validated_at < 0:
|
|
raise CapabilityReportError("'validated_at' must not be negative")
|
|
_require_int(self.duration_ms, "duration_ms", 0)
|
|
_require_int(self.schema_version, "schema_version", 1)
|
|
|
|
@property
|
|
def passed(self) -> bool:
|
|
return self.status == STATUS_PASSED
|
|
|
|
@property
|
|
def fingerprint(self) -> CompatibilityFingerprint | None:
|
|
"""The exact compatibility fingerprint, when this node declares one."""
|
|
return None if self.identity is None else self.identity.fingerprint
|
|
|
|
def identity_key(self) -> tuple[str, int, int, str, str, str, str]:
|
|
"""The tuple a consumer must match to reuse this proof.
|
|
|
|
Startup and the tracker compare on exactly this: a report proves nothing
|
|
about a different model, shard, recipe version, or device.
|
|
"""
|
|
return (
|
|
self.model.model_id,
|
|
self.shard.start,
|
|
self.shard.end,
|
|
self.recipe.recipe_id,
|
|
self.recipe.recipe_version,
|
|
self.backend.backend_id,
|
|
self.backend.device,
|
|
)
|
|
|
|
def age_seconds(self, now: float | None = None) -> float:
|
|
return max(0.0, (time.time() if now is None else now) - self.validated_at)
|
|
|
|
def to_dict(self) -> dict:
|
|
doc = {
|
|
"schema_version": self.schema_version,
|
|
"model": self.model.to_dict(),
|
|
"shard": self.shard.to_dict(),
|
|
"recipe": self.recipe.to_dict(),
|
|
"backend": self.backend.to_dict(),
|
|
"status": self.status,
|
|
"validated_at": self.validated_at,
|
|
"duration_ms": self.duration_ms,
|
|
"diagnostics": list(self.diagnostics),
|
|
}
|
|
if self.identity is not None:
|
|
doc["identity"] = self.identity.to_dict()
|
|
return doc
|
|
|
|
def to_json(self, indent: int | None = None) -> str:
|
|
return json.dumps(self.to_dict(), indent=indent, sort_keys=True)
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: Any) -> CapabilityReport:
|
|
doc = _as_mapping(data, "report")
|
|
|
|
if "schema_version" not in doc:
|
|
raise CapabilityReportError(
|
|
"report is missing 'schema_version'; this node reads capability "
|
|
f"schema version {CAPABILITY_SCHEMA_VERSION}"
|
|
)
|
|
schema_version = _require_int(doc["schema_version"], "schema_version", 1)
|
|
if schema_version != CAPABILITY_SCHEMA_VERSION:
|
|
raise CapabilityReportError(
|
|
f"report declares capability schema version {schema_version}, but this "
|
|
f"node reads version {CAPABILITY_SCHEMA_VERSION}"
|
|
)
|
|
|
|
validated_at = doc.get("validated_at")
|
|
if isinstance(validated_at, bool) or not isinstance(
|
|
validated_at, (int, float)
|
|
):
|
|
raise CapabilityReportError("'validated_at' must be a Unix timestamp")
|
|
|
|
raw_identity = doc.get("identity")
|
|
return cls(
|
|
schema_version=schema_version,
|
|
model=ModelIdentity.from_dict(doc.get("model")),
|
|
shard=ShardRange.from_dict(doc.get("shard")),
|
|
recipe=RecipeIdentity.from_dict(doc.get("recipe")),
|
|
backend=BackendIdentity.from_dict(doc.get("backend")),
|
|
status=_require_text(doc.get("status"), "status"),
|
|
validated_at=float(validated_at),
|
|
duration_ms=_require_int(doc.get("duration_ms"), "duration_ms", 0),
|
|
diagnostics=sanitize_diagnostics(doc.get("diagnostics")),
|
|
identity=(
|
|
None if raw_identity is None else ShardIdentity.from_dict(raw_identity)
|
|
),
|
|
)
|
|
|
|
@classmethod
|
|
def from_json(cls, text: str) -> CapabilityReport:
|
|
try:
|
|
data = json.loads(text)
|
|
except json.JSONDecodeError as exc:
|
|
raise CapabilityReportError(
|
|
f"capability report is not valid JSON: {exc.msg} "
|
|
f"at line {exc.lineno} column {exc.colno}"
|
|
) from exc
|
|
return cls.from_dict(data)
|
|
|
|
|
|
def build_capability_report(
|
|
*,
|
|
model_id: str,
|
|
shard_start: int,
|
|
shard_end: int,
|
|
recipe_id: str,
|
|
recipe_version: str,
|
|
catalogue_version: str,
|
|
backend_id: str,
|
|
device: str,
|
|
status: str,
|
|
duration_ms: int,
|
|
revision: str | None = None,
|
|
model_config: Any = None,
|
|
device_name: str | None = None,
|
|
quantization: str | None = None,
|
|
runtime: Mapping[str, str] | None = None,
|
|
diagnostics: Any = None,
|
|
validated_at: float | None = None,
|
|
environ: Mapping[str, str] | None = None,
|
|
identity: ShardIdentity | None = None,
|
|
) -> CapabilityReport:
|
|
"""Assemble a report from flat validation results.
|
|
|
|
`model_config` may be the loaded config mapping (hashed into a fingerprint)
|
|
or an already-computed ``sha256:…`` string. `validated_at` defaults to now,
|
|
so callers that need determinism pass it explicitly. `identity` is the exact
|
|
DGR-003 artifact/recipe block, when the backend can state one.
|
|
"""
|
|
return CapabilityReport(
|
|
model=ModelIdentity(
|
|
model_id=model_id,
|
|
revision=revision,
|
|
config_fingerprint=config_fingerprint(model_config),
|
|
),
|
|
shard=ShardRange(start=shard_start, end=shard_end),
|
|
recipe=RecipeIdentity(
|
|
recipe_id=recipe_id,
|
|
recipe_version=recipe_version,
|
|
catalogue_version=catalogue_version,
|
|
),
|
|
backend=BackendIdentity(
|
|
backend_id=backend_id,
|
|
device=device,
|
|
device_name=device_name,
|
|
quantization=quantization,
|
|
runtime=dict(runtime or {}),
|
|
),
|
|
status=status,
|
|
validated_at=time.time() if validated_at is None else validated_at,
|
|
duration_ms=duration_ms,
|
|
diagnostics=sanitize_diagnostics(diagnostics, environ),
|
|
identity=identity,
|
|
)
|