feat: checkpoint distributed gguf runtime stories

This commit is contained in:
Dobromir Popov
2026-07-15 23:42:58 +03:00
parent eaf00f6add
commit 1fe31ef38d
60 changed files with 8478 additions and 105 deletions

View File

@@ -36,6 +36,8 @@ from .capability import (
CapabilityReport,
build_capability_report,
)
from . import __version__ as _PACKAGE_VERSION
from .runtime_recipe import build_runtime_recipe_identity
from .recipe_manifest import (
DEFAULT_RECIPE_ID,
Recipe,
@@ -43,6 +45,7 @@ from .recipe_manifest import (
RecipeManifestError,
load_recipe_manifest,
)
from .gguf_ownership import authoritative_dense_llama_ownership
# The probe is deliberately tiny: enough tokens to drive every layer in the
# shard once, small enough that `doctor` costs seconds beyond the model load.
@@ -464,10 +467,28 @@ def _validate_recipe(
duration_ms = int((time.monotonic() - started) * 1000)
device = _backend_device(backend, selection)
ownership = authoritative_dense_llama_ownership(backend, selection)
runtime_recipe = build_runtime_recipe_identity(
model_id=selection.model_id,
revision=getattr(getattr(backend, "model", None), "revision", None),
model_config=_model_config(backend),
recipe_params=recipe.params,
weight_quantization=selection.quantization,
backend_id=recipe.backend_id,
runtime_version=_PACKAGE_VERSION,
activation_dtype="bfloat16",
compute_dtype=_backend_compute_dtype(backend),
kv_dtype=_backend_kv_dtype(backend),
kv_layout=_backend_kv_layout(backend),
tokenizer_revision=_backend_tokenizer_revision(backend, selection),
architecture_adapter=_backend_architecture_adapter(backend, recipe.backend_id),
boundary_schema_version=1,
cache_layout=_backend_cache_layout(backend, recipe.params),
)
report = build_capability_report(
model_id=selection.model_id,
shard_start=selection.shard_start,
shard_end=selection.shard_end,
shard_start=ownership.start_layer,
shard_end=ownership.end_layer,
recipe_id=recipe.id,
recipe_version=recipe.version,
catalogue_version=manifest.catalogue_version,
@@ -477,6 +498,9 @@ def _validate_recipe(
quantization=selection.quantization,
runtime=_runtime_versions(),
model_config=_model_config(backend),
runtime_recipe=runtime_recipe,
owns_embedding=ownership.owns_embedding,
owns_final_head=ownership.owns_final_head,
status=STATUS_FAILED if category else STATUS_PASSED,
duration_ms=duration_ms,
diagnostics=[d for d in diagnostics if d] or None,
@@ -568,6 +592,65 @@ def _runtime_versions() -> dict[str, str]:
return versions
def _backend_compute_dtype(backend: Any) -> str:
config = getattr(getattr(backend, "model", None), "config", None)
for candidate in (config, getattr(config, "text_config", None)):
if candidate is None:
continue
for attr in ("dtype", "torch_dtype"):
value = getattr(candidate, attr, None)
if value is None:
continue
return str(value).removeprefix("torch.")
return "bfloat16"
def _backend_kv_dtype(backend: Any) -> str:
return _backend_compute_dtype(backend)
def _backend_kv_layout(backend: Any) -> str:
return "session-cache" if getattr(backend, "supports_kv_cache", False) else "stateless"
def _backend_tokenizer_revision(backend: Any, selection: DoctorSelection) -> str:
model = getattr(backend, "model", None)
revision = getattr(model, "revision", None)
if isinstance(revision, str) and revision.strip():
return revision
return selection.model_id
def _backend_architecture_adapter(backend: Any, default: str) -> str:
config = getattr(getattr(backend, "model", None), "config", None)
for candidate in (config, getattr(config, "text_config", None)):
if candidate is None:
continue
for attr in ("architecture_adapter", "model_type"):
value = getattr(candidate, attr, None)
if isinstance(value, str) and value.strip():
return value
architectures = getattr(candidate, "architectures", None)
if isinstance(architectures, (list, tuple)) and architectures:
first = architectures[0]
if isinstance(first, str) and first.strip():
return first
return default
def _backend_cache_layout(backend: Any, recipe_params: Mapping[str, Any] | None) -> str:
if getattr(backend, "supports_kv_cache", False) is False:
return "stateless"
if recipe_params is None:
return "local-hot-kv"
if recipe_params.get("use_cache") is False:
return "stateless"
value = recipe_params.get("cache_layout")
if isinstance(value, str) and value.strip():
return value
return "local-hot-kv"
# --- output -----------------------------------------------------------------
DEFAULT_REPORT_FILENAME = "capability.json"