feat: checkpoint distributed gguf runtime stories
This commit is contained in:
@@ -36,6 +36,8 @@ from .capability import (
|
||||
CapabilityReport,
|
||||
build_capability_report,
|
||||
)
|
||||
from . import __version__ as _PACKAGE_VERSION
|
||||
from .runtime_recipe import build_runtime_recipe_identity
|
||||
from .recipe_manifest import (
|
||||
DEFAULT_RECIPE_ID,
|
||||
Recipe,
|
||||
@@ -43,6 +45,7 @@ from .recipe_manifest import (
|
||||
RecipeManifestError,
|
||||
load_recipe_manifest,
|
||||
)
|
||||
from .gguf_ownership import authoritative_dense_llama_ownership
|
||||
|
||||
# The probe is deliberately tiny: enough tokens to drive every layer in the
|
||||
# shard once, small enough that `doctor` costs seconds beyond the model load.
|
||||
@@ -464,10 +467,28 @@ def _validate_recipe(
|
||||
duration_ms = int((time.monotonic() - started) * 1000)
|
||||
|
||||
device = _backend_device(backend, selection)
|
||||
ownership = authoritative_dense_llama_ownership(backend, selection)
|
||||
runtime_recipe = build_runtime_recipe_identity(
|
||||
model_id=selection.model_id,
|
||||
revision=getattr(getattr(backend, "model", None), "revision", None),
|
||||
model_config=_model_config(backend),
|
||||
recipe_params=recipe.params,
|
||||
weight_quantization=selection.quantization,
|
||||
backend_id=recipe.backend_id,
|
||||
runtime_version=_PACKAGE_VERSION,
|
||||
activation_dtype="bfloat16",
|
||||
compute_dtype=_backend_compute_dtype(backend),
|
||||
kv_dtype=_backend_kv_dtype(backend),
|
||||
kv_layout=_backend_kv_layout(backend),
|
||||
tokenizer_revision=_backend_tokenizer_revision(backend, selection),
|
||||
architecture_adapter=_backend_architecture_adapter(backend, recipe.backend_id),
|
||||
boundary_schema_version=1,
|
||||
cache_layout=_backend_cache_layout(backend, recipe.params),
|
||||
)
|
||||
report = build_capability_report(
|
||||
model_id=selection.model_id,
|
||||
shard_start=selection.shard_start,
|
||||
shard_end=selection.shard_end,
|
||||
shard_start=ownership.start_layer,
|
||||
shard_end=ownership.end_layer,
|
||||
recipe_id=recipe.id,
|
||||
recipe_version=recipe.version,
|
||||
catalogue_version=manifest.catalogue_version,
|
||||
@@ -477,6 +498,9 @@ def _validate_recipe(
|
||||
quantization=selection.quantization,
|
||||
runtime=_runtime_versions(),
|
||||
model_config=_model_config(backend),
|
||||
runtime_recipe=runtime_recipe,
|
||||
owns_embedding=ownership.owns_embedding,
|
||||
owns_final_head=ownership.owns_final_head,
|
||||
status=STATUS_FAILED if category else STATUS_PASSED,
|
||||
duration_ms=duration_ms,
|
||||
diagnostics=[d for d in diagnostics if d] or None,
|
||||
@@ -568,6 +592,65 @@ def _runtime_versions() -> dict[str, str]:
|
||||
return versions
|
||||
|
||||
|
||||
def _backend_compute_dtype(backend: Any) -> str:
|
||||
config = getattr(getattr(backend, "model", None), "config", None)
|
||||
for candidate in (config, getattr(config, "text_config", None)):
|
||||
if candidate is None:
|
||||
continue
|
||||
for attr in ("dtype", "torch_dtype"):
|
||||
value = getattr(candidate, attr, None)
|
||||
if value is None:
|
||||
continue
|
||||
return str(value).removeprefix("torch.")
|
||||
return "bfloat16"
|
||||
|
||||
|
||||
def _backend_kv_dtype(backend: Any) -> str:
|
||||
return _backend_compute_dtype(backend)
|
||||
|
||||
|
||||
def _backend_kv_layout(backend: Any) -> str:
|
||||
return "session-cache" if getattr(backend, "supports_kv_cache", False) else "stateless"
|
||||
|
||||
|
||||
def _backend_tokenizer_revision(backend: Any, selection: DoctorSelection) -> str:
|
||||
model = getattr(backend, "model", None)
|
||||
revision = getattr(model, "revision", None)
|
||||
if isinstance(revision, str) and revision.strip():
|
||||
return revision
|
||||
return selection.model_id
|
||||
|
||||
|
||||
def _backend_architecture_adapter(backend: Any, default: str) -> str:
|
||||
config = getattr(getattr(backend, "model", None), "config", None)
|
||||
for candidate in (config, getattr(config, "text_config", None)):
|
||||
if candidate is None:
|
||||
continue
|
||||
for attr in ("architecture_adapter", "model_type"):
|
||||
value = getattr(candidate, attr, None)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value
|
||||
architectures = getattr(candidate, "architectures", None)
|
||||
if isinstance(architectures, (list, tuple)) and architectures:
|
||||
first = architectures[0]
|
||||
if isinstance(first, str) and first.strip():
|
||||
return first
|
||||
return default
|
||||
|
||||
|
||||
def _backend_cache_layout(backend: Any, recipe_params: Mapping[str, Any] | None) -> str:
|
||||
if getattr(backend, "supports_kv_cache", False) is False:
|
||||
return "stateless"
|
||||
if recipe_params is None:
|
||||
return "local-hot-kv"
|
||||
if recipe_params.get("use_cache") is False:
|
||||
return "stateless"
|
||||
value = recipe_params.get("cache_layout")
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value
|
||||
return "local-hot-kv"
|
||||
|
||||
|
||||
# --- output -----------------------------------------------------------------
|
||||
|
||||
DEFAULT_REPORT_FILENAME = "capability.json"
|
||||
|
||||
Reference in New Issue
Block a user