feat: DGR-001 - Lock the safetensors-versus-GGUF performance contract

packages/node/meshnet_node/recipe_benchmark.py

@@ -115,6 +115,8 @@ class BenchmarkPlan:
             raise BenchmarkError("concurrency levels must all be >= 1")
         if self.repeats < 1:
             raise BenchmarkError("repeats must be >= 1")
+        if 1 not in self.concurrency_levels or 4 not in self.concurrency_levels:
+            raise BenchmarkError("a controlled baseline must include concurrency levels 1 and 4")
 
     def to_dict(self) -> dict:
         return {
@@ -145,6 +147,9 @@ class RecipeSpec:
     lane: Lane
     device: str
     artifact_path: str = ""
+    source_model_id: str = ""
+    source_model_revision: str = ""
+    artifact_sha256: str = ""
     is_reference: bool = False
     notes: str = ""
 

packages/node/meshnet_node/recipe_drivers.py

@@ -22,8 +22,10 @@ from __future__ import annotations
 
 import json
 import os
+import platform
 import socket
 import subprocess
+import sys
 import time
 import urllib.error
 import urllib.request
@@ -83,6 +85,56 @@ def _directory_bytes(path: Path) -> int:
     return sum(entry.stat().st_size for entry in path.rglob("*") if entry.is_file())
 
 
+def _host_manifest() -> dict[str, Any]:
+    """Capture non-secret host facts with the report rather than trusting prose."""
+    manifest: dict[str, Any] = {
+        "hostname": socket.gethostname(),
+        "platform": platform.platform(),
+        "python": sys.version.split()[0],
+        "cpu_count": os.cpu_count(),
+    }
+    try:
+        import torch
+
+        manifest["torch_version"] = torch.__version__
+        manifest["cuda_available"] = bool(torch.cuda.is_available())
+        if torch.cuda.is_available():
+            manifest["accelerator_name"] = torch.cuda.get_device_name(0)
+            manifest["accelerator_runtime"] = getattr(torch.version, "cuda", None) or getattr(
+                torch.version, "hip", None
+            )
+    except ImportError:
+        manifest["torch_version"] = None
+    return manifest
+
+
+def _validate_config(config: Mapping[str, Any]) -> None:
+    """Reject a comparison that could silently mix models or use home storage."""
+    try:
+        plan = config["plan"]
+        root = Path(config["artifact_storage_root"]).resolve(strict=True)
+        recipes = config["recipes"]
+    except (KeyError, TypeError, OSError) as exc:
+        raise BenchmarkError(
+            "benchmark config needs an existing artifact_storage_root, plan, and recipes"
+        ) from exc
+    if not root.is_absolute() or root == Path("/home") or Path("/home") in root.parents:
+        raise BenchmarkError("model artifacts must use configured mounted-drive storage, never /home")
+    if not isinstance(recipes, list) or not recipes:
+        raise BenchmarkError("benchmark config needs at least one recipe")
+    for spec in recipes:
+        if spec.get("source_model_id") != plan.get("model_id"):
+            raise BenchmarkError("every recipe must declare the plan's exact source_model_id")
+        if spec.get("source_model_revision") != plan.get("model_revision"):
+            raise BenchmarkError("every recipe must declare the plan's exact source_model_revision")
+        digest = spec.get("artifact_sha256", "")
+        if not isinstance(digest, str) or len(digest) != 64:
+            raise BenchmarkError("every recipe must declare its exact 64-character artifact_sha256")
+        artifact = Path(spec.get("artifact_path", "")).resolve(strict=True)
+        if artifact != root and root not in artifact.parents:
+            raise BenchmarkError("every model artifact must be beneath artifact_storage_root")
+
+
 class TransformersDriver:
     """The current Transformers/safetensors recipe: the correctness reference.
 
@@ -435,6 +487,9 @@ def _recipe_from_config(spec: Mapping[str, Any]) -> RecipeSpec:
         lane=Lane(spec["lane"]),
         device=spec["device"],
         artifact_path=spec.get("artifact_path", ""),
+        source_model_id=spec.get("source_model_id", ""),
+        source_model_revision=spec.get("source_model_revision", ""),
+        artifact_sha256=spec.get("artifact_sha256", ""),
         is_reference=bool(spec.get("is_reference", False)),
         notes=spec.get("notes", ""),
     )
@@ -448,6 +503,7 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
     crashed would read as a clean result.
     """
     require_real_inference()
+    _validate_config(config)
     plan = _plan_from_config(config)
 
     from .recipe_benchmark import RecipeMeasurement  # local import keeps the seam obvious
@@ -468,6 +524,6 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict:
     return build_report(
         plan,
         measurements,
-        host=dict(config.get("host", {})),
+        host={**_host_manifest(), **dict(config.get("host", {}))},
         evidence_class=config.get("evidence_class", "local-real"),
     )