feat: compare safetensors and gguf on cpu and gpu

.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md

@@ -11,9 +11,11 @@
 
 - Locks the DGR-001 benchmark contract in code.
 - Pins the architecture-aligned baseline to **DeepSeek-V2-Lite-Chat** (`deepseek2`).
-- Uses the smallest DeepSeek-family GGUF target selected for this story: **Q2_K** via `second-state/DeepSeek-V2-Lite-Chat-GGUF`.
+- Uses the same model on both sides of the comparison:
+  - **safetensors:** `deepseek-ai/DeepSeek-V2-Lite-Chat` in **BF16**
+  - **GGUF:** `second-state/DeepSeek-V2-Lite-Chat-GGUF` in **Q2_K**
 - Exposes a machine-readable JSON contract with:
-  - benchmark lanes for `transformers` safetensors and `llama.cpp` GGUF
+  - benchmark lanes for `transformers` safetensors and `llama.cpp` GGUF on **CPU** and **GPU**
   - concurrency levels `1` and `4`
   - the required metrics list
   - an explicit stop condition for “no meaningful speed or fit benefit”

.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json

@@ -5,7 +5,8 @@
         1,
         4
       ],
-      "id": "transformers-safetensors",
+      "device": "cpu",
+      "id": "transformers-safetensors-cpu",
       "recipe": "current safetensors recipe",
       "runtime": "transformers"
     },
@@ -14,7 +15,28 @@
         1,
         4
       ],
-      "id": "llama-cpp-gguf",
+      "device": "cpu",
+      "id": "llama-cpp-gguf-cpu",
+      "recipe": "whole-model GGUF recipe",
+      "runtime": "llama.cpp"
+    },
+    {
+      "concurrency_levels": [
+        1,
+        4
+      ],
+      "device": "gpu",
+      "id": "transformers-safetensors-gpu",
+      "recipe": "current safetensors recipe",
+      "runtime": "transformers"
+    },
+    {
+      "concurrency_levels": [
+        1,
+        4
+      ],
+      "device": "gpu",
+      "id": "llama-cpp-gguf-gpu",
       "recipe": "whole-model GGUF recipe",
       "runtime": "llama.cpp"
     }
@@ -34,11 +56,13 @@
   ],
   "model_target": {
     "architecture": "deepseek2",
+    "comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF",
     "gguf_quant": "Q2_K",
     "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
     "gguf_size_gb": 6.43,
     "name": "DeepSeek-V2-Lite-Chat",
     "rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.",
+    "safetensors_precision": "bfloat16",
     "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat"
   },
   "notes": [

.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md

@@ -15,7 +15,12 @@ As a runtime engineer, I need a controlled baseline so that GGUF work proceeds f
 
 ## Baseline model target
 
-Use the smallest *DeepSeek-family* GGUF that still points toward DeepSeek-V4-Flash. Current choice: **DeepSeek-V2-Lite GGUF Q2_K** (~6.5GB, `deepseek2` architecture). Reserve smaller non-DeepSeek fallback models only for loader plumbing smoke tests if needed; they do not count as the DGR-001 architecture-aligned baseline.
+Use the same model on both sides of the comparison, with the closest practical low-footprint precision pair:
+
+- **safetensors:** `deepseek-ai/DeepSeek-V2-Lite-Chat` in **BF16**
+- **GGUF:** `second-state/DeepSeek-V2-Lite-Chat-GGUF` in **Q2_K** (~6.5GB)
+
+Keep the benchmark matrix explicit for **CPU** and **GPU** runs. Reserve smaller non-DeepSeek fallback models only for loader plumbing smoke tests if needed; they do not count as the DGR-001 architecture-aligned baseline.
 
 ## Expected durable outputs
 

packages/node/meshnet_node/performance_contract.py

@@ -24,9 +24,11 @@ class ModelTarget:
     name: str
     architecture: str
     safetensors_repo: str
+    safetensors_precision: str
     gguf_repo: str
     gguf_quant: str
     gguf_size_gb: float
+    comparison_policy: str
     rationale: str
 
     def to_dict(self) -> dict:
@@ -34,9 +36,11 @@ class ModelTarget:
             "name": self.name,
             "architecture": self.architecture,
             "safetensors_repo": self.safetensors_repo,
+            "safetensors_precision": self.safetensors_precision,
             "gguf_repo": self.gguf_repo,
             "gguf_quant": self.gguf_quant,
             "gguf_size_gb": self.gguf_size_gb,
+            "comparison_policy": self.comparison_policy,
             "rationale": self.rationale,
         }
 
@@ -47,6 +51,7 @@ class BenchmarkLane:
 
     id: str
     runtime: str
+    device: str
     recipe: str
     concurrency_levels: tuple[int, ...]
 
@@ -54,6 +59,7 @@ class BenchmarkLane:
         return {
             "id": self.id,
             "runtime": self.runtime,
+            "device": self.device,
             "recipe": self.recipe,
             "concurrency_levels": list(self.concurrency_levels),
         }
@@ -96,9 +102,14 @@ DEFAULT_CONTRACT = PerformanceContract(
         name="DeepSeek-V2-Lite-Chat",
         architecture="deepseek2",
         safetensors_repo="deepseek-ai/DeepSeek-V2-Lite-Chat",
+        safetensors_precision="bfloat16",
         gguf_repo="second-state/DeepSeek-V2-Lite-Chat-GGUF",
         gguf_quant="Q2_K",
         gguf_size_gb=6.43,
+        comparison_policy=(
+            "same model/revision, closest practical low-footprint precision pair: "
+            "BF16 safetensors versus Q2_K GGUF"
+        ),
         rationale=(
             "Smallest DeepSeek-family benchmark anchor that still points toward "
             "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead "
@@ -107,14 +118,30 @@ DEFAULT_CONTRACT = PerformanceContract(
     ),
     benchmark_lanes=(
         BenchmarkLane(
-            id="transformers-safetensors",
+            id="transformers-safetensors-cpu",
             runtime="transformers",
+            device="cpu",
             recipe="current safetensors recipe",
             concurrency_levels=(1, 4),
         ),
         BenchmarkLane(
-            id="llama-cpp-gguf",
+            id="llama-cpp-gguf-cpu",
             runtime="llama.cpp",
+            device="cpu",
+            recipe="whole-model GGUF recipe",
+            concurrency_levels=(1, 4),
+        ),
+        BenchmarkLane(
+            id="transformers-safetensors-gpu",
+            runtime="transformers",
+            device="gpu",
+            recipe="current safetensors recipe",
+            concurrency_levels=(1, 4),
+        ),
+        BenchmarkLane(
+            id="llama-cpp-gguf-gpu",
+            runtime="llama.cpp",
+            device="gpu",
             recipe="whole-model GGUF recipe",
             concurrency_levels=(1, 4),
         ),

tests/test_performance_contract.py

@@ -20,9 +20,14 @@ def test_default_contract_is_architecture_aligned_and_small():
         "name": "DeepSeek-V2-Lite-Chat",
         "architecture": "deepseek2",
         "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat",
+        "safetensors_precision": "bfloat16",
         "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
         "gguf_quant": "Q2_K",
         "gguf_size_gb": 6.43,
+        "comparison_policy": (
+            "same model/revision, closest practical low-footprint precision pair: "
+            "BF16 safetensors versus Q2_K GGUF"
+        ),
         "rationale": (
             "Smallest DeepSeek-family benchmark anchor that still points toward "
             "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead "
@@ -31,14 +36,30 @@ def test_default_contract_is_architecture_aligned_and_small():
     }
     assert payload["benchmark_lanes"] == [
         {
-            "id": "transformers-safetensors",
+            "id": "transformers-safetensors-cpu",
             "runtime": "transformers",
+            "device": "cpu",
             "recipe": "current safetensors recipe",
             "concurrency_levels": [1, 4],
         },
         {
-            "id": "llama-cpp-gguf",
+            "id": "llama-cpp-gguf-cpu",
             "runtime": "llama.cpp",
+            "device": "cpu",
+            "recipe": "whole-model GGUF recipe",
+            "concurrency_levels": [1, 4],
+        },
+        {
+            "id": "transformers-safetensors-gpu",
+            "runtime": "transformers",
+            "device": "gpu",
+            "recipe": "current safetensors recipe",
+            "concurrency_levels": [1, 4],
+        },
+        {
+            "id": "llama-cpp-gguf-gpu",
+            "runtime": "llama.cpp",
+            "device": "gpu",
             "recipe": "whole-model GGUF recipe",
             "concurrency_levels": [1, 4],
         },