feat: compare safetensors and gguf on cpu and gpu

packages/node/meshnet_node/performance_contract.py

@@ -24,9 +24,11 @@ class ModelTarget:
     name: str
     architecture: str
     safetensors_repo: str
+    safetensors_precision: str
     gguf_repo: str
     gguf_quant: str
     gguf_size_gb: float
+    comparison_policy: str
     rationale: str
 
     def to_dict(self) -> dict:
@@ -34,9 +36,11 @@ class ModelTarget:
             "name": self.name,
             "architecture": self.architecture,
             "safetensors_repo": self.safetensors_repo,
+            "safetensors_precision": self.safetensors_precision,
             "gguf_repo": self.gguf_repo,
             "gguf_quant": self.gguf_quant,
             "gguf_size_gb": self.gguf_size_gb,
+            "comparison_policy": self.comparison_policy,
             "rationale": self.rationale,
         }
 
@@ -47,6 +51,7 @@ class BenchmarkLane:
 
     id: str
     runtime: str
+    device: str
     recipe: str
     concurrency_levels: tuple[int, ...]
 
@@ -54,6 +59,7 @@ class BenchmarkLane:
         return {
             "id": self.id,
             "runtime": self.runtime,
+            "device": self.device,
             "recipe": self.recipe,
             "concurrency_levels": list(self.concurrency_levels),
         }
@@ -96,9 +102,14 @@ DEFAULT_CONTRACT = PerformanceContract(
         name="DeepSeek-V2-Lite-Chat",
         architecture="deepseek2",
         safetensors_repo="deepseek-ai/DeepSeek-V2-Lite-Chat",
+        safetensors_precision="bfloat16",
         gguf_repo="second-state/DeepSeek-V2-Lite-Chat-GGUF",
         gguf_quant="Q2_K",
         gguf_size_gb=6.43,
+        comparison_policy=(
+            "same model/revision, closest practical low-footprint precision pair: "
+            "BF16 safetensors versus Q2_K GGUF"
+        ),
         rationale=(
             "Smallest DeepSeek-family benchmark anchor that still points toward "
             "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead "
@@ -107,14 +118,30 @@ DEFAULT_CONTRACT = PerformanceContract(
     ),
     benchmark_lanes=(
         BenchmarkLane(
-            id="transformers-safetensors",
+            id="transformers-safetensors-cpu",
             runtime="transformers",
+            device="cpu",
             recipe="current safetensors recipe",
             concurrency_levels=(1, 4),
         ),
         BenchmarkLane(
-            id="llama-cpp-gguf",
+            id="llama-cpp-gguf-cpu",
             runtime="llama.cpp",
+            device="cpu",
+            recipe="whole-model GGUF recipe",
+            concurrency_levels=(1, 4),
+        ),
+        BenchmarkLane(
+            id="transformers-safetensors-gpu",
+            runtime="transformers",
+            device="gpu",
+            recipe="current safetensors recipe",
+            concurrency_levels=(1, 4),
+        ),
+        BenchmarkLane(
+            id="llama-cpp-gguf-gpu",
+            runtime="llama.cpp",
+            device="gpu",
             recipe="whole-model GGUF recipe",
             concurrency_levels=(1, 4),
         ),