From 5b33bf8b99f4e4fb8a71ad3a6369e9b897a14963 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Tue, 14 Jul 2026 18:45:12 +0300 Subject: [PATCH] feat: compare safetensors and gguf on cpu and gpu --- .../evidence/DGR-001/README.md | 6 ++-- .../DGR-001/performance-contract.json | 28 +++++++++++++++-- ...ensors-versus-gguf-performance-contract.md | 7 ++++- .../node/meshnet_node/performance_contract.py | 31 +++++++++++++++++-- tests/test_performance_contract.py | 25 +++++++++++++-- 5 files changed, 88 insertions(+), 9 deletions(-) diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md index 38f8689..19a6f70 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -11,9 +11,11 @@ - Locks the DGR-001 benchmark contract in code. - Pins the architecture-aligned baseline to **DeepSeek-V2-Lite-Chat** (`deepseek2`). -- Uses the smallest DeepSeek-family GGUF target selected for this story: **Q2_K** via `second-state/DeepSeek-V2-Lite-Chat-GGUF`. +- Uses the same model on both sides of the comparison: + - **safetensors:** `deepseek-ai/DeepSeek-V2-Lite-Chat` in **BF16** + - **GGUF:** `second-state/DeepSeek-V2-Lite-Chat-GGUF` in **Q2_K** - Exposes a machine-readable JSON contract with: - - benchmark lanes for `transformers` safetensors and `llama.cpp` GGUF + - benchmark lanes for `transformers` safetensors and `llama.cpp` GGUF on **CPU** and **GPU** - concurrency levels `1` and `4` - the required metrics list - an explicit stop condition for “no meaningful speed or fit benefit” diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json index 37f412e..02d1562 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json @@ -5,7 +5,8 @@ 1, 4 ], - "id": "transformers-safetensors", + "device": "cpu", + "id": "transformers-safetensors-cpu", "recipe": "current safetensors recipe", "runtime": "transformers" }, @@ -14,7 +15,28 @@ 1, 4 ], - "id": "llama-cpp-gguf", + "device": "cpu", + "id": "llama-cpp-gguf-cpu", + "recipe": "whole-model GGUF recipe", + "runtime": "llama.cpp" + }, + { + "concurrency_levels": [ + 1, + 4 + ], + "device": "gpu", + "id": "transformers-safetensors-gpu", + "recipe": "current safetensors recipe", + "runtime": "transformers" + }, + { + "concurrency_levels": [ + 1, + 4 + ], + "device": "gpu", + "id": "llama-cpp-gguf-gpu", "recipe": "whole-model GGUF recipe", "runtime": "llama.cpp" } @@ -34,11 +56,13 @@ ], "model_target": { "architecture": "deepseek2", + "comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF", "gguf_quant": "Q2_K", "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", "gguf_size_gb": 6.43, "name": "DeepSeek-V2-Lite-Chat", "rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.", + "safetensors_precision": "bfloat16", "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat" }, "notes": [ diff --git a/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md b/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md index 569f174..35904ba 100644 --- a/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md +++ b/.scratch/distributed-gguf-runtime/issues/01-lock-the-safetensors-versus-gguf-performance-contract.md @@ -15,7 +15,12 @@ As a runtime engineer, I need a controlled baseline so that GGUF work proceeds f ## Baseline model target -Use the smallest *DeepSeek-family* GGUF that still points toward DeepSeek-V4-Flash. Current choice: **DeepSeek-V2-Lite GGUF Q2_K** (~6.5GB, `deepseek2` architecture). Reserve smaller non-DeepSeek fallback models only for loader plumbing smoke tests if needed; they do not count as the DGR-001 architecture-aligned baseline. +Use the same model on both sides of the comparison, with the closest practical low-footprint precision pair: + +- **safetensors:** `deepseek-ai/DeepSeek-V2-Lite-Chat` in **BF16** +- **GGUF:** `second-state/DeepSeek-V2-Lite-Chat-GGUF` in **Q2_K** (~6.5GB) + +Keep the benchmark matrix explicit for **CPU** and **GPU** runs. Reserve smaller non-DeepSeek fallback models only for loader plumbing smoke tests if needed; they do not count as the DGR-001 architecture-aligned baseline. ## Expected durable outputs diff --git a/packages/node/meshnet_node/performance_contract.py b/packages/node/meshnet_node/performance_contract.py index fa113a1..8b04067 100644 --- a/packages/node/meshnet_node/performance_contract.py +++ b/packages/node/meshnet_node/performance_contract.py @@ -24,9 +24,11 @@ class ModelTarget: name: str architecture: str safetensors_repo: str + safetensors_precision: str gguf_repo: str gguf_quant: str gguf_size_gb: float + comparison_policy: str rationale: str def to_dict(self) -> dict: @@ -34,9 +36,11 @@ class ModelTarget: "name": self.name, "architecture": self.architecture, "safetensors_repo": self.safetensors_repo, + "safetensors_precision": self.safetensors_precision, "gguf_repo": self.gguf_repo, "gguf_quant": self.gguf_quant, "gguf_size_gb": self.gguf_size_gb, + "comparison_policy": self.comparison_policy, "rationale": self.rationale, } @@ -47,6 +51,7 @@ class BenchmarkLane: id: str runtime: str + device: str recipe: str concurrency_levels: tuple[int, ...] @@ -54,6 +59,7 @@ class BenchmarkLane: return { "id": self.id, "runtime": self.runtime, + "device": self.device, "recipe": self.recipe, "concurrency_levels": list(self.concurrency_levels), } @@ -96,9 +102,14 @@ DEFAULT_CONTRACT = PerformanceContract( name="DeepSeek-V2-Lite-Chat", architecture="deepseek2", safetensors_repo="deepseek-ai/DeepSeek-V2-Lite-Chat", + safetensors_precision="bfloat16", gguf_repo="second-state/DeepSeek-V2-Lite-Chat-GGUF", gguf_quant="Q2_K", gguf_size_gb=6.43, + comparison_policy=( + "same model/revision, closest practical low-footprint precision pair: " + "BF16 safetensors versus Q2_K GGUF" + ), rationale=( "Smallest DeepSeek-family benchmark anchor that still points toward " "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead " @@ -107,14 +118,30 @@ DEFAULT_CONTRACT = PerformanceContract( ), benchmark_lanes=( BenchmarkLane( - id="transformers-safetensors", + id="transformers-safetensors-cpu", runtime="transformers", + device="cpu", recipe="current safetensors recipe", concurrency_levels=(1, 4), ), BenchmarkLane( - id="llama-cpp-gguf", + id="llama-cpp-gguf-cpu", runtime="llama.cpp", + device="cpu", + recipe="whole-model GGUF recipe", + concurrency_levels=(1, 4), + ), + BenchmarkLane( + id="transformers-safetensors-gpu", + runtime="transformers", + device="gpu", + recipe="current safetensors recipe", + concurrency_levels=(1, 4), + ), + BenchmarkLane( + id="llama-cpp-gguf-gpu", + runtime="llama.cpp", + device="gpu", recipe="whole-model GGUF recipe", concurrency_levels=(1, 4), ), diff --git a/tests/test_performance_contract.py b/tests/test_performance_contract.py index 8bd6e90..41f7e79 100644 --- a/tests/test_performance_contract.py +++ b/tests/test_performance_contract.py @@ -20,9 +20,14 @@ def test_default_contract_is_architecture_aligned_and_small(): "name": "DeepSeek-V2-Lite-Chat", "architecture": "deepseek2", "safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat", + "safetensors_precision": "bfloat16", "gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF", "gguf_quant": "Q2_K", "gguf_size_gb": 6.43, + "comparison_policy": ( + "same model/revision, closest practical low-footprint precision pair: " + "BF16 safetensors versus Q2_K GGUF" + ), "rationale": ( "Smallest DeepSeek-family benchmark anchor that still points toward " "DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead " @@ -31,14 +36,30 @@ def test_default_contract_is_architecture_aligned_and_small(): } assert payload["benchmark_lanes"] == [ { - "id": "transformers-safetensors", + "id": "transformers-safetensors-cpu", "runtime": "transformers", + "device": "cpu", "recipe": "current safetensors recipe", "concurrency_levels": [1, 4], }, { - "id": "llama-cpp-gguf", + "id": "llama-cpp-gguf-cpu", "runtime": "llama.cpp", + "device": "cpu", + "recipe": "whole-model GGUF recipe", + "concurrency_levels": [1, 4], + }, + { + "id": "transformers-safetensors-gpu", + "runtime": "transformers", + "device": "gpu", + "recipe": "current safetensors recipe", + "concurrency_levels": [1, 4], + }, + { + "id": "llama-cpp-gguf-gpu", + "runtime": "llama.cpp", + "device": "gpu", "recipe": "whole-model GGUF recipe", "concurrency_levels": [1, 4], },