feat: compare safetensors and gguf on cpu and gpu
This commit is contained in:
@@ -5,7 +5,8 @@
|
||||
1,
|
||||
4
|
||||
],
|
||||
"id": "transformers-safetensors",
|
||||
"device": "cpu",
|
||||
"id": "transformers-safetensors-cpu",
|
||||
"recipe": "current safetensors recipe",
|
||||
"runtime": "transformers"
|
||||
},
|
||||
@@ -14,7 +15,28 @@
|
||||
1,
|
||||
4
|
||||
],
|
||||
"id": "llama-cpp-gguf",
|
||||
"device": "cpu",
|
||||
"id": "llama-cpp-gguf-cpu",
|
||||
"recipe": "whole-model GGUF recipe",
|
||||
"runtime": "llama.cpp"
|
||||
},
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"device": "gpu",
|
||||
"id": "transformers-safetensors-gpu",
|
||||
"recipe": "current safetensors recipe",
|
||||
"runtime": "transformers"
|
||||
},
|
||||
{
|
||||
"concurrency_levels": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"device": "gpu",
|
||||
"id": "llama-cpp-gguf-gpu",
|
||||
"recipe": "whole-model GGUF recipe",
|
||||
"runtime": "llama.cpp"
|
||||
}
|
||||
@@ -34,11 +56,13 @@
|
||||
],
|
||||
"model_target": {
|
||||
"architecture": "deepseek2",
|
||||
"comparison_policy": "same model/revision, closest practical low-footprint precision pair: BF16 safetensors versus Q2_K GGUF",
|
||||
"gguf_quant": "Q2_K",
|
||||
"gguf_repo": "second-state/DeepSeek-V2-Lite-Chat-GGUF",
|
||||
"gguf_size_gb": 6.43,
|
||||
"name": "DeepSeek-V2-Lite-Chat",
|
||||
"rationale": "Smallest DeepSeek-family benchmark anchor that still points toward DeepSeek-V4-Flash; keeps the runtime on the DeepSeek2 path instead of falling back to a tiny but architecture-mismatched smoke model.",
|
||||
"safetensors_precision": "bfloat16",
|
||||
"safetensors_repo": "deepseek-ai/DeepSeek-V2-Lite-Chat"
|
||||
},
|
||||
"notes": [
|
||||
|
||||
Reference in New Issue
Block a user