neuron-tai/.scratch/distributed-gguf-runtime/evidence/DGR-020/benchmark-config.json

{
  "artifact_storage_root": "/run/media/popov/DATA/llm",
  "evidence_class": "local-real",
  "host": {
    "benchmark_lane": "cpu-controlled-baseline",
    "llama_cpp_commit": "e920c523e3b8a0163fe498af5bf90df35ff51d25",
    "llama_cpp_version": "9991",
    "llama_server_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
    "converter_sha256": "c819f18fb22927b49fabc3b35d1c9e21ee638b3817eccd1bd4efbcc7116eeb4d",
    "quantizer_sha256": "bd0cc8c7be6d48aad4755b31062e0e59a887cbadd43dbb8771853d5858bb198f",
    "transformers_version": "5.13.0"
  },
  "plan": {
    "plan_id": "dgr-001-controlled-whole-model-baseline-v1",
    "model_id": "Qwen/Qwen2.5-0.5B-Instruct",
    "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
    "prompts": [
      {
        "id": "short-fact",
        "text": "The capital of France is",
        "context_class": "short"
      },
      {
        "id": "medium-code",
        "text": "Complete this Python function without commentary:\n\ndef fibonacci(n):\n    \"\"\"Return the nth Fibonacci number for n >= 0.\"\"\"\n",
        "context_class": "medium"
      },
      {
        "id": "long-summary",
        "text": "A distributed inference service divides a transformer across consumer machines. The tracker owns admission, routing, cancellation, accounting, and telemetry, while workers own only model execution. Every request carries an immutable model identity and revision. Workers must reject incompatible protocol versions and resource demands before allocating large buffers. Activation tensors are chunked, checksummed, bounded by negotiated limits, and propagated with explicit flow-control credits. A caller may disconnect at any time, so cancellation must release queued work, in-flight transfers, and cache reservations without double billing. Retries can occur after network failures, requiring idempotent request identifiers and deterministic completion accounting. The system keeps the existing safetensors path as a correctness reference while a native GGUF path is measured. Benchmarks compare the same prompts, output lengths, sampling policy, device, and concurrency, and they separate near-lossless quality checks from quantized speed and fit claims. Summarize the design priorities in three concise bullet points.",
        "context_class": "long"
      }
    ],
    "sampling": {
      "temperature": 0.0,
      "top_p": 1.0,
      "top_k": 1,
      "seed": 1234,
      "max_output_tokens": 32
    },
    "concurrency_levels": [1, 4],
    "repeats": 3,
    "warmup_requests": 2
  },
  "recipes": [
    {
      "id": "transformers-safetensors-reference",
      "runtime": "transformers-5.13.0",
      "weight_format": "safetensors",
      "weight_quantization": "bfloat16",
      "lane": "quality",
      "device": "cpu",
      "artifact_path": "/run/media/popov/DATA/llm/safetensor/models/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775",
      "artifact_sha256": "e596e9d6205fdc9177569cccd7f8b471b058f66e3630c8e4326d5aad52bd18b6",
      "source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
      "source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
      "is_reference": true,
      "notes": "artifact_sha256 is the deterministic digest of every snapshot path and file byte",
      "driver": {
        "type": "transformers",
        "model_path": "/run/media/popov/DATA/llm/safetensor/models/models--Qwen--Qwen2.5-0.5B-Instruct/snapshots/7ae557604adf67be50417f59c2c2f167def9a775",
        "device": "cpu",
        "dtype": "bfloat16",
        "threads": 16
      }
    },
    {
      "id": "llama-cpp-near-lossless-quality",
      "runtime": "llama.cpp-9991-e920c523",
      "weight_format": "gguf",
      "weight_quantization": "bfloat16",
      "lane": "quality",
      "device": "cpu",
      "artifact_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-BF16.gguf",
      "artifact_sha256": "e842fdc35d7f00fda95a54e1b51731ba1d196aea45065cc9f46925fdc1d6f862",
      "source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
      "source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
      "is_reference": false,
      "notes": "Converted directly from the exact mounted safetensors revision while preserving BF16 weights with pinned llama.cpp",
      "driver": {
        "type": "llama-cpp-server",
        "binary": "/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server",
        "binary_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
        "gguf_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-BF16.gguf",
        "device": "cpu",
        "threads": 16,
        "n_parallel": 4,
        "context_per_slot": 512,
        "n_gpu_layers": 0
      }
    },
    {
      "id": "llama-cpp-quantized-performance-fit",
      "runtime": "llama.cpp-9991-e920c523",
      "weight_format": "gguf",
      "weight_quantization": "Q4_K_M",
      "lane": "performance-fit",
      "device": "cpu",
      "artifact_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-Q4_K_M.gguf",
      "artifact_sha256": "a88e3f570e2efeaf06b50df9859db2c70d8646aa3a2c94a14e14d5797a2921a5",
      "source_model_id": "Qwen/Qwen2.5-0.5B-Instruct",
      "source_model_revision": "7ae557604adf67be50417f59c2c2f167def9a775",
      "is_reference": false,
      "notes": "Quantized from the exact-revision F16 GGUF with pinned llama-quantize",
      "driver": {
        "type": "llama-cpp-server",
        "binary": "/run/media/popov/d/DEV/llamacpp/llama.cpp/build/bin/llama-server",
        "binary_sha256": "fd8fe612970f23e447f2e717cfa51665be06b8d7315ba60556e010f6bca510dd",
        "gguf_path": "/run/media/popov/DATA/llm/dgr-001/Qwen2.5-0.5B-Instruct-7ae5576-Q4_K_M.gguf",
        "device": "cpu",
        "threads": 16,
        "n_parallel": 4,
        "context_per_slot": 512,
        "n_gpu_layers": 0
      }
    }
  ]
}