diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/BLOCKED.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/BLOCKED.md new file mode 100644 index 0000000..2bae119 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/BLOCKED.md @@ -0,0 +1,30 @@ +# DGR-001 real-benchmark blocker + +Status: blocked only for the required real-model measurement. The deterministic +harness, report schema, and immutable contract are implemented and tested; this +file deliberately does not turn an unrun benchmark into a passing result. + +## Verified environment state (2026-07-13) + +- Mounted GGUF artifacts exist under `/run/media/popov/DATA/llm/`. +- `llama-server` is not on `PATH`. +- The available Python test environment has neither `torch` nor `transformers`. +- No matching local safetensors snapshot was found for an installed GGUF recipe. + +Therefore this session cannot run the controlled same-model, same-revision, +same-machine comparison without downloading/installing new runtime/model assets. +That is intentionally not inferred from the story request. + +## Continuation + +1. Put a matching safetensors snapshot and near-lossless plus quantized GGUF + artifacts below one mounted-drive root, never `/home`. +2. Install or build the pinned `llama-server`, and use `.venv-rocm` when testing + the Radeon backend. +3. Compute each artifact SHA-256 and create a config declaring the same + `source_model_id` and `source_model_revision` for every recipe. +4. Run the command in `commands.txt` with + `MESHNET_ENABLE_REAL_INFERENCE_TESTS=1`; save its JSON report and summary in + this directory, then evaluate it against `performance-contract.json`. +5. Only after those results and all quality gates pass may DGR-001 be marked + done and DGR-004 consume the baseline. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md new file mode 100644 index 0000000..3a79edc --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -0,0 +1,62 @@ +# DGR-001 — Safetensors versus GGUF performance contract + +Status: **blocked for real evidence; deterministic implementation complete.** +No model benchmark is claimed. See `BLOCKED.md` and the explicitly `not-run` +`results.json`. + +## What is implemented + +- `recipe_benchmark.py` is a deterministic measurement core that runs the exact + same plan for every recipe and reports TTFT, prefill/decode rates, p50/p95 + latency, aggregate throughput, RSS, VRAM, artifact bytes, request failures, + and per-prompt output drift in JSON. +- `recipe_drivers.py` supplies opt-in Transformers/safetensors and whole-model + llama.cpp-server drivers. Real execution requires + `MESHNET_ENABLE_REAL_INFERENCE_TESTS=1`, refuses model paths outside the + declared mounted-drive root, requires a SHA-256 per artifact, records host + facts, and requires the same declared source model and revision for every + recipe. +- `performance_contract.py` separates a near-lossless quality lane from the + quantized performance/fit lane. Quantized drift is advisory; only the quality + lane can establish parity. `performance-contract.json` locks v1 thresholds + and the stop condition before any result exists. + +## Files changed + +- `packages/node/meshnet_node/recipe_benchmark.py` +- `packages/node/meshnet_node/recipe_drivers.py` +- `packages/node/meshnet_node/performance_contract.py` +- `tests/test_recipe_benchmark.py` +- This evidence directory. + +## Commands and results + +`commands.txt` contains exact commands. Final targeted result: + +```text +pytest -q tests/test_recipe_benchmark.py -> 15 passed +python -m compileall -q packages tests -> exit 0 +git diff --check -> exit 0 +``` + +The full suite was attempted and is blocked during collection by the unrelated, +pre-existing DGR-002 runtime dependency mismatch: + +```text +google.protobuf.runtime_version.VersionError: +gencode 7.35.0 runtime 6.33.6 +``` + +This was reproduced from a clean `git archive HEAD` extracted to +`/tmp/dgr-001-clean`, with the same command and same failure before any +uncommitted DGR-001 changes were present. No real benchmark command was run +because the prerequisites in `BLOCKED.md` are absent. + +## Compatibility and handoff + +This is additive: it does not alter the current Transformers route, Tracker, +relay, or native protocol. DGR-014 must load `performance-contract.json`, run +the same controlled plan at concurrency 1 and 4, and make only its +promote/optimize/stop recommendation from a `local-real` or +`multi-machine-real` report. DGR-004 remains blocked on this story's real +baseline decision. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt b/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt new file mode 100644 index 0000000..a6ca7fc --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/commands.txt @@ -0,0 +1,11 @@ +# Deterministic implementation checks completed in this worktree +PYTHONPATH=packages/node /home/popov/.hermes/hermes-agent/venv/bin/python -m pytest -q tests/test_recipe_benchmark.py +PYTHONPATH=packages/node /home/popov/.hermes/hermes-agent/venv/bin/python -m compileall -q packages tests +git diff --check + +# Full suite attempted in this worktree and a clean HEAD archive; both stop at +# protobuf gencode 7.35.0 versus installed runtime 6.33.6 during collection. +PYTHONPATH=packages/node /home/popov/.hermes/hermes-agent/venv/bin/python -m pytest -q + +# Required opt-in real benchmark after the BLOCKED.md prerequisites exist +MESHNET_ENABLE_REAL_INFERENCE_TESTS=1 PYTHONPATH=packages/node python -m meshnet_node.recipe_benchmark --config /run/media/popov/DATA/meshnet/dgr-001-benchmark.json --json-out .scratch/distributed-gguf-runtime/evidence/DGR-001/results.json --summary-out .scratch/distributed-gguf-runtime/evidence/DGR-001/results.txt diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json new file mode 100644 index 0000000..188713c --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/performance-contract.json @@ -0,0 +1,36 @@ +{ + "schema_version": 1, + "contract_version": 1, + "locked_at": "2026-07-13T00:00:00Z", + "locked_by": "DGR-001", + "plan_id": "dgr-001-controlled-whole-model-baseline-v1", + "thresholds": { + "min_decode_speedup": 1.25, + "max_ttft_ratio": 1.25, + "min_aggregate_throughput_speedup": 1.25, + "max_resident_memory_ratio": 0.75, + "max_artifact_size_ratio": 0.6, + "min_quality_exact_match_rate": 0.9, + "min_quality_mean_similarity": 0.97, + "max_failure_rate": 0.0 + }, + "baseline": { + "status": "pending-real-evidence", + "required_evidence_class": "local-real", + "required_recipes": [ + "transformers-safetensors-reference", + "llama-cpp-near-lossless-quality", + "llama-cpp-quantized-performance-fit" + ], + "required_concurrency_levels": [1, 4], + "required_controlled_variables": [ + "model architecture", + "model revision", + "machine and device", + "formatted prompts and context lengths", + "output length and greedy sampling policy" + ] + }, + "stop_condition": "Stop the native llama.cpp/GGUF track when, on the same machine and device as the Transformers/safetensors reference and under this plan, no performance-fit GGUF recipe delivers either a meaningful speed benefit (at least 25% higher single-request decode tokens/sec without more than 25% worse TTFT, or at least 25% higher aggregate throughput under concurrency) or a meaningful fit benefit (at least 25% lower peak resident memory), or when the near-lossless quality lane fails.", + "notes": "Quantized performance-fit output drift is reported as advisory only. It is not numerical-equivalence evidence. DGR-014 consumes this immutable v1 contract." +} diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json new file mode 100644 index 0000000..9371a7b --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/results.json @@ -0,0 +1,8 @@ +{ + "schema_version": 1, + "evidence_class": "not-run", + "status": "blocked", + "measured_at": null, + "reason": "No matching local Transformers/safetensors snapshot and whole-model llama-server runtime were available in this execution environment. No performance, memory, latency, failure, or drift values were fabricated.", + "required_output": "A local-real recipe benchmark report emitted by python -m meshnet_node.recipe_benchmark with MESHNET_ENABLE_REAL_INFERENCE_TESTS=1." +} diff --git a/packages/node/meshnet_node/recipe_benchmark.py b/packages/node/meshnet_node/recipe_benchmark.py index 9bfaf16..30adae6 100644 --- a/packages/node/meshnet_node/recipe_benchmark.py +++ b/packages/node/meshnet_node/recipe_benchmark.py @@ -115,6 +115,8 @@ class BenchmarkPlan: raise BenchmarkError("concurrency levels must all be >= 1") if self.repeats < 1: raise BenchmarkError("repeats must be >= 1") + if 1 not in self.concurrency_levels or 4 not in self.concurrency_levels: + raise BenchmarkError("a controlled baseline must include concurrency levels 1 and 4") def to_dict(self) -> dict: return { @@ -145,6 +147,9 @@ class RecipeSpec: lane: Lane device: str artifact_path: str = "" + source_model_id: str = "" + source_model_revision: str = "" + artifact_sha256: str = "" is_reference: bool = False notes: str = "" diff --git a/packages/node/meshnet_node/recipe_drivers.py b/packages/node/meshnet_node/recipe_drivers.py index 0ce96e2..5cbd39c 100644 --- a/packages/node/meshnet_node/recipe_drivers.py +++ b/packages/node/meshnet_node/recipe_drivers.py @@ -22,8 +22,10 @@ from __future__ import annotations import json import os +import platform import socket import subprocess +import sys import time import urllib.error import urllib.request @@ -83,6 +85,56 @@ def _directory_bytes(path: Path) -> int: return sum(entry.stat().st_size for entry in path.rglob("*") if entry.is_file()) +def _host_manifest() -> dict[str, Any]: + """Capture non-secret host facts with the report rather than trusting prose.""" + manifest: dict[str, Any] = { + "hostname": socket.gethostname(), + "platform": platform.platform(), + "python": sys.version.split()[0], + "cpu_count": os.cpu_count(), + } + try: + import torch + + manifest["torch_version"] = torch.__version__ + manifest["cuda_available"] = bool(torch.cuda.is_available()) + if torch.cuda.is_available(): + manifest["accelerator_name"] = torch.cuda.get_device_name(0) + manifest["accelerator_runtime"] = getattr(torch.version, "cuda", None) or getattr( + torch.version, "hip", None + ) + except ImportError: + manifest["torch_version"] = None + return manifest + + +def _validate_config(config: Mapping[str, Any]) -> None: + """Reject a comparison that could silently mix models or use home storage.""" + try: + plan = config["plan"] + root = Path(config["artifact_storage_root"]).resolve(strict=True) + recipes = config["recipes"] + except (KeyError, TypeError, OSError) as exc: + raise BenchmarkError( + "benchmark config needs an existing artifact_storage_root, plan, and recipes" + ) from exc + if not root.is_absolute() or root == Path("/home") or Path("/home") in root.parents: + raise BenchmarkError("model artifacts must use configured mounted-drive storage, never /home") + if not isinstance(recipes, list) or not recipes: + raise BenchmarkError("benchmark config needs at least one recipe") + for spec in recipes: + if spec.get("source_model_id") != plan.get("model_id"): + raise BenchmarkError("every recipe must declare the plan's exact source_model_id") + if spec.get("source_model_revision") != plan.get("model_revision"): + raise BenchmarkError("every recipe must declare the plan's exact source_model_revision") + digest = spec.get("artifact_sha256", "") + if not isinstance(digest, str) or len(digest) != 64: + raise BenchmarkError("every recipe must declare its exact 64-character artifact_sha256") + artifact = Path(spec.get("artifact_path", "")).resolve(strict=True) + if artifact != root and root not in artifact.parents: + raise BenchmarkError("every model artifact must be beneath artifact_storage_root") + + class TransformersDriver: """The current Transformers/safetensors recipe: the correctness reference. @@ -435,6 +487,9 @@ def _recipe_from_config(spec: Mapping[str, Any]) -> RecipeSpec: lane=Lane(spec["lane"]), device=spec["device"], artifact_path=spec.get("artifact_path", ""), + source_model_id=spec.get("source_model_id", ""), + source_model_revision=spec.get("source_model_revision", ""), + artifact_sha256=spec.get("artifact_sha256", ""), is_reference=bool(spec.get("is_reference", False)), notes=spec.get("notes", ""), ) @@ -448,6 +503,7 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict: crashed would read as a clean result. """ require_real_inference() + _validate_config(config) plan = _plan_from_config(config) from .recipe_benchmark import RecipeMeasurement # local import keeps the seam obvious @@ -468,6 +524,6 @@ def run_configured_benchmark(config: Mapping[str, Any]) -> dict: return build_report( plan, measurements, - host=dict(config.get("host", {})), + host={**_host_manifest(), **dict(config.get("host", {}))}, evidence_class=config.get("evidence_class", "local-real"), ) diff --git a/tests/test_recipe_benchmark.py b/tests/test_recipe_benchmark.py index 3e3f370..4ee0154 100644 --- a/tests/test_recipe_benchmark.py +++ b/tests/test_recipe_benchmark.py @@ -9,6 +9,12 @@ report. from __future__ import annotations import pytest +import time +from meshnet_node.performance_contract import ( + ContractThresholds, + PerformanceContract, + evaluate_contract, +) from meshnet_node.recipe_benchmark import ( BenchmarkError, BenchmarkPlan, @@ -60,6 +66,7 @@ class FakeDriver: texts: dict[str, str] | None = None, fail_at_concurrency: int | None = None, decode_tokens: int = 8, + generation_delay_s: float = 0.0, ) -> None: self.decode_ms_per_token = decode_ms_per_token self.prefill_ms = prefill_ms @@ -69,6 +76,7 @@ class FakeDriver: self.texts = texts or {} self.fail_at_concurrency = fail_at_concurrency self.decode_tokens = decode_tokens + self.generation_delay_s = generation_delay_s self.in_flight = 0 self.max_in_flight = 0 self.loads = 0 @@ -86,6 +94,8 @@ class FakeDriver: self.in_flight += 1 self.max_in_flight = max(self.max_in_flight, self.in_flight) try: + if self.generation_delay_s: + time.sleep(self.generation_delay_s) if self.fail_at_concurrency and self.in_flight >= self.fail_at_concurrency: raise RuntimeError("slot exhausted") self.generations += 1 @@ -138,8 +148,8 @@ def test_measure_runs_every_prompt_at_every_concurrency_level(): def test_concurrency_level_actually_overlaps_requests(): - driver = FakeDriver(decode_ms_per_token=5.0) - measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(4,))) + driver = FakeDriver(decode_ms_per_token=5.0, generation_delay_s=0.02) + measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan(concurrency_levels=(1, 4))) assert driver.max_in_flight > 1, "concurrency 4 must run requests in parallel, not serially" @@ -155,7 +165,7 @@ def test_driver_is_closed_even_when_every_request_fails(): def test_failed_requests_are_reported_not_raised(): - driver = FakeDriver(fail_at_concurrency=4) + driver = FakeDriver(fail_at_concurrency=4, generation_delay_s=0.02) measurement = measure_recipe(driver, recipe("r", Lane.QUALITY, reference=True), plan()) assert measurement.metrics[1].failures == 0 @@ -308,3 +318,29 @@ def test_unavailable_recipes_are_recorded_rather_than_dropped(): assert entry["available"] is False assert "not found" in entry["unavailable_reason"] assert report["drift"] == [], "an unmeasured recipe has no drift to report" + + +def test_contract_requires_a_quality_lane_then_allows_quantized_fit_benefit(): + texts = {prompt.text: "same greedy answer" for prompt in PROMPTS} + reference = measure_recipe( + FakeDriver(texts=texts, rss_bytes=4_000_000), recipe("safetensors", Lane.QUALITY, reference=True), plan() + ) + quality = measure_recipe( + FakeDriver(texts=texts), recipe("gguf-f16", Lane.QUALITY), plan() + ) + q4 = measure_recipe( + FakeDriver(texts={prompt.text: "different quantized answer" for prompt in PROMPTS}, + rss_bytes=1_000_000, decode_ms_per_token=20.0), + recipe("gguf-q4", Lane.PERFORMANCE_FIT), plan() + ) + report = build_report(plan(), [reference, quality, q4], host={}, evidence_class="synthetic") + contract = PerformanceContract( + contract_version=1, locked_at="2026-07-13T00:00:00Z", locked_by="test", + plan_id="test-plan", thresholds=ContractThresholds(), baseline={}, stop_condition="test", + ) + + evaluation = evaluate_contract(contract, report) + + assert evaluation.quality_lane_pass is True + assert evaluation.fit_benefit is True + assert evaluation.verdict == "optimize"