feat: wire live benchmark CLI endpoints

.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md

@@ -35,15 +35,30 @@ The report includes per-device comparisons for:
 
 and records the memory metric (`rss_bytes` on CPU, `vram_bytes` on GPU), decode speedup, artifact ratio, and output drift.
 
+## Live endpoint CLI wiring
+
+The contract CLI can now drive the live endpoint runner. Passing one `--live-endpoint LANE_ID=URL` mapping per contract lane (plus `--live-benchmark-out`) invokes `run_real_model_endpoint_benchmark` against already-running OpenAI-compatible servers and writes the report using the same schema as the stub:
+
+```bash
+PYTHONPATH=packages/node python -m meshnet_node.performance_contract \
+  --live-endpoint transformers-safetensors-cpu=http://127.0.0.1:8001 \
+  --live-endpoint llama-cpp-gguf-cpu=http://127.0.0.1:8002 \
+  --live-endpoint transformers-safetensors-gpu=http://127.0.0.1:8003 \
+  --live-endpoint llama-cpp-gguf-gpu=http://127.0.0.1:8004 \
+  --live-benchmark-out .scratch/distributed-gguf-runtime/evidence/DGR-001/live-benchmark-report.json
+```
+
+`--live-model` overrides the model name sent in requests (defaults to the contract's safetensors repo). Without any `--live-endpoint` flags the CLI behaves exactly as before: it writes the contract JSON and, with `--benchmark-out`, the deterministic stub report.
+
 ## Exact commands and real results
 
 ### Targeted tests
 
 ```bash
-pytest -q tests/test_performance_contract.py tests/test_route_session_benchmark.py
+PYTHONPATH=packages/node pytest -q tests/test_performance_contract.py tests/test_route_session_benchmark.py
 ```
 
-Result: `9 passed in 0.14s`
+Result: `19 passed in 0.11s`
 
 ### Contract artifact generation
 
@@ -64,7 +79,7 @@ Result: passed
 ## Limitations
 
 - This slice still uses a deterministic stub backend for the core comparison matrix.
-- It now also includes a live endpoint runner that can fan out one OpenAI-compatible request per lane when the caller provides endpoints.
+- It now also includes a live endpoint runner, reachable from the CLI via `--live-endpoint`/`--live-benchmark-out`, that fans out one OpenAI-compatible request per lane when the caller provides endpoints; the CLI does not start those servers.
 - It does **not** download or run a real model from within the repo.
 - Real safetensors vs GGUF execution, TTFT/prefill/decode measurements, RSS/VRAM capture, and output-drift comparison are still to be implemented against the contract.
 

packages/node/meshnet_node/performance_contract.py

@@ -427,6 +427,21 @@ def run_real_model_endpoint_benchmark(
     }
 
 
+def _parse_lane_endpoints(pairs: list[str], parser: argparse.ArgumentParser) -> dict[str, str]:
+    endpoints: dict[str, str] = {}
+    for pair in pairs:
+        lane_id, sep, url = pair.partition("=")
+        if not sep or not lane_id or not url:
+            parser.error(f"--live-endpoint expects LANE_ID=URL, got {pair!r}")
+        endpoints[lane_id] = url
+    return endpoints
+
+
+def _write_report(report: dict, path: Path) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON")
     parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path")
@@ -436,15 +451,43 @@ def main(argv: list[str] | None = None) -> int:
         default=None,
         help="also run the deterministic stub benchmark and write its JSON report here",
     )
+    parser.add_argument(
+        "--live-endpoint",
+        action="append",
+        default=None,
+        metavar="LANE_ID=URL",
+        help="lane-to-endpoint mapping for the live benchmark; repeat once per contract lane",
+    )
+    parser.add_argument(
+        "--live-model",
+        default=None,
+        help="model name sent to live endpoints (default: contract safetensors repo)",
+    )
+    parser.add_argument(
+        "--live-benchmark-out",
+        type=Path,
+        default=None,
+        help="run the live endpoint benchmark against --live-endpoint lanes and write its JSON report here",
+    )
     args = parser.parse_args(argv)
+    if args.live_endpoint and args.live_benchmark_out is None:
+        parser.error("--live-endpoint requires --live-benchmark-out")
+    if args.live_benchmark_out is not None and not args.live_endpoint:
+        parser.error("--live-benchmark-out requires at least one --live-endpoint")
     contract = build_default_contract()
     path = contract.write_json(args.json_out)
     print(path)
     if args.benchmark_out is not None:
-        report = run_performance_benchmark(contract)
-        args.benchmark_out.parent.mkdir(parents=True, exist_ok=True)
-        args.benchmark_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+        _write_report(run_performance_benchmark(contract), args.benchmark_out)
         print(args.benchmark_out)
+    if args.live_endpoint:
+        report = run_real_model_endpoint_benchmark(
+            _parse_lane_endpoints(args.live_endpoint, parser),
+            model=args.live_model or contract.model_target.safetensors_repo,
+            contract=contract,
+        )
+        _write_report(report, args.live_benchmark_out)
+        print(args.live_benchmark_out)
     return 0
 
 

tests/test_performance_contract.py

@@ -5,6 +5,8 @@ from __future__ import annotations
 import json
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from meshnet_node.performance_contract import (
     BENCHMARK_SCHEMA_VERSION,
     DEFAULT_CONTRACT,
@@ -201,3 +203,84 @@ def test_real_model_endpoint_benchmark_uses_lane_specific_endpoints_and_shared_s
         assert lane["output_tokens"] == ["mesh", "activation"]
     assert report["comparisons"]["cpu"]["gguf_lane"] == "llama-cpp-gguf-cpu"
     assert report["comparisons"]["gpu"]["gguf_lane"] == "llama-cpp-gguf-gpu"
+
+
+def test_contract_cli_runs_live_endpoint_benchmark(tmp_path, capsys):
+    """--live-endpoint mappings drive the live runner and write its report.
+
+    Tags: performance, benchmark, live, artifact
+    """
+    contract_out = tmp_path / "performance-contract.json"
+    live_out = tmp_path / "artifacts" / "live-benchmark-report.json"
+    endpoints = {
+        "transformers-safetensors-cpu": "http://cpu-safetensors",
+        "llama-cpp-gguf-cpu": "http://cpu-gguf",
+        "transformers-safetensors-gpu": "http://gpu-safetensors",
+        "llama-cpp-gguf-gpu": "http://gpu-gguf",
+    }
+    fake_report = {"schema_version": BENCHMARK_SCHEMA_VERSION, "source": "real-model-endpoints"}
+    argv = ["--json-out", str(contract_out), "--live-benchmark-out", str(live_out)]
+    for lane_id, url in endpoints.items():
+        argv += ["--live-endpoint", f"{lane_id}={url}"]
+
+    with patch(
+        "meshnet_node.performance_contract.run_real_model_endpoint_benchmark",
+        return_value=fake_report,
+    ) as runner:
+        assert main(argv) == 0
+
+    runner.assert_called_once_with(
+        endpoints,
+        model=DEFAULT_CONTRACT.model_target.safetensors_repo,
+        contract=DEFAULT_CONTRACT,
+    )
+    assert json.loads(live_out.read_text(encoding="utf-8")) == fake_report
+    output = capsys.readouterr().out
+    assert str(contract_out) in output
+    assert str(live_out) in output
+
+
+def test_contract_cli_passes_explicit_live_model(tmp_path):
+    """--live-model overrides the contract's safetensors repo default.
+
+    Tags: performance, benchmark, live
+    """
+    live_out = tmp_path / "live-benchmark-report.json"
+    argv = [
+        "--json-out", str(tmp_path / "performance-contract.json"),
+        "--live-benchmark-out", str(live_out),
+        "--live-endpoint", "transformers-safetensors-cpu=http://cpu-safetensors",
+        "--live-model", "local/DeepSeek-V2-Lite-Chat-Q2_K",
+    ]
+
+    with patch(
+        "meshnet_node.performance_contract.run_real_model_endpoint_benchmark",
+        return_value={},
+    ) as runner:
+        assert main(argv) == 0
+
+    assert runner.call_args.kwargs["model"] == "local/DeepSeek-V2-Lite-Chat-Q2_K"
+
+
+@pytest.mark.parametrize(
+    "argv",
+    [
+        ["--live-endpoint", "transformers-safetensors-cpu=http://cpu"],
+        ["--live-benchmark-out", "live-report.json"],
+        [
+            "--live-endpoint", "not-a-mapping",
+            "--live-benchmark-out", "live-report.json",
+        ],
+    ],
+    ids=["endpoint-without-out", "out-without-endpoint", "malformed-mapping"],
+)
+def test_contract_cli_rejects_incomplete_live_arguments(tmp_path, argv, capsys):
+    """Live flags must arrive as a consistent LANE_ID=URL + output-path set.
+
+    Tags: performance, benchmark, live, cli
+    """
+    with pytest.raises(SystemExit) as excinfo:
+        main(["--json-out", str(tmp_path / "performance-contract.json"), *argv])
+
+    assert excinfo.value.code == 2
+    assert "--live-" in capsys.readouterr().err