From c035bad5b7a89bd6b44bdd6bff8e8043c96f7153 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Wed, 15 Jul 2026 10:34:20 +0300 Subject: [PATCH] feat: wire live benchmark CLI endpoints --- .../evidence/DGR-001/README.md | 21 ++++- .../node/meshnet_node/performance_contract.py | 49 ++++++++++- tests/test_performance_contract.py | 83 +++++++++++++++++++ 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md index 8f0f5ea..a9e7221 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -35,15 +35,30 @@ The report includes per-device comparisons for: and records the memory metric (`rss_bytes` on CPU, `vram_bytes` on GPU), decode speedup, artifact ratio, and output drift. +## Live endpoint CLI wiring + +The contract CLI can now drive the live endpoint runner. Passing one `--live-endpoint LANE_ID=URL` mapping per contract lane (plus `--live-benchmark-out`) invokes `run_real_model_endpoint_benchmark` against already-running OpenAI-compatible servers and writes the report using the same schema as the stub: + +```bash +PYTHONPATH=packages/node python -m meshnet_node.performance_contract \ + --live-endpoint transformers-safetensors-cpu=http://127.0.0.1:8001 \ + --live-endpoint llama-cpp-gguf-cpu=http://127.0.0.1:8002 \ + --live-endpoint transformers-safetensors-gpu=http://127.0.0.1:8003 \ + --live-endpoint llama-cpp-gguf-gpu=http://127.0.0.1:8004 \ + --live-benchmark-out .scratch/distributed-gguf-runtime/evidence/DGR-001/live-benchmark-report.json +``` + +`--live-model` overrides the model name sent in requests (defaults to the contract's safetensors repo). Without any `--live-endpoint` flags the CLI behaves exactly as before: it writes the contract JSON and, with `--benchmark-out`, the deterministic stub report. + ## Exact commands and real results ### Targeted tests ```bash -pytest -q tests/test_performance_contract.py tests/test_route_session_benchmark.py +PYTHONPATH=packages/node pytest -q tests/test_performance_contract.py tests/test_route_session_benchmark.py ``` -Result: `9 passed in 0.14s` +Result: `19 passed in 0.11s` ### Contract artifact generation @@ -64,7 +79,7 @@ Result: passed ## Limitations - This slice still uses a deterministic stub backend for the core comparison matrix. -- It now also includes a live endpoint runner that can fan out one OpenAI-compatible request per lane when the caller provides endpoints. +- It now also includes a live endpoint runner, reachable from the CLI via `--live-endpoint`/`--live-benchmark-out`, that fans out one OpenAI-compatible request per lane when the caller provides endpoints; the CLI does not start those servers. - It does **not** download or run a real model from within the repo. - Real safetensors vs GGUF execution, TTFT/prefill/decode measurements, RSS/VRAM capture, and output-drift comparison are still to be implemented against the contract. diff --git a/packages/node/meshnet_node/performance_contract.py b/packages/node/meshnet_node/performance_contract.py index 724765f..c0105d6 100644 --- a/packages/node/meshnet_node/performance_contract.py +++ b/packages/node/meshnet_node/performance_contract.py @@ -427,6 +427,21 @@ def run_real_model_endpoint_benchmark( } +def _parse_lane_endpoints(pairs: list[str], parser: argparse.ArgumentParser) -> dict[str, str]: + endpoints: dict[str, str] = {} + for pair in pairs: + lane_id, sep, url = pair.partition("=") + if not sep or not lane_id or not url: + parser.error(f"--live-endpoint expects LANE_ID=URL, got {pair!r}") + endpoints[lane_id] = url + return endpoints + + +def _write_report(report: dict, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON") parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path") @@ -436,15 +451,43 @@ def main(argv: list[str] | None = None) -> int: default=None, help="also run the deterministic stub benchmark and write its JSON report here", ) + parser.add_argument( + "--live-endpoint", + action="append", + default=None, + metavar="LANE_ID=URL", + help="lane-to-endpoint mapping for the live benchmark; repeat once per contract lane", + ) + parser.add_argument( + "--live-model", + default=None, + help="model name sent to live endpoints (default: contract safetensors repo)", + ) + parser.add_argument( + "--live-benchmark-out", + type=Path, + default=None, + help="run the live endpoint benchmark against --live-endpoint lanes and write its JSON report here", + ) args = parser.parse_args(argv) + if args.live_endpoint and args.live_benchmark_out is None: + parser.error("--live-endpoint requires --live-benchmark-out") + if args.live_benchmark_out is not None and not args.live_endpoint: + parser.error("--live-benchmark-out requires at least one --live-endpoint") contract = build_default_contract() path = contract.write_json(args.json_out) print(path) if args.benchmark_out is not None: - report = run_performance_benchmark(contract) - args.benchmark_out.parent.mkdir(parents=True, exist_ok=True) - args.benchmark_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + _write_report(run_performance_benchmark(contract), args.benchmark_out) print(args.benchmark_out) + if args.live_endpoint: + report = run_real_model_endpoint_benchmark( + _parse_lane_endpoints(args.live_endpoint, parser), + model=args.live_model or contract.model_target.safetensors_repo, + contract=contract, + ) + _write_report(report, args.live_benchmark_out) + print(args.live_benchmark_out) return 0 diff --git a/tests/test_performance_contract.py b/tests/test_performance_contract.py index 537d2e3..a0d03b9 100644 --- a/tests/test_performance_contract.py +++ b/tests/test_performance_contract.py @@ -5,6 +5,8 @@ from __future__ import annotations import json from unittest.mock import MagicMock, patch +import pytest + from meshnet_node.performance_contract import ( BENCHMARK_SCHEMA_VERSION, DEFAULT_CONTRACT, @@ -201,3 +203,84 @@ def test_real_model_endpoint_benchmark_uses_lane_specific_endpoints_and_shared_s assert lane["output_tokens"] == ["mesh", "activation"] assert report["comparisons"]["cpu"]["gguf_lane"] == "llama-cpp-gguf-cpu" assert report["comparisons"]["gpu"]["gguf_lane"] == "llama-cpp-gguf-gpu" + + +def test_contract_cli_runs_live_endpoint_benchmark(tmp_path, capsys): + """--live-endpoint mappings drive the live runner and write its report. + + Tags: performance, benchmark, live, artifact + """ + contract_out = tmp_path / "performance-contract.json" + live_out = tmp_path / "artifacts" / "live-benchmark-report.json" + endpoints = { + "transformers-safetensors-cpu": "http://cpu-safetensors", + "llama-cpp-gguf-cpu": "http://cpu-gguf", + "transformers-safetensors-gpu": "http://gpu-safetensors", + "llama-cpp-gguf-gpu": "http://gpu-gguf", + } + fake_report = {"schema_version": BENCHMARK_SCHEMA_VERSION, "source": "real-model-endpoints"} + argv = ["--json-out", str(contract_out), "--live-benchmark-out", str(live_out)] + for lane_id, url in endpoints.items(): + argv += ["--live-endpoint", f"{lane_id}={url}"] + + with patch( + "meshnet_node.performance_contract.run_real_model_endpoint_benchmark", + return_value=fake_report, + ) as runner: + assert main(argv) == 0 + + runner.assert_called_once_with( + endpoints, + model=DEFAULT_CONTRACT.model_target.safetensors_repo, + contract=DEFAULT_CONTRACT, + ) + assert json.loads(live_out.read_text(encoding="utf-8")) == fake_report + output = capsys.readouterr().out + assert str(contract_out) in output + assert str(live_out) in output + + +def test_contract_cli_passes_explicit_live_model(tmp_path): + """--live-model overrides the contract's safetensors repo default. + + Tags: performance, benchmark, live + """ + live_out = tmp_path / "live-benchmark-report.json" + argv = [ + "--json-out", str(tmp_path / "performance-contract.json"), + "--live-benchmark-out", str(live_out), + "--live-endpoint", "transformers-safetensors-cpu=http://cpu-safetensors", + "--live-model", "local/DeepSeek-V2-Lite-Chat-Q2_K", + ] + + with patch( + "meshnet_node.performance_contract.run_real_model_endpoint_benchmark", + return_value={}, + ) as runner: + assert main(argv) == 0 + + assert runner.call_args.kwargs["model"] == "local/DeepSeek-V2-Lite-Chat-Q2_K" + + +@pytest.mark.parametrize( + "argv", + [ + ["--live-endpoint", "transformers-safetensors-cpu=http://cpu"], + ["--live-benchmark-out", "live-report.json"], + [ + "--live-endpoint", "not-a-mapping", + "--live-benchmark-out", "live-report.json", + ], + ], + ids=["endpoint-without-out", "out-without-endpoint", "malformed-mapping"], +) +def test_contract_cli_rejects_incomplete_live_arguments(tmp_path, argv, capsys): + """Live flags must arrive as a consistent LANE_ID=URL + output-path set. + + Tags: performance, benchmark, live, cli + """ + with pytest.raises(SystemExit) as excinfo: + main(["--json-out", str(tmp_path / "performance-contract.json"), *argv]) + + assert excinfo.value.code == 2 + assert "--live-" in capsys.readouterr().err