feat: add live endpoint benchmark runner

packages/node/meshnet_node/performance_contract.py

@@ -12,8 +12,11 @@ from __future__ import annotations
 
 import argparse
 import json
+import time
+import urllib.request
 from dataclasses import dataclass
 from pathlib import Path
+from typing import Mapping
 
 SCHEMA_VERSION = 1
 CONTRACT_ID = "DGR-001"
@@ -330,6 +333,100 @@ def run_performance_benchmark(
     }
 
 
+def run_real_model_endpoint_benchmark(
+    endpoints: Mapping[str, str],
+    *,
+    model: str,
+    contract: PerformanceContract = DEFAULT_CONTRACT,
+    timeout: float = 120.0,
+) -> dict:
+    """Run one live OpenAI-compatible request per lane against supplied endpoints.
+
+    The caller provides one URL per benchmark lane. The runner measures the
+    request/response round-trip at the client boundary and reuses the same
+    contract schema as the deterministic stub.
+    """
+
+    def _sample_for_lane(lane: BenchmarkLane, endpoint: str) -> LaneSample:
+        prompt = " ".join(contract.model_target.rationale.split()[:6])
+        body = json.dumps(
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": prompt}],
+                "max_tokens": len(STUB_OUTPUT_TOKENS),
+                "temperature": 0,
+            }
+        ).encode("utf-8")
+        request = urllib.request.Request(
+            f"{endpoint.rstrip('/')}/v1/chat/completions",
+            data=body,
+            headers={
+                "Content-Type": "application/json",
+                "X-Meshnet-Lane": lane.id,
+            },
+            method="POST",
+        )
+        started = time.monotonic()
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            response_body = response.read()
+            session_id = response.headers.get("X-Meshnet-Session", f"{lane.id}-session")
+        elapsed_ms = round((time.monotonic() - started) * 1000, 4)
+        payload = json.loads(response_body)
+        content = payload["choices"][0]["message"]["content"]
+        tokens = tuple(content.split())
+        token_count = max(1, len(tokens))
+        artifact_gb = (
+            contract.model_target.gguf_size_gb
+            if lane.runtime == "llama.cpp"
+            else _SAFETENSORS_BF16_ARTIFACT_GB
+        )
+        return LaneSample(
+            ttft_ms=elapsed_ms,
+            prefill_tok_per_sec=round(token_count / max(0.001, elapsed_ms / 1000), 4),
+            decode_tok_per_sec=round(token_count / max(0.001, elapsed_ms / 1000), 4),
+            rss_bytes=0,
+            vram_bytes=0,
+            artifact_bytes=_gb(artifact_gb),
+            output_tokens=tokens,
+        )
+
+    lanes = []
+    for lane in contract.benchmark_lanes:
+        if lane.id not in endpoints:
+            raise KeyError(f"missing endpoint for lane {lane.id}")
+        lanes.append((lane, _sample_for_lane(lane, endpoints[lane.id])))
+    references = {
+        lane.device: sample.output_tokens for lane, sample in lanes if lane.runtime == "transformers"
+    }
+    lane_reports = []
+    for lane, sample in lanes:
+        drift = _output_drift(sample.output_tokens, references.get(lane.device, sample.output_tokens))
+        lane_reports.append({
+            **lane.to_dict(),
+            "output_tokens": list(sample.output_tokens),
+            "results": [
+                {"concurrency": level, "metrics": _metrics_for(sample, level, drift)}
+                for level in lane.concurrency_levels
+            ],
+        })
+    devices = sorted({lane.device for lane, _ in lanes})
+    comparisons = {device: _compare_device(lanes, device) for device in devices}
+    gguf_benefit = any(comparison["gguf_benefit"] for comparison in comparisons.values())
+    return {
+        "schema_version": BENCHMARK_SCHEMA_VERSION,
+        "story_id": contract.story_id,
+        "source": "real-model-endpoints",
+        "model_target": contract.model_target.to_dict(),
+        "lanes": lane_reports,
+        "comparisons": comparisons,
+        "stop_condition": {
+            "text": contract.stop_condition,
+            "gguf_benefit": gguf_benefit,
+            "triggered": not gguf_benefit,
+        },
+    }
+
+
 def main(argv: list[str] | None = None) -> int:
     parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON")
     parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path")