popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: wire live benchmark CLI endpoints

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-15 10:34:20 +03:00
parent a508768e8a
commit c035bad5b7
3 changed files with 147 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
packages/node/meshnet_node/performance_contract.py
View File

@@ -427,6 +427,21 @@ def run_real_model_endpoint_benchmark(
}
def _parse_lane_endpoints(pairs: list[str], parser: argparse.ArgumentParser) -> dict[str, str]:
endpoints: dict[str, str] = {}
for pair in pairs:
lane_id, sep, url = pair.partition("=")
if not sep or not lane_id or not url:
parser.error(f"--live-endpoint expects LANE_ID=URL, got {pair!r}")
endpoints[lane_id] = url
return endpoints
def _write_report(report: dict, path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Write the DGR-001 performance contract JSON")
parser.add_argument("--json-out", type=Path, default=DEFAULT_OUTPUT_PATH, help="output JSON path")
@@ -436,15 +451,43 @@ def main(argv: list[str] | None = None) -> int:
default=None,
help="also run the deterministic stub benchmark and write its JSON report here",
)
parser.add_argument(
"--live-endpoint",
action="append",
default=None,
metavar="LANE_ID=URL",
help="lane-to-endpoint mapping for the live benchmark; repeat once per contract lane",
)
parser.add_argument(
"--live-model",
default=None,
help="model name sent to live endpoints (default: contract safetensors repo)",
)
parser.add_argument(
"--live-benchmark-out",
type=Path,
default=None,
help="run the live endpoint benchmark against --live-endpoint lanes and write its JSON report here",
)
args = parser.parse_args(argv)
if args.live_endpoint and args.live_benchmark_out is None:
parser.error("--live-endpoint requires --live-benchmark-out")
if args.live_benchmark_out is not None and not args.live_endpoint:
parser.error("--live-benchmark-out requires at least one --live-endpoint")
contract = build_default_contract()
path = contract.write_json(args.json_out)
print(path)
if args.benchmark_out is not None:
report = run_performance_benchmark(contract)
args.benchmark_out.parent.mkdir(parents=True, exist_ok=True)
args.benchmark_out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
_write_report(run_performance_benchmark(contract), args.benchmark_out)
print(args.benchmark_out)
if args.live_endpoint:
report = run_real_model_endpoint_benchmark(
_parse_lane_endpoints(args.live_endpoint, parser),
model=args.live_model or contract.model_target.safetensors_repo,
contract=contract,
)
_write_report(report, args.live_benchmark_out)
print(args.live_benchmark_out)
return 0