127 lines
5.0 KiB
Python
127 lines
5.0 KiB
Python
"""DIP-001 deterministic Route Session benchmark coverage."""
|
|
|
|
import json
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from meshnet_node.route_session_benchmark import (
|
|
BenchmarkScenario,
|
|
PerformanceThresholds,
|
|
assert_benchmark,
|
|
assert_performance_gate,
|
|
format_summary,
|
|
main,
|
|
run_benchmark_matrix,
|
|
run_real_model_lan_benchmark,
|
|
run_route_session_benchmark,
|
|
)
|
|
|
|
|
|
def test_matrix_reports_direct_relay_prefill_decode_and_machine_readable_metrics():
|
|
"""The baseline reports every required scenario and transport metric.
|
|
|
|
Tags: performance, routing
|
|
"""
|
|
report = run_benchmark_matrix()
|
|
assert {(run["mode"], run["cache_mode"]) for run in report["runs"]} == {
|
|
("direct", "cached"), ("direct", "stateless"),
|
|
("relay", "cached"), ("relay", "stateless"),
|
|
}
|
|
for run in report["runs"]:
|
|
assert set(run["phases"]) == {"prefill", "decode"}
|
|
assert "head->tail" in run["seams"]
|
|
assert {"p50_latency_ms", "p95_latency_ms", "payload_bytes", "compression_ratio",
|
|
"connection_attempts", "p95_queue_wait_ms"} <= set(run["phases"]["decode"])
|
|
sample = run["samples"][0]
|
|
assert sample["framing_ms"] > 0
|
|
assert sample["metadata_ms"] > 0
|
|
assert sample["copy_allocation_ms"] > 0
|
|
assert sample["copy_allocation_bytes"] >= sample["payload_bytes"]
|
|
assert len(run["samples"]) == 1 + len(run["output_tokens"])
|
|
assert {"tokens_per_sec", "bytes_per_token", "compression_cpu_ms", "peak_buffered_bytes"} <= set(run["phases"]["decode"])
|
|
|
|
|
|
def test_cached_sessions_reuse_one_connection_and_preserve_stub_tokens():
|
|
"""Cached Route Sessions keep one direct or relay connection per seam.
|
|
|
|
Tags: performance, relay
|
|
"""
|
|
scenario = BenchmarkScenario(output_tokens=(" one", " two", " three"))
|
|
for mode in ("direct", "relay"):
|
|
run = run_route_session_benchmark(mode, "cached", scenario)
|
|
assert_benchmark(run, expected_tokens=scenario.output_tokens, expected_connection_attempts=1)
|
|
if mode == "relay":
|
|
assert all(sample.queue_wait_ms > 0 for sample in run.samples)
|
|
|
|
|
|
def test_stateless_baselines_make_each_activation_a_connection_attempt():
|
|
"""Stateless comparison mode does not accidentally inherit Route Session state.
|
|
|
|
Tags: performance, routing
|
|
"""
|
|
scenario = BenchmarkScenario(output_tokens=(" one", " two"))
|
|
for mode in ("direct", "relay"):
|
|
run = run_route_session_benchmark(mode, "stateless", scenario)
|
|
assert_benchmark(run, expected_tokens=scenario.output_tokens, expected_connection_attempts=3)
|
|
|
|
|
|
def test_cli_writes_json_artifact_and_human_summary(tmp_path, capsys):
|
|
"""The CLI emits both CI-ready JSON and an operator-readable summary.
|
|
|
|
Tags: performance
|
|
"""
|
|
output = tmp_path / "route-session-benchmark.json"
|
|
assert main(["--json-out", str(output)]) == 0
|
|
report = json.loads(output.read_text())
|
|
assert report["schema_version"] == 1
|
|
assert "Route Session benchmark" in capsys.readouterr().out
|
|
assert "relay" in format_summary(report)
|
|
|
|
|
|
def test_performance_gate_checks_comparison_identity_session_and_cleanup():
|
|
"""CI gate accepts the fixed matrix and rejects a meaningful slowdown.
|
|
|
|
Tags: performance, routing
|
|
"""
|
|
report = run_benchmark_matrix()
|
|
assert_performance_gate(report)
|
|
report["runs"][0]["phases"]["decode"]["tokens_per_sec"] = 0.1
|
|
try:
|
|
assert_performance_gate(report, thresholds=PerformanceThresholds())
|
|
except AssertionError as exc:
|
|
assert "throughput regressed" in str(exc)
|
|
else: # pragma: no cover - makes the intended threshold explicit
|
|
raise AssertionError("gate did not catch the throughput regression")
|
|
|
|
|
|
def test_performance_gate_rejects_session_or_cleanup_leaks():
|
|
"""Exact resource/session invariants are not subject to variance tolerance.
|
|
|
|
Tags: performance, routing
|
|
"""
|
|
report = run_benchmark_matrix()
|
|
report["runs"][0]["samples"][1]["session_id"] = "wrong-session"
|
|
try:
|
|
assert_performance_gate(report)
|
|
except AssertionError as exc:
|
|
assert "Route Session changed" in str(exc)
|
|
else: # pragma: no cover
|
|
raise AssertionError("gate did not catch session instability")
|
|
|
|
|
|
def test_real_model_lan_capture_uses_the_shared_report_schema():
|
|
"""The opt-in LAN command is client-measurable and needs no real model in CI.
|
|
|
|
Tags: performance
|
|
"""
|
|
response = MagicMock()
|
|
response.read.return_value = json.dumps({"choices": [{"message": {"content": "amber birch"}}]}).encode()
|
|
response.headers.get.return_value = "lan-session"
|
|
response.__enter__.return_value = response
|
|
with patch("meshnet_node.route_session_benchmark.urllib.request.urlopen", return_value=response):
|
|
report = run_real_model_lan_benchmark("http://lan-node:7000", model="test-model")
|
|
run = report["runs"][0]
|
|
assert report["source"] == "real-model-lan-client"
|
|
assert run["session_id"] == "lan-session"
|
|
assert run["phases"]["decode"]["tokens_per_sec"] > 0
|
|
assert run["cleanup"]["open_connections"] == 0
|