From eaf00f6add936d189aa7051dbb219a530d01ddd0 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Wed, 15 Jul 2026 13:42:22 +0300 Subject: [PATCH] test: record public relay smoke benchmark --- .../evidence/DGR-001/README.md | 28 +++++++ .../DGR-001/public-relay-smoke-benchmark.json | 83 +++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 .scratch/distributed-gguf-runtime/evidence/DGR-001/public-relay-smoke-benchmark.json diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md index a9e7221..a29623d 100644 --- a/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/README.md @@ -76,6 +76,34 @@ python -m compileall packages/node/meshnet_node/performance_contract.py tests/te Result: passed +## Public relay smoke benchmark (2026-07-15) + +A real streamed request was run through the public tracker — **not** by connecting directly to the private node address: + +```text +https://meshnet.2.d-popov.com/v1/chat/completions + -> wss://meshnet.2.d-popov.com/ws + -> wss://meshnet.2.d-popov.com/rpc/7j77FsPY1evV8tuf-7000 + -> local CUDA node, Qwen/Qwen2.5-0.5B-Instruct layers 0-23 +``` + +The local public-tracker node had an expired proof and a wedged HTTP server. A graceful restart refreshed its CUDA capability proof in `336 ms`, restored `admitted`/`routable` status, and reconnected its relay endpoint. + +Measured streaming results after recovery: + +| metric | result | +| --- | ---: | +| warm-up TTFT | 420.80 ms | +| warm-up elapsed | 610.23 ms | +| p50 TTFT (3 runs) | 288.26 ms | +| p50 elapsed (3 runs) | 363.20 ms | +| tracker-recorded relay throughput | 58.18-65.25 tok/s | +| HTTP status | 200 for all runs | + +The tracker recorded `relay: true` and the local node ID `7j77FsPY-b32476219492` for each completion. Full redacted evidence is in `public-relay-smoke-benchmark.json`. + +The other connected node is still alive but **not routable** because its capability proof is stale. It must revalidate before a multi-node shard/relay test can run. + ## Limitations - This slice still uses a deterministic stub backend for the core comparison matrix. diff --git a/.scratch/distributed-gguf-runtime/evidence/DGR-001/public-relay-smoke-benchmark.json b/.scratch/distributed-gguf-runtime/evidence/DGR-001/public-relay-smoke-benchmark.json new file mode 100644 index 0000000..32668c6 --- /dev/null +++ b/.scratch/distributed-gguf-runtime/evidence/DGR-001/public-relay-smoke-benchmark.json @@ -0,0 +1,83 @@ +{ + "schema_version": 1, + "executed_at_utc": "2026-07-15T10:41:14Z", + "test_kind": "public-relay-single-node-streaming-smoke-benchmark", + "target": { + "public_chat_endpoint": "https://meshnet.2.d-popov.com/v1/chat/completions", + "relay_url": "wss://meshnet.2.d-popov.com/ws", + "model": "qwen2.5-0.5b-instruct", + "quantization": "bfloat16" + }, + "recovery": { + "problem": "The local node's capability proof had expired and its port-7000 HTTP server had wedged with CLOSE-WAIT sockets.", + "action": "Gracefully restarted the local public-tracker meshnet-node process on port 7000.", + "startup_validation": { + "device": "cuda", + "capability_proof_ms": 336, + "node_id": "7j77FsPY-b32476219492", + "relay_addr": "wss://meshnet.2.d-popov.com/rpc/7j77FsPY1evV8tuf-7000" + } + }, + "tracker_admission_after_recovery": { + "node_id": "7j77FsPY-b32476219492", + "alive": true, + "status": "ready", + "capability_state": "admitted", + "routable": true, + "route_hops": 1 + }, + "client_measurements": { + "warmup": { + "http_status": 200, + "ttft_ms": 420.8, + "elapsed_ms": 610.23, + "response_text": "MeshNet Relay Benchmark Passed" + }, + "runs": [ + { + "run": 1, + "ttft_ms": 376.04, + "elapsed_ms": 458.65, + "response_text": "relay benchmark pass" + }, + { + "run": 2, + "ttft_ms": 258.33, + "elapsed_ms": 336.71, + "response_text": "relay benchmark pass" + }, + { + "run": 3, + "ttft_ms": 288.26, + "elapsed_ms": 363.2, + "response_text": "relay benchmark pass" + } + ], + "p50_ttft_ms": 288.26, + "p50_elapsed_ms": 363.2 + }, + "tracker_relay_evidence": [ + { + "status": 200, + "relay": true, + "node_id": "7j77FsPY-b32476219492", + "tokens": 11, + "elapsed_seconds": 0.1686, + "tokens_per_sec": 65.2541 + }, + { + "status": 200, + "relay": true, + "node_id": "7j77FsPY-b32476219492", + "tokens": 11, + "elapsed_seconds": 0.1891, + "tokens_per_sec": 58.1799 + } + ], + "scope_and_remaining_work": { + "validated": "Public HTTPS chat endpoint routed a streaming request through the tracker relay to the local CUDA node and completed with HTTP 200.", + "not_validated": "Two-node shard routing was not run because the remote node 5gMLrmyB-88f5cba044d0 still had an expired capability proof and was not routable.", + "next_gate": "Refresh the remote node capability proof, then load a multi-node-compatible assignment and repeat the benchmark through the public tracker relay." + }, + "reproduction": "Use a valid bearer API key with the public /v1/chat/completions endpoint and stream a short qwen2.5-0.5b-instruct request. Do not connect directly to private node HTTP endpoints; the tracker relay is the required path." +}