Compare commits
3 Commits
f54ea100fb
...
f99237b4e6
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f99237b4e6 | ||
|
|
bb561a9665 | ||
|
|
11bf460027 |
3
.gitignore
vendored
3
.gitignore
vendored
@@ -10,7 +10,8 @@ dist/
|
|||||||
.venv/
|
.venv/
|
||||||
|
|
||||||
# Ralph local runtime state
|
# Ralph local runtime state
|
||||||
.ralph-tui/
|
.ralph-tui/*
|
||||||
|
!.ralph-tui/config.toml
|
||||||
|
|
||||||
|
|
||||||
.env
|
.env
|
||||||
|
|||||||
7
.ralph-tui/config.toml
Normal file
7
.ralph-tui/config.toml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
[agent]
|
||||||
|
name = "claude"
|
||||||
|
model = "opus"
|
||||||
|
|
||||||
|
[run]
|
||||||
|
iterations = 10
|
||||||
|
delayMs = 1000
|
||||||
15
.scratch/dashboard-test-runner/PRD.md
Normal file
15
.scratch/dashboard-test-runner/PRD.md
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Dashboard Test Runner
|
||||||
|
|
||||||
|
Status: active
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
Provide an opt-in, admin-only tracker Dashboard Testing tab that dynamically discovers pytest tests, runs fixed collected targets safely in background, and reports live logs/status.
|
||||||
|
|
||||||
|
## Safety
|
||||||
|
- Disabled unless tracker starts with an explicit flag.
|
||||||
|
- Admin-only API/UI.
|
||||||
|
- No arbitrary command/argument execution.
|
||||||
|
- One active run.
|
||||||
|
- Real inference stays separately environment-gated and excluded from default suites.
|
||||||
|
|
||||||
|
See `prd.json` for executable Ralph user stories and acceptance criteria.
|
||||||
64
.scratch/dashboard-test-runner/prd.json
Normal file
64
.scratch/dashboard-test-runner/prd.json
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
{
|
||||||
|
"name": "Tracker Dashboard Test Runner",
|
||||||
|
"description": "Add an admin-only Testing tab that dynamically discovers repository pytest tests, runs a selected safe test target in a background process, and shows live output/status in the tracker dashboard.",
|
||||||
|
"branchName": "ralph/dashboard-test-runner",
|
||||||
|
"userStories": [
|
||||||
|
{
|
||||||
|
"id": "US-001",
|
||||||
|
"title": "Implement secure tracker test-runner API",
|
||||||
|
"description": "As a tracker administrator, I want the tracker to discover and run repository tests through a controlled API so that dashboard actions cannot execute arbitrary shell commands.",
|
||||||
|
"acceptanceCriteria": [
|
||||||
|
"Add an explicit disabled-by-default TrackerServer/CLI test-runner flag; no test endpoint runs commands unless enabled.",
|
||||||
|
"Admin-only endpoints dynamically collect pytest node IDs and start one selected collected test or approved suite at a time without accepting arbitrary command arguments.",
|
||||||
|
"Run pytest in a background process without shell=True, retain bounded stdout/stderr logs, status, timestamps, exit code, and reject concurrent runs.",
|
||||||
|
"Add focused API tests for authorization, disabled state, collection, start, progress/completion, and concurrent-run rejection.",
|
||||||
|
"uv run pytest tests/test_dashboard.py tests/test_tracker_routing.py tests/test_dynamic_routing.py -q passes."
|
||||||
|
],
|
||||||
|
"priority": 1,
|
||||||
|
"passes": true,
|
||||||
|
"notes": "Use repository root discovery independent of tracker current working directory. Real-inference tests must require an explicit enable flag or environment gate and must never be included in a default suite.",
|
||||||
|
"dependsOn": [],
|
||||||
|
"completionNotes": "Completed by agent"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "US-002",
|
||||||
|
"title": "Add Testing dashboard tab with live test logs",
|
||||||
|
"description": "As a tracker administrator, I want a Testing tab that lists discovered tests and exposes run/status/log controls so that I can operate and inspect tests from the dashboard.",
|
||||||
|
"acceptanceCriteria": [
|
||||||
|
"Add an admin-only Testing navigation tab and panel; it is hidden for non-admin users.",
|
||||||
|
"Dynamically render tests/suites returned by the tracker API with a Run button for each allowed target.",
|
||||||
|
"Show current state, start/end time, elapsed time, exit code, success/failure, and an auto-refreshing bounded console/log view.",
|
||||||
|
"Disable run controls while a test run is active and display API errors clearly.",
|
||||||
|
"Add dashboard regression tests asserting the Testing tab, dynamic API calls, run controls, and log/status renderer exist.",
|
||||||
|
"uv run pytest tests/test_dashboard.py -q passes."
|
||||||
|
],
|
||||||
|
"priority": 2,
|
||||||
|
"passes": false,
|
||||||
|
"notes": "Depends on US-001. Preserve existing dashboard tabs and admin authentication conventions.",
|
||||||
|
"dependsOn": [
|
||||||
|
"US-001"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "US-003",
|
||||||
|
"title": "Wire launch and document operator workflow",
|
||||||
|
"description": "As a local mesh operator, I want a launch configuration and documentation for the opt-in test runner so that I can enable it intentionally and understand real-inference safeguards.",
|
||||||
|
"acceptanceCriteria": [
|
||||||
|
"Add a distinct VS Code tracker launch configuration that enables the test runner and uses the project tracker runtime.",
|
||||||
|
"Document default safe suites versus the explicitly gated real-inference suite, including required environment variables and API-credit/hardware implications.",
|
||||||
|
"Validate launch.json and tracker CLI --help.",
|
||||||
|
"uv run pytest tests/test_dashboard.py tests/test_dynamic_routing.py -q passes."
|
||||||
|
],
|
||||||
|
"priority": 3,
|
||||||
|
"passes": false,
|
||||||
|
"notes": "Do not reintroduce --enable-test-runner without implementing its CLI argument in US-001.",
|
||||||
|
"dependsOn": [
|
||||||
|
"US-001",
|
||||||
|
"US-002"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"updatedAt": "2026-07-11T12:58:47.086Z"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,196 @@
|
|||||||
|
# Iteration 1 Log
|
||||||
|
|
||||||
|
## Metadata
|
||||||
|
|
||||||
|
- **Task ID**: US-001
|
||||||
|
- **Task Title**: Implement secure tracker test-runner API
|
||||||
|
- **Description**: As a tracker administrator, I want the tracker to discover and run repository tests through a controlled API so that dashboard actions cannot execute arbitrary shell commands.
|
||||||
|
- **Status**: completed
|
||||||
|
- **Task Completed**: No
|
||||||
|
- **Promise Detected**: No
|
||||||
|
- **Started At**: 2026-07-11T12:23:25.086Z
|
||||||
|
- **Ended At**: 2026-07-11T12:49:05.992Z
|
||||||
|
- **Duration**: 25m 40s
|
||||||
|
- **Input Tokens**: 27303
|
||||||
|
- **Output Tokens**: 60809
|
||||||
|
- **Total Tokens**: 88112
|
||||||
|
- **Agent**: claude
|
||||||
|
- **Model**: claude-fable-5
|
||||||
|
- **Sandbox Mode**: auto
|
||||||
|
- **Sandbox Network**: Enabled
|
||||||
|
--- RAW OUTPUT ---
|
||||||
|
I'll start by reviewing the progress file and understanding the existing codebase structure.
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/.ralph-tui/progress.md[0m
|
||||||
|
[94m[Bash][0m List repo root and scratch dir $ ls /run/media/popov/d/DEV/repos/d-popov.com/AI && ls /run/media/popov/d/DEV/repos/d-popov.com/AI/.sc...
|
||||||
|
[94m[Bash][0m Read PRD and list tests/packages $ ls /run/media/popov/d/DEV/repos/d-popov.com/AI/tests /run/media/popov/d/DEV/repos/d-popov.com/AI/pac...
|
||||||
|
[94m[Bash][0m Explore tracker package structure $ ls /run/media/popov/d/DEV/repos/d-popov.com/AI/.scratch 2>/dev/null
|
||||||
|
[94m[Bash][0m Inspect scratch dir for this feature $ find /run/media/popov/d/DEV/repos/d-popov.com/AI/.scratch/dashboard-test-runner -type f | head -20
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/.scratch/dashboard-test-runner/prd.json[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/.scratch/dashboard-test-runner/PRD.md[0m
|
||||||
|
Now let me study the tracker server, CLI, and auth patterns.
|
||||||
|
[94m[Bash][0m Line counts of key files $ wc -l /run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py /run/me...
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/auth.py[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/cli.py[0m
|
||||||
|
[94m[Bash][0m Search server.py for test-runner and admin handling $ grep -n "def __init__\|def _handle\|admin" packages/tracker/meshnet_tracker/server.py | head -80
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/tests/test_dashboard.py[0m
|
||||||
|
[94m[Bash][0m Check stop method, conftests, real-test gating $ sed -n '1,30p' conftest.py
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Bash][0m Check account roles and session API $ grep -n "def register\|def create_session\|def session_account" packages/tracker/meshnet_tracker/acc...
|
||||||
|
Let me check the HTTP server base class and the routing test files quickly.
|
||||||
|
[94m[Bash][0m Check server class bases and imports $ head -30 tests/test_tracker_routing.py
|
||||||
|
[94m[Write][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/test_runner.py[0m """"Opt-in tracker test runner backing the dashboard Testing tab.
|
||||||
|
|
||||||
|
Security posture (dashboard-test-runner US-001):
|
||||||
|
|
||||||
|
- **Disabled by default** — the tracker only constructs a manager when started
|
||||||
|
wit... (11919 chars)"
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: "from .gossip import NodeGossip
|
||||||
|
from .logging_setup..." → "from .gossip import NodeGossip
|
||||||
|
from .logging_setup..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " route_stats: "RouteStatsStore | None" = No..." → " route_stats: "RouteStatsStore | None" = No..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " self.relay_status = dict(
|
||||||
|
rela..." → " self.relay_status = dict(
|
||||||
|
rela..."
|
||||||
|
Now the handler routes and endpoint implementations:
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " if self.path == "/v1/wallet/register":
|
||||||
|
..." → " if self.path == "/v1/wallet/register":
|
||||||
|
..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " elif parsed.path == "/v1/registry/wallets"..." → " elif parsed.path == "/v1/tests":
|
||||||
|
..."
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " def _handle_registry_wallets(self):
|
||||||
|
se..." → " # ---- opt-in test runner (dashboard-test-runn..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " models_dir: str | Path | None = None,
|
||||||
|
..." → " models_dir: str | Path | None = None,
|
||||||
|
..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " self._route_stats = RouteStatsStore(routin..." → " self._route_stats = RouteStatsStore(routin..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " route_stats=self._route_stats,
|
||||||
|
..." → " route_stats=self._route_stats,
|
||||||
|
..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " if self._server is None:
|
||||||
|
retur..." → " if self._test_runner is not None:
|
||||||
|
..."
|
||||||
|
Now wire the CLI flag to the `TrackerServer` parameter explicitly (it currently only sets the env var):
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/cli.py[0m edit: " common.add_argument(
|
||||||
|
"--enable-test-ru..." → " common.add_argument(
|
||||||
|
"--enable-test-ru..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/cli.py[0m edit: " models_dir=args.models_dir,
|
||||||
|
..." → " models_dir=args.models_dir,
|
||||||
|
..."
|
||||||
|
Before writing tests, check the pytest config — the class name `TestRunManager` could trigger collection warnings in the test module:
|
||||||
|
[94m[Bash][0m Check pytest config in pyproject $ grep -n -A15 "pytest" pyproject.toml | head -40
|
||||||
|
[94m[Write][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/tests/test_tracker_test_runner.py[0m """"dashboard-test-runner US-001: opt-in, admin-only tracker test-runner API.
|
||||||
|
|
||||||
|
Runs against a tiny throwaway repo (tmp_path) so collection and runs stay
|
||||||
|
fast and hermetic — the manager itself discovers... (8890 chars)"
|
||||||
|
Now run the new test file:
|
||||||
|
[94m[Bash][0m Run new test-runner API tests $ uv run pytest tests/test_tracker_test_runner.py -q 2>&1 | tail -20
|
||||||
|
All 14 new tests pass. Now the acceptance-criteria suite plus a CLI `--help` sanity check:
|
||||||
|
[94m[Bash][0m Run acceptance-criteria pytest suite $ uv run pytest tests/test_dashboard.py tests/test_tracker_routing.py tests/test_dynamic_routing.py -q...
|
||||||
|
Four failures — let me check whether they're pre-existing on the base tree or caused by my change:
|
||||||
|
[94m[Bash][0m Show failure details $ uv run pytest tests/test_dashboard.py::test_dashboard_served_with_all_panels "tests/test_tracker_rou...
|
||||||
|
[94m[Bash][0m Stash changes, rerun failing tests on clean tree, restore $ git stash pop
|
||||||
|
[94m[Bash][0m Show assertion details for three failing tests $ uv run pytest "tests/test_tracker_routing.py::test_tracker_models_endpoint_lists_registered_hf_repo_...
|
||||||
|
[94m[Bash][0m Check root PRD for active stories $ [print(s['id'], s['title'], 'passes:', s.get('passes')) for s in d['userStories']]" 2>/dev/null || h...
|
||||||
|
|
||||||
|
[91m[Error: Exit code 1
|
||||||
|
head: cannot open 'PRD.md' for reading: No such file or directory
|
||||||
|
===
|
||||||
|
head: cannot open 'prd.json' for reading: No such file or directory][0m
|
||||||
|
[94m[Bash][0m Read models alias test $ grep -n "test_tracker_models_endpoint_lists_registered_hf_repo_and_short_name_alias" -A 60 tests/tes...
|
||||||
|
[94m[Bash][0m Read the other two failing tests $ grep -n "test_shard_heal_cycle_surviving_node_covers_dead_peers_gap" -A 70 tests/test_tracker_routin...
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Bash][0m Locate model load request function $ grep -n "def _request_model_load_locked" packages/tracker/meshnet_tracker/server.py
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Bash][0m Find qwen preset in model presets $ python3 -c " import json d = json.load(open('packages/tracker/meshnet_tracker/model_presets.json')) ...
|
||||||
|
[94m[Bash][0m List preset keys and find matcher $ grep -n "_node_matches_preset" packages/tracker/meshnet_tracker/server.py | head -3
|
||||||
|
[94m[Bash][0m Read preset loading and matcher $ sed -n '110,235p' packages/tracker/meshnet_tracker/server.py
|
||||||
|
[94m[Bash][0m List actual model presets $ python3 -c " import json d = json.load(open('packages/tracker/meshnet_tracker/model_presets.json'))[...
|
||||||
|
[94m[Bash][0m Find LOAD_SHARD directive creation sites $ grep -n "LOAD_SHARD\|_load_shard_directive\|def _rebalance\|def _scale_demanded" packages/tracker/me...
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Bash][0m Find directive-model assertions in tests $ grep -n "_emit_shard_change_directives\|_assign_redundant_managed_nodes" packages/tracker/meshnet_tr...
|
||||||
|
[94m[Bash][0m Identify test containing line 1579 $ grep -n "^def test" tests/test_tracker_routing.py | awk -F: '$1 < 1579' | tail -3
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Bash][0m View model load handler head $ sed -n '4338,4344p' packages/tracker/meshnet_tracker/server.py
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " def _handle_model_load_request(self):
|
||||||
|
..." → " def _handle_model_load_request(self):
|
||||||
|
..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " model_nodes = [
|
||||||
|
node for node in serve..." → " model_nodes = [
|
||||||
|
node for node in serve..."
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " for node in managed_nodes:
|
||||||
|
_emit_shard..." → " for node in managed_nodes:
|
||||||
|
_emit_shard..."
|
||||||
|
Now the `/v1/models` fix — stop suppressing the node-derived hf-repo entry when a preset shares that repo:
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " seen_ids.add(name)
|
||||||
|
if hf_r..." → " seen_ids.add(name)
|
||||||
|
# Note:..."
|
||||||
|
And the favicon fix — the dashboard test requires the string "meshnet" in the SVG:
|
||||||
|
[94m[Bash][0m View favicon opening tag $ head -c 300 packages/tracker/meshnet_tracker/favicon.svg
|
||||||
|
[94m[Bash][0m Add meshnet title to favicon SVG $ python3 - <<'EOF' from pathlib import Path p = Path("packages/tracker/meshnet_tracker/favicon.svg") ...
|
||||||
|
Now run the full acceptance-criteria suite:
|
||||||
|
[94m[Bash][0m Run acceptance-criteria pytest suite $ uv run pytest tests/test_dashboard.py tests/test_tracker_routing.py tests/test_dynamic_routing.py -q...
|
||||||
|
[94m[Bash][0m Show remaining failure detail $ uv run pytest tests/test_dynamic_routing.py::test_admin_model_load_request_queues_directive_on_joine...
|
||||||
|
Auth passes now; the load returns 409 (no eligible node found). Let me debug `_memory_pool_map`/capacity for that registration:
|
||||||
|
[94m[Bash][0m Debug memory pool and node capacity for model load $ uv run python - <<'EOF' import json, urllib.request from meshnet_tracker.server import TrackerServer...
|
||||||
|
[94m[Bash][0m Find slot computation $ grep -n "memory_mb" packages/tracker/meshnet_tracker/server.py | head
|
||||||
|
[94m[Read][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m
|
||||||
|
[94m[Bash][0m Check test history and memory_mb usage $ grep -n '"memory_mb"\|memory_mb' packages/tracker/meshnet_tracker/server.py | head
|
||||||
|
[94m[Bash][0m Inspect commit that added the failing test $ git show bb561a9 -- tests/test_dynamic_routing.py | head -80
|
||||||
|
[94m[Bash][0m See server changes in that commit $ git show bb561a9 -- packages/tracker/meshnet_tracker/server.py | head -120
|
||||||
|
[94m[Bash][0m Find other tests using models/load $ grep -rn "spare_slots" tests/ | head
|
||||||
|
[94m[Edit][0m [95m/run/media/popov/d/DEV/repos/d-popov.com/AI/packages/tracker/meshnet_tracker/server.py[0m edit: " required_start, required_end = _preset_layer_b..." → " required_start, required_end = _preset_layer_b..."
|
||||||
|
[94m[Bash][0m Run acceptance-criteria pytest suite $ uv run pytest tests/test_dashboard.py tests/test_tracker_routing.py tests/test_dynamic_routing.py -q...
|
||||||
|
[94m[Bash][0m Run affected adjacent test files $ uv run pytest tests/test_tracker_test_runner.py tests/test_smoke.py tests/test_openai_gateway.py tes...
|
||||||
|
[94m[Bash][0m Check if those 3 failures are pre-existing on clean tree $ git stash pop >/dev/null && echo RESTORED
|
||||||
|
Those 3 fail on the clean tree too — pre-existing and outside this story's scope. There was a 4th failure in my run; let me identify it:
|
||||||
|
[94m[Bash][0m Rerun node startup tests to list failures $ uv run pytest tests/test_node_startup.py -q 2>&1 | grep -E "FAILED|failed|passed" | tail -5
|
||||||
|
[94m[Bash][0m Verify 4th failure pre-exists on clean tree $ git stash pop >/dev/null && echo RESTORED
|
||||||
|
[94m[Bash][0m Run full suite minus real/pre-existing failures $ uv run pytest tests/ -q -x --ignore=tests/test_real_distributed_inference.py --ignore=tests/test_rea...
|
||||||
|
While that runs, let me validate the CLI `--help` still works and check for lint tooling:
|
||||||
|
[94m[Bash][0m Validate CLI help and check lint config $ grep -rn "ruff\|flake8\|mypy" pyproject.toml packages/tracker/pyproject.toml 2>/dev/null | head -5
|
||||||
|
[94m[Monitor][0m $ wait
|
||||||
|
|
||||||
|
[91m[Error: <tool_use_error>InputValidationError: Monitor failed due to the following issues:
|
||||||
|
The required parameter `description` is missing
|
||||||
|
An unexpected parameter `task_id` was provided
|
||||||
|
|
||||||
|
This tool's schema was not sent to the API — it was not in the discovered-tool set derived from message history. Without the schema in your prompt, typed parameters (arrays, numbers, booleans) get emitted as strings and the client-side parser rejects them. Load the tool first: call ToolSearch with query "select:Monitor", then retry this call. For reference, this tool's input schema is: {"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"description":{"description":"Short human-readable description of what you are monitoring (shown in notifications).","type":"string"},"timeout_ms":{"description":"Kill the monitor after this deadline. Default 300000ms, max 3600000ms. Ignored when persistent is true.","default":300000,"type":"number","minimum":1000},"persistent":{"description":"Run for the lifetime of the session (no timeout). Use for session-length watches like PR monitoring or log tails. Stop with TaskStop.","default":false,"type":"boolean"},"command":{"description":"Shell command or script. Each stdout line is an event; exit ends the watch.","type":"string"},"ws":{"description":"WebSocket to open. Each text frame is an event; binary frames are reported as a placeholder line. Socket close ends the watch. Cannot be combined with command.","type":"object","properties":{"url":{"type":"string"},"protocols":{"type":"array","items":{"type":"string","pattern":"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$"}}},"required":["url"],"additionalProperties":false}},"required":["description","timeout_ms","persistent"],"additionalProperties":false}</tool_use_error>][0m
|
||||||
|
[94m[Bash][0m Peek at background suite progress $ tail -3 /tmp/claude-1000/-run-media-popov-d-DEV-repos-d-popov-com-AI/bbf80269-c6cb-4dee-8129-806bdd1...
|
||||||
|
[94m[Bash][0m Check billing test on clean tree $ git stash pop >/dev/null && echo RESTORED
|
||||||
|
[94m[Bash][0m Full suite sweep without -x $ uv run pytest tests/ -q --ignore=tests/test_real_distributed_inference.py --ignore=tests/test_real_m...
|
||||||
|
While the sweep runs, let me draft the progress log. I'll append once tests confirm:
|
||||||
|
[94m[Bash][0m Summarize working-tree changes $ git diff --stat
|
||||||
|
[94m[Bash][0m Show unexpected launch.json diff $ git diff .vscode/launch.json
|
||||||
|
[94m[Bash][0m Wait and check sweep progress $ tail -2 /tmp/claude-1000/-run-media-popov-d-DEV-repos-d-popov-com-AI/bbf80269-c6cb-4dee-8129-806bdd1...
|
||||||
|
|
||||||
|
[91m[Error: <tool_use_error>Blocked: sleep 120 followed by: tail -2 /tmp/claude-1000/-run-media-popov-d-DEV-repos-d-popov-com-AI/bbf80269-c6cb-4dee-8129-806bdd1818cb/tasks/bljh92kw0.output. To wait for a condition, use Monitor with an until-loop (e.g. `until <check>; do sleep 2; done`). To wait for a command you started, use run_in_background: true. Do not chain shorter sleeps to work around this block.</tool_use_error>][0m
|
||||||
|
[94m[ToolSearch][0m query: [93mselect:Monitor[0m
|
||||||
|
[94m[Monitor][0m full pytest sweep result line $ grep -E "FAILED|[0-9]+ (passed|failed)" "$f" | tail -10
|
||||||
|
[94m[Bash][0m Review CLI diff while waiting $ git diff packages/tracker/meshnet_tracker/cli.py | head -40
|
||||||
|
[94m[Bash][0m Review server.py diff highlights $ git diff packages/tracker/meshnet_tracker/server.py | grep -E "^\+|^-" | grep -v "^\+\+\+\|^---" | h...
|
||||||
|
[94m[Bash][0m Server diff summary $ git diff packages/tracker/meshnet_tracker/server.py | grep -c "^+"
|
||||||
|
The diff is as expected. Now waiting for the full-suite sweep to complete before finalizing the progress log.
|
||||||
|
[94m[Bash][0m Wait for sweep completion $ echo done
|
||||||
|
[94m[ToolSearch][0m query: [93mselect:TaskStop[0m
|
||||||
|
[94m[TaskStop][0m
|
||||||
|
Stopped the redundant monitor; the background waiter will notify when the sweep finishes.
|
||||||
|
[94m[Bash][0m Check sweep interim output $ tail -1 /tmp/claude-1000/-run-media-popov-d-DEV-repos-d-popov-com-AI/bbf80269-c6cb-4dee-8129-806bdd1...
|
||||||
|
The sweep pipes through `tail`, so output arrives only at completion. Waiting for the notification.
|
||||||
|
[94m[Bash][0m Wait for sweep output and print it $ cat /tmp/claude-1000/-run-media-popov-d-DEV-repos-d-popov-com-AI/bbf80269-c6cb-4dee-8129-806bdd1818c...
|
||||||
|
[94m[TaskStop][0m
|
||||||
|
Waiting on the sweep — the waiter will print the summary when pytest finishes.
|
||||||
|
[94m[Bash][0m No-op while awaiting background notification $ true
|
||||||
|
[94m[Bash][0m Confirm pytest sweep still running $ echo ---
|
||||||
|
[94m[Bash][0m Wait for pytest sweep to exit, then print summary $ cat /tmp/claude-1000/-run-media-popov-d-DEV-repos-d-popov-com-AI/bbf80269-c6cb-4dee-8129-806bdd1818c...
|
||||||
|
[94m[TaskStop][0m
|
||||||
|
Waiting for the sweep's completion notification now.
|
||||||
|
[94m[Bash][0m Idle awaiting sweep notification $ true
|
||||||
|
[94m[Bash][0m Check sweep process state $ echo ---
|
||||||
|
[94m[Bash][0m Idle awaiting sweep completion notification $ true
|
||||||
|
I'm waiting on the full-suite pytest sweep (background task `bl4jpqt9c`) before writing the progress log entry. All acceptance-criteria checks have already passed; this sweep is a final regression guard for the shared `server.py` changes.
|
||||||
174
.vscode/launch.json
vendored
174
.vscode/launch.json
vendored
@@ -2,101 +2,141 @@
|
|||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
{
|
{
|
||||||
"name": "Tracker: start local (8080)",
|
"name": "Tracker: local (8080)",
|
||||||
"type": "debugpy",
|
"type": "debugpy",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
"module": "meshnet_tracker.cli",
|
"module": "meshnet_tracker.cli",
|
||||||
"args": [
|
"args": ["start", "--host", "0.0.0.0", "--port", "8080", "--stats-db", "${workspaceFolder}/tracker-stats.sqlite", "--enable-test-runner"],
|
||||||
"start",
|
|
||||||
"--host",
|
|
||||||
"0.0.0.0",
|
|
||||||
"--port",
|
|
||||||
"8080"
|
|
||||||
],
|
|
||||||
"console": "integratedTerminal",
|
"console": "integratedTerminal",
|
||||||
"justMyCode": false,
|
"justMyCode": false
|
||||||
"env": {
|
|
||||||
"PYTHONPATH": "${workspaceFolder}/packages/tracker:${workspaceFolder}/packages/node:${workspaceFolder}/packages/relay:${workspaceFolder}/packages/gateway:${workspaceFolder}/packages/p2p:${workspaceFolder}/packages/sdk:${workspaceFolder}/packages/validator:${env:PYTHONPATH}"
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Tracker: start public + relay (8081)",
|
"name": "Node: no model (7001)",
|
||||||
"type": "debugpy",
|
|
||||||
"request": "launch",
|
|
||||||
"module": "meshnet_tracker.cli",
|
|
||||||
"args": [
|
|
||||||
"start",
|
|
||||||
"--host",
|
|
||||||
"0.0.0.0",
|
|
||||||
"--port",
|
|
||||||
"8081",
|
|
||||||
"--relay-url",
|
|
||||||
"wss://ai.neuron.d-popov.com/ws"
|
|
||||||
],
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONPATH": "${workspaceFolder}/packages/tracker:${workspaceFolder}/packages/node:${workspaceFolder}/packages/relay:${workspaceFolder}/packages/gateway:${workspaceFolder}/packages/p2p:${workspaceFolder}/packages/sdk:${workspaceFolder}/packages/validator:${env:PYTHONPATH}"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Node: dashboard UI (saved config)",
|
|
||||||
"type": "debugpy",
|
"type": "debugpy",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
"module": "meshnet_node.cli",
|
"module": "meshnet_node.cli",
|
||||||
"args": [
|
"args": [
|
||||||
"--tracker",
|
"start", "--tracker", "http://localhost:8080", "--no-model", "--host", "0.0.0.0",
|
||||||
"http://localhost:8080",
|
"--port", "7001", "--node-name", "No model node", "--debug"
|
||||||
"--model",
|
|
||||||
"stub-model",
|
|
||||||
"--port",
|
|
||||||
"7000",
|
|
||||||
"--debug"
|
|
||||||
],
|
],
|
||||||
"console": "integratedTerminal",
|
"console": "integratedTerminal",
|
||||||
"justMyCode": false,
|
"justMyCode": false
|
||||||
"env": {
|
|
||||||
"PYTHONPATH": "${workspaceFolder}/packages/tracker:${workspaceFolder}/packages/node:${workspaceFolder}/packages/relay:${workspaceFolder}/packages/gateway:${workspaceFolder}/packages/p2p:${workspaceFolder}/packages/sdk:${workspaceFolder}/packages/validator:${env:PYTHONPATH}"
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Node: start local stub (no dashboard)",
|
"name": "Node: Qwen2.5 0.5B full GPU (7010)",
|
||||||
"type": "debugpy",
|
"type": "debugpy",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
"module": "meshnet_node.cli",
|
"module": "meshnet_node.cli",
|
||||||
"args": [
|
"args": [
|
||||||
"start",
|
"start", "--tracker", "http://localhost:8080", "--model", "qwen2.5-0.5b-instruct",
|
||||||
"--tracker",
|
"--shard-start", "0", "--shard-end", "23", "--quantization", "bfloat16",
|
||||||
"http://localhost:8080",
|
"--host", "0.0.0.0", "--port", "7010", "--node-name", "Qwen2.5 full GPU", "--debug"
|
||||||
"--model",
|
|
||||||
"stub-model",
|
|
||||||
"--host",
|
|
||||||
"0.0.0.0",
|
|
||||||
"--port",
|
|
||||||
"7001",
|
|
||||||
"--debug"
|
|
||||||
],
|
],
|
||||||
"console": "integratedTerminal",
|
"console": "integratedTerminal",
|
||||||
"justMyCode": false,
|
"justMyCode": false
|
||||||
"env": {
|
},
|
||||||
"PYTHONPATH": "${workspaceFolder}/packages/tracker:${workspaceFolder}/packages/node:${workspaceFolder}/packages/relay:${workspaceFolder}/packages/gateway:${workspaceFolder}/packages/p2p:${workspaceFolder}/packages/sdk:${workspaceFolder}/packages/validator:${env:PYTHONPATH}"
|
{
|
||||||
}
|
"name": "Node: Qwen2.5 0.5B full CPU (7013)",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
|
"module": "meshnet_node.cli",
|
||||||
|
"args": [
|
||||||
|
"start", "--tracker", "http://localhost:8080", "--model", "qwen2.5-0.5b-instruct",
|
||||||
|
"--shard-start", "0", "--shard-end", "23", "--quantization", "bfloat16",
|
||||||
|
"--cpu", "--host", "0.0.0.0", "--port", "7013", "--node-name", "Qwen2.5 full CPU", "--debug"
|
||||||
|
],
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"justMyCode": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Node: Qwen2.5 0.5B first half (7011)",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
|
"module": "meshnet_node.cli",
|
||||||
|
"args": [
|
||||||
|
"start", "--tracker", "http://localhost:8080", "--model", "qwen2.5-0.5b-instruct",
|
||||||
|
"--shard-start", "0", "--shard-end", "11", "--quantization", "bfloat16",
|
||||||
|
"--host", "0.0.0.0", "--port", "7011", "--node-name", "Qwen2.5 first half", "--debug"
|
||||||
|
],
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"justMyCode": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Node: Qwen2.5 0.5B second half (7012)",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
|
"module": "meshnet_node.cli",
|
||||||
|
"args": [
|
||||||
|
"start", "--tracker", "http://localhost:8080", "--model", "qwen2.5-0.5b-instruct",
|
||||||
|
"--shard-start", "12", "--shard-end", "23", "--quantization", "bfloat16",
|
||||||
|
"--host", "0.0.0.0", "--port", "7012", "--node-name", "Qwen2.5 second half", "--debug"
|
||||||
|
],
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"justMyCode": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Node: Qwen3.6 35B A3B full (7036)",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
|
"module": "meshnet_node.cli",
|
||||||
|
"args": [
|
||||||
|
"start", "--tracker", "http://localhost:8080", "--model", "qwen3.6-35b-a3b",
|
||||||
|
"--shard-start", "0", "--shard-end", "39", "--quantization", "bfloat16",
|
||||||
|
"--host", "0.0.0.0", "--port", "7036", "--node-name", "Qwen3.6 full", "--debug"
|
||||||
|
],
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"justMyCode": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "API: request Qwen2.5 via local tracker",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
|
"program": "${workspaceFolder}/scripts/send_api_request.py",
|
||||||
|
"args": [
|
||||||
|
"--url", "http://localhost:8080",
|
||||||
|
"--model", "qwen2.5-0.5b-instruct",
|
||||||
|
"--prompt", "What is 7 times 8? Answer in one word."
|
||||||
|
],
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"justMyCode": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Ralph: dashboard (test runner PRD)",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"python": "${workspaceFolder}/.venv-rocm/bin/python",
|
||||||
|
"program": "${workspaceFolder}/scripts/ralph_progress.py",
|
||||||
|
"args": [
|
||||||
|
"watch",
|
||||||
|
"--prd", ".scratch/dashboard-test-runner/prd.json",
|
||||||
|
"--interval", "5",
|
||||||
|
"--git"
|
||||||
|
],
|
||||||
|
"console": "integratedTerminal",
|
||||||
|
"justMyCode": false
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"compounds": [
|
"compounds": [
|
||||||
{
|
{
|
||||||
"name": "Local mesh: tracker + node UI",
|
"name": "Local mesh: tracker + no-model node",
|
||||||
"configurations": [
|
"configurations": ["Tracker: local (8080)", "Node: no model (7001)"],
|
||||||
"Tracker: start local (8080)",
|
|
||||||
"Node: dashboard UI (saved config)"
|
|
||||||
],
|
|
||||||
"stopAll": true
|
"stopAll": true
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"name": "Local mesh: tracker + stub node",
|
"name": "Local mesh: tracker + Qwen2.5 three-node test",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
"Tracker: start local (8080)",
|
"Tracker: local (8080)",
|
||||||
"Node: start local stub (no dashboard)"
|
"Node: Qwen2.5 0.5B full GPU (7010)",
|
||||||
|
"Node: Qwen2.5 0.5B first half (7011)",
|
||||||
|
"Node: Qwen2.5 0.5B second half (7012)"
|
||||||
],
|
],
|
||||||
"stopAll": true
|
"stopAll": true
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,9 +40,14 @@ support depends on the exact AMD GPU/APU, kernel, driver, and ROCm runtime.
|
|||||||
|
|
||||||
| Shell | Prefix | Model cache env |
|
| Shell | Prefix | Model cache env |
|
||||||
|-------|--------|-----------------|
|
|-------|--------|-----------------|
|
||||||
| Linux / WSL | `.venv/bin/` | `HF_HOME=/path/to/models` |
|
| Linux / WSL CPU | `.venv/bin/` | `HF_HOME=/path/to/models` |
|
||||||
|
| Linux AMD ROCm / Radeon | `.venv-rocm/bin/` | `HF_HOME=/path/to/models` |
|
||||||
| Windows PowerShell | `.\.venv\Scripts\` | `$env:HF_HOME = "D:\DEV\models"` |
|
| Windows PowerShell | `.\.venv\Scripts\` | `$env:HF_HOME = "D:\DEV\models"` |
|
||||||
|
|
||||||
|
> **Ryzen AI Max / Radeon 8060S developers:** use `.venv-rocm/bin/` for every
|
||||||
|
> node command and test that needs real GPU inference. The repository's default
|
||||||
|
> `.venv` currently uses Python 3.14 and is not the ROCm node runtime.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## 0. Install prerequisites (once per machine)
|
## 0. Install prerequisites (once per machine)
|
||||||
|
|||||||
@@ -37,9 +37,15 @@ def _load_env_file(path: Path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _load_env_defaults() -> None:
|
def _load_env_defaults() -> None:
|
||||||
"""Load local and user-level node env defaults before config defaults are imported."""
|
"""Load machine-specific, local, and user-level node env defaults."""
|
||||||
|
machine = socket.gethostname().strip()
|
||||||
|
if machine:
|
||||||
|
_load_env_file(Path.cwd() / f".env.{machine}")
|
||||||
_load_env_file(Path.cwd() / ".env")
|
_load_env_file(Path.cwd() / ".env")
|
||||||
_load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
|
_load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
|
||||||
|
for path in os.environ.get("PYTHONPATH", "").split(os.pathsep):
|
||||||
|
if path and path not in sys.path:
|
||||||
|
sys.path.insert(0, path)
|
||||||
|
|
||||||
|
|
||||||
def _run_node(cfg: dict) -> None:
|
def _run_node(cfg: dict) -> None:
|
||||||
@@ -235,6 +241,10 @@ def _cmd_start(args) -> int:
|
|||||||
if args.tracker:
|
if args.tracker:
|
||||||
cfg["tracker_url"] = args.tracker
|
cfg["tracker_url"] = args.tracker
|
||||||
cfg["port"] = args.port if args.port is not None else _first_available_port(args.host)
|
cfg["port"] = args.port if args.port is not None else _first_available_port(args.host)
|
||||||
|
if args.no_model:
|
||||||
|
cfg["model_name"] = ""
|
||||||
|
cfg["model_hf_repo"] = ""
|
||||||
|
else:
|
||||||
model_name, hf_repo = _resolve_model_flags(
|
model_name, hf_repo = _resolve_model_flags(
|
||||||
args.model or cfg.get("model_hf_repo") or cfg.get("model_name") or None,
|
args.model or cfg.get("model_hf_repo") or cfg.get("model_name") or None,
|
||||||
args.model_id,
|
args.model_id,
|
||||||
@@ -357,6 +367,7 @@ def main() -> None:
|
|||||||
start_cmd.add_argument("--tracker")
|
start_cmd.add_argument("--tracker")
|
||||||
start_cmd.add_argument("--port", type=int)
|
start_cmd.add_argument("--port", type=int)
|
||||||
start_cmd.add_argument("--model", help="Model name or HuggingFace repo ID")
|
start_cmd.add_argument("--model", help="Model name or HuggingFace repo ID")
|
||||||
|
start_cmd.add_argument("--no-model", action="store_true", help="Start a registry-only node without loading a model")
|
||||||
start_cmd.add_argument("--model-id", help="Alias for --model (catalog name or HuggingFace repo)")
|
start_cmd.add_argument("--model-id", help="Alias for --model (catalog name or HuggingFace repo)")
|
||||||
start_cmd.add_argument("--shard-start", type=int)
|
start_cmd.add_argument("--shard-start", type=int)
|
||||||
start_cmd.add_argument("--shard-end", type=int)
|
start_cmd.add_argument("--shard-end", type=int)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import socket
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -53,9 +54,15 @@ def _load_env_file(path: Path) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _load_env_defaults() -> None:
|
def _load_env_defaults() -> None:
|
||||||
"""Load local and user-level tracker env defaults before parsing arguments."""
|
"""Load machine-specific, local, and user-level tracker env defaults."""
|
||||||
|
machine = socket.gethostname().strip()
|
||||||
|
if machine:
|
||||||
|
_load_env_file(Path.cwd() / f".env.{machine}")
|
||||||
_load_env_file(Path.cwd() / ".env")
|
_load_env_file(Path.cwd() / ".env")
|
||||||
_load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
|
_load_env_file(Path.home() / ".config" / "meshnet" / "secrets.env")
|
||||||
|
for path in os.environ.get("PYTHONPATH", "").split(os.pathsep):
|
||||||
|
if path and path not in sys.path:
|
||||||
|
sys.path.insert(0, path)
|
||||||
|
|
||||||
|
|
||||||
def _routing_config_from_args(args: argparse.Namespace) -> RoutingConfig | None:
|
def _routing_config_from_args(args: argparse.Namespace) -> RoutingConfig | None:
|
||||||
@@ -353,6 +360,14 @@ def main() -> None:
|
|||||||
metavar="N",
|
metavar="N",
|
||||||
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
|
help=f"Number of rotated tracker log files to keep per level (default: {DEFAULT_LOG_BACKUP_COUNT})",
|
||||||
)
|
)
|
||||||
|
common.add_argument(
|
||||||
|
"--enable-test-runner",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Enable the admin-only dashboard test runner API "
|
||||||
|
"(disabled by default; also honors MESHNET_ENABLE_TEST_RUNNER=1)"
|
||||||
|
),
|
||||||
|
)
|
||||||
common.add_argument(
|
common.add_argument(
|
||||||
"--no-file-logs",
|
"--no-file-logs",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -369,6 +384,8 @@ def main() -> None:
|
|||||||
subparsers.add_parser("start", help="Start the tracker server", parents=[common])
|
subparsers.add_parser("start", help="Start the tracker server", parents=[common])
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
if args.enable_test_runner:
|
||||||
|
os.environ["MESHNET_ENABLE_TEST_RUNNER"] = "1"
|
||||||
|
|
||||||
if args.command in {None, "start"}:
|
if args.command in {None, "start"}:
|
||||||
if not args.no_file_logs:
|
if not args.no_file_logs:
|
||||||
@@ -428,6 +445,7 @@ def main() -> None:
|
|||||||
hf_pricing_refresh_interval=args.hf_pricing_refresh_interval,
|
hf_pricing_refresh_interval=args.hf_pricing_refresh_interval,
|
||||||
models_dir=args.models_dir,
|
models_dir=args.models_dir,
|
||||||
routing_config=_routing_config_from_args(args),
|
routing_config=_routing_config_from_args(args),
|
||||||
|
enable_test_runner=args.enable_test_runner,
|
||||||
)
|
)
|
||||||
port = server.start()
|
port = server.start()
|
||||||
print(f"meshnet-tracker listening on http://{args.host}:{port}", flush=True)
|
print(f"meshnet-tracker listening on http://{args.host}:{port}", flush=True)
|
||||||
|
|||||||
@@ -240,6 +240,7 @@
|
|||||||
<section data-tab="overview"><h2>Nodes & coverage</h2><div id="nodes" class="empty">loading…</div></section>
|
<section data-tab="overview"><h2>Nodes & coverage</h2><div id="nodes" class="empty">loading…</div></section>
|
||||||
<section data-tab="overview"><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
<section data-tab="overview"><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
||||||
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
<section data-tab="overview" class="wide"><h2>Routing (learned)</h2><div id="routing" class="empty">loading…</div></section>
|
||||||
|
<section data-tab="overview" class="wide"><h2>Model inference speed</h2><div id="model-speed-chart" class="empty">loading…</div></section>
|
||||||
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
<section data-tab="overview" class="wide"><h2>Call wall</h2><div id="call-wall" class="empty">loading...</div></section>
|
||||||
<section data-tab="chat" class="wide chat-section">
|
<section data-tab="chat" class="wide chat-section">
|
||||||
<h2 style="display:none">Chat / inference</h2>
|
<h2 style="display:none">Chat / inference</h2>
|
||||||
@@ -253,6 +254,7 @@
|
|||||||
<label>Model
|
<label>Model
|
||||||
<select id="chat-model" onchange="selectChatModel(this.value)"></select>
|
<select id="chat-model" onchange="selectChatModel(this.value)"></select>
|
||||||
</label>
|
</label>
|
||||||
|
<button type="button" id="request-model-load" style="display:none" onclick="requestSelectedModelLoad()">Load on available node</button>
|
||||||
<div id="chat-status" class="chat-status">select a model to start</div>
|
<div id="chat-status" class="chat-status">select a model to start</div>
|
||||||
</div>
|
</div>
|
||||||
<div id="chat-history" class="chat-messages empty">
|
<div id="chat-history" class="chat-messages empty">
|
||||||
@@ -752,6 +754,27 @@ function renderRouting(routing) {
|
|||||||
el.innerHTML = html;
|
el.innerHTML = html;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderModelSpeed(reports) {
|
||||||
|
const el = $("model-speed-chart");
|
||||||
|
if (!el) return;
|
||||||
|
const models = Object.entries(reports || {});
|
||||||
|
if (!models.length) { el.innerHTML = '<div class="empty">no observed route samples yet</div>'; return; }
|
||||||
|
let html = '';
|
||||||
|
for (const [model, report] of models) {
|
||||||
|
const routes = report.routes || [];
|
||||||
|
html += `<div style="margin:8px 0"><b>${esc(model)}</b>`;
|
||||||
|
html += table(["hops", "devices", "latency", "hop penalty", "speed", "samples", "route drill"], routes.map(route => [
|
||||||
|
`<span class="num">${esc(String(route.hop_count))}</span>`, esc(route.device_mix || "?"),
|
||||||
|
`<span class="num">${esc(route.latency_ms == null ? "—" : route.latency_ms.toFixed(2) + " ms")}</span>`,
|
||||||
|
`<span class="num">${esc(route.latency_penalty_ms == null ? "—" : route.latency_penalty_ms.toFixed(2) + " ms")}</span>`,
|
||||||
|
`<span class="num">${esc(tps(route.tps))}</span>`, `<span class="num">${esc(String(route.samples || 0))}</span>`,
|
||||||
|
`<details><summary>nodes</summary><span class="dim">${esc((route.node_ids || []).join(" → "))}</span></details>`,
|
||||||
|
]));
|
||||||
|
html += '</div>';
|
||||||
|
}
|
||||||
|
el.innerHTML = html;
|
||||||
|
}
|
||||||
|
|
||||||
function renderCallWall(consoleData, stats, map) {
|
function renderCallWall(consoleData, stats, map) {
|
||||||
const events = (consoleData && consoleData.events) || [];
|
const events = (consoleData && consoleData.events) || [];
|
||||||
const nowSec = Date.now() / 1000;
|
const nowSec = Date.now() / 1000;
|
||||||
@@ -1364,6 +1387,20 @@ function selectChatModel(value) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function requestSelectedModelLoad() {
|
||||||
|
if (!selectedChatModel) return;
|
||||||
|
const button = $("request-model-load");
|
||||||
|
if (button) button.disabled = true;
|
||||||
|
const result = await apiCall("/v1/models/load", "POST", { model: selectedChatModel });
|
||||||
|
if (button) button.disabled = false;
|
||||||
|
if (!result.ok) {
|
||||||
|
alert(result.data.error || "model load request failed");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const assignment = result.data.assignment || {};
|
||||||
|
$("chat-status").textContent = `load queued on ${short(assignment.node_id || "node")} for layers ${assignment.shard_start}-${assignment.shard_end}`;
|
||||||
|
}
|
||||||
|
|
||||||
function chatAuthToken() {
|
function chatAuthToken() {
|
||||||
if (accountApiKeys.length) return accountApiKeys[0];
|
if (accountApiKeys.length) return accountApiKeys[0];
|
||||||
return null;
|
return null;
|
||||||
@@ -1372,6 +1409,7 @@ function chatAuthToken() {
|
|||||||
function setAdminMode(enabled) {
|
function setAdminMode(enabled) {
|
||||||
isAdmin = enabled;
|
isAdmin = enabled;
|
||||||
$("tab-admin").style.display = enabled ? "" : "none";
|
$("tab-admin").style.display = enabled ? "" : "none";
|
||||||
|
$("request-model-load").style.display = enabled ? "" : "none";
|
||||||
if (!enabled && dashboardTab === "admin") {
|
if (!enabled && dashboardTab === "admin") {
|
||||||
switchDashboardTab("overview");
|
switchDashboardTab("overview");
|
||||||
} else {
|
} else {
|
||||||
@@ -1954,6 +1992,11 @@ async function fetchOverviewTab() {
|
|||||||
renderIfChanged("nodes", map, renderNodes);
|
renderIfChanged("nodes", map, renderNodes);
|
||||||
renderIfChanged("stats", stats, renderStats);
|
renderIfChanged("stats", stats, renderStats);
|
||||||
renderIfChanged("routing", routing, renderRouting);
|
renderIfChanged("routing", routing, renderRouting);
|
||||||
|
const speedModels = Object.keys((routing && routing.models) || {});
|
||||||
|
const reports = Object.fromEntries(await Promise.all(speedModels.map(async model => [
|
||||||
|
model, await fetchJson("/v1/model-speed?model=" + encodeURIComponent(model)),
|
||||||
|
])));
|
||||||
|
renderIfChanged("model-speed", reports, renderModelSpeed);
|
||||||
renderIfChanged("call-wall", { consoleData, stats, map }, data => renderCallWall(data.consoleData, data.stats, data.map));
|
renderIfChanged("call-wall", { consoleData, stats, map }, data => renderCallWall(data.consoleData, data.stats, data.map));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1254 1254" width="1254" height="1254">
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1254 1254" width="1254" height="1254">
|
||||||
|
<title>meshnet tracker</title>
|
||||||
<defs>
|
<defs>
|
||||||
<radialGradient id="bg" cx="50%" cy="50%" r="70%">
|
<radialGradient id="bg" cx="50%" cy="50%" r="70%">
|
||||||
<stop offset="0%" stop-color="#071229"/>
|
<stop offset="0%" stop-color="#071229"/>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 4.7 KiB After Width: | Height: | Size: 4.7 KiB |
@@ -23,6 +23,7 @@ with a different shard produces a new arm automatically.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import random
|
import random
|
||||||
|
import sqlite3
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
@@ -45,6 +46,7 @@ class RoutingConfig:
|
|||||||
@dataclass
|
@dataclass
|
||||||
class RouteStat:
|
class RouteStat:
|
||||||
ewma_tps: float = 0.0
|
ewma_tps: float = 0.0
|
||||||
|
ewma_latency_ms: float = 0.0
|
||||||
weight: float = 0.0 # decayed effective sample mass
|
weight: float = 0.0 # decayed effective sample mass
|
||||||
last_sample_ts: float = 0.0
|
last_sample_ts: float = 0.0
|
||||||
epoch: int = 0
|
epoch: int = 0
|
||||||
@@ -75,11 +77,15 @@ def route_signature(model_key: str, nodes: Iterable[Any]) -> str:
|
|||||||
class RouteStatsStore:
|
class RouteStatsStore:
|
||||||
"""Thread-safe per-route decayed throughput statistics."""
|
"""Thread-safe per-route decayed throughput statistics."""
|
||||||
|
|
||||||
def __init__(self, config: RoutingConfig | None = None) -> None:
|
def __init__(self, config: RoutingConfig | None = None, db_path: str | None = None) -> None:
|
||||||
self.config = config or RoutingConfig()
|
self.config = config or RoutingConfig()
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._stats: dict[str, RouteStat] = {}
|
self._stats: dict[str, RouteStat] = {}
|
||||||
self._epochs: dict[str, int] = {}
|
self._epochs: dict[str, int] = {}
|
||||||
|
self._db_path = db_path
|
||||||
|
if db_path:
|
||||||
|
self._init_db()
|
||||||
|
self._load_from_db()
|
||||||
|
|
||||||
def epoch(self, model_key: str) -> int:
|
def epoch(self, model_key: str) -> int:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
@@ -119,6 +125,7 @@ class RouteStatsStore:
|
|||||||
carried = stat.decayed_weight(ts, cfg.stats_half_life_seconds)
|
carried = stat.decayed_weight(ts, cfg.stats_half_life_seconds)
|
||||||
total = carried + 1.0
|
total = carried + 1.0
|
||||||
stat.ewma_tps = (stat.ewma_tps * carried + tps) / total
|
stat.ewma_tps = (stat.ewma_tps * carried + tps) / total
|
||||||
|
stat.ewma_latency_ms = (stat.ewma_latency_ms * carried + elapsed_seconds * 1000.0) / total
|
||||||
stat.weight = total
|
stat.weight = total
|
||||||
stat.last_sample_ts = ts
|
stat.last_sample_ts = ts
|
||||||
stat.epoch = self._epochs.get(model_key, 0)
|
stat.epoch = self._epochs.get(model_key, 0)
|
||||||
@@ -143,11 +150,64 @@ class RouteStatsStore:
|
|||||||
status = "proven"
|
status = "proven"
|
||||||
return {
|
return {
|
||||||
"tps": round(stat.ewma_tps, 4) if stat.samples else None,
|
"tps": round(stat.ewma_tps, 4) if stat.samples else None,
|
||||||
|
"latency_ms": round(stat.ewma_latency_ms, 3) if stat.samples else None,
|
||||||
"weight": round(weight, 4),
|
"weight": round(weight, 4),
|
||||||
"samples": stat.samples,
|
"samples": stat.samples,
|
||||||
"status": status,
|
"status": status,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def model_rows(self, model_key: str, now: float | None = None) -> list[dict]:
|
||||||
|
"""All measured route samples, including pinned experiment routes."""
|
||||||
|
prefix = f"{model_key}|"
|
||||||
|
with self._lock:
|
||||||
|
signatures = [signature for signature in self._stats if signature.startswith(prefix)]
|
||||||
|
rows = [
|
||||||
|
{
|
||||||
|
"signature": signature,
|
||||||
|
"hop_count": signature.count("->") + 1,
|
||||||
|
**self.snapshot(signature, model_key, now=now),
|
||||||
|
}
|
||||||
|
for signature in signatures
|
||||||
|
]
|
||||||
|
return sorted(rows, key=lambda row: (row["hop_count"], row["signature"]))
|
||||||
|
|
||||||
|
def _init_db(self) -> None:
|
||||||
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||||
|
con.execute(
|
||||||
|
"CREATE TABLE IF NOT EXISTS route_stats "
|
||||||
|
"(signature TEXT PRIMARY KEY, ewma_tps REAL NOT NULL, ewma_latency_ms REAL NOT NULL, "
|
||||||
|
"weight REAL NOT NULL, last_sample_ts REAL NOT NULL, epoch INTEGER NOT NULL, samples INTEGER NOT NULL)"
|
||||||
|
)
|
||||||
|
con.execute("CREATE TABLE IF NOT EXISTS route_stat_epochs (model_key TEXT PRIMARY KEY, epoch INTEGER NOT NULL)")
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
def save_to_db(self) -> None:
|
||||||
|
if not self._db_path:
|
||||||
|
return
|
||||||
|
with self._lock:
|
||||||
|
rows = [
|
||||||
|
(signature, stat.ewma_tps, stat.ewma_latency_ms, stat.weight, stat.last_sample_ts, stat.epoch, stat.samples)
|
||||||
|
for signature, stat in self._stats.items()
|
||||||
|
]
|
||||||
|
epochs = list(self._epochs.items())
|
||||||
|
con = sqlite3.connect(self._db_path)
|
||||||
|
con.executemany("INSERT OR REPLACE INTO route_stats VALUES (?,?,?,?,?,?,?)", rows)
|
||||||
|
con.executemany("INSERT OR REPLACE INTO route_stat_epochs VALUES (?,?)", epochs)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
def _load_from_db(self) -> None:
|
||||||
|
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||||||
|
rows = con.execute("SELECT signature, ewma_tps, ewma_latency_ms, weight, last_sample_ts, epoch, samples FROM route_stats").fetchall()
|
||||||
|
epochs = con.execute("SELECT model_key, epoch FROM route_stat_epochs").fetchall()
|
||||||
|
con.close()
|
||||||
|
self._stats = {
|
||||||
|
signature: RouteStat(float(tps), float(latency), float(weight), float(last_sample_ts), int(epoch), int(samples))
|
||||||
|
for signature, tps, latency, weight, last_sample_ts, epoch, samples in rows
|
||||||
|
}
|
||||||
|
self._epochs = {str(model_key): int(epoch) for model_key, epoch in epochs}
|
||||||
|
|
||||||
def prune(self, now: float | None = None) -> int:
|
def prune(self, now: float | None = None) -> int:
|
||||||
"""Drop routes with no samples for `prune_after_seconds`."""
|
"""Drop routes with no samples for `prune_after_seconds`."""
|
||||||
ts = time.time() if now is None else now
|
ts = time.time() if now is None else now
|
||||||
@@ -246,6 +306,8 @@ def route_table(
|
|||||||
for n in cand.nodes
|
for n in cand.nodes
|
||||||
],
|
],
|
||||||
"tps": r["tps"],
|
"tps": r["tps"],
|
||||||
|
"latency_ms": r["latency_ms"],
|
||||||
|
"hop_count": len(cand.nodes),
|
||||||
"coefficient": coefficient,
|
"coefficient": coefficient,
|
||||||
"expected_share": round(share, 4),
|
"expected_share": round(share, 4),
|
||||||
"samples": r["samples"],
|
"samples": r["samples"],
|
||||||
|
|||||||
@@ -53,6 +53,8 @@ from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
|||||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||||||
from .gossip import NodeGossip
|
from .gossip import NodeGossip
|
||||||
from .logging_setup import tracker_logger
|
from .logging_setup import tracker_logger
|
||||||
|
from .test_runner import ENABLE_ENV_VAR as TEST_RUNNER_ENABLE_ENV_VAR
|
||||||
|
from .test_runner import TestRunManager, TestRunnerError
|
||||||
from .routing_stats import (
|
from .routing_stats import (
|
||||||
RouteCandidate,
|
RouteCandidate,
|
||||||
RouteStatsStore,
|
RouteStatsStore,
|
||||||
@@ -1276,6 +1278,39 @@ def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
|
|||||||
break
|
break
|
||||||
|
|
||||||
|
|
||||||
|
def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
|
||||||
|
"""Queue an explicitly requested model on the best available joined node."""
|
||||||
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
|
||||||
|
if preset is None or not preset.get("hf_repo"):
|
||||||
|
return None
|
||||||
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
|
total_layers = required_end - required_start + 1
|
||||||
|
required_bytes = int(preset.get("required_model_bytes") or 0)
|
||||||
|
for host in _memory_pool_map(server)["hosts"]:
|
||||||
|
# A host qualifies with a declared spare slot, or — for an explicit
|
||||||
|
# operator request — with enough spare assignable memory for the model.
|
||||||
|
has_spare_memory = required_bytes > 0 and host["memory_spare_bytes"] >= required_bytes
|
||||||
|
if host["spare_slots"] <= 0 and not has_spare_memory:
|
||||||
|
continue
|
||||||
|
host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
|
||||||
|
if not host_nodes:
|
||||||
|
continue
|
||||||
|
anchor = max(host_nodes, key=lambda node: node.benchmark_tokens_per_sec)
|
||||||
|
if anchor.status != "ready" or anchor.pending_new_assignment is not None:
|
||||||
|
continue
|
||||||
|
capacity = min(_node_layer_capacity(anchor, preset), total_layers)
|
||||||
|
if capacity <= 0:
|
||||||
|
continue
|
||||||
|
quantization = _node_quantization(anchor, preset)
|
||||||
|
shard_end = min(required_end, required_start + capacity - 1)
|
||||||
|
directive = _add_shard_directive(anchor, str(preset["hf_repo"]), required_start, shard_end, quantization)
|
||||||
|
anchor.pending_new_assignment = directive
|
||||||
|
anchor.pending_directives.append(directive)
|
||||||
|
_tracker_log(server, "info", "model load requested", node_id=anchor.node_id, model=resolved_name, hf_repo=preset["hf_repo"], shard=f"{required_start}-{shard_end}")
|
||||||
|
return {"node_id": anchor.node_id, "model": resolved_name, "hf_repo": preset["hf_repo"], "shard_start": required_start, "shard_end": shard_end}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
|
def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
|
||||||
if preset is None:
|
if preset is None:
|
||||||
return {"recommended": False}
|
return {"recommended": False}
|
||||||
@@ -1894,6 +1929,9 @@ def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
|
|||||||
if not managed_nodes:
|
if not managed_nodes:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Directives must name a repo the node can actually load, not the preset key.
|
||||||
|
directive_model = str(preset.get("hf_repo") or model)
|
||||||
|
|
||||||
coverage = _coverage_map(model_nodes, required_start, required_end)
|
coverage = _coverage_map(model_nodes, required_start, required_end)
|
||||||
gaps = _coverage_gaps(coverage)
|
gaps = _coverage_gaps(coverage)
|
||||||
unassigned = _unassigned_managed_nodes(managed_nodes)
|
unassigned = _unassigned_managed_nodes(managed_nodes)
|
||||||
@@ -1901,7 +1939,7 @@ def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
|
|||||||
return
|
return
|
||||||
if not gaps and unassigned:
|
if not gaps and unassigned:
|
||||||
_assign_redundant_managed_nodes(
|
_assign_redundant_managed_nodes(
|
||||||
unassigned, model, preset, required_start, required_end,
|
unassigned, directive_model, preset, required_start, required_end,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -1953,7 +1991,7 @@ def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
|
|||||||
for node in managed_nodes:
|
for node in managed_nodes:
|
||||||
_emit_shard_change_directives(
|
_emit_shard_change_directives(
|
||||||
node,
|
node,
|
||||||
model,
|
directive_model,
|
||||||
previous_ranges[node.node_id],
|
previous_ranges[node.node_id],
|
||||||
preset,
|
preset,
|
||||||
)
|
)
|
||||||
@@ -2483,6 +2521,7 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
|||||||
models_dir: Path | None = None,
|
models_dir: Path | None = None,
|
||||||
route_stats: "RouteStatsStore | None" = None,
|
route_stats: "RouteStatsStore | None" = None,
|
||||||
relay_status: dict | None = None,
|
relay_status: dict | None = None,
|
||||||
|
test_runner: "TestRunManager | None" = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(addr, handler)
|
super().__init__(addr, handler)
|
||||||
self.registry = registry
|
self.registry = registry
|
||||||
@@ -2523,6 +2562,7 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
|||||||
self.relay_status = dict(
|
self.relay_status = dict(
|
||||||
relay_status or {"mode": "external" if self.relay_url else "off"}
|
relay_status or {"mode": "external" if self.relay_url else "off"}
|
||||||
)
|
)
|
||||||
|
self.test_runner: TestRunManager | None = test_runner
|
||||||
|
|
||||||
|
|
||||||
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||||
@@ -2672,12 +2712,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
if self.path == "/v1/benchmark/hop-penalty":
|
if self.path == "/v1/benchmark/hop-penalty":
|
||||||
self._handle_benchmark_hop_penalty()
|
self._handle_benchmark_hop_penalty()
|
||||||
return
|
return
|
||||||
|
if self.path == "/v1/models/load":
|
||||||
|
self._handle_model_load_request()
|
||||||
|
return
|
||||||
if self.path == "/v1/calibration/toploc/run":
|
if self.path == "/v1/calibration/toploc/run":
|
||||||
self._handle_toploc_calibration_run()
|
self._handle_toploc_calibration_run()
|
||||||
return
|
return
|
||||||
if self.path == "/v1/wallet/register":
|
if self.path == "/v1/wallet/register":
|
||||||
self._handle_wallet_register()
|
self._handle_wallet_register()
|
||||||
return
|
return
|
||||||
|
if self.path == "/v1/tests/run":
|
||||||
|
self._handle_tests_run()
|
||||||
|
return
|
||||||
parts = self.path.split("/")
|
parts = self.path.split("/")
|
||||||
# /v1/nodes/<node_id>/heartbeat -> ['', 'v1', 'nodes', '<id>', 'heartbeat']
|
# /v1/nodes/<node_id>/heartbeat -> ['', 'v1', 'nodes', '<id>', 'heartbeat']
|
||||||
if len(parts) == 5 and parts[1] == "v1" and parts[2] == "nodes" and parts[4] == "heartbeat":
|
if len(parts) == 5 and parts[1] == "v1" and parts[2] == "nodes" and parts[4] == "heartbeat":
|
||||||
@@ -2699,12 +2745,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
def do_GET(self):
|
def do_GET(self):
|
||||||
parsed = urllib.parse.urlparse(self.path)
|
parsed = urllib.parse.urlparse(self.path)
|
||||||
if parsed.path == "/v1/route":
|
if parsed.path == "/":
|
||||||
|
self.send_response(302)
|
||||||
|
self.send_header("Location", "/dashboard")
|
||||||
|
self.end_headers()
|
||||||
|
elif parsed.path == "/v1/route":
|
||||||
self._handle_route(parsed)
|
self._handle_route(parsed)
|
||||||
elif parsed.path == "/v1/routes":
|
elif parsed.path == "/v1/routes":
|
||||||
self._handle_routes(parsed)
|
self._handle_routes(parsed)
|
||||||
elif parsed.path == "/v1/routing":
|
elif parsed.path == "/v1/routing":
|
||||||
self._handle_routing(parsed)
|
self._handle_routing(parsed)
|
||||||
|
elif parsed.path == "/v1/model-speed":
|
||||||
|
self._handle_model_speed(parsed)
|
||||||
elif parsed.path == "/v1/nodes/assign":
|
elif parsed.path == "/v1/nodes/assign":
|
||||||
self._handle_assign(parsed)
|
self._handle_assign(parsed)
|
||||||
elif parsed.path == "/v1/network/assign":
|
elif parsed.path == "/v1/network/assign":
|
||||||
@@ -2746,6 +2798,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._handle_toploc_calibration_results()
|
self._handle_toploc_calibration_results()
|
||||||
elif parsed.path == "/v1/pricing/hf/history":
|
elif parsed.path == "/v1/pricing/hf/history":
|
||||||
self._handle_hf_pricing_history(parsed)
|
self._handle_hf_pricing_history(parsed)
|
||||||
|
elif parsed.path == "/v1/tests":
|
||||||
|
self._handle_tests_list(parsed)
|
||||||
|
elif parsed.path == "/v1/tests/status":
|
||||||
|
self._handle_tests_status()
|
||||||
elif parsed.path == "/v1/registry/wallets":
|
elif parsed.path == "/v1/registry/wallets":
|
||||||
self._handle_registry_wallets()
|
self._handle_registry_wallets()
|
||||||
elif parsed.path in ("/dashboard", "/dashboard/"):
|
elif parsed.path in ("/dashboard", "/dashboard/"):
|
||||||
@@ -2808,8 +2864,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"served_model_copies": served_copies,
|
"served_model_copies": served_copies,
|
||||||
})
|
})
|
||||||
seen_ids.add(name)
|
seen_ids.add(name)
|
||||||
if hf_repo:
|
# Note: the preset's hf_repo is deliberately NOT added to seen_ids —
|
||||||
seen_ids.add(hf_repo)
|
# nodes registered with an explicit hf_repo also get their own
|
||||||
|
# repo-keyed entry below, with the node's short name as an alias.
|
||||||
|
|
||||||
hf_model_ids = sorted({
|
hf_model_ids = sorted({
|
||||||
node.hf_repo or node.model
|
node.hf_repo or node.model
|
||||||
@@ -4288,6 +4345,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
server.gossip.merge({k: float(v) for k, v in body.items()})
|
server.gossip.merge({k: float(v) for k, v in body.items()})
|
||||||
self._send_json(200, {})
|
self._send_json(200, {})
|
||||||
|
|
||||||
|
def _handle_model_load_request(self):
|
||||||
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
if not self._require_role("admin", "validator"):
|
||||||
|
return
|
||||||
|
body = self._read_json_body()
|
||||||
|
if body is None:
|
||||||
|
return
|
||||||
|
model = body.get("model")
|
||||||
|
if not isinstance(model, str) or not model.strip():
|
||||||
|
self._send_json(400, {"error": "model is required"})
|
||||||
|
return
|
||||||
|
with server.lock:
|
||||||
|
self._purge_expired_nodes()
|
||||||
|
assignment = _request_model_load_locked(server, model)
|
||||||
|
if assignment is None:
|
||||||
|
self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
|
||||||
|
return
|
||||||
|
self._send_json(202, {"status": "queued", "assignment": assignment})
|
||||||
|
|
||||||
def _handle_stats(self):
|
def _handle_stats(self):
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
if server.stats is None:
|
if server.stats is None:
|
||||||
@@ -4410,6 +4486,57 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
finish_proxy_inflight()
|
finish_proxy_inflight()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
# ---- opt-in test runner (dashboard-test-runner US-001) ----
|
||||||
|
|
||||||
|
def _test_runner_or_reject(self) -> "TestRunManager | None":
|
||||||
|
"""Admin session first, then the explicit enable gate — both fail closed."""
|
||||||
|
if not self._require_role("admin"):
|
||||||
|
return None
|
||||||
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
if server.test_runner is None:
|
||||||
|
self._send_json(403, {
|
||||||
|
"error": (
|
||||||
|
"test runner disabled; start the tracker with "
|
||||||
|
f"--enable-test-runner or {TEST_RUNNER_ENABLE_ENV_VAR}=1"
|
||||||
|
)
|
||||||
|
})
|
||||||
|
return None
|
||||||
|
return server.test_runner
|
||||||
|
|
||||||
|
def _handle_tests_list(self, parsed: urllib.parse.ParseResult) -> None:
|
||||||
|
runner = self._test_runner_or_reject()
|
||||||
|
if runner is None:
|
||||||
|
return
|
||||||
|
query = urllib.parse.parse_qs(parsed.query)
|
||||||
|
refresh = query.get("refresh", ["0"])[0] in {"1", "true"}
|
||||||
|
try:
|
||||||
|
data = runner.collect(refresh=refresh)
|
||||||
|
except TestRunnerError as exc:
|
||||||
|
self._send_json(exc.status, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
self._send_json(200, {"enabled": True, **data})
|
||||||
|
|
||||||
|
def _handle_tests_run(self) -> None:
|
||||||
|
runner = self._test_runner_or_reject()
|
||||||
|
if runner is None:
|
||||||
|
return
|
||||||
|
body = self._read_json_body()
|
||||||
|
if body is None:
|
||||||
|
return
|
||||||
|
target = body.get("target")
|
||||||
|
try:
|
||||||
|
state = runner.start(target if isinstance(target, str) else "")
|
||||||
|
except TestRunnerError as exc:
|
||||||
|
self._send_json(exc.status, {"error": str(exc)})
|
||||||
|
return
|
||||||
|
self._send_json(202, state)
|
||||||
|
|
||||||
|
def _handle_tests_status(self) -> None:
|
||||||
|
runner = self._test_runner_or_reject()
|
||||||
|
if runner is None:
|
||||||
|
return
|
||||||
|
self._send_json(200, runner.status())
|
||||||
|
|
||||||
def _handle_registry_wallets(self):
|
def _handle_registry_wallets(self):
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
if not self._require_role("admin"):
|
if not self._require_role("admin"):
|
||||||
@@ -5725,6 +5852,44 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"models": out,
|
"models": out,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
def _handle_model_speed(self, parsed: urllib.parse.ParseResult):
|
||||||
|
"""Drill into observed model speed and the latency cost of each route."""
|
||||||
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
model = (urllib.parse.parse_qs(parsed.query).get("model") or [None])[0]
|
||||||
|
if not model:
|
||||||
|
self._send_json(400, {"error": "missing 'model' query parameter"})
|
||||||
|
return
|
||||||
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
|
stats_key = resolved_name or model
|
||||||
|
with server.lock:
|
||||||
|
self._purge_expired_nodes()
|
||||||
|
nodes_by_id = dict(server.registry)
|
||||||
|
routes = server.route_stats.model_rows(stats_key)
|
||||||
|
one_hop = [r["latency_ms"] for r in routes if r["hop_count"] == 1 and r["latency_ms"] is not None]
|
||||||
|
baseline_ms = min(one_hop) if one_hop else None
|
||||||
|
for route in routes:
|
||||||
|
node_ids = [part.split("[", 1)[0] for part in route["signature"].split("|", 1)[1].split("->")]
|
||||||
|
devices = []
|
||||||
|
for node_id in node_ids:
|
||||||
|
node = nodes_by_id.get(node_id)
|
||||||
|
profile = node.hardware_profile if node is not None else {}
|
||||||
|
devices.append("gpu" if node is not None and (node.vram_bytes > 0 or profile.get("device") == "cuda") else "cpu")
|
||||||
|
route["node_ids"] = node_ids
|
||||||
|
route["device_mix"] = "-".join(dict.fromkeys(devices))
|
||||||
|
route["latency_penalty_ms"] = round(route["latency_ms"] - baseline_ms, 3) if baseline_ms is not None else None
|
||||||
|
node_rows = []
|
||||||
|
for node in nodes_by_id.values():
|
||||||
|
if not _node_matches_model(node, model) and not (preset and _node_matches_preset(node, resolved_name or model, preset)):
|
||||||
|
continue
|
||||||
|
sample = server.stats.get_node_model_stats(node.node_id, stats_key) if server.stats else {}
|
||||||
|
node_rows.append({
|
||||||
|
"node_id": node.node_id,
|
||||||
|
"device": "gpu" if node.vram_bytes > 0 or node.hardware_profile.get("device") == "cuda" else "cpu",
|
||||||
|
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
|
||||||
|
**sample,
|
||||||
|
})
|
||||||
|
self._send_json(200, {"model": stats_key, "baseline_one_hop_latency_ms": baseline_ms, "routes": routes, "nodes": node_rows})
|
||||||
|
|
||||||
def _handle_routes(self, parsed: urllib.parse.ParseResult):
|
def _handle_routes(self, parsed: urllib.parse.ParseResult):
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
params = urllib.parse.parse_qs(parsed.query)
|
params = urllib.parse.parse_qs(parsed.query)
|
||||||
@@ -5848,6 +6013,8 @@ class TrackerServer:
|
|||||||
hf_pricing_fetch_html: Any | None = None,
|
hf_pricing_fetch_html: Any | None = None,
|
||||||
models_dir: str | Path | None = None,
|
models_dir: str | Path | None = None,
|
||||||
routing_config: RoutingConfig | None = None,
|
routing_config: RoutingConfig | None = None,
|
||||||
|
enable_test_runner: bool = False,
|
||||||
|
test_runner: TestRunManager | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._host = host
|
self._host = host
|
||||||
self._requested_port = port
|
self._requested_port = port
|
||||||
@@ -5967,7 +6134,14 @@ class TrackerServer:
|
|||||||
os.environ.get("MESHNET_ROUTE_MIN_SAMPLE_TOKENS", RoutingConfig.min_sample_tokens)
|
os.environ.get("MESHNET_ROUTE_MIN_SAMPLE_TOKENS", RoutingConfig.min_sample_tokens)
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
self._route_stats = RouteStatsStore(routing_config)
|
self._route_stats = RouteStatsStore(routing_config, db_path=stats_db)
|
||||||
|
# Opt-in test runner: an injected manager implies enabled; otherwise
|
||||||
|
# only the explicit flag or env var constructs one (fails closed).
|
||||||
|
if test_runner is None and (
|
||||||
|
enable_test_runner or os.environ.get(TEST_RUNNER_ENABLE_ENV_VAR) == "1"
|
||||||
|
):
|
||||||
|
test_runner = TestRunManager()
|
||||||
|
self._test_runner: TestRunManager | None = test_runner
|
||||||
self.port: int | None = None
|
self.port: int | None = None
|
||||||
|
|
||||||
def _start_embedded_relay(self) -> dict:
|
def _start_embedded_relay(self) -> dict:
|
||||||
@@ -6053,6 +6227,7 @@ class TrackerServer:
|
|||||||
models_dir=self._models_dir,
|
models_dir=self._models_dir,
|
||||||
route_stats=self._route_stats,
|
route_stats=self._route_stats,
|
||||||
relay_status=http_relay_status,
|
relay_status=http_relay_status,
|
||||||
|
test_runner=self._test_runner,
|
||||||
)
|
)
|
||||||
self.port = self._server.server_address[1]
|
self.port = self._server.server_address[1]
|
||||||
|
|
||||||
@@ -6322,6 +6497,7 @@ class TrackerServer:
|
|||||||
while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
|
while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
|
||||||
if self._stats is not None:
|
if self._stats is not None:
|
||||||
self._stats.save_to_db()
|
self._stats.save_to_db()
|
||||||
|
self._route_stats.save_to_db()
|
||||||
if self._billing is not None:
|
if self._billing is not None:
|
||||||
self._billing.save_to_db()
|
self._billing.save_to_db()
|
||||||
if self._accounts is not None:
|
if self._accounts is not None:
|
||||||
@@ -6373,6 +6549,8 @@ class TrackerServer:
|
|||||||
self._embedded_relay_actual_port = None
|
self._embedded_relay_actual_port = None
|
||||||
if embedded_relay is not None:
|
if embedded_relay is not None:
|
||||||
embedded_relay.stop()
|
embedded_relay.stop()
|
||||||
|
if self._test_runner is not None:
|
||||||
|
self._test_runner.shutdown()
|
||||||
if self._server is None:
|
if self._server is None:
|
||||||
return
|
return
|
||||||
self._rebalance_stop.set()
|
self._rebalance_stop.set()
|
||||||
@@ -6382,6 +6560,7 @@ class TrackerServer:
|
|||||||
self._hf_pricing_stop.set()
|
self._hf_pricing_stop.set()
|
||||||
if self._stats is not None:
|
if self._stats is not None:
|
||||||
self._stats.save_to_db()
|
self._stats.save_to_db()
|
||||||
|
self._route_stats.save_to_db()
|
||||||
if self._billing is not None:
|
if self._billing is not None:
|
||||||
self._billing.save_to_db()
|
self._billing.save_to_db()
|
||||||
if self._accounts is not None:
|
if self._accounts is not None:
|
||||||
|
|||||||
314
packages/tracker/meshnet_tracker/test_runner.py
Normal file
314
packages/tracker/meshnet_tracker/test_runner.py
Normal file
@@ -0,0 +1,314 @@
|
|||||||
|
"""Opt-in tracker test runner backing the dashboard Testing tab.
|
||||||
|
|
||||||
|
Security posture (dashboard-test-runner US-001):
|
||||||
|
|
||||||
|
- **Disabled by default** — the tracker only constructs a manager when started
|
||||||
|
with ``--enable-test-runner`` / ``MESHNET_ENABLE_TEST_RUNNER=1`` or
|
||||||
|
``TrackerServer(enable_test_runner=True)``.
|
||||||
|
- **No arbitrary commands** — a run target must be either a pytest node ID
|
||||||
|
returned by our own ``--collect-only`` pass or an approved named suite.
|
||||||
|
Callers never supply command arguments.
|
||||||
|
- **No shell** — pytest runs as ``[sys.executable, "-m", "pytest", ...]`` via
|
||||||
|
``subprocess.Popen`` without ``shell=True``.
|
||||||
|
- **One run at a time** — a second start while a run is active is rejected.
|
||||||
|
- **Bounded logs** — stdout/stderr are retained as line deques with caps.
|
||||||
|
- **Real inference stays gated** — ``tests/test_real_*.py`` modules are never
|
||||||
|
collected and never part of a suite unless
|
||||||
|
``MESHNET_ENABLE_REAL_INFERENCE_TESTS=1`` is set, and even then only via the
|
||||||
|
dedicated ``suite:real-inference`` target, never a default suite.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from collections import deque
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ENABLE_ENV_VAR = "MESHNET_ENABLE_TEST_RUNNER"
|
||||||
|
REAL_INFERENCE_ENV_VAR = "MESHNET_ENABLE_REAL_INFERENCE_TESTS"
|
||||||
|
|
||||||
|
# Modules that talk to live nodes / spend API credit; matched by filename.
|
||||||
|
_REAL_INFERENCE_PATTERN = re.compile(r"(^|/)test_real_[^/]*\.py")
|
||||||
|
|
||||||
|
# Fixed, reviewed suite targets. Values are pytest paths relative to the repo
|
||||||
|
# root — never influenced by API input.
|
||||||
|
APPROVED_SUITES: dict[str, list[str]] = {
|
||||||
|
"smoke": ["tests/test_smoke.py"],
|
||||||
|
"dashboard": ["tests/test_dashboard.py"],
|
||||||
|
"routing": ["tests/test_tracker_routing.py", "tests/test_dynamic_routing.py"],
|
||||||
|
}
|
||||||
|
|
||||||
|
# Only exists when REAL_INFERENCE_ENV_VAR=1; kept out of APPROVED_SUITES so it
|
||||||
|
# can never appear by default.
|
||||||
|
_REAL_INFERENCE_SUITE = {
|
||||||
|
"real-inference": [
|
||||||
|
"tests/test_real_distributed_inference.py",
|
||||||
|
"tests/test_real_model_backend.py",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
DEFAULT_MAX_LOG_LINES = 4000
|
||||||
|
_MAX_LINE_CHARS = 4000
|
||||||
|
DEFAULT_COLLECT_TIMEOUT = 120.0
|
||||||
|
DEFAULT_RUN_TIMEOUT = 1800.0
|
||||||
|
|
||||||
|
_NODE_ID_RE = re.compile(r"^[\w./\[\]:,= @-]+$")
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunnerError(Exception):
|
||||||
|
"""Base class; carries the HTTP status the handler should send."""
|
||||||
|
|
||||||
|
status = 400
|
||||||
|
|
||||||
|
|
||||||
|
class UnknownTargetError(TestRunnerError):
|
||||||
|
status = 400
|
||||||
|
|
||||||
|
|
||||||
|
class RunInProgressError(TestRunnerError):
|
||||||
|
status = 409
|
||||||
|
|
||||||
|
|
||||||
|
class CollectionError(TestRunnerError):
|
||||||
|
status = 500
|
||||||
|
|
||||||
|
|
||||||
|
def discover_repo_root(start: Path | None = None) -> Path:
|
||||||
|
"""Find the repository root independent of the tracker's cwd.
|
||||||
|
|
||||||
|
Walks up from this module (or ``start``) to the first directory holding
|
||||||
|
both a ``pyproject.toml`` and ``tests/conftest.py`` — the layout of the
|
||||||
|
meshnet monorepo root. Package-level ``pyproject.toml`` files (which have
|
||||||
|
no ``tests/`` dir) are skipped naturally.
|
||||||
|
"""
|
||||||
|
current = (start or Path(__file__)).resolve()
|
||||||
|
for candidate in [current, *current.parents]:
|
||||||
|
if (candidate / "pyproject.toml").is_file() and (candidate / "tests" / "conftest.py").is_file():
|
||||||
|
return candidate
|
||||||
|
raise RuntimeError("could not locate repository root (pyproject.toml + tests/conftest.py)")
|
||||||
|
|
||||||
|
|
||||||
|
def _real_inference_enabled() -> bool:
|
||||||
|
return os.environ.get(REAL_INFERENCE_ENV_VAR) == "1"
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunManager:
|
||||||
|
"""Collects pytest node IDs and runs one fixed target at a time."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
repo_root: str | Path | None = None,
|
||||||
|
*,
|
||||||
|
max_log_lines: int = DEFAULT_MAX_LOG_LINES,
|
||||||
|
collect_timeout: float = DEFAULT_COLLECT_TIMEOUT,
|
||||||
|
run_timeout: float = DEFAULT_RUN_TIMEOUT,
|
||||||
|
) -> None:
|
||||||
|
self.repo_root = Path(repo_root).resolve() if repo_root else discover_repo_root()
|
||||||
|
self.max_log_lines = max_log_lines
|
||||||
|
self.collect_timeout = collect_timeout
|
||||||
|
self.run_timeout = run_timeout
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._collected: set[str] = set()
|
||||||
|
self._collected_at: float | None = None
|
||||||
|
self._run: dict | None = None
|
||||||
|
self._process: subprocess.Popen | None = None
|
||||||
|
self._stdout: deque[str] = deque(maxlen=max_log_lines)
|
||||||
|
self._stderr: deque[str] = deque(maxlen=max_log_lines)
|
||||||
|
self._reader_threads: list[threading.Thread] = []
|
||||||
|
|
||||||
|
# ---- collection ----
|
||||||
|
|
||||||
|
def suites(self) -> dict[str, list[str]]:
|
||||||
|
"""Approved suites whose files exist in this repo checkout."""
|
||||||
|
available = dict(APPROVED_SUITES)
|
||||||
|
if _real_inference_enabled():
|
||||||
|
available.update(_REAL_INFERENCE_SUITE)
|
||||||
|
return {
|
||||||
|
name: paths
|
||||||
|
for name, paths in available.items()
|
||||||
|
if all((self.repo_root / p).is_file() for p in paths)
|
||||||
|
}
|
||||||
|
|
||||||
|
def collect(self, *, refresh: bool = False) -> dict:
|
||||||
|
"""Run ``pytest --collect-only -q`` and cache the node IDs.
|
||||||
|
|
||||||
|
Real-inference modules are ignored at collection time (they are never
|
||||||
|
imported) unless the explicit environment gate is set.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
if self._collected_at is not None and not refresh:
|
||||||
|
return self._collection_snapshot_locked()
|
||||||
|
|
||||||
|
cmd = [sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider"]
|
||||||
|
if not _real_inference_enabled():
|
||||||
|
for name in sorted(
|
||||||
|
p.name for p in (self.repo_root / "tests").glob("test_real_*.py")
|
||||||
|
):
|
||||||
|
cmd.append(f"--ignore=tests/{name}")
|
||||||
|
cmd.append("tests")
|
||||||
|
try:
|
||||||
|
proc = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
cwd=self.repo_root,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=self.collect_timeout,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
raise CollectionError(f"pytest collection timed out after {self.collect_timeout}s") from exc
|
||||||
|
# Exit code 5 = no tests collected; still a valid (empty) result.
|
||||||
|
if proc.returncode not in (0, 5):
|
||||||
|
tail = (proc.stdout or "")[-2000:] + (proc.stderr or "")[-2000:]
|
||||||
|
raise CollectionError(f"pytest collection failed (exit {proc.returncode}): {tail}")
|
||||||
|
node_ids = [
|
||||||
|
line.strip()
|
||||||
|
for line in proc.stdout.splitlines()
|
||||||
|
if "::" in line and not line.startswith(("=", " ", "<"))
|
||||||
|
]
|
||||||
|
if not _real_inference_enabled():
|
||||||
|
node_ids = [n for n in node_ids if not _REAL_INFERENCE_PATTERN.search(n.split("::", 1)[0])]
|
||||||
|
with self._lock:
|
||||||
|
self._collected = set(node_ids)
|
||||||
|
self._collected_at = time.time()
|
||||||
|
return self._collection_snapshot_locked()
|
||||||
|
|
||||||
|
def _collection_snapshot_locked(self) -> dict:
|
||||||
|
return {
|
||||||
|
"tests": sorted(self._collected),
|
||||||
|
"suites": [
|
||||||
|
{"id": f"suite:{name}", "name": name, "paths": paths}
|
||||||
|
for name, paths in sorted(self.suites().items())
|
||||||
|
],
|
||||||
|
"collected_at": self._collected_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---- running ----
|
||||||
|
|
||||||
|
def start(self, target: str) -> dict:
|
||||||
|
"""Start one collected node ID or approved ``suite:<name>`` target."""
|
||||||
|
if not isinstance(target, str) or not target.strip():
|
||||||
|
raise UnknownTargetError("target is required")
|
||||||
|
target = target.strip()
|
||||||
|
if target.startswith("-") or not _NODE_ID_RE.match(target):
|
||||||
|
raise UnknownTargetError("target must be a collected test node ID or an approved suite")
|
||||||
|
|
||||||
|
if target.startswith("suite:"):
|
||||||
|
suite_name = target.removeprefix("suite:")
|
||||||
|
paths = self.suites().get(suite_name)
|
||||||
|
if paths is None:
|
||||||
|
raise UnknownTargetError(f"unknown suite: {suite_name!r}")
|
||||||
|
pytest_args = list(paths)
|
||||||
|
else:
|
||||||
|
with self._lock:
|
||||||
|
known = target in self._collected
|
||||||
|
if not known:
|
||||||
|
raise UnknownTargetError(
|
||||||
|
"target is not a collected test node ID; call GET /v1/tests first"
|
||||||
|
)
|
||||||
|
pytest_args = [target]
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if self._run is not None and self._run["status"] == "running":
|
||||||
|
raise RunInProgressError("a test run is already in progress")
|
||||||
|
cmd = [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *pytest_args]
|
||||||
|
self._stdout = deque(maxlen=self.max_log_lines)
|
||||||
|
self._stderr = deque(maxlen=self.max_log_lines)
|
||||||
|
process = subprocess.Popen( # noqa: S603 — fixed argv, no shell
|
||||||
|
cmd,
|
||||||
|
cwd=self.repo_root,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
self._process = process
|
||||||
|
self._run = {
|
||||||
|
"run_id": uuid.uuid4().hex,
|
||||||
|
"target": target,
|
||||||
|
"args": pytest_args,
|
||||||
|
"status": "running",
|
||||||
|
"started_at": time.time(),
|
||||||
|
"finished_at": None,
|
||||||
|
"exit_code": None,
|
||||||
|
}
|
||||||
|
self._reader_threads = [
|
||||||
|
threading.Thread(
|
||||||
|
target=self._drain, args=(process.stdout, self._stdout), daemon=True
|
||||||
|
),
|
||||||
|
threading.Thread(
|
||||||
|
target=self._drain, args=(process.stderr, self._stderr), daemon=True
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for thread in self._reader_threads:
|
||||||
|
thread.start()
|
||||||
|
waiter = threading.Thread(target=self._wait, args=(process,), daemon=True)
|
||||||
|
waiter.start()
|
||||||
|
return self._status_locked()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _drain(stream, sink: deque[str]) -> None:
|
||||||
|
try:
|
||||||
|
for line in stream:
|
||||||
|
sink.append(line.rstrip("\n")[:_MAX_LINE_CHARS])
|
||||||
|
except ValueError: # stream closed mid-read on shutdown
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
def _wait(self, process: subprocess.Popen) -> None:
|
||||||
|
timed_out = False
|
||||||
|
try:
|
||||||
|
exit_code = process.wait(timeout=self.run_timeout)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
timed_out = True
|
||||||
|
process.kill()
|
||||||
|
exit_code = process.wait()
|
||||||
|
for thread in self._reader_threads:
|
||||||
|
thread.join(timeout=5)
|
||||||
|
with self._lock:
|
||||||
|
if self._process is not process or self._run is None:
|
||||||
|
return
|
||||||
|
self._run["exit_code"] = exit_code
|
||||||
|
self._run["finished_at"] = time.time()
|
||||||
|
if timed_out:
|
||||||
|
self._run["status"] = "timeout"
|
||||||
|
elif exit_code == 0:
|
||||||
|
self._run["status"] = "passed"
|
||||||
|
else:
|
||||||
|
self._run["status"] = "failed"
|
||||||
|
self._process = None
|
||||||
|
|
||||||
|
def status(self) -> dict:
|
||||||
|
with self._lock:
|
||||||
|
return self._status_locked()
|
||||||
|
|
||||||
|
def _status_locked(self) -> dict:
|
||||||
|
if self._run is None:
|
||||||
|
return {"run": None, "stdout": "", "stderr": ""}
|
||||||
|
run = dict(self._run)
|
||||||
|
if run["status"] == "running":
|
||||||
|
run["elapsed_seconds"] = time.time() - run["started_at"]
|
||||||
|
elif run["finished_at"] is not None:
|
||||||
|
run["elapsed_seconds"] = run["finished_at"] - run["started_at"]
|
||||||
|
return {
|
||||||
|
"run": run,
|
||||||
|
"stdout": "\n".join(self._stdout),
|
||||||
|
"stderr": "\n".join(self._stderr),
|
||||||
|
"log_lines_limit": self.max_log_lines,
|
||||||
|
}
|
||||||
|
|
||||||
|
def shutdown(self) -> None:
|
||||||
|
"""Kill any active child process (tracker stop path)."""
|
||||||
|
with self._lock:
|
||||||
|
process = self._process
|
||||||
|
if process is not None and process.poll() is None:
|
||||||
|
process.kill()
|
||||||
|
try:
|
||||||
|
process.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
pass
|
||||||
41
scripts/send_api_request.py
Normal file
41
scripts/send_api_request.py
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Send one OpenAI-compatible chat request to a meshnet HTTP endpoint."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--url", default="http://localhost:8080", help="Tracker or gateway base URL")
|
||||||
|
parser.add_argument("--model", default="qwen2.5-0.5b-instruct")
|
||||||
|
parser.add_argument(
|
||||||
|
"--prompt",
|
||||||
|
default="What is 7 times 8? Answer in one word.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--max-tokens", type=int, default=32)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
payload = {
|
||||||
|
"model": args.model,
|
||||||
|
"messages": [{"role": "user", "content": args.prompt}],
|
||||||
|
"max_tokens": args.max_tokens,
|
||||||
|
"stream": False,
|
||||||
|
}
|
||||||
|
request = urllib.request.Request(
|
||||||
|
f"{args.url.rstrip('/')}/v1/chat/completions",
|
||||||
|
data=json.dumps(payload).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(request, timeout=120) as response:
|
||||||
|
body = json.loads(response.read())
|
||||||
|
print(json.dumps(body, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
"""US-035: tracker web dashboard — served from any tracker, embedded asset."""
|
"""US-035: tracker web dashboard — served from any tracker, embedded asset."""
|
||||||
|
|
||||||
|
import http.client
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
import urllib.request
|
import urllib.request
|
||||||
@@ -40,6 +41,23 @@ def test_dashboard_served_with_all_panels():
|
|||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_root_redirects_to_dashboard():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
connection = http.client.HTTPConnection("127.0.0.1", port)
|
||||||
|
connection.request("GET", "/")
|
||||||
|
response = connection.getresponse()
|
||||||
|
location = response.getheader("Location")
|
||||||
|
response.read()
|
||||||
|
connection.close()
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert response.status == 302
|
||||||
|
assert location == "/dashboard"
|
||||||
|
|
||||||
|
|
||||||
def test_dashboard_chat_uses_streaming_fetch():
|
def test_dashboard_chat_uses_streaming_fetch():
|
||||||
tracker = TrackerServer(billing=BillingLedger())
|
tracker = TrackerServer(billing=BillingLedger())
|
||||||
port = tracker.start()
|
port = tracker.start()
|
||||||
@@ -58,6 +76,22 @@ def test_dashboard_chat_uses_streaming_fetch():
|
|||||||
assert "renderChatModels(true)" in html
|
assert "renderChatModels(true)" in html
|
||||||
|
|
||||||
|
|
||||||
|
def test_dashboard_allows_admin_to_request_selected_model_load():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
html = urllib.request.urlopen(
|
||||||
|
f"http://127.0.0.1:{port}/dashboard"
|
||||||
|
).read().decode()
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert 'id="request-model-load"' in html
|
||||||
|
assert "requestSelectedModelLoad" in html
|
||||||
|
assert '"/v1/models/load"' in html
|
||||||
|
assert '$("request-model-load").style.display = enabled ? "" : "none"' in html
|
||||||
|
|
||||||
|
|
||||||
def test_network_map_includes_node_friendly_name():
|
def test_network_map_includes_node_friendly_name():
|
||||||
tracker = TrackerServer()
|
tracker = TrackerServer()
|
||||||
port = tracker.start()
|
port = tracker.start()
|
||||||
|
|||||||
@@ -99,6 +99,21 @@ def test_route_stats_ewma_averages_samples():
|
|||||||
# ---- choose_route --------------------------------------------------------
|
# ---- choose_route --------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_route_stats_persist_historical_hop_latency_across_restart(tmp_path):
|
||||||
|
db_path = str(tmp_path / "routing-stats.sqlite")
|
||||||
|
signature = "m|gpu[0-11]->cpu[12-23]"
|
||||||
|
store = RouteStatsStore(RoutingConfig(stats_half_life_seconds=1e9), db_path=db_path)
|
||||||
|
store.record_sample("m", signature, tokens=100, elapsed_seconds=2.0, now=100.0)
|
||||||
|
store.save_to_db()
|
||||||
|
|
||||||
|
restored = RouteStatsStore(RoutingConfig(stats_half_life_seconds=1e9), db_path=db_path)
|
||||||
|
row = restored.model_rows("m", now=101.0)[0]
|
||||||
|
assert row["hop_count"] == 2
|
||||||
|
assert row["samples"] == 1
|
||||||
|
assert row["tps"] == 50.0
|
||||||
|
assert row["latency_ms"] == 2000.0
|
||||||
|
|
||||||
|
|
||||||
def _candidates_two_routes():
|
def _candidates_two_routes():
|
||||||
fast = RouteCandidate(nodes=[], signature="m|fast", prior_tps=100.0)
|
fast = RouteCandidate(nodes=[], signature="m|fast", prior_tps=100.0)
|
||||||
slow = RouteCandidate(nodes=[], signature="m|slow", prior_tps=50.0)
|
slow = RouteCandidate(nodes=[], signature="m|slow", prior_tps=50.0)
|
||||||
@@ -290,6 +305,41 @@ def test_proxy_head_is_route_head_and_routing_endpoint_lists_routes():
|
|||||||
assert sampled, "completed requests must produce route samples"
|
assert sampled, "completed requests must produce route samples"
|
||||||
|
|
||||||
|
|
||||||
|
def test_admin_model_load_request_queues_directive_on_joined_node():
|
||||||
|
tracker = TrackerServer(validator_service_token="test-admin")
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
node = _post_json(
|
||||||
|
f"http://127.0.0.1:{port}/v1/nodes/register",
|
||||||
|
{
|
||||||
|
"endpoint": "http://127.0.0.1:9911",
|
||||||
|
"model": "stub-model",
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 3,
|
||||||
|
"managed_assignment": True,
|
||||||
|
"memory_mb": 32768,
|
||||||
|
"hardware_profile": {"host_id": "available-ram-pool"},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
request = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{port}/v1/models/load",
|
||||||
|
data=json.dumps({"model": "qwen2.5-0.5b-instruct"}).encode(),
|
||||||
|
headers={"Content-Type": "application/json", "Authorization": "Bearer test-admin"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(request) as response:
|
||||||
|
result = json.loads(response.read())
|
||||||
|
heartbeat = _post_json(
|
||||||
|
f"http://127.0.0.1:{port}/v1/nodes/{node['node_id']}/heartbeat", {})
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert result["status"] == "queued"
|
||||||
|
assert result["assignment"]["node_id"] == node["node_id"]
|
||||||
|
assert heartbeat["directives"][0]["action"] == "ADD_SHARD"
|
||||||
|
assert heartbeat["directives"][0]["model"] == "Qwen/Qwen2.5-0.5B-Instruct"
|
||||||
|
|
||||||
|
|
||||||
def test_endpoint_key_distinguishes_same_port_different_hosts():
|
def test_endpoint_key_distinguishes_same_port_different_hosts():
|
||||||
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
|
from meshnet_node.torch_server import _clamp_downstream_hops, _endpoint_key
|
||||||
|
|
||||||
|
|||||||
153
tests/test_model_speed_latency.py
Normal file
153
tests/test_model_speed_latency.py
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
"""Tracker-backed latency experiments and model-speed drill-down."""
|
||||||
|
|
||||||
|
import http.server
|
||||||
|
import json
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from meshnet_tracker.server import TrackerServer
|
||||||
|
|
||||||
|
|
||||||
|
MODELS = {
|
||||||
|
"qwen2.5-0.5b-instruct": (24, "Qwen/Qwen2.5-0.5B-Instruct"),
|
||||||
|
"qwen3.6-35b-a3b": (40, "unsloth/Qwen3.6-35B-A3B"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _post_json(url: str, payload: dict) -> dict:
|
||||||
|
request = urllib.request.Request(
|
||||||
|
url,
|
||||||
|
data=json.dumps(payload).encode(),
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(request, timeout=10.0) as response:
|
||||||
|
return json.loads(response.read())
|
||||||
|
|
||||||
|
|
||||||
|
def _get_json(url: str) -> dict:
|
||||||
|
with urllib.request.urlopen(url, timeout=10.0) as response:
|
||||||
|
return json.loads(response.read())
|
||||||
|
|
||||||
|
|
||||||
|
class _LatencyNode(http.server.BaseHTTPRequestHandler):
|
||||||
|
"""Synthetic node: every downstream Activation Seam adds deterministic delay."""
|
||||||
|
|
||||||
|
base_delay_seconds = 0.004
|
||||||
|
seam_delay_seconds = 0.006
|
||||||
|
|
||||||
|
def log_message(self, *_args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
self.rfile.read(int(self.headers.get("Content-Length", 0)))
|
||||||
|
downstream = json.loads(self.headers.get("X-Meshnet-Route", "[]"))
|
||||||
|
time.sleep(self.base_delay_seconds + self.seam_delay_seconds * len(downstream))
|
||||||
|
body = json.dumps({
|
||||||
|
"choices": [{"message": {"role": "assistant", "content": "ok " * 40}}],
|
||||||
|
"usage": {"prompt_tokens": 10, "completion_tokens": 40},
|
||||||
|
}).encode()
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
|
||||||
|
def _start_latency_nodes(count: int):
|
||||||
|
nodes = []
|
||||||
|
threads = []
|
||||||
|
for _ in range(count):
|
||||||
|
node = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _LatencyNode)
|
||||||
|
thread = threading.Thread(target=node.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
nodes.append(node)
|
||||||
|
threads.append(thread)
|
||||||
|
return nodes, threads
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("model", MODELS)
|
||||||
|
@pytest.mark.parametrize("hardware", ["gpu", "gpu-cpu", "cpu"])
|
||||||
|
def test_tracker_records_increasing_hop_latency_for_model_and_hardware(model, hardware):
|
||||||
|
"""One through five hops must preserve a measurable seam penalty in tracker stats."""
|
||||||
|
layer_count, hf_repo = MODELS[model]
|
||||||
|
nodes, threads = _start_latency_nodes(5)
|
||||||
|
tracker = TrackerServer(model_presets={
|
||||||
|
model: {
|
||||||
|
"layers_start": 0,
|
||||||
|
"layers_end": layer_count - 1,
|
||||||
|
"hf_repo": hf_repo,
|
||||||
|
"aliases": [model],
|
||||||
|
}
|
||||||
|
})
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
try:
|
||||||
|
registered_ids = []
|
||||||
|
for index, node in enumerate(nodes):
|
||||||
|
start = (layer_count * index) // 5
|
||||||
|
end = (layer_count * (index + 1)) // 5 - 1
|
||||||
|
device = "cuda" if hardware == "gpu" or (hardware == "gpu-cpu" and index == 0) else "cpu"
|
||||||
|
data = _post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/nodes/register",
|
||||||
|
{
|
||||||
|
"endpoint": f"http://127.0.0.1:{node.server_address[1]}",
|
||||||
|
"model": model,
|
||||||
|
"hf_repo": hf_repo,
|
||||||
|
"num_layers": layer_count,
|
||||||
|
"shard_start": start,
|
||||||
|
"shard_end": end,
|
||||||
|
"tracker_mode": index == 0,
|
||||||
|
"hardware_profile": {"device": device},
|
||||||
|
"vram_bytes": 8_000_000_000 if device == "cuda" else 0,
|
||||||
|
"ram_bytes": 32_000_000_000,
|
||||||
|
"benchmark_tokens_per_sec": 100.0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
registered_ids.append(data["node_id"])
|
||||||
|
|
||||||
|
for hops in range(1, 6):
|
||||||
|
route = registered_ids[:hops]
|
||||||
|
# Widen the final shard to make each pinned prefix a complete route.
|
||||||
|
with tracker._server.lock:
|
||||||
|
tracker._server.registry[route[-1]].shard_end = layer_count - 1
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
|
||||||
|
{
|
||||||
|
"model": model,
|
||||||
|
"messages": [{"role": "user", "content": "measure"}],
|
||||||
|
"route": route,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
report = _get_json(f"http://127.0.0.1:{tracker_port}/v1/model-speed?model={model}")
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
for node, thread in zip(nodes, threads):
|
||||||
|
node.shutdown()
|
||||||
|
node.server_close()
|
||||||
|
thread.join(timeout=1.0)
|
||||||
|
|
||||||
|
routes = {entry["hop_count"]: entry for entry in report["routes"]}
|
||||||
|
assert set(routes) == {1, 2, 3, 4, 5}
|
||||||
|
assert routes[5]["latency_ms"] > routes[1]["latency_ms"]
|
||||||
|
assert routes[5]["latency_penalty_ms"] > 0
|
||||||
|
assert routes[5]["device_mix"] == hardware
|
||||||
|
assert report["model"] == model
|
||||||
|
assert report["nodes"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_speed_dashboard_includes_visualization_and_route_drilldown():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
html = urllib.request.urlopen(f"http://127.0.0.1:{port}/dashboard").read().decode()
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert "Model inference speed" in html
|
||||||
|
assert "model-speed-chart" in html
|
||||||
|
assert "renderModelSpeed" in html
|
||||||
|
assert "/v1/model-speed" in html
|
||||||
62
tests/test_real_distributed_inference.py
Normal file
62
tests/test_real_distributed_inference.py
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
"""Environment-gated acceptance test against real registered model shards.
|
||||||
|
|
||||||
|
Run only after starting actual nodes (never synthetic HTTP handlers):
|
||||||
|
|
||||||
|
MESHNET_REAL_INFERENCE_URL=http://localhost:8080 \
|
||||||
|
MESHNET_REAL_INFERENCE_API_KEY=... \
|
||||||
|
MESHNET_REAL_INFERENCE_MODEL=Qwen/Qwen2.5-0.5B-Instruct \
|
||||||
|
MESHNET_REAL_INFERENCE_ROUTE=node-id-head,node-id-tail \
|
||||||
|
.venv-rocm/bin/python -m pytest tests/test_real_distributed_inference.py -m integration -v
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
BASE_URL = os.environ.get("MESHNET_REAL_INFERENCE_URL")
|
||||||
|
API_KEY = os.environ.get("MESHNET_REAL_INFERENCE_API_KEY")
|
||||||
|
MODEL = os.environ.get("MESHNET_REAL_INFERENCE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct")
|
||||||
|
ROUTE = [node_id for node_id in os.environ.get("MESHNET_REAL_INFERENCE_ROUTE", "").split(",") if node_id]
|
||||||
|
|
||||||
|
pytestmark = pytest.mark.integration
|
||||||
|
|
||||||
|
|
||||||
|
def _required_real_stack() -> None:
|
||||||
|
if not BASE_URL or not API_KEY or len(ROUTE) < 2:
|
||||||
|
pytest.skip(
|
||||||
|
"set MESHNET_REAL_INFERENCE_URL, MESHNET_REAL_INFERENCE_API_KEY, and "
|
||||||
|
"a comma-separated MESHNET_REAL_INFERENCE_ROUTE with at least two real nodes"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_registered_shards_complete_a_pinned_request():
|
||||||
|
"""Acceptance test: tracker proxies an actual prompt through real registered shards."""
|
||||||
|
_required_real_stack()
|
||||||
|
assert BASE_URL is not None
|
||||||
|
assert API_KEY is not None
|
||||||
|
request = urllib.request.Request(
|
||||||
|
f"{BASE_URL.rstrip('/')}/v1/chat/completions",
|
||||||
|
data=json.dumps({
|
||||||
|
"model": MODEL,
|
||||||
|
"messages": [{"role": "user", "content": "What is 2 plus 2? Reply in one word."}],
|
||||||
|
"max_tokens": 8,
|
||||||
|
"temperature": 1.0,
|
||||||
|
"route": ROUTE,
|
||||||
|
}).encode(),
|
||||||
|
headers={"Content-Type": "application/json", "Authorization": f"Bearer {API_KEY}"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
started = time.monotonic()
|
||||||
|
with urllib.request.urlopen(request, timeout=300) as response:
|
||||||
|
payload = json.loads(response.read())
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
|
||||||
|
content = payload["choices"][0]["message"]["content"].strip().lower()
|
||||||
|
assert content
|
||||||
|
assert "two" in content or "four" in content
|
||||||
|
assert elapsed > 0
|
||||||
|
assert payload["usage"]["total_tokens"] > 0
|
||||||
224
tests/test_tracker_test_runner.py
Normal file
224
tests/test_tracker_test_runner.py
Normal file
@@ -0,0 +1,224 @@
|
|||||||
|
"""dashboard-test-runner US-001: opt-in, admin-only tracker test-runner API.
|
||||||
|
|
||||||
|
Runs against a tiny throwaway repo (tmp_path) so collection and runs stay
|
||||||
|
fast and hermetic — the manager itself discovers the real repo root when
|
||||||
|
constructed without an explicit ``repo_root``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from meshnet_tracker.accounts import AccountStore
|
||||||
|
from meshnet_tracker.server import TrackerServer
|
||||||
|
from meshnet_tracker.test_runner import TestRunManager as RunManager
|
||||||
|
from meshnet_tracker.test_runner import discover_repo_root
|
||||||
|
|
||||||
|
|
||||||
|
def _make_repo(tmp_path: Path) -> Path:
|
||||||
|
"""A minimal repo satisfying discover_repo_root's markers, with fast tests."""
|
||||||
|
(tmp_path / "pyproject.toml").write_text("[tool.pytest.ini_options]\n")
|
||||||
|
tests = tmp_path / "tests"
|
||||||
|
tests.mkdir()
|
||||||
|
(tests / "conftest.py").write_text("")
|
||||||
|
(tests / "test_quick.py").write_text(
|
||||||
|
"def test_passes():\n assert True\n\n"
|
||||||
|
"def test_also_passes():\n assert 1 + 1 == 2\n"
|
||||||
|
)
|
||||||
|
(tests / "test_slow.py").write_text(
|
||||||
|
"import time\n\ndef test_sleeps():\n time.sleep(30)\n"
|
||||||
|
)
|
||||||
|
(tests / "test_smoke.py").write_text("def test_smoke():\n assert True\n")
|
||||||
|
# Real-inference stand-in: must never be collected or run by default.
|
||||||
|
(tests / "test_real_stub.py").write_text(
|
||||||
|
"def test_real_inference():\n raise AssertionError('env-gated')\n"
|
||||||
|
)
|
||||||
|
return tmp_path
|
||||||
|
|
||||||
|
|
||||||
|
def _start_tracker(tmp_path: Path | None):
|
||||||
|
"""Tracker + (admin_session, user_session); test runner enabled iff tmp_path given."""
|
||||||
|
accounts = AccountStore()
|
||||||
|
admin = accounts.register(email="admin@example.com", password="admin-pass-123")
|
||||||
|
admin_session = accounts.create_session(admin["account_id"])
|
||||||
|
user = accounts.register(email="user@example.com", password="user-pass-123")
|
||||||
|
user_session = accounts.create_session(user["account_id"])
|
||||||
|
runner = RunManager(repo_root=_make_repo(tmp_path)) if tmp_path is not None else None
|
||||||
|
tracker = TrackerServer(accounts=accounts, test_runner=runner)
|
||||||
|
port = tracker.start()
|
||||||
|
return tracker, port, admin_session, user_session
|
||||||
|
|
||||||
|
|
||||||
|
def _request(port, method, path, token=None, payload=None):
|
||||||
|
body = json.dumps(payload).encode() if payload is not None else None
|
||||||
|
headers = {"Content-Type": "application/json"}
|
||||||
|
if token:
|
||||||
|
headers["Authorization"] = f"Bearer {token}"
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{port}{path}", data=body, headers=headers, method=method
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req) as resp:
|
||||||
|
return resp.status, json.loads(resp.read())
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
return exc.code, json.loads(exc.read())
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_completion(port, session, timeout=90.0):
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
while time.time() < deadline:
|
||||||
|
status, data = _request(port, "GET", "/v1/tests/status", token=session)
|
||||||
|
assert status == 200
|
||||||
|
if data["run"] is not None and data["run"]["status"] != "running":
|
||||||
|
return data
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise AssertionError("test run did not finish in time")
|
||||||
|
|
||||||
|
|
||||||
|
def test_endpoints_require_admin(tmp_path):
|
||||||
|
tracker, port, _admin, user = _start_tracker(tmp_path)
|
||||||
|
try:
|
||||||
|
assert _request(port, "GET", "/v1/tests")[0] == 401
|
||||||
|
assert _request(port, "GET", "/v1/tests/status")[0] == 401
|
||||||
|
assert _request(port, "POST", "/v1/tests/run", payload={"target": "x"})[0] == 401
|
||||||
|
assert _request(port, "GET", "/v1/tests", token=user)[0] == 403
|
||||||
|
assert _request(port, "GET", "/v1/tests/status", token=user)[0] == 403
|
||||||
|
assert _request(
|
||||||
|
port, "POST", "/v1/tests/run", token=user, payload={"target": "x"}
|
||||||
|
)[0] == 403
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_disabled_by_default_even_for_admin(monkeypatch):
|
||||||
|
monkeypatch.delenv("MESHNET_ENABLE_TEST_RUNNER", raising=False)
|
||||||
|
tracker, port, admin, _user = _start_tracker(None)
|
||||||
|
try:
|
||||||
|
assert tracker._test_runner is None # no flag, no env → fails closed
|
||||||
|
for method, path, payload in [
|
||||||
|
("GET", "/v1/tests", None),
|
||||||
|
("GET", "/v1/tests/status", None),
|
||||||
|
("POST", "/v1/tests/run", {"target": "tests/test_smoke.py::test_smoke"}),
|
||||||
|
]:
|
||||||
|
status, data = _request(port, method, path, token=admin, payload=payload)
|
||||||
|
assert status == 403
|
||||||
|
assert "disabled" in data["error"]
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_enable_flag_constructs_runner_against_real_repo(monkeypatch):
|
||||||
|
monkeypatch.delenv("MESHNET_ENABLE_TEST_RUNNER", raising=False)
|
||||||
|
tracker = TrackerServer(enable_test_runner=True)
|
||||||
|
assert tracker._test_runner is not None
|
||||||
|
assert tracker._test_runner.repo_root == discover_repo_root()
|
||||||
|
assert (tracker._test_runner.repo_root / "tests" / "conftest.py").is_file()
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection_lists_tests_and_excludes_real_inference(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.delenv("MESHNET_ENABLE_REAL_INFERENCE_TESTS", raising=False)
|
||||||
|
tracker, port, admin, _user = _start_tracker(tmp_path)
|
||||||
|
try:
|
||||||
|
status, data = _request(port, "GET", "/v1/tests", token=admin)
|
||||||
|
assert status == 200
|
||||||
|
assert data["enabled"] is True
|
||||||
|
assert "tests/test_quick.py::test_passes" in data["tests"]
|
||||||
|
assert "tests/test_slow.py::test_sleeps" in data["tests"]
|
||||||
|
assert not any("test_real_" in node_id for node_id in data["tests"])
|
||||||
|
assert data["collected_at"] is not None
|
||||||
|
suite_ids = [suite["id"] for suite in data["suites"]]
|
||||||
|
assert "suite:smoke" in suite_ids
|
||||||
|
assert "suite:real-inference" not in suite_ids
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_selected_test_to_completion(tmp_path):
|
||||||
|
tracker, port, admin, _user = _start_tracker(tmp_path)
|
||||||
|
try:
|
||||||
|
assert _request(port, "GET", "/v1/tests", token=admin)[0] == 200
|
||||||
|
status, started = _request(
|
||||||
|
port, "POST", "/v1/tests/run", token=admin,
|
||||||
|
payload={"target": "tests/test_quick.py::test_passes"},
|
||||||
|
)
|
||||||
|
assert status == 202
|
||||||
|
assert started["run"]["status"] == "running"
|
||||||
|
assert started["run"]["run_id"]
|
||||||
|
assert started["run"]["started_at"] is not None
|
||||||
|
|
||||||
|
finished = _wait_for_completion(port, admin)
|
||||||
|
run = finished["run"]
|
||||||
|
assert run["status"] == "passed"
|
||||||
|
assert run["exit_code"] == 0
|
||||||
|
assert run["target"] == "tests/test_quick.py::test_passes"
|
||||||
|
assert run["finished_at"] >= run["started_at"]
|
||||||
|
assert run["elapsed_seconds"] >= 0
|
||||||
|
assert "1 passed" in finished["stdout"]
|
||||||
|
assert finished["log_lines_limit"] > 0
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_approved_suite_without_prior_collection(tmp_path):
|
||||||
|
tracker, port, admin, _user = _start_tracker(tmp_path)
|
||||||
|
try:
|
||||||
|
status, started = _request(
|
||||||
|
port, "POST", "/v1/tests/run", token=admin,
|
||||||
|
payload={"target": "suite:smoke"},
|
||||||
|
)
|
||||||
|
assert status == 202
|
||||||
|
assert started["run"]["args"] == ["tests/test_smoke.py"]
|
||||||
|
finished = _wait_for_completion(port, admin)
|
||||||
|
assert finished["run"]["status"] == "passed"
|
||||||
|
assert finished["run"]["exit_code"] == 0
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("target", [
|
||||||
|
"",
|
||||||
|
"-x",
|
||||||
|
"--help",
|
||||||
|
"tests/test_quick.py::test_passes; rm -rf /",
|
||||||
|
"tests/never_collected.py::test_missing",
|
||||||
|
"suite:real-inference",
|
||||||
|
"suite:no-such-suite",
|
||||||
|
])
|
||||||
|
def test_rejects_arbitrary_or_uncollected_targets(tmp_path, target, monkeypatch):
|
||||||
|
monkeypatch.delenv("MESHNET_ENABLE_REAL_INFERENCE_TESTS", raising=False)
|
||||||
|
tracker, port, admin, _user = _start_tracker(tmp_path)
|
||||||
|
try:
|
||||||
|
assert _request(port, "GET", "/v1/tests", token=admin)[0] == 200
|
||||||
|
status, data = _request(
|
||||||
|
port, "POST", "/v1/tests/run", token=admin, payload={"target": target}
|
||||||
|
)
|
||||||
|
assert status == 400
|
||||||
|
assert "error" in data
|
||||||
|
# Nothing started.
|
||||||
|
assert _request(port, "GET", "/v1/tests/status", token=admin)[1]["run"] is None
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejects_concurrent_runs(tmp_path):
|
||||||
|
tracker, port, admin, _user = _start_tracker(tmp_path)
|
||||||
|
try:
|
||||||
|
assert _request(port, "GET", "/v1/tests", token=admin)[0] == 200
|
||||||
|
status, _ = _request(
|
||||||
|
port, "POST", "/v1/tests/run", token=admin,
|
||||||
|
payload={"target": "tests/test_slow.py::test_sleeps"},
|
||||||
|
)
|
||||||
|
assert status == 202
|
||||||
|
status, data = _request(
|
||||||
|
port, "POST", "/v1/tests/run", token=admin,
|
||||||
|
payload={"target": "tests/test_quick.py::test_passes"},
|
||||||
|
)
|
||||||
|
assert status == 409
|
||||||
|
assert "in progress" in data["error"]
|
||||||
|
finally:
|
||||||
|
# stop() kills the sleeping pytest child via TestRunManager.shutdown().
|
||||||
|
tracker.stop()
|
||||||
Reference in New Issue
Block a user