popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dash, tests

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-14 12:26:10 +02:00
parent d1a1400db9
commit 13d82f8032
11 changed files with 93 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
tests/test_tracker_routing.py
View File

@@ -28,6 +28,11 @@ from meshnet_tracker.server import (
_TEST_HIVE_SECRET = "test-hive-secret"
def _tracker(**kwargs):
kwargs.setdefault("enable_billing", False)
return TrackerServer(**kwargs)
def _post_json(url: str, payload: dict) -> dict:
data = json.dumps(payload).encode()
req = urllib.request.Request(
@@ -67,7 +72,7 @@ def test_tracker_send_json_ignores_broken_pipe_after_client_disconnect():
def test_tracker_exposes_registered_model_metadata():
"Tracker exposes registered model metadata\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
@@ -114,7 +119,7 @@ def test_tracker_exposes_registered_model_metadata():
def test_tracker_lists_recommended_kimi_before_nodes_register():
"Tracker lists recommended kimi before nodes register\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
@@ -138,7 +143,7 @@ def test_tracker_lists_recommended_qwen_27b_before_nodes_register():
Tags: http, routing, tracker
"""
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
@@ -177,7 +182,7 @@ def test_quantization_coverage_uses_higher_precision_shards():
def test_network_map_exposes_pool_size_and_speed_summary():
"Network map exposes pool size and speed summary\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
@@ -219,7 +224,7 @@ def test_network_map_exposes_pool_size_and_speed_summary():
def test_network_map_exposes_served_model_copy_count():
"Network map exposes served model copy count\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
@@ -273,7 +278,7 @@ def test_network_map_exposes_served_model_copy_count():
def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
"Recommended kimi becomes deployable when pool is large enough\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
gib = 1024 * 1024 * 1024
@@ -302,7 +307,7 @@ def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
def test_network_assign_can_start_recommended_kimi_by_short_name():
"Network assign can start recommended kimi by short name\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
@@ -344,7 +349,7 @@ def test_tracker_serves_health_while_proxy_request_is_in_flight():
slow_node = http.server.HTTPServer(("127.0.0.1", 0), SlowChatHandler)
slow_thread = threading.Thread(target=slow_node.serve_forever, daemon=True)
slow_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker = _tracker(heartbeat_timeout=60.0)
tracker_port = tracker.start()
proxy_error = []
try:
@@ -410,7 +415,7 @@ def test_tracker_route_log_counts_proxy_inflight_requests():
slow_node = http.server.HTTPServer(("127.0.0.1", 0), SlowChatHandler)
slow_thread = threading.Thread(target=slow_node.serve_forever, daemon=True)
slow_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker = _tracker(heartbeat_timeout=60.0)
tracker_port = tracker.start()
errors = []
try:
@@ -498,7 +503,7 @@ def test_tracker_logs_stream_progress_before_request_completes():
node = http.server.HTTPServer(("127.0.0.1", 0), StreamingChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker = _tracker(heartbeat_timeout=60.0)
tracker_port = tracker.start()
response = None
try:
@@ -590,7 +595,7 @@ def test_tracker_stream_survives_idle_gap_between_sse_chunks():
node = http.server.HTTPServer(("127.0.0.1", 0), IdleStreamingChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker = _tracker(heartbeat_timeout=60.0)
tracker_port = tracker.start()
response = None
try:
@@ -657,7 +662,7 @@ def test_tracker_dashboard_can_cancel_inflight_proxy():
node = http.server.HTTPServer(("127.0.0.1", 0), StreamingChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(heartbeat_timeout=60.0)
tracker = _tracker(heartbeat_timeout=60.0)
tracker_port = tracker.start()
response = None
request_id = None
@@ -723,7 +728,7 @@ def test_tracker_dashboard_can_cancel_inflight_proxy():
def test_tracker_routes_hf_model_alias_from_quickstart():
"The documented qwen2.5-0.5b alias resolves a full HF repo registration.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
_post_json(
@@ -775,7 +780,7 @@ def test_tracker_proxy_accepts_hf_model_alias_from_quickstart():
node = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
_post_json(
@@ -833,7 +838,7 @@ def test_tracker_proxy_routes_split_preset_nodes_by_alias():
head = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
head_thread = threading.Thread(target=head.serve_forever, daemon=True)
head_thread.start()
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
@@ -889,7 +894,7 @@ def test_tracker_proxy_routes_split_preset_nodes_by_alias():
def test_tracker_route_endpoint_routes_split_preset_nodes_by_alias():
"Tracker route endpoint routes split preset nodes by alias\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
@@ -936,7 +941,7 @@ def test_tracker_route_endpoint_routes_split_preset_nodes_by_alias():
def test_tracker_route_endpoint_ignores_model_case_and_outer_whitespace():
"Tracker route endpoint ignores model case and outer whitespace\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
@@ -970,7 +975,7 @@ def test_tracker_route_endpoint_ignores_model_case_and_outer_whitespace():
def test_tracker_route_prefers_distributed_over_single_full_shard():
"When a full 0-39 node and a partial 0-21 head coexist, /v1/route\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
@@ -1044,7 +1049,7 @@ def test_tracker_proxy_ignores_model_case_and_outer_whitespace():
node = http.server.HTTPServer(("127.0.0.1", 0), ChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"qwen3.6-35b-a3b": {
"layers_start": 0,
"layers_end": 39,
@@ -1083,7 +1088,7 @@ def test_tracker_proxy_ignores_model_case_and_outer_whitespace():
def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix():
"Tracker registration node id includes wallet prefix and stable suffix\n\nTags: http, routing, security, tracker, wallet"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
wallet = "7j77FsPY1evV8tuf4Z73AVrWwxBEW1pvKwi4EvcRD3g"
try:
@@ -1124,7 +1129,7 @@ def test_tracker_registration_node_id_includes_wallet_prefix_and_stable_suffix()
def test_tracker_node_registration():
"A node can register with the tracker and receives a node_id.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
resp = _post_json(
@@ -1146,7 +1151,7 @@ def test_tracker_node_registration():
def test_tracker_route_selection():
"Tracker returns ordered route when nodes collectively cover all layers.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
_post_json(
@@ -1174,7 +1179,7 @@ def test_tracker_route_selection():
def test_tracker_route_error_no_coverage():
"Tracker returns 503 when registered nodes do not cover all required layers.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
# Only half the layers covered — route must fail.
@@ -1197,7 +1202,7 @@ def test_tracker_route_error_no_coverage():
def test_tracker_coverage_endpoint_reports_uncovered_ranges():
"Coverage endpoint returns compressed layer ranges with node counts.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 6,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1242,7 +1247,7 @@ def test_tracker_coverage_endpoint_reports_uncovered_ranges():
def test_tracker_coverage_endpoint_accepts_registered_hf_repo_or_short_name():
"Coverage endpoint supports real HF models registered outside preset catalog.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
_post_json(
@@ -1279,7 +1284,7 @@ def test_tracker_coverage_endpoint_accepts_registered_hf_repo_or_short_name():
def test_tracker_models_endpoint_lists_registered_hf_repo_and_short_name_alias():
"Tracker models endpoint lists registered hf repo and short name alias\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
_post_json(
@@ -1302,7 +1307,7 @@ def test_tracker_models_endpoint_lists_registered_hf_repo_and_short_name_alias()
def test_tracker_auto_assigns_new_node_to_uncovered_range_first():
"Capability-driven registration fills the first uncovered layer gap.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 8,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1331,7 +1336,7 @@ def test_tracker_auto_assigns_new_node_to_uncovered_range_first():
def test_tracker_speed_weighted_vram_assignment_covers_model():
"Three auto-assigned nodes reach 100% coverage with widest range on largest VRAM.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 12,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1375,7 +1380,7 @@ def test_tracker_speed_weighted_vram_assignment_covers_model():
def test_tracker_speed_is_primary_when_both_nodes_can_cover_gap():
"Tracker speed is primary when both nodes can cover gap\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 12,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1410,7 +1415,7 @@ def test_tracker_speed_is_primary_when_both_nodes_can_cover_gap():
def test_tracker_registration_directive_is_not_replayed_on_heartbeat():
"Tracker registration directive is not replayed on heartbeat\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 2,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1434,7 +1439,7 @@ def test_tracker_registration_directive_is_not_replayed_on_heartbeat():
def test_tracker_pool_join_adds_redundant_copy_without_splitting_incumbent():
"A second managed node with capacity for the full model keeps the first copy intact.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 4,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1472,7 +1477,7 @@ def test_tracker_pool_join_adds_redundant_copy_without_splitting_incumbent():
def test_tracker_explicit_full_copy_join_preserves_existing_serving_node():
"--model style joins with explicit shards add redundancy instead of reshuffling.\n\nTags: http, routing, tracker"
tracker = TrackerServer(heartbeat_timeout=10.0)
tracker = _tracker(heartbeat_timeout=10.0)
tracker_port = tracker.start()
try:
base_reg = {
@@ -1511,7 +1516,7 @@ def test_tracker_explicit_full_copy_join_preserves_existing_serving_node():
def test_tracker_periodic_rebalance_purges_expired_nodes_without_requests():
"Tracker periodic rebalance purges expired nodes without requests\n\nTags: http, routing, tracker"
tracker = TrackerServer(
tracker = _tracker(
heartbeat_timeout=0.05,
rebalance_interval=0.02,
model_presets={"tiny-model": {"total_layers": 1, "bytes_per_layer": {"bfloat16": 1_000}}},
@@ -1533,7 +1538,7 @@ def test_tracker_periodic_rebalance_purges_expired_nodes_without_requests():
def test_tracker_faster_node_receives_wider_range_when_capacity_ties():
"Tracker faster node receives wider range when capacity ties\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 12,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1564,7 +1569,7 @@ def test_tracker_faster_node_receives_wider_range_when_capacity_ties():
def test_tracker_rebalances_after_middle_range_node_timeout():
"Killing a middle shard queues LOAD_SHARD and restores coverage.\n\nTags: http, routing, tracker"
tracker = TrackerServer(
tracker = _tracker(
heartbeat_timeout=0.15,
model_presets={
"tiny-model": {
@@ -1612,7 +1617,7 @@ def test_tracker_rebalances_after_middle_range_node_timeout():
def test_tracker_rebalances_managed_hf_node_after_peer_timeout():
"HF nodes auto-assigned by the tracker receive LOAD_SHARD after a peer dies.\n\nTags: http, routing, tracker"
tracker = TrackerServer(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker = _tracker(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker_port = tracker.start()
try:
managed = _post_json(
@@ -1648,7 +1653,7 @@ def test_tracker_rebalances_managed_hf_node_after_peer_timeout():
def test_tracker_route_error_no_nodes():
"Tracker returns 503 with clear error when the registry is empty.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
try:
@@ -1664,7 +1669,7 @@ def test_tracker_route_error_no_nodes():
def test_tracker_heartbeat_updates_node():
"Sending a heartbeat for a registered node succeeds.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
reg = _post_json(
@@ -1684,7 +1689,7 @@ def test_tracker_heartbeat_updates_node():
def test_tracker_heartbeat_stores_current_requests():
"Node-reported in-flight request snapshots appear on the network map.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
reg = _post_json(
@@ -1749,7 +1754,7 @@ def test_normalize_current_requests_sanitizes_payload():
def test_tracker_heartbeat_expiry():
"Nodes that miss their heartbeat window are excluded from routes.\n\nTags: http, routing, tracker"
tracker = TrackerServer(heartbeat_timeout=0.05) # 50 ms
tracker = _tracker(heartbeat_timeout=0.05) # 50 ms
tracker_port = tracker.start()
try:
_post_json(
@@ -1778,7 +1783,7 @@ def test_tracker_heartbeat_expiry():
def test_tracker_heartbeat_expiry_removes_node_from_registry():
"Expired nodes are removed, not merely hidden from route responses.\n\nTags: http, routing, tracker"
tracker = TrackerServer(heartbeat_timeout=0.05)
tracker = _tracker(heartbeat_timeout=0.05)
tracker_port = tracker.start()
try:
reg = _post_json(
@@ -1806,7 +1811,7 @@ def test_tracker_heartbeat_expiry_removes_node_from_registry():
def test_tracker_route_rejects_non_extending_overlap():
"Overlapping shards that do not extend coverage cannot form a complete route.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
_post_json(
@@ -1831,7 +1836,7 @@ def test_tracker_route_rejects_non_extending_overlap():
def test_tracker_console_records_model_not_available():
"Tracker console records model not available\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
try:
@@ -1856,7 +1861,7 @@ def test_tracker_console_records_model_not_available():
def test_tracker_registration_rejects_invalid_payload():
"Registration errors return a defined JSON 400 response.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
try:
@@ -1876,7 +1881,7 @@ def test_tracker_registration_rejects_invalid_payload():
def test_tracker_registration_rejects_invalid_max_loaded_shards():
"Tracker registration rejects invalid max loaded shards\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
try:
@@ -1896,7 +1901,7 @@ def test_tracker_registration_rejects_invalid_max_loaded_shards():
def test_network_map_exposes_node_capacity_limits():
"Network map exposes node capacity limits\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 12,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1927,7 +1932,7 @@ def test_network_map_exposes_node_capacity_limits():
def test_tracker_capacity_uses_ram_when_node_has_no_vram():
"CPU-only nodes should expose RAM-backed shard capacity, not default GPU capacity.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 20,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1956,7 +1961,7 @@ def test_tracker_capacity_uses_ram_when_node_has_no_vram():
def test_rebalance_keeps_one_active_range_even_when_multiple_slots_advertised():
"max_loaded_shards is exposed but reserved until node runtime supports multi-range serving.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 8,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -1983,7 +1988,7 @@ def test_rebalance_keeps_one_active_range_even_when_multiple_slots_advertised():
def test_tracker_routes_only_nodes_for_requested_model():
"A node registered for one model cannot satisfy another model's route.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"model-a": {"layers_start": 0, "layers_end": 31},
"model-b": {"layers_start": 0, "layers_end": 31},
})
@@ -2006,7 +2011,7 @@ def test_tracker_routes_only_nodes_for_requested_model():
def test_tracker_assignment_coverage_is_model_scoped():
"Shard assignment gaps are computed only from nodes serving the same model.\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"model-a": {"layers_start": 0, "layers_end": 31},
"model-b": {"layers_start": 0, "layers_end": 31},
})
@@ -2031,7 +2036,7 @@ def test_tracker_excludes_banned_wallets_from_routes():
contracts.registry.submit_stake("wallet-a", 1_000)
contracts.registry.submit_stake("wallet-b", 1_000)
contracts.registry.ban_wallet("wallet-b")
tracker = TrackerServer(contracts=contracts, minimum_stake=100)
tracker = _tracker(contracts=contracts, minimum_stake=100)
tracker_port = tracker.start()
try:
_post_json(
@@ -2065,7 +2070,7 @@ def test_tracker_rejects_banned_wallet_registration():
contracts = LocalSolanaContracts()
contracts.registry.submit_stake("wallet-b", 1_000)
contracts.registry.ban_wallet("wallet-b")
tracker = TrackerServer(contracts=contracts, minimum_stake=100)
tracker = _tracker(contracts=contracts, minimum_stake=100)
tracker_port = tracker.start()
try:
try:
@@ -2111,7 +2116,7 @@ def test_gateway_routes_even_when_compute_node_has_low_stake():
contracts = LocalSolanaContracts()
contracts.registry.submit_stake("wallet-a", 1_000)
contracts.registry.submit_stake("wallet-b", 5)
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
@@ -2166,7 +2171,7 @@ def test_gateway_records_compute_attribution_after_inference_session():
contracts.registry.submit_stake("wallet-a", 1_000)
contracts.registry.submit_stake("wallet-b", 1_000)
contracts.payment.fund_api_key("api-key-a", lamports=10_000)
tracker = TrackerServer(contracts=contracts, minimum_stake=100)
tracker = _tracker(contracts=contracts, minimum_stake=100)
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
@@ -2294,7 +2299,7 @@ def test_gateway_returns_json_503_for_malformed_tracker_route():
def test_two_node_pipeline_via_tracker():
"End-to-end: nodes register with tracker; gateway discovers route dynamically.\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
tracker_url = f"http://127.0.0.1:{tracker_port}"
@@ -2351,7 +2356,7 @@ def test_two_node_pipeline_via_tracker():
def test_stats_endpoint_returns_empty_models_initially():
"Stats endpoint returns empty models initially\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
port = tracker.start()
try:
resp = _get_json(f"http://127.0.0.1:{port}/v1/stats")
@@ -2449,7 +2454,7 @@ def test_stats_sqlite_persistence_survives_restart(tmp_path):
def test_stats_gossip_endpoint_merges_peer_slice():
"Stats gossip endpoint merges peer slice\n\nTags: gossip, http, network, routing, tracker"
tracker = TrackerServer(hive_secret=_TEST_HIVE_SECRET)
tracker = _tracker(hive_secret=_TEST_HIVE_SECRET)
port = tracker.start()
try:
peer_payload = {
@@ -2610,7 +2615,7 @@ def test_select_route_reputation_never_overrides_coverage():
def test_two_stub_nodes_complete_pipeline_via_tracker():
"Integration: two StubNodeServer instances serving complementary shards\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"two-node-model": {"layers_start": 0, "layers_end": 3}
})
tracker_port = tracker.start()
@@ -2732,7 +2737,7 @@ def test_shard_heal_cycle_surviving_node_covers_dead_peers_gap(monkeypatch):
monkeypatch.setattr(torch_server, "_load_backend", fake_load)
# Use a very short timeout so Node A expires quickly.
tracker = TrackerServer(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker = _tracker(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker_port = tracker.start()
node_b = TorchNodeServer(backend=_FakeBackend(shard_start=12, shard_end=23))
@@ -2807,7 +2812,7 @@ def test_shard_heal_cycle_surviving_node_covers_dead_peers_gap(monkeypatch):
def test_network_map_exposes_memory_pool():
"Network map exposes memory pool\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"tiny-model": {
"total_layers": 8,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -2836,7 +2841,7 @@ def test_network_map_exposes_memory_pool():
def test_same_endpoint_can_register_multiple_models():
"Same endpoint can register multiple models\n\nTags: http, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker_port = tracker.start()
try:
base = {
@@ -2866,7 +2871,7 @@ def test_same_endpoint_can_register_multiple_models():
def test_scale_demanded_models_queues_add_shard_on_spare_host():
"Scale demanded models queues add shard on spare host\n\nTags: http, routing, tracker"
tracker = TrackerServer(model_presets={
tracker = _tracker(model_presets={
"model-a": {
"total_layers": 4,
"bytes_per_layer": {"bfloat16": 1_000},
@@ -2957,7 +2962,7 @@ def _proxy_chat_status(policy: str, quantization: str | None) -> tuple[int, dict
node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(capability_policy=policy)
tracker = _tracker(capability_policy=policy)
tracker_port = tracker.start()
try:
endpoint = f"http://127.0.0.1:{node.server_address[1]}"
@@ -3005,7 +3010,7 @@ def test_a_node_declaring_a_null_quantization_is_never_routed():
node = http.server.HTTPServer(("127.0.0.1", 0), _EchoChatHandler)
node_thread = threading.Thread(target=node.serve_forever, daemon=True)
node_thread.start()
tracker = TrackerServer(capability_policy=POLICY_COMPAT)
tracker = _tracker(capability_policy=POLICY_COMPAT)
tracker_port = tracker.start()
try:
endpoint = f"http://127.0.0.1:{node.server_address[1]}"
@@ -3028,7 +3033,7 @@ def test_a_node_declaring_a_null_quantization_is_never_routed():
def test_raft_apply_preserves_a_declared_precision_for_follower_routing():
"A follower that drops the declared precision would route unlike the leader.\n\nTags: raft, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker.start()
try:
tracker._raft_apply("register", {
@@ -3051,7 +3056,7 @@ def test_raft_apply_preserves_a_declared_precision_for_follower_routing():
def test_raft_apply_grandfathers_a_legacy_registration_that_omits_quantization():
"The follower must reach the same legacy fallback the leader did, not a bare None.\n\nTags: raft, routing, tracker"
tracker = TrackerServer()
tracker = _tracker()
tracker.start()
try:
tracker._raft_apply("register", {
@@ -3071,7 +3076,7 @@ def test_raft_apply_grandfathers_a_legacy_registration_that_omits_quantization()
def test_rebalancing_does_not_resurrect_a_node_that_declared_an_unusable_precision():
"Managed assignment picks a node's precision, so it must not overwrite an explicit unusable one.\n\nTags: http, routing, tracker"
tracker = TrackerServer(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker = _tracker(heartbeat_timeout=0.15, rebalance_interval=10.0)
tracker_port = tracker.start()
try:
managed = _post_json(
@@ -3120,7 +3125,7 @@ def _register_shard(tracker_port: int, port: int, **overrides) -> dict:
def test_a_node_with_an_unusable_precision_covers_no_layers():
"A node no request can reach must not report its shards as served coverage.\n\nTags: http, routing, tracker"
tracker = TrackerServer(rebalance_interval=10.0)
tracker = _tracker(rebalance_interval=10.0)
tracker_port = tracker.start()
try:
_register_shard(
@@ -3136,7 +3141,7 @@ def test_a_node_with_an_unusable_precision_covers_no_layers():
def test_rebalancing_fills_the_layers_an_unusable_node_only_appears_to_cover():
"Counting a dark node as coverage hides the gap, so the rebalancer never fills it.\n\nTags: http, routing, tracker"
tracker = TrackerServer(rebalance_interval=10.0)
tracker = _tracker(rebalance_interval=10.0)
tracker_port = tracker.start()
try:
# The dark node claims the model's tail; the managed node holds the head.