models on tracker
This commit is contained in:
@@ -34,7 +34,7 @@ HF_HOME=/run/media/popov/d/DEV/models .venv/bin/meshnet-node start --m
|
|||||||
HF_HOME=/run/media/popov/d/DEV/models .venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --shard-start 10
|
HF_HOME=/run/media/popov/d/DEV/models .venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --shard-start 10
|
||||||
meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 0 --shard-end 20
|
meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 0 --shard-end 20
|
||||||
|
|
||||||
.venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 10
|
HF_HOME=/run/media/popov/d/DEV/models .venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct --shard-start 10
|
||||||
.venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen3.6-27B
|
.venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen3.6-27B
|
||||||
|
|
||||||
meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --cpu
|
meshnet-node start --tracker https://meshnet.2.d-popov.com --model qwen3.6-35b-a3b --cpu
|
||||||
@@ -71,3 +71,9 @@ meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen/Qwen2.5-
|
|||||||
1. no benchmark at node start
|
1. no benchmark at node start
|
||||||
2. CUDA stopped working on windows PS
|
2. CUDA stopped working on windows PS
|
||||||
3. solana/crypto does not work on linux tracker. does it still work on windows?
|
3. solana/crypto does not work on linux tracker. does it still work on windows?
|
||||||
|
|
||||||
|
# corrected malformed node command (original notes preserved above)
|
||||||
|
HF_HOME=/run/media/popov/d/DEV/models .venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen3.6-27B
|
||||||
|
|
||||||
|
# explicit Hugging Face model (namespace required to bypass tracker preset lookup)
|
||||||
|
HF_HOME=/run/media/popov/d/DEV/models .venv-rocm/bin/meshnet-node start --tracker https://meshnet.2.d-popov.com --model Qwen/Qwen3.6-27B --download-dir /run/media/popov/d/DEV/models
|
||||||
|
|||||||
@@ -106,6 +106,11 @@ def _resolve_model_flags(
|
|||||||
explicit = model_id or model
|
explicit = model_id or model
|
||||||
if not explicit:
|
if not explicit:
|
||||||
return None, None
|
return None, None
|
||||||
|
from .model_catalog import resolve_model_alias
|
||||||
|
|
||||||
|
preset = resolve_model_alias(explicit)
|
||||||
|
if preset is not None:
|
||||||
|
return preset.name, preset.hf_repo
|
||||||
if "/" in explicit:
|
if "/" in explicit:
|
||||||
return explicit.split("/")[-1], explicit
|
return explicit.split("/")[-1], explicit
|
||||||
return explicit, None
|
return explicit, None
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ class ModelPreset:
|
|||||||
vram_bf16: float
|
vram_bf16: float
|
||||||
description: str
|
description: str
|
||||||
metadata: dict | None = None
|
metadata: dict | None = None
|
||||||
|
aliases: tuple[str, ...] = ()
|
||||||
|
|
||||||
def vram_for_quant(self, quant: str) -> float:
|
def vram_for_quant(self, quant: str) -> float:
|
||||||
"""Return VRAM requirement in GB for the given quantization."""
|
"""Return VRAM requirement in GB for the given quantization."""
|
||||||
@@ -167,9 +168,29 @@ CURATED_MODELS: list[ModelPreset] = [
|
|||||||
description="Large coding-focused MoE model",
|
description="Large coding-focused MoE model",
|
||||||
metadata=_MODEL_METADATA.get("unsloth/Kimi-K2.7-Code"),
|
metadata=_MODEL_METADATA.get("unsloth/Kimi-K2.7-Code"),
|
||||||
),
|
),
|
||||||
|
ModelPreset(
|
||||||
|
name="Qwen3.6-27B",
|
||||||
|
hf_repo="Qwen/Qwen3.6-27B",
|
||||||
|
num_layers=64,
|
||||||
|
vram_nf4=13.0,
|
||||||
|
vram_int8=26.0,
|
||||||
|
vram_bf16=52.0,
|
||||||
|
description="Qwen 27B hybrid linear-attention model",
|
||||||
|
aliases=("qwen3.6-27b",),
|
||||||
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_model_alias(value: str) -> ModelPreset | None:
|
||||||
|
"""Resolve a curated name, repository, or alias case-insensitively."""
|
||||||
|
normalized = value.strip().casefold()
|
||||||
|
for model in CURATED_MODELS:
|
||||||
|
candidates = (model.name, model.hf_repo, *model.aliases)
|
||||||
|
if normalized in {candidate.casefold() for candidate in candidates}:
|
||||||
|
return model
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def layers_from_config(cfg) -> int | None:
|
def layers_from_config(cfg) -> int | None:
|
||||||
"""Extract the transformer layer count from a HuggingFace config object.
|
"""Extract the transformer layer count from a HuggingFace config object.
|
||||||
|
|
||||||
|
|||||||
@@ -631,6 +631,21 @@ def _registration_display_fields(node_name: str | None) -> dict[str, str]:
|
|||||||
return {"friendly_name": name}
|
return {"friendly_name": name}
|
||||||
|
|
||||||
|
|
||||||
|
def _tracker_http_error_message(exc: urllib.error.HTTPError) -> str:
|
||||||
|
"""Describe an HTTP rejection from the tracker, including its JSON error."""
|
||||||
|
detail = exc.reason or "request rejected"
|
||||||
|
try:
|
||||||
|
payload = json.loads(exc.read().decode("utf-8", errors="replace"))
|
||||||
|
error = payload.get("error") if isinstance(payload, dict) else None
|
||||||
|
if isinstance(error, dict):
|
||||||
|
detail = error.get("message") or error.get("code") or detail
|
||||||
|
elif error:
|
||||||
|
detail = error
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return f"Tracker rejected shard assignment (HTTP {exc.code}): {detail}"
|
||||||
|
|
||||||
|
|
||||||
def run_startup(
|
def run_startup(
|
||||||
tracker_url: str,
|
tracker_url: str,
|
||||||
port: int = 0,
|
port: int = 0,
|
||||||
@@ -1108,6 +1123,8 @@ def run_startup(
|
|||||||
})
|
})
|
||||||
try:
|
try:
|
||||||
assignment = _get_json(f"{tracker_url}/v1/nodes/assign?{assign_qs}")
|
assignment = _get_json(f"{tracker_url}/v1/nodes/assign?{assign_qs}")
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
raise RuntimeError(_tracker_http_error_message(exc)) from exc
|
||||||
except urllib.error.URLError as exc:
|
except urllib.error.URLError as exc:
|
||||||
print(f" ERROR: Cannot reach tracker at {tracker_url}: {exc}", file=sys.stderr, flush=True)
|
print(f" ERROR: Cannot reach tracker at {tracker_url}: {exc}", file=sys.stderr, flush=True)
|
||||||
raise
|
raise
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ import random
|
|||||||
import select
|
import select
|
||||||
import socketserver
|
import socketserver
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
import sys
|
||||||
import tarfile
|
import tarfile
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
@@ -6738,19 +6739,32 @@ class TrackerServer:
|
|||||||
delivered_all = False
|
delivered_all = False
|
||||||
return delivered_all
|
return delivered_all
|
||||||
|
|
||||||
|
def _save_stats_once(self) -> None:
|
||||||
|
"""Persist each store independently so one DB failure cannot kill the loop."""
|
||||||
|
registry_log = getattr(self._contracts, "registry_log", None)
|
||||||
|
stores = (
|
||||||
|
("request stats", self._stats),
|
||||||
|
("route stats", self._route_stats),
|
||||||
|
("billing", self._billing),
|
||||||
|
("accounts", self._accounts),
|
||||||
|
("registry", registry_log),
|
||||||
|
)
|
||||||
|
for label, store in stores:
|
||||||
|
if store is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
store.save_to_db()
|
||||||
|
except Exception as exc:
|
||||||
|
print(
|
||||||
|
f"[tracker] warn: {label} persistence failed: {exc}",
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|
||||||
def _stats_loop(self) -> None:
|
def _stats_loop(self) -> None:
|
||||||
"""Periodically save stats/billing to DB and push local slices to cluster peers."""
|
"""Periodically save stats/billing to DB and push local slices to cluster peers."""
|
||||||
while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
|
while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
|
||||||
if self._stats is not None:
|
self._save_stats_once()
|
||||||
self._stats.save_to_db()
|
|
||||||
self._route_stats.save_to_db()
|
|
||||||
if self._billing is not None:
|
|
||||||
self._billing.save_to_db()
|
|
||||||
if self._accounts is not None:
|
|
||||||
self._accounts.save_to_db()
|
|
||||||
registry_log = getattr(self._contracts, "registry_log", None)
|
|
||||||
if registry_log is not None:
|
|
||||||
registry_log.save_to_db()
|
|
||||||
if self._cluster_peers and not self._hive_secret:
|
if self._cluster_peers and not self._hive_secret:
|
||||||
print(
|
print(
|
||||||
"[tracker] WARNING: cluster peers configured without --hive-secret — "
|
"[tracker] WARNING: cluster peers configured without --hive-secret — "
|
||||||
|
|||||||
@@ -16,12 +16,14 @@ import pytest
|
|||||||
|
|
||||||
from meshnet_node.downloader import download_shard, write_shard_archive
|
from meshnet_node.downloader import download_shard, write_shard_archive
|
||||||
from meshnet_node.hardware import detect_hardware, benchmark_throughput
|
from meshnet_node.hardware import detect_hardware, benchmark_throughput
|
||||||
|
from meshnet_node.cli import _resolve_model_flags
|
||||||
from meshnet_node.startup import (
|
from meshnet_node.startup import (
|
||||||
_configure_torch_threads,
|
_configure_torch_threads,
|
||||||
_hardware_label,
|
_hardware_label,
|
||||||
_infer_relay_url_from_tracker,
|
_infer_relay_url_from_tracker,
|
||||||
_memory_budget,
|
_memory_budget,
|
||||||
_probationary_status_line,
|
_probationary_status_line,
|
||||||
|
_tracker_http_error_message,
|
||||||
run_startup,
|
run_startup,
|
||||||
)
|
)
|
||||||
from meshnet_node.wallet import _b58encode, load_or_create_wallet
|
from meshnet_node.wallet import _b58encode, load_or_create_wallet
|
||||||
@@ -34,6 +36,30 @@ from meshnet_tracker.server import TrackerServer
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_short_curated_model_alias_resolves_to_huggingface_repo():
|
||||||
|
"""A known short model name must use the explicit Hugging Face path."""
|
||||||
|
assert _resolve_model_flags("Qwen3.6-27B", None) == (
|
||||||
|
"Qwen3.6-27B",
|
||||||
|
"Qwen/Qwen3.6-27B",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_http_error_message_includes_rejection_body():
|
||||||
|
"""HTTP responses are tracker rejections, not connectivity failures."""
|
||||||
|
error = urllib.error.HTTPError(
|
||||||
|
"https://tracker.example/v1/nodes/assign",
|
||||||
|
404,
|
||||||
|
"Not Found",
|
||||||
|
{},
|
||||||
|
io.BytesIO(b'{"error": "unknown model preset: \'missing-model\'"}'),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert _tracker_http_error_message(error) == (
|
||||||
|
"Tracker rejected shard assignment (HTTP 404): "
|
||||||
|
"unknown model preset: 'missing-model'"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_with_forced_cpu_overrides_device_but_keeps_gpu_inventory():
|
def test_with_forced_cpu_overrides_device_but_keeps_gpu_inventory():
|
||||||
"--cpu should register and run on CPU while preserving detected GPU metadata.\n\nTags: node, startup"
|
"--cpu should register and run on CPU while preserving detected GPU metadata.\n\nTags: node, startup"
|
||||||
import meshnet_node.hardware as hardware_mod
|
import meshnet_node.hardware as hardware_mod
|
||||||
|
|||||||
31
tests/test_tracker_persistence.py
Normal file
31
tests/test_tracker_persistence.py
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
from meshnet_tracker.server import TrackerServer
|
||||||
|
|
||||||
|
|
||||||
|
class _FailingStore:
|
||||||
|
def save_to_db(self) -> None:
|
||||||
|
raise OSError("database or disk is full")
|
||||||
|
|
||||||
|
|
||||||
|
class _RecordingStore:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.saved = False
|
||||||
|
|
||||||
|
def save_to_db(self) -> None:
|
||||||
|
self.saved = True
|
||||||
|
|
||||||
|
|
||||||
|
def test_stats_persistence_failure_does_not_abort_remaining_stores(capsys):
|
||||||
|
"Tracker persistence isolates one full-disk failure from other stores\n\nTags: tracker, persistence"
|
||||||
|
tracker = TrackerServer()
|
||||||
|
tracker._stats = _FailingStore()
|
||||||
|
tracker._route_stats = _RecordingStore()
|
||||||
|
tracker._billing = None
|
||||||
|
tracker._accounts = None
|
||||||
|
tracker._contracts = None
|
||||||
|
|
||||||
|
tracker._save_stats_once()
|
||||||
|
|
||||||
|
assert tracker._route_stats.saved is True
|
||||||
|
stderr = capsys.readouterr().err
|
||||||
|
assert "stats persistence failed" in stderr
|
||||||
|
assert "database or disk is full" in stderr
|
||||||
Reference in New Issue
Block a user