Load recommended model metadata from JSON
This commit is contained in:
@@ -86,12 +86,18 @@ python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
|
|||||||
|
|
||||||
If you get `ModuleNotFoundError: No module named 'torch'` even though `pip install torch`
|
If you get `ModuleNotFoundError: No module named 'torch'` even though `pip install torch`
|
||||||
says "already satisfied", the `torch/` package directory is missing while the metadata
|
says "already satisfied", the `torch/` package directory is missing while the metadata
|
||||||
stub remains (can happen after a conda-managed install). Force-reinstall via pip:
|
stub remains (can happen after a conda-managed install). Force-reinstall all three
|
||||||
|
PyTorch packages together so their versions stay in sync:
|
||||||
|
|
||||||
```powershell
|
```powershell
|
||||||
pip install --force-reinstall torch --index-url https://download.pytorch.org/whl/cu118
|
pip install --force-reinstall torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
|
||||||
```
|
```
|
||||||
|
|
||||||
|
> **Important:** always reinstall `torch`, `torchvision`, and `torchaudio` as a group.
|
||||||
|
> Upgrading only `torch` leaves `torchvision` on an incompatible version, which causes
|
||||||
|
> `RuntimeError: operator torchvision::nms does not exist` and makes transformers fail
|
||||||
|
> to import any model class (the error surfaces as `Could not import module 'Qwen2ForCausalLM'`).
|
||||||
|
|
||||||
Then re-run the verify step above.
|
Then re-run the verify step above.
|
||||||
|
|
||||||
If that prints `True` but `meshnet-node` still can't find torch, the venv entry point
|
If that prints `True` but `meshnet-node` still can't find torch, the venv entry point
|
||||||
|
|||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from importlib.resources import files
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
@@ -43,6 +45,25 @@ class ModelPreset:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _load_model_metadata() -> dict[str, dict]:
|
||||||
|
try:
|
||||||
|
raw = files("meshnet_node").joinpath("model_metadata.json").read_text()
|
||||||
|
data = json.loads(raw)
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
models = data.get("models", {})
|
||||||
|
if not isinstance(models, dict):
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
str(repo): metadata
|
||||||
|
for repo, metadata in models.items()
|
||||||
|
if isinstance(metadata, dict)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
_MODEL_METADATA = _load_model_metadata()
|
||||||
|
|
||||||
|
|
||||||
CURATED_MODELS: list[ModelPreset] = [
|
CURATED_MODELS: list[ModelPreset] = [
|
||||||
ModelPreset(
|
ModelPreset(
|
||||||
name="Qwen2.5-0.5B-Instruct",
|
name="Qwen2.5-0.5B-Instruct",
|
||||||
@@ -132,29 +153,8 @@ CURATED_MODELS: list[ModelPreset] = [
|
|||||||
vram_nf4=500.0,
|
vram_nf4=500.0,
|
||||||
vram_int8=1000.0,
|
vram_int8=1000.0,
|
||||||
vram_bf16=2000.0,
|
vram_bf16=2000.0,
|
||||||
description="Moonshot/Unsloth coding-focused MoE model; 1T total, 32B activated",
|
description="Large coding-focused MoE model",
|
||||||
metadata={
|
metadata=_MODEL_METADATA.get("unsloth/Kimi-K2.7-Code"),
|
||||||
"architecture": "Mixture-of-Experts (MoE)",
|
|
||||||
"total_parameters": "1T",
|
|
||||||
"activated_parameters": "32B",
|
|
||||||
"num_layers": 61,
|
|
||||||
"dense_layers": 1,
|
|
||||||
"attention_hidden_dimension": 7168,
|
|
||||||
"moe_hidden_dimension_per_expert": 2048,
|
|
||||||
"attention_heads": 64,
|
|
||||||
"experts": 384,
|
|
||||||
"selected_experts_per_token": 8,
|
|
||||||
"shared_experts": 1,
|
|
||||||
"vocabulary_size": 160000,
|
|
||||||
"context_length": 256000,
|
|
||||||
"attention_mechanism": "MLA",
|
|
||||||
"activation_function": "SwiGLU",
|
|
||||||
"vision_encoder": "MoonViT",
|
|
||||||
"vision_encoder_parameters": "400M",
|
|
||||||
"license": "modified-mit",
|
|
||||||
"native_quantization": "int4",
|
|
||||||
"recommended_engines": ["vLLM", "SGLang", "KTransformers"],
|
|
||||||
},
|
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
32
packages/node/meshnet_node/model_metadata.json
Normal file
32
packages/node/meshnet_node/model_metadata.json
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
{
|
||||||
|
"models": {
|
||||||
|
"unsloth/Kimi-K2.7-Code": {
|
||||||
|
"architecture": "Mixture-of-Experts (MoE)",
|
||||||
|
"total_parameters": "1T",
|
||||||
|
"activated_parameters": "32B",
|
||||||
|
"num_layers": 61,
|
||||||
|
"dense_layers": 1,
|
||||||
|
"attention_hidden_dimension": 7168,
|
||||||
|
"moe_hidden_dimension_per_expert": 2048,
|
||||||
|
"attention_heads": 64,
|
||||||
|
"experts": 384,
|
||||||
|
"selected_experts_per_token": 8,
|
||||||
|
"shared_experts": 1,
|
||||||
|
"vocabulary_size": 160000,
|
||||||
|
"context_length": 256000,
|
||||||
|
"attention_mechanism": "MLA",
|
||||||
|
"activation_function": "SwiGLU",
|
||||||
|
"vision_encoder": "MoonViT",
|
||||||
|
"vision_encoder_parameters": "400M",
|
||||||
|
"license": "modified-mit",
|
||||||
|
"native_quantization": "int4",
|
||||||
|
"download_size_gb": 595,
|
||||||
|
"recommended_short_name": "kimi-k2.7",
|
||||||
|
"recommended_engines": [
|
||||||
|
"vLLM",
|
||||||
|
"SGLang",
|
||||||
|
"KTransformers"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,3 +27,6 @@ meshnet-node = "meshnet_node.cli:main"
|
|||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
where = ["."]
|
where = ["."]
|
||||||
include = ["meshnet_node*"]
|
include = ["meshnet_node*"]
|
||||||
|
|
||||||
|
[tool.setuptools.package-data]
|
||||||
|
meshnet_node = ["*.json"]
|
||||||
|
|||||||
37
packages/tracker/meshnet_tracker/model_presets.json
Normal file
37
packages/tracker/meshnet_tracker/model_presets.json
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
{
|
||||||
|
"models": {
|
||||||
|
"kimi-k2.7": {
|
||||||
|
"layers_start": 0,
|
||||||
|
"layers_end": 60,
|
||||||
|
"hf_repo": "unsloth/Kimi-K2.7-Code",
|
||||||
|
"aliases": [
|
||||||
|
"kimi-k2.7",
|
||||||
|
"Kimi-K2.7-Code",
|
||||||
|
"unsloth/Kimi-K2.7-Code"
|
||||||
|
],
|
||||||
|
"recommended": true,
|
||||||
|
"deployment_status": "recommended",
|
||||||
|
"required_model_bytes": 638876385280,
|
||||||
|
"download_size_bytes": 638876385280,
|
||||||
|
"native_quantization": "int4",
|
||||||
|
"bytes_per_layer": {
|
||||||
|
"int4": 10473383366
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"architecture": "Mixture-of-Experts (MoE)",
|
||||||
|
"total_parameters": "1T",
|
||||||
|
"activated_parameters": "32B",
|
||||||
|
"num_layers": 61,
|
||||||
|
"context_length": 256000,
|
||||||
|
"native_quantization": "int4",
|
||||||
|
"download_size_gb": 595,
|
||||||
|
"recommended_short_name": "kimi-k2.7",
|
||||||
|
"recommended_engines": [
|
||||||
|
"vLLM",
|
||||||
|
"SGLang",
|
||||||
|
"KTransformers"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -31,6 +31,7 @@ import time
|
|||||||
import urllib.parse
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
import uuid
|
import uuid
|
||||||
|
from importlib.resources import files
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from .gossip import NodeGossip
|
from .gossip import NodeGossip
|
||||||
@@ -50,6 +51,23 @@ def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
|
|||||||
return f"wss://{parsed.netloc}/ws"
|
return f"wss://{parsed.netloc}/ws"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_model_presets_from_data() -> dict[str, dict]:
|
||||||
|
"""Load recommended model presets from package JSON data."""
|
||||||
|
try:
|
||||||
|
raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
|
||||||
|
data = json.loads(raw)
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
models = data.get("models", {})
|
||||||
|
if not isinstance(models, dict):
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
str(name): preset
|
||||||
|
for name, preset in models.items()
|
||||||
|
if isinstance(preset, dict)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_MODEL_PRESETS: dict[str, dict] = {
|
DEFAULT_MODEL_PRESETS: dict[str, dict] = {
|
||||||
"stub-model": {
|
"stub-model": {
|
||||||
"layers_start": 0,
|
"layers_start": 0,
|
||||||
@@ -61,6 +79,7 @@ DEFAULT_MODEL_PRESETS: dict[str, dict] = {
|
|||||||
"layers_end": 11,
|
"layers_end": 11,
|
||||||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
|
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
|
||||||
},
|
},
|
||||||
|
**_load_model_presets_from_data(),
|
||||||
}
|
}
|
||||||
|
|
||||||
DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
|
DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
|
||||||
@@ -83,6 +102,27 @@ def _model_aliases(model: str | None) -> set[str]:
|
|||||||
return aliases
|
return aliases
|
||||||
|
|
||||||
|
|
||||||
|
def _preset_aliases(name: str, preset: dict | None) -> set[str]:
|
||||||
|
aliases = _model_aliases(name)
|
||||||
|
if not preset:
|
||||||
|
return aliases
|
||||||
|
hf_repo = preset.get("hf_repo")
|
||||||
|
if isinstance(hf_repo, str):
|
||||||
|
aliases |= _model_aliases(hf_repo)
|
||||||
|
for alias in preset.get("aliases", []) or []:
|
||||||
|
if isinstance(alias, str):
|
||||||
|
aliases |= _model_aliases(alias)
|
||||||
|
return aliases
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
|
||||||
|
requested = _model_aliases(model)
|
||||||
|
for name, preset in model_presets.items():
|
||||||
|
if requested & _preset_aliases(name, preset):
|
||||||
|
return name, preset
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
|
||||||
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
|
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
|
||||||
requested = _model_aliases(model)
|
requested = _model_aliases(model)
|
||||||
if not requested:
|
if not requested:
|
||||||
@@ -90,6 +130,11 @@ def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
|
|||||||
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
|
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
|
||||||
|
|
||||||
|
|
||||||
|
def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
|
||||||
|
requested = _preset_aliases(name, preset)
|
||||||
|
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
|
||||||
|
|
||||||
|
|
||||||
class _RollingCounter:
|
class _RollingCounter:
|
||||||
"""Circular-bucket request counter.
|
"""Circular-bucket request counter.
|
||||||
|
|
||||||
@@ -469,6 +514,60 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
|
|||||||
return summary
|
return summary
|
||||||
|
|
||||||
|
|
||||||
|
def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
|
||||||
|
budget, _source = _node_memory_budget_bytes(node)
|
||||||
|
if preset is None:
|
||||||
|
return int(budget * 0.8)
|
||||||
|
return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))
|
||||||
|
|
||||||
|
|
||||||
|
def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
|
||||||
|
total_vram = sum(max(0, node.vram_bytes) for node in nodes)
|
||||||
|
total_ram = sum(max(0, node.ram_bytes) for node in nodes)
|
||||||
|
total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
|
||||||
|
effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
|
||||||
|
return {
|
||||||
|
"node_count": len(nodes),
|
||||||
|
"total_vram_bytes": total_vram,
|
||||||
|
"total_ram_bytes": total_ram,
|
||||||
|
"total_memory_budget_bytes": total_budget,
|
||||||
|
"effective_assignable_memory_bytes": effective_budget,
|
||||||
|
"total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
|
||||||
|
"total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
|
||||||
|
if preset is None:
|
||||||
|
return {"recommended": False}
|
||||||
|
pool = _pool_summary(nodes, preset)
|
||||||
|
required = int(preset.get("required_model_bytes", 0) or 0)
|
||||||
|
deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
|
||||||
|
missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
|
||||||
|
return {
|
||||||
|
"recommended": bool(preset.get("recommended", False)),
|
||||||
|
"status": preset.get("deployment_status", "available"),
|
||||||
|
"required_model_bytes": required or None,
|
||||||
|
"download_size_bytes": preset.get("download_size_bytes"),
|
||||||
|
"native_quantization": preset.get("native_quantization"),
|
||||||
|
"pool": pool,
|
||||||
|
"deployable": deployable,
|
||||||
|
"missing_effective_memory_bytes": missing,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
|
||||||
|
if total_layers <= 0:
|
||||||
|
return 0
|
||||||
|
if memory_mb <= 0:
|
||||||
|
return max(1, total_layers // 2)
|
||||||
|
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
|
||||||
|
return min(
|
||||||
|
total_layers,
|
||||||
|
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
|
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
|
||||||
metadata: dict = {}
|
metadata: dict = {}
|
||||||
for node in nodes:
|
for node in nodes:
|
||||||
@@ -622,10 +721,13 @@ def _nodes_and_bounds_for_model(
|
|||||||
server: "_TrackerHTTPServer",
|
server: "_TrackerHTTPServer",
|
||||||
model: str,
|
model: str,
|
||||||
) -> tuple[list[_NodeEntry], int, int] | None:
|
) -> tuple[list[_NodeEntry], int, int] | None:
|
||||||
preset = server.model_presets.get(model)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
if preset is not None:
|
if preset is not None:
|
||||||
required_start, required_end = _preset_layer_bounds(preset)
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
return [node for node in server.registry.values() if node.model == model], required_start, required_end
|
return [
|
||||||
|
node for node in server.registry.values()
|
||||||
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
|
], required_start, required_end
|
||||||
|
|
||||||
nodes = [
|
nodes = [
|
||||||
node for node in server.registry.values()
|
node for node in server.registry.values()
|
||||||
@@ -682,12 +784,15 @@ def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
|
|||||||
|
|
||||||
|
|
||||||
def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
|
def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
|
||||||
preset = server.model_presets.get(model)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
if preset is None:
|
if preset is None:
|
||||||
return
|
return
|
||||||
required_start, required_end = _preset_layer_bounds(preset)
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
total_layers = required_end - required_start + 1
|
total_layers = required_end - required_start + 1
|
||||||
model_nodes = [node for node in server.registry.values() if node.model == model]
|
model_nodes = [
|
||||||
|
node for node in server.registry.values()
|
||||||
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
|
]
|
||||||
managed_nodes = [node for node in model_nodes if node.managed_assignment]
|
managed_nodes = [node for node in model_nodes if node.managed_assignment]
|
||||||
if not managed_nodes:
|
if not managed_nodes:
|
||||||
return
|
return
|
||||||
@@ -1032,8 +1137,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
data = []
|
data = []
|
||||||
seen_ids: set[str] = set()
|
seen_ids: set[str] = set()
|
||||||
for name, preset in server.model_presets.items():
|
for name, preset in server.model_presets.items():
|
||||||
model_nodes = [node for node in alive if node.model == name]
|
model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
|
||||||
if not model_nodes:
|
if not model_nodes and not preset.get("recommended"):
|
||||||
continue
|
continue
|
||||||
required_start, required_end = _preset_layer_bounds(preset)
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
coverage = _coverage_percentage(
|
coverage = _coverage_percentage(
|
||||||
@@ -1045,6 +1150,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
hf_repo = preset.get("hf_repo")
|
hf_repo = preset.get("hf_repo")
|
||||||
if hf_repo and hf_repo not in aliases:
|
if hf_repo and hf_repo not in aliases:
|
||||||
aliases.append(hf_repo)
|
aliases.append(hf_repo)
|
||||||
|
for alias in preset.get("aliases", []) or []:
|
||||||
|
if isinstance(alias, str) and alias not in aliases:
|
||||||
|
aliases.append(alias)
|
||||||
data.append({
|
data.append({
|
||||||
"id": name,
|
"id": name,
|
||||||
"object": "model",
|
"object": "model",
|
||||||
@@ -1054,9 +1162,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"hf_repo": hf_repo,
|
"hf_repo": hf_repo,
|
||||||
"aliases": aliases,
|
"aliases": aliases,
|
||||||
"metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
|
"metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
|
||||||
|
"recommended": bool(preset.get("recommended", False)),
|
||||||
|
"deployment": _deployment_summary(alive, preset),
|
||||||
"shard_coverage_percentage": coverage,
|
"shard_coverage_percentage": coverage,
|
||||||
})
|
})
|
||||||
seen_ids.add(name)
|
seen_ids.add(name)
|
||||||
|
if hf_repo:
|
||||||
|
seen_ids.add(hf_repo)
|
||||||
|
|
||||||
hf_model_ids = sorted({
|
hf_model_ids = sorted({
|
||||||
node.hf_repo or node.model
|
node.hf_repo or node.model
|
||||||
@@ -1122,14 +1234,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
def _handle_tracker_nodes(self, model: str):
|
def _handle_tracker_nodes(self, model: str):
|
||||||
"""Return nodes registered with tracker_mode=True whose shard starts at layer 0."""
|
"""Return nodes registered with tracker_mode=True whose shard starts at layer 0."""
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
preset = server.model_presets.get(model)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
if preset is None:
|
if preset is None:
|
||||||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||||||
return
|
return
|
||||||
required_start, _ = _preset_layer_bounds(preset)
|
required_start, _ = _preset_layer_bounds(preset)
|
||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
alive = [node for node in server.registry.values() if node.model == model]
|
alive = [
|
||||||
|
node for node in server.registry.values()
|
||||||
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
|
]
|
||||||
if server.contracts is not None:
|
if server.contracts is not None:
|
||||||
alive = [
|
alive = [
|
||||||
node for node in alive
|
node for node in alive
|
||||||
@@ -1142,7 +1257,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
and node.tracker_mode
|
and node.tracker_mode
|
||||||
]
|
]
|
||||||
self._send_json(200, {
|
self._send_json(200, {
|
||||||
"model": model,
|
"model": resolved_name,
|
||||||
"tracker_nodes": [
|
"tracker_nodes": [
|
||||||
{
|
{
|
||||||
"node_id": node.node_id,
|
"node_id": node.node_id,
|
||||||
@@ -1171,6 +1286,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
self._send_json(200, {
|
self._send_json(200, {
|
||||||
"relay_url": server.relay_url,
|
"relay_url": server.relay_url,
|
||||||
|
"pool": _pool_summary(nodes),
|
||||||
|
"recommended_models": [
|
||||||
|
{
|
||||||
|
"id": name,
|
||||||
|
"hf_repo": preset.get("hf_repo"),
|
||||||
|
"aliases": list(preset.get("aliases", []) or []),
|
||||||
|
"metadata": dict(preset.get("metadata") or {}),
|
||||||
|
"deployment": _deployment_summary(nodes, preset),
|
||||||
|
}
|
||||||
|
for name, preset in server.model_presets.items()
|
||||||
|
if preset.get("recommended")
|
||||||
|
],
|
||||||
"nodes": [
|
"nodes": [
|
||||||
{
|
{
|
||||||
"node_id": node.node_id,
|
"node_id": node.node_id,
|
||||||
@@ -1768,7 +1895,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
else:
|
else:
|
||||||
model = model_list[0]
|
model = model_list[0]
|
||||||
|
|
||||||
preset = server.model_presets.get(model)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
if preset is None:
|
if preset is None:
|
||||||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||||||
return
|
return
|
||||||
@@ -1777,7 +1904,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
alive = [node for node in server.registry.values() if node.model == model]
|
alive = [
|
||||||
|
node for node in server.registry.values()
|
||||||
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
|
]
|
||||||
if server.contracts is not None:
|
if server.contracts is not None:
|
||||||
alive = [
|
alive = [
|
||||||
node for node in alive
|
node for node in alive
|
||||||
@@ -1830,7 +1960,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
peers = [
|
peers = [
|
||||||
{"endpoint": node.endpoint, "checksum": node.shard_checksum}
|
{"endpoint": node.endpoint, "checksum": node.shard_checksum}
|
||||||
for node in alive
|
for node in alive
|
||||||
if node.model == model
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
and node.shard_start == shard_start
|
and node.shard_start == shard_start
|
||||||
and node.shard_end == shard_end
|
and node.shard_end == shard_end
|
||||||
and node.shard_checksum
|
and node.shard_checksum
|
||||||
@@ -1839,7 +1969,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._send_json(200, {
|
self._send_json(200, {
|
||||||
"shard_start": shard_start,
|
"shard_start": shard_start,
|
||||||
"shard_end": shard_end,
|
"shard_end": shard_end,
|
||||||
"model": model,
|
"model": resolved_name,
|
||||||
"model_layers_end": required_end,
|
"model_layers_end": required_end,
|
||||||
"peers": peers,
|
"peers": peers,
|
||||||
**({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
|
**({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
|
||||||
@@ -1886,6 +2016,37 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
]
|
]
|
||||||
|
|
||||||
if not hf_nodes:
|
if not hf_nodes:
|
||||||
|
resolved_name = None
|
||||||
|
preset = None
|
||||||
|
if filter_repo:
|
||||||
|
resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
|
||||||
|
else:
|
||||||
|
deployable = [
|
||||||
|
(name, preset)
|
||||||
|
for name, preset in server.model_presets.items()
|
||||||
|
if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"]
|
||||||
|
]
|
||||||
|
if deployable:
|
||||||
|
resolved_name, preset = deployable[0]
|
||||||
|
if preset is not None and preset.get("hf_repo"):
|
||||||
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
|
total_l = required_end - required_start + 1
|
||||||
|
memory_mb = vram_mb if vram_mb > 0 else ram_mb
|
||||||
|
max_layers = _max_layers_for_memory(memory_mb, total_l, preset)
|
||||||
|
shard_start = required_start
|
||||||
|
shard_end = min(required_end, shard_start + max_layers - 1)
|
||||||
|
self._send_json(200, {
|
||||||
|
"hf_repo": preset["hf_repo"],
|
||||||
|
"model": resolved_name,
|
||||||
|
"shard_start": shard_start,
|
||||||
|
"shard_end": shard_end,
|
||||||
|
"num_layers": total_l,
|
||||||
|
"gap_found": True,
|
||||||
|
"price_per_token": 0.0,
|
||||||
|
"deployment": _deployment_summary(all_nodes, preset),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
|
||||||
msg = (
|
msg = (
|
||||||
f"no HF-model nodes registered for {filter_repo!r}"
|
f"no HF-model nodes registered for {filter_repo!r}"
|
||||||
if filter_repo
|
if filter_repo
|
||||||
@@ -1954,15 +2115,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
# Capacity: use the same 80%-of-memory rule as registered node planning.
|
# Capacity: use the same 80%-of-memory rule as registered node planning.
|
||||||
total_l = best_num_layers
|
total_l = best_num_layers
|
||||||
memory_mb = vram_mb if vram_mb > 0 else ram_mb
|
memory_mb = vram_mb if vram_mb > 0 else ram_mb
|
||||||
|
_resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
|
||||||
if memory_mb > 0:
|
if memory_mb > 0:
|
||||||
max_layers = min(
|
max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
|
||||||
total_l,
|
|
||||||
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // (30 * 1024 * 1024))),
|
|
||||||
)
|
|
||||||
elif device == "cuda" and vram_mb >= 8192:
|
elif device == "cuda" and vram_mb >= 8192:
|
||||||
max_layers = total_l
|
max_layers = total_l
|
||||||
else:
|
else:
|
||||||
max_layers = max(1, total_l // 2)
|
max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
|
||||||
|
|
||||||
shard_start = best_gap_start
|
shard_start = best_gap_start
|
||||||
shard_end = min(total_l - 1, shard_start + max_layers - 1)
|
shard_end = min(total_l - 1, shard_start + max_layers - 1)
|
||||||
@@ -1985,13 +2144,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
|
|
||||||
model = model_list[0]
|
model = model_list[0]
|
||||||
preset = server.model_presets.get(model)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
|
|
||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
if preset is not None:
|
if preset is not None:
|
||||||
# Preset-based routing (stub-model system).
|
# Preset-based routing (stub-model system).
|
||||||
alive = [node for node in server.registry.values() if node.model == model]
|
alive = [
|
||||||
|
node for node in server.registry.values()
|
||||||
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
|
]
|
||||||
required_start, required_end = _preset_layer_bounds(preset)
|
required_start, required_end = _preset_layer_bounds(preset)
|
||||||
else:
|
else:
|
||||||
# HF model routing: match by hf_repo (full) or model short name.
|
# HF model routing: match by hf_repo (full) or model short name.
|
||||||
@@ -2066,7 +2228,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
|
|
||||||
model = model_list[0]
|
model = model_list[0]
|
||||||
preset = server.model_presets.get(model)
|
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||||||
if preset is None:
|
if preset is None:
|
||||||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||||||
return
|
return
|
||||||
@@ -2075,7 +2237,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
|
|
||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
candidates = [node for node in server.registry.values() if node.model == model]
|
candidates = [
|
||||||
|
node for node in server.registry.values()
|
||||||
|
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||||||
|
]
|
||||||
if server.contracts is not None:
|
if server.contracts is not None:
|
||||||
candidates = [
|
candidates = [
|
||||||
node for node in candidates
|
node for node in candidates
|
||||||
|
|||||||
@@ -18,3 +18,6 @@ meshnet-tracker = "meshnet_tracker.cli:main"
|
|||||||
[tool.setuptools.packages.find]
|
[tool.setuptools.packages.find]
|
||||||
where = ["."]
|
where = ["."]
|
||||||
include = ["meshnet_tracker*"]
|
include = ["meshnet_tracker*"]
|
||||||
|
|
||||||
|
[tool.setuptools.package-data]
|
||||||
|
meshnet_tracker = ["*.json"]
|
||||||
|
|||||||
@@ -78,7 +78,8 @@ def test_tracker_exposes_registered_model_metadata():
|
|||||||
finally:
|
finally:
|
||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
kimi = next(model for model in models["data"] if model["id"] == "unsloth/Kimi-K2.7-Code")
|
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
|
||||||
|
assert kimi["hf_repo"] == "unsloth/Kimi-K2.7-Code"
|
||||||
assert kimi["metadata"]["total_parameters"] == "1T"
|
assert kimi["metadata"]["total_parameters"] == "1T"
|
||||||
assert kimi["metadata"]["activated_parameters"] == "32B"
|
assert kimi["metadata"]["activated_parameters"] == "32B"
|
||||||
assert kimi["metadata"]["num_layers"] == 61
|
assert kimi["metadata"]["num_layers"] == 61
|
||||||
@@ -87,6 +88,113 @@ def test_tracker_exposes_registered_model_metadata():
|
|||||||
assert registered["model_metadata"]["context_length"] == 256000
|
assert registered["model_metadata"]["context_length"] == 256000
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_lists_recommended_kimi_before_nodes_register():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
url = f"http://127.0.0.1:{port}"
|
||||||
|
try:
|
||||||
|
models = _get_json(f"{url}/v1/models")
|
||||||
|
network_map = _get_json(f"{url}/v1/network/map")
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
|
||||||
|
assert kimi["hf_repo"] == "unsloth/Kimi-K2.7-Code"
|
||||||
|
assert "Kimi-K2.7-Code" in kimi["aliases"]
|
||||||
|
assert kimi["metadata"]["download_size_gb"] == 595
|
||||||
|
assert kimi["deployment"]["native_quantization"] == "int4"
|
||||||
|
assert kimi["deployment"]["deployable"] is False
|
||||||
|
assert network_map["pool"]["node_count"] == 0
|
||||||
|
assert network_map["recommended_models"][0]["id"] == "kimi-k2.7"
|
||||||
|
|
||||||
|
|
||||||
|
def test_network_map_exposes_pool_size_and_speed_summary():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
url = f"http://127.0.0.1:{port}"
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"{url}/v1/nodes/register",
|
||||||
|
{
|
||||||
|
"endpoint": "http://127.0.0.1:7101",
|
||||||
|
"model": "inventory-a",
|
||||||
|
"vram_bytes": 10_000,
|
||||||
|
"ram_bytes": 20_000,
|
||||||
|
"benchmark_tokens_per_sec": 4.0,
|
||||||
|
"hardware_profile": {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_post_json(
|
||||||
|
f"{url}/v1/nodes/register",
|
||||||
|
{
|
||||||
|
"endpoint": "http://127.0.0.1:7102",
|
||||||
|
"model": "inventory-b",
|
||||||
|
"vram_bytes": 0,
|
||||||
|
"ram_bytes": 30_000,
|
||||||
|
"benchmark_tokens_per_sec": 6.0,
|
||||||
|
"hardware_profile": {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
network_map = _get_json(f"{url}/v1/network/map")
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
pool = network_map["pool"]
|
||||||
|
assert pool["node_count"] == 2
|
||||||
|
assert pool["total_vram_bytes"] == 10_000
|
||||||
|
assert pool["total_ram_bytes"] == 50_000
|
||||||
|
assert pool["total_memory_budget_bytes"] == 40_000
|
||||||
|
assert pool["total_benchmark_tokens_per_sec"] == 10.0
|
||||||
|
assert pool["total_effective_throughput"] == 10.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
url = f"http://127.0.0.1:{port}"
|
||||||
|
gib = 1024 * 1024 * 1024
|
||||||
|
try:
|
||||||
|
for idx in range(2):
|
||||||
|
_post_json(
|
||||||
|
f"{url}/v1/nodes/register",
|
||||||
|
{
|
||||||
|
"endpoint": f"http://127.0.0.1:{7200 + idx}",
|
||||||
|
"model": f"inventory-{idx}",
|
||||||
|
"vram_bytes": 0,
|
||||||
|
"ram_bytes": 400 * gib,
|
||||||
|
"benchmark_tokens_per_sec": 5.0,
|
||||||
|
"hardware_profile": {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
models = _get_json(f"{url}/v1/models")
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
|
||||||
|
assert kimi["deployment"]["deployable"] is True
|
||||||
|
assert kimi["deployment"]["missing_effective_memory_bytes"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_network_assign_can_start_recommended_kimi_by_short_name():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
url = f"http://127.0.0.1:{port}"
|
||||||
|
try:
|
||||||
|
assignment = _get_json(
|
||||||
|
f"{url}/v1/network/assign?device=cpu&ram_mb=204800&hf_repo=kimi-k2.7"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert assignment["model"] == "kimi-k2.7"
|
||||||
|
assert assignment["hf_repo"] == "unsloth/Kimi-K2.7-Code"
|
||||||
|
assert assignment["num_layers"] == 61
|
||||||
|
assert assignment["shard_start"] == 0
|
||||||
|
assert 0 <= assignment["shard_end"] < 60
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_serves_health_while_proxy_request_is_in_flight():
|
def test_tracker_serves_health_while_proxy_request_is_in_flight():
|
||||||
"""Long inference proxy requests must not block heartbeats/health checks."""
|
"""Long inference proxy requests must not block heartbeats/health checks."""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user