Load recommended model metadata from JSON

This commit is contained in:
Dobromir Popov
2026-07-01 12:57:23 +02:00
parent bc760c1694
commit b035338e58
8 changed files with 402 additions and 48 deletions

View File

@@ -86,12 +86,18 @@ python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
If you get `ModuleNotFoundError: No module named 'torch'` even though `pip install torch` If you get `ModuleNotFoundError: No module named 'torch'` even though `pip install torch`
says "already satisfied", the `torch/` package directory is missing while the metadata says "already satisfied", the `torch/` package directory is missing while the metadata
stub remains (can happen after a conda-managed install). Force-reinstall via pip: stub remains (can happen after a conda-managed install). Force-reinstall all three
PyTorch packages together so their versions stay in sync:
```powershell ```powershell
pip install --force-reinstall torch --index-url https://download.pytorch.org/whl/cu118 pip install --force-reinstall torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118
``` ```
> **Important:** always reinstall `torch`, `torchvision`, and `torchaudio` as a group.
> Upgrading only `torch` leaves `torchvision` on an incompatible version, which causes
> `RuntimeError: operator torchvision::nms does not exist` and makes transformers fail
> to import any model class (the error surfaces as `Could not import module 'Qwen2ForCausalLM'`).
Then re-run the verify step above. Then re-run the verify step above.
If that prints `True` but `meshnet-node` still can't find torch, the venv entry point If that prints `True` but `meshnet-node` still can't find torch, the venv entry point

View File

@@ -2,7 +2,9 @@
from __future__ import annotations from __future__ import annotations
import json
from dataclasses import dataclass from dataclasses import dataclass
from importlib.resources import files
from pathlib import Path from pathlib import Path
@@ -43,6 +45,25 @@ class ModelPreset:
return None return None
def _load_model_metadata() -> dict[str, dict]:
try:
raw = files("meshnet_node").joinpath("model_metadata.json").read_text()
data = json.loads(raw)
except Exception:
return {}
models = data.get("models", {})
if not isinstance(models, dict):
return {}
return {
str(repo): metadata
for repo, metadata in models.items()
if isinstance(metadata, dict)
}
_MODEL_METADATA = _load_model_metadata()
CURATED_MODELS: list[ModelPreset] = [ CURATED_MODELS: list[ModelPreset] = [
ModelPreset( ModelPreset(
name="Qwen2.5-0.5B-Instruct", name="Qwen2.5-0.5B-Instruct",
@@ -132,29 +153,8 @@ CURATED_MODELS: list[ModelPreset] = [
vram_nf4=500.0, vram_nf4=500.0,
vram_int8=1000.0, vram_int8=1000.0,
vram_bf16=2000.0, vram_bf16=2000.0,
description="Moonshot/Unsloth coding-focused MoE model; 1T total, 32B activated", description="Large coding-focused MoE model",
metadata={ metadata=_MODEL_METADATA.get("unsloth/Kimi-K2.7-Code"),
"architecture": "Mixture-of-Experts (MoE)",
"total_parameters": "1T",
"activated_parameters": "32B",
"num_layers": 61,
"dense_layers": 1,
"attention_hidden_dimension": 7168,
"moe_hidden_dimension_per_expert": 2048,
"attention_heads": 64,
"experts": 384,
"selected_experts_per_token": 8,
"shared_experts": 1,
"vocabulary_size": 160000,
"context_length": 256000,
"attention_mechanism": "MLA",
"activation_function": "SwiGLU",
"vision_encoder": "MoonViT",
"vision_encoder_parameters": "400M",
"license": "modified-mit",
"native_quantization": "int4",
"recommended_engines": ["vLLM", "SGLang", "KTransformers"],
},
), ),
] ]

View File

@@ -0,0 +1,32 @@
{
"models": {
"unsloth/Kimi-K2.7-Code": {
"architecture": "Mixture-of-Experts (MoE)",
"total_parameters": "1T",
"activated_parameters": "32B",
"num_layers": 61,
"dense_layers": 1,
"attention_hidden_dimension": 7168,
"moe_hidden_dimension_per_expert": 2048,
"attention_heads": 64,
"experts": 384,
"selected_experts_per_token": 8,
"shared_experts": 1,
"vocabulary_size": 160000,
"context_length": 256000,
"attention_mechanism": "MLA",
"activation_function": "SwiGLU",
"vision_encoder": "MoonViT",
"vision_encoder_parameters": "400M",
"license": "modified-mit",
"native_quantization": "int4",
"download_size_gb": 595,
"recommended_short_name": "kimi-k2.7",
"recommended_engines": [
"vLLM",
"SGLang",
"KTransformers"
]
}
}
}

View File

@@ -27,3 +27,6 @@ meshnet-node = "meshnet_node.cli:main"
[tool.setuptools.packages.find] [tool.setuptools.packages.find]
where = ["."] where = ["."]
include = ["meshnet_node*"] include = ["meshnet_node*"]
[tool.setuptools.package-data]
meshnet_node = ["*.json"]

View File

@@ -0,0 +1,37 @@
{
"models": {
"kimi-k2.7": {
"layers_start": 0,
"layers_end": 60,
"hf_repo": "unsloth/Kimi-K2.7-Code",
"aliases": [
"kimi-k2.7",
"Kimi-K2.7-Code",
"unsloth/Kimi-K2.7-Code"
],
"recommended": true,
"deployment_status": "recommended",
"required_model_bytes": 638876385280,
"download_size_bytes": 638876385280,
"native_quantization": "int4",
"bytes_per_layer": {
"int4": 10473383366
},
"metadata": {
"architecture": "Mixture-of-Experts (MoE)",
"total_parameters": "1T",
"activated_parameters": "32B",
"num_layers": 61,
"context_length": 256000,
"native_quantization": "int4",
"download_size_gb": 595,
"recommended_short_name": "kimi-k2.7",
"recommended_engines": [
"vLLM",
"SGLang",
"KTransformers"
]
}
}
}
}

View File

@@ -31,6 +31,7 @@ import time
import urllib.parse import urllib.parse
import urllib.request import urllib.request
import uuid import uuid
from importlib.resources import files
from typing import Any from typing import Any
from .gossip import NodeGossip from .gossip import NodeGossip
@@ -50,6 +51,23 @@ def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
return f"wss://{parsed.netloc}/ws" return f"wss://{parsed.netloc}/ws"
def _load_model_presets_from_data() -> dict[str, dict]:
"""Load recommended model presets from package JSON data."""
try:
raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
data = json.loads(raw)
except Exception:
return {}
models = data.get("models", {})
if not isinstance(models, dict):
return {}
return {
str(name): preset
for name, preset in models.items()
if isinstance(preset, dict)
}
DEFAULT_MODEL_PRESETS: dict[str, dict] = { DEFAULT_MODEL_PRESETS: dict[str, dict] = {
"stub-model": { "stub-model": {
"layers_start": 0, "layers_start": 0,
@@ -61,6 +79,7 @@ DEFAULT_MODEL_PRESETS: dict[str, dict] = {
"layers_end": 11, "layers_end": 11,
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024}, "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
}, },
**_load_model_presets_from_data(),
} }
DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024 DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
@@ -83,6 +102,27 @@ def _model_aliases(model: str | None) -> set[str]:
return aliases return aliases
def _preset_aliases(name: str, preset: dict | None) -> set[str]:
aliases = _model_aliases(name)
if not preset:
return aliases
hf_repo = preset.get("hf_repo")
if isinstance(hf_repo, str):
aliases |= _model_aliases(hf_repo)
for alias in preset.get("aliases", []) or []:
if isinstance(alias, str):
aliases |= _model_aliases(alias)
return aliases
def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
requested = _model_aliases(model)
for name, preset in model_presets.items():
if requested & _preset_aliases(name, preset):
return name, preset
return None, None
def _node_matches_model(node: "_NodeEntry", model: str) -> bool: def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
requested = _model_aliases(model) requested = _model_aliases(model)
if not requested: if not requested:
@@ -90,6 +130,11 @@ def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo))) return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
requested = _preset_aliases(name, preset)
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
class _RollingCounter: class _RollingCounter:
"""Circular-bucket request counter. """Circular-bucket request counter.
@@ -469,6 +514,60 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
return summary return summary
def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
budget, _source = _node_memory_budget_bytes(node)
if preset is None:
return int(budget * 0.8)
return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))
def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
total_vram = sum(max(0, node.vram_bytes) for node in nodes)
total_ram = sum(max(0, node.ram_bytes) for node in nodes)
total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
return {
"node_count": len(nodes),
"total_vram_bytes": total_vram,
"total_ram_bytes": total_ram,
"total_memory_budget_bytes": total_budget,
"effective_assignable_memory_bytes": effective_budget,
"total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
"total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
}
def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
if preset is None:
return {"recommended": False}
pool = _pool_summary(nodes, preset)
required = int(preset.get("required_model_bytes", 0) or 0)
deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
return {
"recommended": bool(preset.get("recommended", False)),
"status": preset.get("deployment_status", "available"),
"required_model_bytes": required or None,
"download_size_bytes": preset.get("download_size_bytes"),
"native_quantization": preset.get("native_quantization"),
"pool": pool,
"deployable": deployable,
"missing_effective_memory_bytes": missing,
}
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
if total_layers <= 0:
return 0
if memory_mb <= 0:
return max(1, total_layers // 2)
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
return min(
total_layers,
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
)
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict: def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
metadata: dict = {} metadata: dict = {}
for node in nodes: for node in nodes:
@@ -622,10 +721,13 @@ def _nodes_and_bounds_for_model(
server: "_TrackerHTTPServer", server: "_TrackerHTTPServer",
model: str, model: str,
) -> tuple[list[_NodeEntry], int, int] | None: ) -> tuple[list[_NodeEntry], int, int] | None:
preset = server.model_presets.get(model) resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is not None: if preset is not None:
required_start, required_end = _preset_layer_bounds(preset) required_start, required_end = _preset_layer_bounds(preset)
return [node for node in server.registry.values() if node.model == model], required_start, required_end return [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
], required_start, required_end
nodes = [ nodes = [
node for node in server.registry.values() node for node in server.registry.values()
@@ -682,12 +784,15 @@ def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None: def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
preset = server.model_presets.get(model) resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None: if preset is None:
return return
required_start, required_end = _preset_layer_bounds(preset) required_start, required_end = _preset_layer_bounds(preset)
total_layers = required_end - required_start + 1 total_layers = required_end - required_start + 1
model_nodes = [node for node in server.registry.values() if node.model == model] model_nodes = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
managed_nodes = [node for node in model_nodes if node.managed_assignment] managed_nodes = [node for node in model_nodes if node.managed_assignment]
if not managed_nodes: if not managed_nodes:
return return
@@ -1032,8 +1137,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
data = [] data = []
seen_ids: set[str] = set() seen_ids: set[str] = set()
for name, preset in server.model_presets.items(): for name, preset in server.model_presets.items():
model_nodes = [node for node in alive if node.model == name] model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
if not model_nodes: if not model_nodes and not preset.get("recommended"):
continue continue
required_start, required_end = _preset_layer_bounds(preset) required_start, required_end = _preset_layer_bounds(preset)
coverage = _coverage_percentage( coverage = _coverage_percentage(
@@ -1045,6 +1150,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
hf_repo = preset.get("hf_repo") hf_repo = preset.get("hf_repo")
if hf_repo and hf_repo not in aliases: if hf_repo and hf_repo not in aliases:
aliases.append(hf_repo) aliases.append(hf_repo)
for alias in preset.get("aliases", []) or []:
if isinstance(alias, str) and alias not in aliases:
aliases.append(alias)
data.append({ data.append({
"id": name, "id": name,
"object": "model", "object": "model",
@@ -1054,9 +1162,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"hf_repo": hf_repo, "hf_repo": hf_repo,
"aliases": aliases, "aliases": aliases,
"metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)), "metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
"recommended": bool(preset.get("recommended", False)),
"deployment": _deployment_summary(alive, preset),
"shard_coverage_percentage": coverage, "shard_coverage_percentage": coverage,
}) })
seen_ids.add(name) seen_ids.add(name)
if hf_repo:
seen_ids.add(hf_repo)
hf_model_ids = sorted({ hf_model_ids = sorted({
node.hf_repo or node.model node.hf_repo or node.model
@@ -1122,14 +1234,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def _handle_tracker_nodes(self, model: str): def _handle_tracker_nodes(self, model: str):
"""Return nodes registered with tracker_mode=True whose shard starts at layer 0.""" """Return nodes registered with tracker_mode=True whose shard starts at layer 0."""
server: _TrackerHTTPServer = self.server # type: ignore[assignment] server: _TrackerHTTPServer = self.server # type: ignore[assignment]
preset = server.model_presets.get(model) resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None: if preset is None:
self._send_json(404, {"error": f"unknown model preset: {model!r}"}) self._send_json(404, {"error": f"unknown model preset: {model!r}"})
return return
required_start, _ = _preset_layer_bounds(preset) required_start, _ = _preset_layer_bounds(preset)
with server.lock: with server.lock:
self._purge_expired_nodes() self._purge_expired_nodes()
alive = [node for node in server.registry.values() if node.model == model] alive = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
if server.contracts is not None: if server.contracts is not None:
alive = [ alive = [
node for node in alive node for node in alive
@@ -1142,7 +1257,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
and node.tracker_mode and node.tracker_mode
] ]
self._send_json(200, { self._send_json(200, {
"model": model, "model": resolved_name,
"tracker_nodes": [ "tracker_nodes": [
{ {
"node_id": node.node_id, "node_id": node.node_id,
@@ -1171,6 +1286,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, { self._send_json(200, {
"relay_url": server.relay_url, "relay_url": server.relay_url,
"pool": _pool_summary(nodes),
"recommended_models": [
{
"id": name,
"hf_repo": preset.get("hf_repo"),
"aliases": list(preset.get("aliases", []) or []),
"metadata": dict(preset.get("metadata") or {}),
"deployment": _deployment_summary(nodes, preset),
}
for name, preset in server.model_presets.items()
if preset.get("recommended")
],
"nodes": [ "nodes": [
{ {
"node_id": node.node_id, "node_id": node.node_id,
@@ -1768,7 +1895,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else: else:
model = model_list[0] model = model_list[0]
preset = server.model_presets.get(model) resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None: if preset is None:
self._send_json(404, {"error": f"unknown model preset: {model!r}"}) self._send_json(404, {"error": f"unknown model preset: {model!r}"})
return return
@@ -1777,7 +1904,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock: with server.lock:
self._purge_expired_nodes() self._purge_expired_nodes()
alive = [node for node in server.registry.values() if node.model == model] alive = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
if server.contracts is not None: if server.contracts is not None:
alive = [ alive = [
node for node in alive node for node in alive
@@ -1830,7 +1960,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
peers = [ peers = [
{"endpoint": node.endpoint, "checksum": node.shard_checksum} {"endpoint": node.endpoint, "checksum": node.shard_checksum}
for node in alive for node in alive
if node.model == model if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
and node.shard_start == shard_start and node.shard_start == shard_start
and node.shard_end == shard_end and node.shard_end == shard_end
and node.shard_checksum and node.shard_checksum
@@ -1839,7 +1969,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, { self._send_json(200, {
"shard_start": shard_start, "shard_start": shard_start,
"shard_end": shard_end, "shard_end": shard_end,
"model": model, "model": resolved_name,
"model_layers_end": required_end, "model_layers_end": required_end,
"peers": peers, "peers": peers,
**({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}), **({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
@@ -1886,6 +2016,37 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
] ]
if not hf_nodes: if not hf_nodes:
resolved_name = None
preset = None
if filter_repo:
resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
else:
deployable = [
(name, preset)
for name, preset in server.model_presets.items()
if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"]
]
if deployable:
resolved_name, preset = deployable[0]
if preset is not None and preset.get("hf_repo"):
required_start, required_end = _preset_layer_bounds(preset)
total_l = required_end - required_start + 1
memory_mb = vram_mb if vram_mb > 0 else ram_mb
max_layers = _max_layers_for_memory(memory_mb, total_l, preset)
shard_start = required_start
shard_end = min(required_end, shard_start + max_layers - 1)
self._send_json(200, {
"hf_repo": preset["hf_repo"],
"model": resolved_name,
"shard_start": shard_start,
"shard_end": shard_end,
"num_layers": total_l,
"gap_found": True,
"price_per_token": 0.0,
"deployment": _deployment_summary(all_nodes, preset),
})
return
msg = ( msg = (
f"no HF-model nodes registered for {filter_repo!r}" f"no HF-model nodes registered for {filter_repo!r}"
if filter_repo if filter_repo
@@ -1954,15 +2115,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Capacity: use the same 80%-of-memory rule as registered node planning. # Capacity: use the same 80%-of-memory rule as registered node planning.
total_l = best_num_layers total_l = best_num_layers
memory_mb = vram_mb if vram_mb > 0 else ram_mb memory_mb = vram_mb if vram_mb > 0 else ram_mb
_resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
if memory_mb > 0: if memory_mb > 0:
max_layers = min( max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
total_l,
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // (30 * 1024 * 1024))),
)
elif device == "cuda" and vram_mb >= 8192: elif device == "cuda" and vram_mb >= 8192:
max_layers = total_l max_layers = total_l
else: else:
max_layers = max(1, total_l // 2) max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
shard_start = best_gap_start shard_start = best_gap_start
shard_end = min(total_l - 1, shard_start + max_layers - 1) shard_end = min(total_l - 1, shard_start + max_layers - 1)
@@ -1985,13 +2144,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return return
model = model_list[0] model = model_list[0]
preset = server.model_presets.get(model) resolved_name, preset = _resolve_model_preset(server.model_presets, model)
with server.lock: with server.lock:
self._purge_expired_nodes() self._purge_expired_nodes()
if preset is not None: if preset is not None:
# Preset-based routing (stub-model system). # Preset-based routing (stub-model system).
alive = [node for node in server.registry.values() if node.model == model] alive = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
required_start, required_end = _preset_layer_bounds(preset) required_start, required_end = _preset_layer_bounds(preset)
else: else:
# HF model routing: match by hf_repo (full) or model short name. # HF model routing: match by hf_repo (full) or model short name.
@@ -2066,7 +2228,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return return
model = model_list[0] model = model_list[0]
preset = server.model_presets.get(model) resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None: if preset is None:
self._send_json(404, {"error": f"unknown model preset: {model!r}"}) self._send_json(404, {"error": f"unknown model preset: {model!r}"})
return return
@@ -2075,7 +2237,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock: with server.lock:
self._purge_expired_nodes() self._purge_expired_nodes()
candidates = [node for node in server.registry.values() if node.model == model] candidates = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
if server.contracts is not None: if server.contracts is not None:
candidates = [ candidates = [
node for node in candidates node for node in candidates

View File

@@ -18,3 +18,6 @@ meshnet-tracker = "meshnet_tracker.cli:main"
[tool.setuptools.packages.find] [tool.setuptools.packages.find]
where = ["."] where = ["."]
include = ["meshnet_tracker*"] include = ["meshnet_tracker*"]
[tool.setuptools.package-data]
meshnet_tracker = ["*.json"]

View File

@@ -78,7 +78,8 @@ def test_tracker_exposes_registered_model_metadata():
finally: finally:
tracker.stop() tracker.stop()
kimi = next(model for model in models["data"] if model["id"] == "unsloth/Kimi-K2.7-Code") kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
assert kimi["hf_repo"] == "unsloth/Kimi-K2.7-Code"
assert kimi["metadata"]["total_parameters"] == "1T" assert kimi["metadata"]["total_parameters"] == "1T"
assert kimi["metadata"]["activated_parameters"] == "32B" assert kimi["metadata"]["activated_parameters"] == "32B"
assert kimi["metadata"]["num_layers"] == 61 assert kimi["metadata"]["num_layers"] == 61
@@ -87,6 +88,113 @@ def test_tracker_exposes_registered_model_metadata():
assert registered["model_metadata"]["context_length"] == 256000 assert registered["model_metadata"]["context_length"] == 256000
def test_tracker_lists_recommended_kimi_before_nodes_register():
tracker = TrackerServer()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
models = _get_json(f"{url}/v1/models")
network_map = _get_json(f"{url}/v1/network/map")
finally:
tracker.stop()
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
assert kimi["hf_repo"] == "unsloth/Kimi-K2.7-Code"
assert "Kimi-K2.7-Code" in kimi["aliases"]
assert kimi["metadata"]["download_size_gb"] == 595
assert kimi["deployment"]["native_quantization"] == "int4"
assert kimi["deployment"]["deployable"] is False
assert network_map["pool"]["node_count"] == 0
assert network_map["recommended_models"][0]["id"] == "kimi-k2.7"
def test_network_map_exposes_pool_size_and_speed_summary():
tracker = TrackerServer()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
_post_json(
f"{url}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:7101",
"model": "inventory-a",
"vram_bytes": 10_000,
"ram_bytes": 20_000,
"benchmark_tokens_per_sec": 4.0,
"hardware_profile": {},
},
)
_post_json(
f"{url}/v1/nodes/register",
{
"endpoint": "http://127.0.0.1:7102",
"model": "inventory-b",
"vram_bytes": 0,
"ram_bytes": 30_000,
"benchmark_tokens_per_sec": 6.0,
"hardware_profile": {},
},
)
network_map = _get_json(f"{url}/v1/network/map")
finally:
tracker.stop()
pool = network_map["pool"]
assert pool["node_count"] == 2
assert pool["total_vram_bytes"] == 10_000
assert pool["total_ram_bytes"] == 50_000
assert pool["total_memory_budget_bytes"] == 40_000
assert pool["total_benchmark_tokens_per_sec"] == 10.0
assert pool["total_effective_throughput"] == 10.0
def test_recommended_kimi_becomes_deployable_when_pool_is_large_enough():
tracker = TrackerServer()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
gib = 1024 * 1024 * 1024
try:
for idx in range(2):
_post_json(
f"{url}/v1/nodes/register",
{
"endpoint": f"http://127.0.0.1:{7200 + idx}",
"model": f"inventory-{idx}",
"vram_bytes": 0,
"ram_bytes": 400 * gib,
"benchmark_tokens_per_sec": 5.0,
"hardware_profile": {},
},
)
models = _get_json(f"{url}/v1/models")
finally:
tracker.stop()
kimi = next(model for model in models["data"] if model["id"] == "kimi-k2.7")
assert kimi["deployment"]["deployable"] is True
assert kimi["deployment"]["missing_effective_memory_bytes"] == 0
def test_network_assign_can_start_recommended_kimi_by_short_name():
tracker = TrackerServer()
port = tracker.start()
url = f"http://127.0.0.1:{port}"
try:
assignment = _get_json(
f"{url}/v1/network/assign?device=cpu&ram_mb=204800&hf_repo=kimi-k2.7"
)
finally:
tracker.stop()
assert assignment["model"] == "kimi-k2.7"
assert assignment["hf_repo"] == "unsloth/Kimi-K2.7-Code"
assert assignment["num_layers"] == 61
assert assignment["shard_start"] == 0
assert 0 <= assignment["shard_end"] < 60
def test_tracker_serves_health_while_proxy_request_is_in_flight(): def test_tracker_serves_health_while_proxy_request_is_in_flight():
"""Long inference proxy requests must not block heartbeats/health checks.""" """Long inference proxy requests must not block heartbeats/health checks."""