popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Load recommended model metadata from JSON

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-01 12:57:23 +02:00
parent bc760c1694
commit b035338e58
8 changed files with 402 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
packages/tracker/meshnet_tracker/model_presets.json Normal file
View File

@@ -0,0 +1,37 @@
{
"models": {
"kimi-k2.7": {
"layers_start": 0,
"layers_end": 60,
"hf_repo": "unsloth/Kimi-K2.7-Code",
"aliases": [
"kimi-k2.7",
"Kimi-K2.7-Code",
"unsloth/Kimi-K2.7-Code"
],
"recommended": true,
"deployment_status": "recommended",
"required_model_bytes": 638876385280,
"download_size_bytes": 638876385280,
"native_quantization": "int4",
"bytes_per_layer": {
"int4": 10473383366
},
"metadata": {
"architecture": "Mixture-of-Experts (MoE)",
"total_parameters": "1T",
"activated_parameters": "32B",
"num_layers": 61,
"context_length": 256000,
"native_quantization": "int4",
"download_size_gb": 595,
"recommended_short_name": "kimi-k2.7",
"recommended_engines": [
"vLLM",
"SGLang",
"KTransformers"
]
}
}
}
}

209
packages/tracker/meshnet_tracker/server.py
View File

@@ -31,6 +31,7 @@ import time
import urllib.parse
import urllib.request
import uuid
from importlib.resources import files
from typing import Any
from .gossip import NodeGossip
@@ -50,6 +51,23 @@ def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
return f"wss://{parsed.netloc}/ws"
def _load_model_presets_from_data() -> dict[str, dict]:
"""Load recommended model presets from package JSON data."""
try:
raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
data = json.loads(raw)
except Exception:
return {}
models = data.get("models", {})
if not isinstance(models, dict):
return {}
return {
str(name): preset
for name, preset in models.items()
if isinstance(preset, dict)
}
DEFAULT_MODEL_PRESETS: dict[str, dict] = {
"stub-model": {
"layers_start": 0,
@@ -61,6 +79,7 @@ DEFAULT_MODEL_PRESETS: dict[str, dict] = {
"layers_end": 11,
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
},
**_load_model_presets_from_data(),
}
DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
@@ -83,6 +102,27 @@ def _model_aliases(model: str | None) -> set[str]:
return aliases
def _preset_aliases(name: str, preset: dict | None) -> set[str]:
aliases = _model_aliases(name)
if not preset:
return aliases
hf_repo = preset.get("hf_repo")
if isinstance(hf_repo, str):
aliases |= _model_aliases(hf_repo)
for alias in preset.get("aliases", []) or []:
if isinstance(alias, str):
aliases |= _model_aliases(alias)
return aliases
def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
requested = _model_aliases(model)
for name, preset in model_presets.items():
if requested & _preset_aliases(name, preset):
return name, preset
return None, None
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
requested = _model_aliases(model)
if not requested:
@@ -90,6 +130,11 @@ def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
requested = _preset_aliases(name, preset)
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
class _RollingCounter:
"""Circular-bucket request counter.
@@ -469,6 +514,60 @@ def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict
return summary
def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
budget, _source = _node_memory_budget_bytes(node)
if preset is None:
return int(budget * 0.8)
return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))
def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
total_vram = sum(max(0, node.vram_bytes) for node in nodes)
total_ram = sum(max(0, node.ram_bytes) for node in nodes)
total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
return {
"node_count": len(nodes),
"total_vram_bytes": total_vram,
"total_ram_bytes": total_ram,
"total_memory_budget_bytes": total_budget,
"effective_assignable_memory_bytes": effective_budget,
"total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
"total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
}
def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
if preset is None:
return {"recommended": False}
pool = _pool_summary(nodes, preset)
required = int(preset.get("required_model_bytes", 0) or 0)
deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
return {
"recommended": bool(preset.get("recommended", False)),
"status": preset.get("deployment_status", "available"),
"required_model_bytes": required or None,
"download_size_bytes": preset.get("download_size_bytes"),
"native_quantization": preset.get("native_quantization"),
"pool": pool,
"deployable": deployable,
"missing_effective_memory_bytes": missing,
}
def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
if total_layers <= 0:
return 0
if memory_mb <= 0:
return max(1, total_layers // 2)
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
return min(
total_layers,
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
)
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
metadata: dict = {}
for node in nodes:
@@ -622,10 +721,13 @@ def _nodes_and_bounds_for_model(
server: "_TrackerHTTPServer",
model: str,
) -> tuple[list[_NodeEntry], int, int] | None:
preset = server.model_presets.get(model)
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is not None:
required_start, required_end = _preset_layer_bounds(preset)
return [node for node in server.registry.values() if node.model == model], required_start, required_end
return [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
], required_start, required_end
nodes = [
node for node in server.registry.values()
@@ -682,12 +784,15 @@ def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
preset = server.model_presets.get(model)
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None:
return
required_start, required_end = _preset_layer_bounds(preset)
total_layers = required_end - required_start + 1
model_nodes = [node for node in server.registry.values() if node.model == model]
model_nodes = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
managed_nodes = [node for node in model_nodes if node.managed_assignment]
if not managed_nodes:
return
@@ -1032,8 +1137,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
data = []
seen_ids: set[str] = set()
for name, preset in server.model_presets.items():
model_nodes = [node for node in alive if node.model == name]
if not model_nodes:
model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
if not model_nodes and not preset.get("recommended"):
continue
required_start, required_end = _preset_layer_bounds(preset)
coverage = _coverage_percentage(
@@ -1045,6 +1150,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
hf_repo = preset.get("hf_repo")
if hf_repo and hf_repo not in aliases:
aliases.append(hf_repo)
for alias in preset.get("aliases", []) or []:
if isinstance(alias, str) and alias not in aliases:
aliases.append(alias)
data.append({
"id": name,
"object": "model",
@@ -1054,9 +1162,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"hf_repo": hf_repo,
"aliases": aliases,
"metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
"recommended": bool(preset.get("recommended", False)),
"deployment": _deployment_summary(alive, preset),
"shard_coverage_percentage": coverage,
})
seen_ids.add(name)
if hf_repo:
seen_ids.add(hf_repo)
hf_model_ids = sorted({
node.hf_repo or node.model
@@ -1122,14 +1234,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
def _handle_tracker_nodes(self, model: str):
"""Return nodes registered with tracker_mode=True whose shard starts at layer 0."""
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
preset = server.model_presets.get(model)
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None:
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
return
required_start, _ = _preset_layer_bounds(preset)
with server.lock:
self._purge_expired_nodes()
alive = [node for node in server.registry.values() if node.model == model]
alive = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
if server.contracts is not None:
alive = [
node for node in alive
@@ -1142,7 +1257,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
and node.tracker_mode
]
self._send_json(200, {
"model": model,
"model": resolved_name,
"tracker_nodes": [
{
"node_id": node.node_id,
@@ -1171,6 +1286,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, {
"relay_url": server.relay_url,
"pool": _pool_summary(nodes),
"recommended_models": [
{
"id": name,
"hf_repo": preset.get("hf_repo"),
"aliases": list(preset.get("aliases", []) or []),
"metadata": dict(preset.get("metadata") or {}),
"deployment": _deployment_summary(nodes, preset),
}
for name, preset in server.model_presets.items()
if preset.get("recommended")
],
"nodes": [
{
"node_id": node.node_id,
@@ -1768,7 +1895,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
else:
model = model_list[0]
preset = server.model_presets.get(model)
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None:
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
return
@@ -1777,7 +1904,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
self._purge_expired_nodes()
alive = [node for node in server.registry.values() if node.model == model]
alive = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
if server.contracts is not None:
alive = [
node for node in alive
@@ -1830,7 +1960,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
peers = [
{"endpoint": node.endpoint, "checksum": node.shard_checksum}
for node in alive
if node.model == model
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
and node.shard_start == shard_start
and node.shard_end == shard_end
and node.shard_checksum
@@ -1839,7 +1969,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, {
"shard_start": shard_start,
"shard_end": shard_end,
"model": model,
"model": resolved_name,
"model_layers_end": required_end,
"peers": peers,
**({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
@@ -1886,6 +2016,37 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
]
if not hf_nodes:
resolved_name = None
preset = None
if filter_repo:
resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
else:
deployable = [
(name, preset)
for name, preset in server.model_presets.items()
if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"]
]
if deployable:
resolved_name, preset = deployable[0]
if preset is not None and preset.get("hf_repo"):
required_start, required_end = _preset_layer_bounds(preset)
total_l = required_end - required_start + 1
memory_mb = vram_mb if vram_mb > 0 else ram_mb
max_layers = _max_layers_for_memory(memory_mb, total_l, preset)
shard_start = required_start
shard_end = min(required_end, shard_start + max_layers - 1)
self._send_json(200, {
"hf_repo": preset["hf_repo"],
"model": resolved_name,
"shard_start": shard_start,
"shard_end": shard_end,
"num_layers": total_l,
"gap_found": True,
"price_per_token": 0.0,
"deployment": _deployment_summary(all_nodes, preset),
})
return
msg = (
f"no HF-model nodes registered for {filter_repo!r}"
if filter_repo
@@ -1954,15 +2115,13 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
# Capacity: use the same 80%-of-memory rule as registered node planning.
total_l = best_num_layers
memory_mb = vram_mb if vram_mb > 0 else ram_mb
_resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
if memory_mb > 0:
max_layers = min(
total_l,
max(1, int(((memory_mb * 1024 * 1024) * 0.8) // (30 * 1024 * 1024))),
)
max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
elif device == "cuda" and vram_mb >= 8192:
max_layers = total_l
else:
max_layers = max(1, total_l // 2)
max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
shard_start = best_gap_start
shard_end = min(total_l - 1, shard_start + max_layers - 1)
@@ -1985,13 +2144,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return
model = model_list[0]
preset = server.model_presets.get(model)
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
with server.lock:
self._purge_expired_nodes()
if preset is not None:
# Preset-based routing (stub-model system).
alive = [node for node in server.registry.values() if node.model == model]
alive = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
required_start, required_end = _preset_layer_bounds(preset)
else:
# HF model routing: match by hf_repo (full) or model short name.
@@ -2066,7 +2228,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
return
model = model_list[0]
preset = server.model_presets.get(model)
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
if preset is None:
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
return
@@ -2075,7 +2237,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
with server.lock:
self._purge_expired_nodes()
candidates = [node for node in server.registry.values() if node.model == model]
candidates = [
node for node in server.registry.values()
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
]
if server.contracts is not None:
candidates = [
node for node in candidates

3
packages/tracker/pyproject.toml
View File

@@ -18,3 +18,6 @@ meshnet-tracker = "meshnet_tracker.cli:main"
[tool.setuptools.packages.find]
where = ["."]
include = ["meshnet_tracker*"]
[tool.setuptools.package-data]
meshnet_tracker = ["*.json"]