fix model registration and anouncement. added console panel
This commit is contained in:
@@ -35,3 +35,10 @@ Historical handoff note: `/mnt/c/Users/popov/Downloads/neuron-tai-alpha-handoff-
|
|||||||
- Run: `meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct`
|
- Run: `meshnet-node start --tracker https://ai.neuron.d-popov.com --model Qwen/Qwen2.5-0.5B-Instruct`
|
||||||
- Known: tracker registration fails with `http://` — must use `https://`
|
- Known: tracker registration fails with `http://` — must use `https://`
|
||||||
- pynvml deprecation warning is harmless (use nvidia-ml-py to silence it)
|
- pynvml deprecation warning is harmless (use nvidia-ml-py to silence it)
|
||||||
|
|
||||||
|
## Model cache layout (2026-07-07)
|
||||||
|
- Node downloads now cache files directly under `<download_dir>/<model>/`, not `<model>/layers_<start>-<end>/`, so a wider cached layer assignment can satisfy a later narrower assignment without duplicate shard folders.
|
||||||
|
- Downloader checks tracker-advertised `files` + `file_sizes` before peer/HF download; complete local files return immediately and preserve any extra files already in the model folder.
|
||||||
|
- Verification: downloader/startup targeted subset passes (`pytest tests/test_node_startup.py -k "download_shard or same_shard"`). Full `tests/test_node_startup.py` has 46 passed and 4 unrelated Windows chmod/path separator failures.
|
||||||
|
- Live Windows confirmation: `meshnet-node start --tracker http://192.168.0.179:8080 --model Qwen3.6-35B-A3B` reuses `F:\_STORAGE\models\qwen3.6-35b-a3b`, prints `Cached at`, registers, and reaches ready as node `5gMLrmyB-26b1f8a4204a`.
|
||||||
|
- Follow-up fix: preset-model startup now starts the heartbeat thread after registration; without this, the node appeared briefly on the dashboard and was purged on first inference/route after heartbeat expiry. Tracker dashboard now has a "Console output" panel backed by `/v1/console` for node register/expiry, routing failures, and proxy events.
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"""Shard downloader — fetches model shards from peers or HuggingFace Hub.
|
"""Shard downloader — fetches model shards from peers or HuggingFace Hub.
|
||||||
|
|
||||||
Cache layout: ~/.cache/meshnet/shards/<model>/layers_<start>-<end>/
|
Cache layout: ~/.cache/meshnet/shards/<model>/
|
||||||
|
|
||||||
For "stub-model" (no HF repo), a placeholder JSON file is written so the
|
For "stub-model" (no HF repo), a placeholder JSON file is written so the
|
||||||
test suite never touches the network.
|
test suite never touches the network.
|
||||||
@@ -106,9 +106,7 @@ def _download_shard_from_peer(
|
|||||||
_safe_extract_shard(archive_path, extract_dir)
|
_safe_extract_shard(archive_path, extract_dir)
|
||||||
if compute_shard_checksum(extract_dir) != checksum:
|
if compute_shard_checksum(extract_dir) != checksum:
|
||||||
return False
|
return False
|
||||||
if shard_dir.exists():
|
_merge_tree(extract_dir, shard_dir)
|
||||||
shutil.rmtree(shard_dir)
|
|
||||||
shutil.move(str(extract_dir), str(shard_dir))
|
|
||||||
return True
|
return True
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
@@ -217,6 +215,38 @@ def _reuse_local_file(expected: int, dest: Path, final: Path) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_source_rel_files(source: dict) -> list[str]:
|
||||||
|
return [
|
||||||
|
rel for rel in (source.get("files") or [])
|
||||||
|
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _source_files_cached(source: dict, shard_dir: Path) -> bool:
|
||||||
|
rel_files = _valid_source_rel_files(source)
|
||||||
|
if not rel_files:
|
||||||
|
return False
|
||||||
|
sizes = source.get("file_sizes")
|
||||||
|
if not isinstance(sizes, dict):
|
||||||
|
return False
|
||||||
|
for rel in rel_files:
|
||||||
|
expected = sizes.get(rel)
|
||||||
|
path = shard_dir / rel
|
||||||
|
if not isinstance(expected, int) or not path.exists() or path.stat().st_size != expected:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _merge_tree(src: Path, dest: Path) -> None:
|
||||||
|
dest.mkdir(parents=True, exist_ok=True)
|
||||||
|
for path in sorted(p for p in src.rglob("*") if p.is_file()):
|
||||||
|
target = dest / path.relative_to(src)
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if target.exists():
|
||||||
|
target.unlink()
|
||||||
|
shutil.move(str(path), str(target))
|
||||||
|
|
||||||
|
|
||||||
def _download_source_files(
|
def _download_source_files(
|
||||||
source: dict,
|
source: dict,
|
||||||
shard_dir: Path,
|
shard_dir: Path,
|
||||||
@@ -231,10 +261,7 @@ def _download_source_files(
|
|||||||
the download is retried or the node is restarted.
|
the download is retried or the node is restarted.
|
||||||
"""
|
"""
|
||||||
url = source.get("url")
|
url = source.get("url")
|
||||||
rel_files = [
|
rel_files = _valid_source_rel_files(source)
|
||||||
rel for rel in (source.get("files") or [])
|
|
||||||
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts
|
|
||||||
]
|
|
||||||
if not isinstance(url, str) or not url or not rel_files:
|
if not isinstance(url, str) or not url or not rel_files:
|
||||||
return None
|
return None
|
||||||
sizes = source.get("file_sizes")
|
sizes = source.get("file_sizes")
|
||||||
@@ -283,9 +310,11 @@ def _download_source_files(
|
|||||||
time.sleep(1.0 * attempt)
|
time.sleep(1.0 * attempt)
|
||||||
finally:
|
finally:
|
||||||
tracker_bar.close()
|
tracker_bar.close()
|
||||||
if shard_dir.exists():
|
_merge_tree(partial_dir, shard_dir)
|
||||||
shutil.rmtree(shard_dir)
|
try:
|
||||||
shutil.move(str(partial_dir), str(shard_dir))
|
partial_dir.rmdir()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
return shard_dir
|
return shard_dir
|
||||||
|
|
||||||
|
|
||||||
@@ -331,9 +360,7 @@ def _download_model_source(
|
|||||||
if progress:
|
if progress:
|
||||||
print(f" {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
|
print(f" {label}: transfer complete ({received / 1e9:.2f} GB), extracting ...", flush=True)
|
||||||
_safe_extract_shard(archive_path, extract_dir)
|
_safe_extract_shard(archive_path, extract_dir)
|
||||||
if shard_dir.exists():
|
_merge_tree(extract_dir, shard_dir)
|
||||||
shutil.rmtree(shard_dir)
|
|
||||||
shutil.move(str(extract_dir), str(shard_dir))
|
|
||||||
return shard_dir
|
return shard_dir
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if progress:
|
if progress:
|
||||||
@@ -431,10 +458,17 @@ def download_shard(
|
|||||||
the test suite hermetic while the real download path is exercised by
|
the test suite hermetic while the real download path is exercised by
|
||||||
passing a non-stub *hf_repo*.
|
passing a non-stub *hf_repo*.
|
||||||
"""
|
"""
|
||||||
shard_dir = cache_dir / model / f"layers_{shard_start}-{shard_end}"
|
shard_dir = cache_dir / model
|
||||||
if progress:
|
if progress:
|
||||||
print(f" Target location: {shard_dir}", flush=True)
|
print(f" Target location: {shard_dir}", flush=True)
|
||||||
|
|
||||||
|
for source in model_sources or []:
|
||||||
|
label = str(source.get("type") or "model-source")
|
||||||
|
if _source_files_cached(source, shard_dir):
|
||||||
|
if progress:
|
||||||
|
print(f" [{label}] requested files already cached at {shard_dir}", flush=True)
|
||||||
|
return shard_dir
|
||||||
|
|
||||||
for peer in peers or []:
|
for peer in peers or []:
|
||||||
if progress:
|
if progress:
|
||||||
print(f" Trying peer shard download from {peer.get('endpoint')} ...", flush=True)
|
print(f" Trying peer shard download from {peer.get('endpoint')} ...", flush=True)
|
||||||
|
|||||||
@@ -64,6 +64,17 @@ def _load_model_metadata() -> dict[str, dict]:
|
|||||||
_MODEL_METADATA = _load_model_metadata()
|
_MODEL_METADATA = _load_model_metadata()
|
||||||
|
|
||||||
|
|
||||||
|
def _local_model_path(hf_repo: str, cache_dir: Path | None) -> Path | None:
|
||||||
|
if cache_dir is None:
|
||||||
|
return None
|
||||||
|
if (cache_dir / "config.json").exists():
|
||||||
|
return cache_dir
|
||||||
|
candidate = cache_dir / hf_repo.split("/")[-1]
|
||||||
|
if (candidate / "config.json").exists():
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
CURATED_MODELS: list[ModelPreset] = [
|
CURATED_MODELS: list[ModelPreset] = [
|
||||||
ModelPreset(
|
ModelPreset(
|
||||||
name="Qwen2.5-0.5B-Instruct",
|
name="Qwen2.5-0.5B-Instruct",
|
||||||
@@ -215,9 +226,11 @@ def model_metadata_for(
|
|||||||
try:
|
try:
|
||||||
from transformers import AutoConfig # type: ignore[import]
|
from transformers import AutoConfig # type: ignore[import]
|
||||||
|
|
||||||
|
local_model = _local_model_path(hf_repo, cache_dir)
|
||||||
|
load_source = str(local_model) if local_model is not None else hf_repo
|
||||||
cfg = AutoConfig.from_pretrained(
|
cfg = AutoConfig.from_pretrained(
|
||||||
hf_repo,
|
load_source,
|
||||||
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
cache_dir=str(cache_dir) if cache_dir is not None and local_model is None else None,
|
||||||
)
|
)
|
||||||
# Composite configs (VLM/MoE) nest decoder fields in text_config.
|
# Composite configs (VLM/MoE) nest decoder fields in text_config.
|
||||||
text_cfg = getattr(cfg, "text_config", None) or cfg
|
text_cfg = getattr(cfg, "text_config", None) or cfg
|
||||||
|
|||||||
@@ -25,6 +25,86 @@ from .wallet import load_or_create_wallet
|
|||||||
_DEFAULT_BYTES_PER_LAYER = 30 * 1024 * 1024
|
_DEFAULT_BYTES_PER_LAYER = 30 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def _downloaded_model_inventory(
|
||||||
|
model: str,
|
||||||
|
shard_start: int,
|
||||||
|
shard_end: int,
|
||||||
|
shard_path: Path,
|
||||||
|
hf_repo: str | None = None,
|
||||||
|
model_sources: list[dict] | None = None,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Return a cheap local inventory record without reading model file contents."""
|
||||||
|
file_count = 0
|
||||||
|
total_bytes = 0
|
||||||
|
existing_rel_files: set[str] = set()
|
||||||
|
if shard_path.exists():
|
||||||
|
for path in shard_path.rglob("*"):
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
file_count += 1
|
||||||
|
try:
|
||||||
|
existing_rel_files.add(path.relative_to(shard_path).as_posix())
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
total_bytes += path.stat().st_size
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
record = {
|
||||||
|
"model": model,
|
||||||
|
"shard_start": shard_start,
|
||||||
|
"shard_end": shard_end,
|
||||||
|
"path": str(shard_path),
|
||||||
|
"file_count": file_count,
|
||||||
|
"total_bytes": total_bytes,
|
||||||
|
}
|
||||||
|
if hf_repo is not None:
|
||||||
|
record["hf_repo"] = hf_repo
|
||||||
|
expected_files: set[str] = set()
|
||||||
|
file_sizes: dict[str, int] = {}
|
||||||
|
for source in model_sources or []:
|
||||||
|
for rel in source.get("full_files") or source.get("files") or []:
|
||||||
|
if isinstance(rel, str) and rel and not rel.startswith("/") and ".." not in Path(rel).parts:
|
||||||
|
expected_files.add(rel)
|
||||||
|
sizes = source.get("file_sizes")
|
||||||
|
if isinstance(sizes, dict):
|
||||||
|
for rel, size in sizes.items():
|
||||||
|
if isinstance(rel, str) and isinstance(size, int):
|
||||||
|
file_sizes[rel] = size
|
||||||
|
if expected_files:
|
||||||
|
expected_bytes = sum(file_sizes.get(rel, 0) for rel in expected_files)
|
||||||
|
local_expected_files = existing_rel_files & expected_files
|
||||||
|
local_expected_bytes = sum(file_sizes.get(rel, 0) for rel in local_expected_files)
|
||||||
|
record["expected_file_count"] = len(expected_files)
|
||||||
|
record["local_expected_file_count"] = len(local_expected_files)
|
||||||
|
record["expected_bytes"] = expected_bytes
|
||||||
|
record["local_expected_bytes"] = local_expected_bytes
|
||||||
|
record["local_model_percentage"] = (
|
||||||
|
round((local_expected_bytes / expected_bytes) * 100, 4)
|
||||||
|
if expected_bytes > 0
|
||||||
|
else 0.0
|
||||||
|
)
|
||||||
|
return [record]
|
||||||
|
|
||||||
|
|
||||||
|
def _registration_shard_checksum(model: str, shard_path: Path) -> str | None:
|
||||||
|
"""Only checksum tiny stub shards; real model folders are too large to hash at startup."""
|
||||||
|
if model != "stub-model":
|
||||||
|
return None
|
||||||
|
return compute_shard_checksum(shard_path)
|
||||||
|
|
||||||
|
|
||||||
|
def _model_cache_path(model_id: str, cache_dir: Path | None) -> Path | None:
|
||||||
|
if cache_dir is None:
|
||||||
|
return None
|
||||||
|
if (cache_dir / "config.json").exists():
|
||||||
|
return cache_dir
|
||||||
|
candidate = cache_dir / model_id.split("/")[-1]
|
||||||
|
if candidate.exists():
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _memory_budget(device: str, vram_mb: int, ram_mb: int, shared_vram_mb: int = 0) -> tuple[int, str]:
|
def _memory_budget(device: str, vram_mb: int, ram_mb: int, shared_vram_mb: int = 0) -> tuple[int, str]:
|
||||||
"""Return the capacity budget in MB and whether it came from VRAM or RAM."""
|
"""Return the capacity budget in MB and whether it came from VRAM or RAM."""
|
||||||
if device == "cuda" and vram_mb > 0:
|
if device == "cuda" and vram_mb > 0:
|
||||||
@@ -440,6 +520,7 @@ def run_startup(
|
|||||||
|
|
||||||
if model_id: # treat "" the same as None — no explicit model given
|
if model_id: # treat "" the same as None — no explicit model given
|
||||||
user_pinned_shard = shard_start is not None or shard_end is not None
|
user_pinned_shard = shard_start is not None or shard_end is not None
|
||||||
|
full_sources: list[dict] = []
|
||||||
# Auto-detect shard range from model config if not explicitly provided
|
# Auto-detect shard range from model config if not explicitly provided
|
||||||
if shard_start is None or shard_end is None:
|
if shard_start is None or shard_end is None:
|
||||||
try:
|
try:
|
||||||
@@ -519,6 +600,7 @@ def run_startup(
|
|||||||
_attach_relay_bridge(node, relay_bridge)
|
_attach_relay_bridge(node, relay_bridge)
|
||||||
# Register with tracker so other nodes can auto-join this model.
|
# Register with tracker so other nodes can auto-join this model.
|
||||||
total_layers = getattr(getattr(node, "backend", None), "total_layers", None)
|
total_layers = getattr(getattr(node, "backend", None), "total_layers", None)
|
||||||
|
model_cache_path = _model_cache_path(model_id, cache_dir)
|
||||||
reg_payload = {
|
reg_payload = {
|
||||||
"endpoint": endpoint,
|
"endpoint": endpoint,
|
||||||
"model": model_id.split("/")[-1],
|
"model": model_id.split("/")[-1],
|
||||||
@@ -533,6 +615,18 @@ def run_startup(
|
|||||||
"tracker_mode": (shard_start == 0),
|
"tracker_mode": (shard_start == 0),
|
||||||
"managed_assignment": not user_pinned_shard,
|
"managed_assignment": not user_pinned_shard,
|
||||||
"model_metadata": model_metadata_for(model_id, total_layers, cache_dir=cache_dir),
|
"model_metadata": model_metadata_for(model_id, total_layers, cache_dir=cache_dir),
|
||||||
|
"downloaded_models": (
|
||||||
|
_downloaded_model_inventory(
|
||||||
|
model_id.split("/")[-1],
|
||||||
|
shard_start,
|
||||||
|
shard_end,
|
||||||
|
model_cache_path,
|
||||||
|
hf_repo=model_id,
|
||||||
|
model_sources=full_sources,
|
||||||
|
)
|
||||||
|
if model_cache_path is not None
|
||||||
|
else []
|
||||||
|
),
|
||||||
**registration_capabilities,
|
**registration_capabilities,
|
||||||
**relay_fields,
|
**relay_fields,
|
||||||
}
|
}
|
||||||
@@ -631,6 +725,7 @@ def run_startup(
|
|||||||
relay_url=relay_url,
|
relay_url=relay_url,
|
||||||
)
|
)
|
||||||
_attach_relay_bridge(node, relay_bridge)
|
_attach_relay_bridge(node, relay_bridge)
|
||||||
|
model_cache_path = _model_cache_path(assigned_hf_repo, cache_dir)
|
||||||
auto_reg_payload = {
|
auto_reg_payload = {
|
||||||
"endpoint": endpoint,
|
"endpoint": endpoint,
|
||||||
"model": assigned_hf_repo.split("/")[-1],
|
"model": assigned_hf_repo.split("/")[-1],
|
||||||
@@ -645,6 +740,18 @@ def run_startup(
|
|||||||
"tracker_mode": (assigned_shard_start == 0),
|
"tracker_mode": (assigned_shard_start == 0),
|
||||||
"managed_assignment": True,
|
"managed_assignment": True,
|
||||||
"model_metadata": model_metadata_for(assigned_hf_repo, assigned_num_layers, cache_dir=cache_dir),
|
"model_metadata": model_metadata_for(assigned_hf_repo, assigned_num_layers, cache_dir=cache_dir),
|
||||||
|
"downloaded_models": (
|
||||||
|
_downloaded_model_inventory(
|
||||||
|
assigned_hf_repo.split("/")[-1],
|
||||||
|
assigned_shard_start,
|
||||||
|
assigned_shard_end,
|
||||||
|
model_cache_path,
|
||||||
|
hf_repo=assigned_hf_repo,
|
||||||
|
model_sources=full_sources,
|
||||||
|
)
|
||||||
|
if model_cache_path is not None
|
||||||
|
else []
|
||||||
|
),
|
||||||
**registration_capabilities,
|
**registration_capabilities,
|
||||||
**relay_fields,
|
**relay_fields,
|
||||||
}
|
}
|
||||||
@@ -714,7 +821,15 @@ def run_startup(
|
|||||||
if model_sources:
|
if model_sources:
|
||||||
dl_kwargs["model_sources"] = model_sources
|
dl_kwargs["model_sources"] = model_sources
|
||||||
shard_path = download_shard(assigned_model, shard_start, shard_end, **dl_kwargs)
|
shard_path = download_shard(assigned_model, shard_start, shard_end, **dl_kwargs)
|
||||||
shard_checksum = compute_shard_checksum(shard_path)
|
shard_checksum = _registration_shard_checksum(assigned_model, shard_path)
|
||||||
|
downloaded_models = _downloaded_model_inventory(
|
||||||
|
assigned_model,
|
||||||
|
shard_start,
|
||||||
|
shard_end,
|
||||||
|
shard_path,
|
||||||
|
hf_repo=hf_repo,
|
||||||
|
model_sources=model_sources,
|
||||||
|
)
|
||||||
print(f" Cached at: {shard_path}", flush=True)
|
print(f" Cached at: {shard_path}", flush=True)
|
||||||
|
|
||||||
# 5. Start HTTP server
|
# 5. Start HTTP server
|
||||||
@@ -728,6 +843,7 @@ def run_startup(
|
|||||||
model=assigned_model,
|
model=assigned_model,
|
||||||
shard_path=shard_path,
|
shard_path=shard_path,
|
||||||
)
|
)
|
||||||
|
_node_start_time = time.monotonic()
|
||||||
actual_port = node.start()
|
actual_port = node.start()
|
||||||
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
public_host = advertise_host or (socket.getfqdn() if host == "0.0.0.0" else host)
|
||||||
endpoint = f"http://{public_host}:{actual_port}"
|
endpoint = f"http://{public_host}:{actual_port}"
|
||||||
@@ -743,24 +859,27 @@ def run_startup(
|
|||||||
|
|
||||||
# 6. Register with tracker
|
# 6. Register with tracker
|
||||||
print("Registering with tracker...", flush=True)
|
print("Registering with tracker...", flush=True)
|
||||||
|
reg_payload = {
|
||||||
|
"endpoint": endpoint,
|
||||||
|
"model": assigned_model,
|
||||||
|
"shard_start": shard_start,
|
||||||
|
"shard_end": shard_end,
|
||||||
|
"shard_checksum": shard_checksum,
|
||||||
|
"downloaded_models": downloaded_models,
|
||||||
|
"hardware_profile": hw,
|
||||||
|
"wallet_address": address,
|
||||||
|
"score": 1.0,
|
||||||
|
**registration_capabilities,
|
||||||
|
**relay_fields,
|
||||||
|
}
|
||||||
try:
|
try:
|
||||||
reg_resp = _post_json(
|
reg_resp = _post_json(
|
||||||
f"{tracker_url}/v1/nodes/register",
|
f"{tracker_url}/v1/nodes/register",
|
||||||
{
|
reg_payload,
|
||||||
"endpoint": endpoint,
|
|
||||||
"model": assigned_model,
|
|
||||||
"shard_start": shard_start,
|
|
||||||
"shard_end": shard_end,
|
|
||||||
"shard_checksum": shard_checksum,
|
|
||||||
"hardware_profile": hw,
|
|
||||||
"wallet_address": address,
|
|
||||||
"score": 1.0,
|
|
||||||
**registration_capabilities,
|
|
||||||
**relay_fields,
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
node_id = str(reg_resp["node_id"])
|
node_id = str(reg_resp["node_id"])
|
||||||
setattr(node, "tracker_node_id", node_id)
|
setattr(node, "tracker_node_id", node_id)
|
||||||
|
_start_heartbeat(tracker_url, node_id, reg_payload, node_ref=node, start_time=_node_start_time)
|
||||||
except Exception:
|
except Exception:
|
||||||
node.stop()
|
node.stop()
|
||||||
raise
|
raise
|
||||||
@@ -793,9 +912,11 @@ def _detect_num_layers(model_id: str, cache_dir: Path | None = None) -> int | No
|
|||||||
|
|
||||||
from .model_catalog import layers_from_config
|
from .model_catalog import layers_from_config
|
||||||
|
|
||||||
|
local_model = _model_cache_path(model_id, cache_dir)
|
||||||
|
load_source = str(local_model) if local_model is not None else model_id
|
||||||
cfg = AutoConfig.from_pretrained(
|
cfg = AutoConfig.from_pretrained(
|
||||||
model_id,
|
load_source,
|
||||||
cache_dir=str(cache_dir) if cache_dir is not None else None,
|
cache_dir=str(cache_dir) if cache_dir is not None and local_model is None else None,
|
||||||
)
|
)
|
||||||
layers = layers_from_config(cfg)
|
layers = layers_from_config(cfg)
|
||||||
if layers is None:
|
if layers is None:
|
||||||
|
|||||||
@@ -41,10 +41,21 @@
|
|||||||
.error-msg { color:var(--bad); font-size:12px; min-height:16px; }
|
.error-msg { color:var(--bad); font-size:12px; min-height:16px; }
|
||||||
.keybox { word-break:break-all; background:var(--bg); border:1px solid var(--border);
|
.keybox { word-break:break-all; background:var(--bg); border:1px solid var(--border);
|
||||||
border-radius:6px; padding:4px 8px; margin:4px 0; font-size:11px; }
|
border-radius:6px; padding:4px 8px; margin:4px 0; font-size:11px; }
|
||||||
.tabs { display:flex; gap:10px; margin-bottom:8px; }
|
.tabs { display:flex; gap:10px; margin-bottom:8px; }
|
||||||
.tabs a { color:var(--dim); cursor:pointer; }
|
.tabs a { color:var(--dim); cursor:pointer; }
|
||||||
.tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
|
.tabs a.active { color:var(--accent); border-bottom:1px solid var(--accent); }
|
||||||
</style>
|
.wide { grid-column:1 / -1; }
|
||||||
|
.console {
|
||||||
|
background:var(--bg); border:1px solid var(--border); border-radius:6px;
|
||||||
|
min-height:160px; max-height:280px; overflow:auto; padding:7px 9px;
|
||||||
|
white-space:pre-wrap; word-break:break-word; font-size:11px;
|
||||||
|
}
|
||||||
|
.console-line { padding:1px 0; border-bottom:1px solid #161b22; }
|
||||||
|
.console-time { color:var(--dim); }
|
||||||
|
.console-level-info { color:var(--accent); }
|
||||||
|
.console-level-warn { color:var(--warn); }
|
||||||
|
.console-level-error { color:var(--bad); }
|
||||||
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<header>
|
<header>
|
||||||
@@ -61,9 +72,10 @@
|
|||||||
<section><h2>Node pending payouts</h2><div id="pending" class="empty">loading…</div></section>
|
<section><h2>Node pending payouts</h2><div id="pending" class="empty">loading…</div></section>
|
||||||
<section><h2>Settlement history</h2><div id="settlements" class="empty">loading…</div></section>
|
<section><h2>Settlement history</h2><div id="settlements" class="empty">loading…</div></section>
|
||||||
<section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
|
<section><h2>Strikes / bans / forfeitures</h2><div id="fraud" class="empty">loading…</div></section>
|
||||||
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
<section><h2>Model usage (RPM)</h2><div id="stats" class="empty">loading…</div></section>
|
||||||
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
|
<section><h2>Node throughput</h2><div id="throughput" class="empty">loading…</div></section>
|
||||||
</main>
|
<section class="wide"><h2>Console output</h2><div id="console" class="console empty">loading…</div></section>
|
||||||
|
</main>
|
||||||
<script>
|
<script>
|
||||||
"use strict";
|
"use strict";
|
||||||
const $ = id => document.getElementById(id);
|
const $ = id => document.getElementById(id);
|
||||||
@@ -196,7 +208,7 @@ function renderStats(stats) {
|
|||||||
$("stats").innerHTML = table(["model", "rpm (1h)", "rpm (24h)", "rpm (30d)"], rows);
|
$("stats").innerHTML = table(["model", "rpm (1h)", "rpm (24h)", "rpm (30d)"], rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
function renderThroughput(stats) {
|
function renderThroughput(stats) {
|
||||||
const nodes = (stats && stats.nodes) || {};
|
const nodes = (stats && stats.nodes) || {};
|
||||||
const rows = [];
|
const rows = [];
|
||||||
for (const [nodeId, nodeStats] of Object.entries(nodes)) {
|
for (const [nodeId, nodeStats] of Object.entries(nodes)) {
|
||||||
@@ -209,8 +221,23 @@ function renderThroughput(stats) {
|
|||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
|
$("throughput").innerHTML = table(["node", "model", "tps (1h)", "samples"], rows);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function renderConsole(data) {
|
||||||
|
const events = (data && data.events) || [];
|
||||||
|
if (!events.length) {
|
||||||
|
$("console").innerHTML = '<div class="empty">no console events</div>';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
$("console").innerHTML = events.slice(-120).map(e => {
|
||||||
|
const level = String(e.level || "info");
|
||||||
|
const cls = level === "error" ? "console-level-error" : level === "warn" ? "console-level-warn" : "console-level-info";
|
||||||
|
const fields = e.fields && Object.keys(e.fields).length ? " " + JSON.stringify(e.fields) : "";
|
||||||
|
return `<div class="console-line"><span class="console-time">${new Date((e.ts || 0) * 1000).toLocaleTimeString()}</span> ` +
|
||||||
|
`<span class="${cls}">${esc(level.toUpperCase())}</span> ${esc(e.message || "")}${esc(fields)}</div>`;
|
||||||
|
}).join("");
|
||||||
|
}
|
||||||
|
|
||||||
// ---- account panel (registration / login / balance / usage / API keys) ----
|
// ---- account panel (registration / login / balance / usage / API keys) ----
|
||||||
|
|
||||||
@@ -366,21 +393,23 @@ async function renderAdminPanel() {
|
|||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
$("self-url").textContent = location.host;
|
$("self-url").textContent = location.host;
|
||||||
const [raft, map, summary, settlements, wallets, stats] = await Promise.all([
|
const [raft, map, summary, settlements, wallets, stats, consoleData] = await Promise.all([
|
||||||
fetchJson("/v1/raft/status"),
|
fetchJson("/v1/raft/status"),
|
||||||
fetchJson("/v1/network/map"),
|
fetchJson("/v1/network/map"),
|
||||||
fetchJson("/v1/billing/summary"),
|
fetchJson("/v1/billing/summary"),
|
||||||
fetchJson("/v1/billing/settlements"),
|
fetchJson("/v1/billing/settlements"),
|
||||||
fetchJson("/v1/registry/wallets"),
|
fetchJson("/v1/registry/wallets"),
|
||||||
fetchJson("/v1/stats"),
|
fetchJson("/v1/stats"),
|
||||||
]);
|
fetchJson("/v1/console"),
|
||||||
|
]);
|
||||||
renderHive(raft);
|
renderHive(raft);
|
||||||
renderNodes(map);
|
renderNodes(map);
|
||||||
renderBilling(summary);
|
renderBilling(summary);
|
||||||
renderSettlements(settlements);
|
renderSettlements(settlements);
|
||||||
renderFraud(wallets, summary);
|
renderFraud(wallets, summary);
|
||||||
renderStats(stats);
|
renderStats(stats);
|
||||||
renderThroughput(stats);
|
renderThroughput(stats);
|
||||||
|
renderConsole(consoleData);
|
||||||
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
|
$("refreshed").textContent = "refreshed " + new Date().toLocaleTimeString();
|
||||||
}
|
}
|
||||||
refresh();
|
refresh();
|
||||||
|
|||||||
@@ -32,9 +32,10 @@ import tarfile
|
|||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
import urllib.request
|
import urllib.request
|
||||||
import uuid
|
import uuid
|
||||||
from importlib.resources import files
|
from collections import deque
|
||||||
|
from importlib.resources import files
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -46,7 +47,10 @@ from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
|||||||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||||||
from .gossip import NodeGossip
|
from .gossip import NodeGossip
|
||||||
from .model_files import files_for_layer_range, snapshot_dir_for_repo
|
from .model_files import files_for_layer_range, snapshot_dir_for_repo
|
||||||
from .raft import RaftNode
|
from .raft import RaftNode
|
||||||
|
|
||||||
|
|
||||||
|
_CONSOLE_LIMIT = 300
|
||||||
|
|
||||||
|
|
||||||
def _preset_price_keys(name: str, preset: dict) -> set[str]:
|
def _preset_price_keys(name: str, preset: dict) -> set[str]:
|
||||||
@@ -529,7 +533,7 @@ class _StatsCollector:
|
|||||||
class _NodeEntry:
|
class _NodeEntry:
|
||||||
__slots__ = (
|
__slots__ = (
|
||||||
"node_id", "endpoint", "shard_start", "shard_end",
|
"node_id", "endpoint", "shard_start", "shard_end",
|
||||||
"model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "hardware_profile", "wallet_address",
|
"model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "downloaded_models", "hardware_profile", "wallet_address",
|
||||||
"score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
|
"score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
|
||||||
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
|
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
|
||||||
"model_tokens_per_sec",
|
"model_tokens_per_sec",
|
||||||
@@ -562,10 +566,11 @@ class _NodeEntry:
|
|||||||
quantization: str | None = None,
|
quantization: str | None = None,
|
||||||
managed_assignment: bool = False,
|
managed_assignment: bool = False,
|
||||||
tracker_mode: bool = False,
|
tracker_mode: bool = False,
|
||||||
hf_repo: str | None = None,
|
hf_repo: str | None = None,
|
||||||
num_layers: int | None = None,
|
num_layers: int | None = None,
|
||||||
model_metadata: dict | None = None,
|
model_metadata: dict | None = None,
|
||||||
relay_addr: str | None = None,
|
downloaded_models: list[dict] | None = None,
|
||||||
|
relay_addr: str | None = None,
|
||||||
cert_fingerprint: str | None = None,
|
cert_fingerprint: str | None = None,
|
||||||
peer_id: str | None = None,
|
peer_id: str | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
@@ -588,9 +593,10 @@ class _NodeEntry:
|
|||||||
self.model_tokens_per_sec: dict[str, float] = {}
|
self.model_tokens_per_sec: dict[str, float] = {}
|
||||||
self.tracker_mode = tracker_mode
|
self.tracker_mode = tracker_mode
|
||||||
self.hf_repo = hf_repo
|
self.hf_repo = hf_repo
|
||||||
self.num_layers = num_layers
|
self.num_layers = num_layers
|
||||||
self.model_metadata = dict(model_metadata or {})
|
self.model_metadata = dict(model_metadata or {})
|
||||||
self.relay_addr = relay_addr
|
self.downloaded_models = [dict(item) for item in (downloaded_models or []) if isinstance(item, dict)]
|
||||||
|
self.relay_addr = relay_addr
|
||||||
self.cert_fingerprint = cert_fingerprint
|
self.cert_fingerprint = cert_fingerprint
|
||||||
self.peer_id = peer_id
|
self.peer_id = peer_id
|
||||||
self.pending_directives: list[dict] = []
|
self.pending_directives: list[dict] = []
|
||||||
@@ -1245,13 +1251,18 @@ def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
|
|||||||
node_id for node_id, entry in server.registry.items()
|
node_id for node_id, entry in server.registry.items()
|
||||||
if (now - entry.last_heartbeat) > server.heartbeat_timeout
|
if (now - entry.last_heartbeat) > server.heartbeat_timeout
|
||||||
]
|
]
|
||||||
for node_id in expired_ids:
|
for node_id in expired_ids:
|
||||||
entry = server.registry.pop(node_id)
|
entry = server.registry.pop(node_id)
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] node expired: {node_id} {entry.endpoint} "
|
server,
|
||||||
f"(no heartbeat for >{server.heartbeat_timeout:.0f}s)",
|
"warn",
|
||||||
flush=True,
|
"node expired",
|
||||||
)
|
node_id=node_id,
|
||||||
|
endpoint=entry.endpoint,
|
||||||
|
model=entry.model,
|
||||||
|
shard=f"{entry.shard_start}-{entry.shard_end}",
|
||||||
|
heartbeat_timeout_seconds=server.heartbeat_timeout,
|
||||||
|
)
|
||||||
if expired_ids:
|
if expired_ids:
|
||||||
_rebalance_all_locked(server)
|
_rebalance_all_locked(server)
|
||||||
return expired_ids
|
return expired_ids
|
||||||
@@ -1609,15 +1620,33 @@ def _billable_stream_tokens(observed_tokens: int, reported_tokens: int | None) -
|
|||||||
return min(observed, max(0, reported_tokens))
|
return min(observed, max(0, reported_tokens))
|
||||||
|
|
||||||
|
|
||||||
def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
|
def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
|
||||||
if contracts is None or not wallet_address:
|
if contracts is None or not wallet_address:
|
||||||
return None
|
return None
|
||||||
if contracts.registry.get_wallet(wallet_address).banned:
|
if contracts.registry.get_wallet(wallet_address).banned:
|
||||||
return "wallet is banned"
|
return "wallet is banned"
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _node_id_for_registration(
|
def _tracker_log(server: "_TrackerHTTPServer", level: str, message: str, **fields: Any) -> None:
|
||||||
|
event = {
|
||||||
|
"ts": time.time(),
|
||||||
|
"level": level,
|
||||||
|
"message": message,
|
||||||
|
"fields": {
|
||||||
|
key: value
|
||||||
|
for key, value in fields.items()
|
||||||
|
if value is not None
|
||||||
|
},
|
||||||
|
}
|
||||||
|
with server.console_lock:
|
||||||
|
server.console_events.append(event)
|
||||||
|
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
|
||||||
|
suffix = f" {extras}" if extras else ""
|
||||||
|
print(f"[tracker] {level}: {message}{suffix}", flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _node_id_for_registration(
|
||||||
endpoint: str,
|
endpoint: str,
|
||||||
model: str,
|
model: str,
|
||||||
wallet_address: str | None,
|
wallet_address: str | None,
|
||||||
@@ -1698,8 +1727,10 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
|||||||
)
|
)
|
||||||
self.toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
|
self.toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
|
||||||
self.toploc_backend = toploc_backend
|
self.toploc_backend = toploc_backend
|
||||||
self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
|
self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
|
||||||
self.models_dir = models_dir
|
self.models_dir = models_dir
|
||||||
|
self.console_events = deque(maxlen=_CONSOLE_LIMIT)
|
||||||
|
self.console_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||||
@@ -1885,10 +1916,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self._handle_tracker_nodes(model)
|
self._handle_tracker_nodes(model)
|
||||||
elif parsed.path == "/v1/raft/status":
|
elif parsed.path == "/v1/raft/status":
|
||||||
self._handle_raft_status()
|
self._handle_raft_status()
|
||||||
elif parsed.path == "/v1/stats":
|
elif parsed.path == "/v1/stats":
|
||||||
self._handle_stats()
|
self._handle_stats()
|
||||||
elif parsed.path == "/v1/billing/summary":
|
elif parsed.path == "/v1/console":
|
||||||
self._handle_billing_summary()
|
self._handle_console()
|
||||||
|
elif parsed.path == "/v1/billing/summary":
|
||||||
|
self._handle_billing_summary()
|
||||||
elif parsed.path == "/v1/billing/settlements":
|
elif parsed.path == "/v1/billing/settlements":
|
||||||
self._handle_billing_settlements()
|
self._handle_billing_settlements()
|
||||||
elif parsed.path == "/v1/account":
|
elif parsed.path == "/v1/account":
|
||||||
@@ -2116,9 +2149,10 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"peer_id": node.peer_id,
|
"peer_id": node.peer_id,
|
||||||
"model": node.model,
|
"model": node.model,
|
||||||
"hf_repo": node.hf_repo,
|
"hf_repo": node.hf_repo,
|
||||||
"num_layers": node.num_layers,
|
"num_layers": node.num_layers,
|
||||||
"model_metadata": dict(node.model_metadata),
|
"model_metadata": dict(node.model_metadata),
|
||||||
"shard_start": node.shard_start,
|
"downloaded_models": [dict(item) for item in node.downloaded_models],
|
||||||
|
"shard_start": node.shard_start,
|
||||||
"shard_end": node.shard_end,
|
"shard_end": node.shard_end,
|
||||||
"tracker_mode": node.tracker_mode,
|
"tracker_mode": node.tracker_mode,
|
||||||
"last_heartbeat": node.last_heartbeat,
|
"last_heartbeat": node.last_heartbeat,
|
||||||
@@ -2263,9 +2297,27 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
if n.shard_start == 0 and _node_matches_model(n, model)
|
if n.shard_start == 0 and _node_matches_model(n, model)
|
||||||
]
|
]
|
||||||
|
|
||||||
if not candidates:
|
if not candidates:
|
||||||
self._send_json(503, {"error": {
|
with server.lock:
|
||||||
"message": f"no nodes available for model {model!r}",
|
registered = [
|
||||||
|
{
|
||||||
|
"node_id": n.node_id,
|
||||||
|
"model": n.model,
|
||||||
|
"hf_repo": n.hf_repo,
|
||||||
|
"shard": f"{n.shard_start}-{n.shard_end}",
|
||||||
|
"tracker_mode": n.tracker_mode,
|
||||||
|
}
|
||||||
|
for n in server.registry.values()
|
||||||
|
]
|
||||||
|
_tracker_log(
|
||||||
|
server,
|
||||||
|
"warn",
|
||||||
|
"no nodes available for model",
|
||||||
|
model=model,
|
||||||
|
registered_nodes=registered,
|
||||||
|
)
|
||||||
|
self._send_json(503, {"error": {
|
||||||
|
"message": f"no nodes available for model {model!r}",
|
||||||
"type": "service_unavailable",
|
"type": "service_unavailable",
|
||||||
"code": "model_not_available",
|
"code": "model_not_available",
|
||||||
}})
|
}})
|
||||||
@@ -2294,9 +2346,18 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
if pinned_nodes is not None:
|
if pinned_nodes is not None:
|
||||||
route_nodes = pinned_nodes
|
route_nodes = pinned_nodes
|
||||||
else:
|
else:
|
||||||
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
|
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
|
||||||
if route_error:
|
if route_error:
|
||||||
self._send_json(503, {"error": {
|
_tracker_log(
|
||||||
|
server,
|
||||||
|
"warn",
|
||||||
|
"route unavailable",
|
||||||
|
model=model,
|
||||||
|
route_model=route_model,
|
||||||
|
error=route_error,
|
||||||
|
candidate_count=len(all_nodes),
|
||||||
|
)
|
||||||
|
self._send_json(503, {"error": {
|
||||||
"message": route_error,
|
"message": route_error,
|
||||||
"type": "service_unavailable",
|
"type": "service_unavailable",
|
||||||
"code": "route_not_available",
|
"code": "route_not_available",
|
||||||
@@ -2322,12 +2383,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
|
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
|
||||||
for n in route_nodes
|
for n in route_nodes
|
||||||
)
|
)
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] proxy route {request_id}: model={model!r} "
|
server,
|
||||||
f"head={node.node_id}@{node.endpoint} downstream={downstream_urls} "
|
"info",
|
||||||
f"route={route_debug or '<empty>'}",
|
"proxy route selected",
|
||||||
flush=True,
|
request_id=request_id,
|
||||||
)
|
model=model,
|
||||||
|
head_node_id=node.node_id,
|
||||||
|
head_endpoint=node.endpoint,
|
||||||
|
downstream=downstream_urls,
|
||||||
|
route=route_debug or "<empty>",
|
||||||
|
)
|
||||||
|
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
target_url,
|
target_url,
|
||||||
@@ -2350,11 +2416,14 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
}
|
}
|
||||||
|
|
||||||
if node.relay_addr:
|
if node.relay_addr:
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] proxy via relay {request_id}: {node.relay_addr} "
|
server,
|
||||||
f"(direct endpoint {target_url})",
|
"info",
|
||||||
flush=True,
|
"proxy via relay",
|
||||||
)
|
request_id=request_id,
|
||||||
|
relay_addr=node.relay_addr,
|
||||||
|
direct_endpoint=target_url,
|
||||||
|
)
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
frames = _relay_http_request_frames(
|
frames = _relay_http_request_frames(
|
||||||
node.relay_addr,
|
node.relay_addr,
|
||||||
@@ -2388,16 +2457,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
input_tokens=in_tokens, output_tokens=out_tokens,
|
input_tokens=in_tokens, output_tokens=out_tokens,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] relay proxy failed {request_id}: {node.relay_addr}; "
|
server,
|
||||||
f"trying direct {target_url}",
|
"warn",
|
||||||
flush=True,
|
"relay proxy failed, trying direct",
|
||||||
)
|
request_id=request_id,
|
||||||
|
relay_addr=node.relay_addr,
|
||||||
|
direct_endpoint=target_url,
|
||||||
|
)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
upstream = urllib.request.urlopen(req, timeout=300.0)
|
upstream = urllib.request.urlopen(req, timeout=300.0)
|
||||||
print(f"[tracker] proxy connected {request_id}: {target_url}", flush=True)
|
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
|
||||||
except urllib.error.HTTPError as exc:
|
except urllib.error.HTTPError as exc:
|
||||||
# Relay error status + body from node
|
# Relay error status + body from node
|
||||||
err_body = exc.read()
|
err_body = exc.read()
|
||||||
@@ -2411,14 +2483,25 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
pass
|
pass
|
||||||
return
|
return
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if node.relay_addr:
|
if node.relay_addr:
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] direct proxy failed {request_id}: {target_url}: {exc}; "
|
server,
|
||||||
f"relay already attempted for {node.relay_addr}",
|
"error",
|
||||||
flush=True,
|
"direct proxy failed after relay",
|
||||||
)
|
request_id=request_id,
|
||||||
else:
|
target_url=target_url,
|
||||||
print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True)
|
relay_addr=node.relay_addr,
|
||||||
|
error=repr(exc),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_tracker_log(
|
||||||
|
server,
|
||||||
|
"error",
|
||||||
|
"proxy failed",
|
||||||
|
request_id=request_id,
|
||||||
|
target_url=target_url,
|
||||||
|
error=repr(exc),
|
||||||
|
)
|
||||||
self._send_json(503, {"error": {
|
self._send_json(503, {"error": {
|
||||||
"message": f"upstream node unreachable: {exc}",
|
"message": f"upstream node unreachable: {exc}",
|
||||||
"type": "service_unavailable",
|
"type": "service_unavailable",
|
||||||
@@ -2464,13 +2547,17 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
input_tokens=in_tokens, output_tokens=out_tokens,
|
input_tokens=in_tokens, output_tokens=out_tokens,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# Non-streaming: buffer and relay
|
# Non-streaming: buffer and relay
|
||||||
resp_body = upstream.read()
|
resp_body = upstream.read()
|
||||||
elapsed = time.monotonic() - started
|
elapsed = time.monotonic() - started
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
|
server,
|
||||||
flush=True,
|
"info",
|
||||||
)
|
"proxy complete",
|
||||||
|
request_id=request_id,
|
||||||
|
target_url=target_url,
|
||||||
|
bytes=len(resp_body),
|
||||||
|
)
|
||||||
observed_output = ""
|
observed_output = ""
|
||||||
try:
|
try:
|
||||||
response_payload = json.loads(resp_body)
|
response_payload = json.loads(resp_body)
|
||||||
@@ -2809,10 +2896,19 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
model_metadata = body.get("model_metadata", {})
|
model_metadata = body.get("model_metadata", {})
|
||||||
if model_metadata is None:
|
if model_metadata is None:
|
||||||
model_metadata = {}
|
model_metadata = {}
|
||||||
if not isinstance(model_metadata, dict):
|
if not isinstance(model_metadata, dict):
|
||||||
self._send_json(400, {"error": "model_metadata must be an object"})
|
self._send_json(400, {"error": "model_metadata must be an object"})
|
||||||
return
|
return
|
||||||
relay_addr = body.get("relay_addr") or None
|
downloaded_models = body.get("downloaded_models", [])
|
||||||
|
if downloaded_models is None:
|
||||||
|
downloaded_models = []
|
||||||
|
if not (
|
||||||
|
isinstance(downloaded_models, list)
|
||||||
|
and all(isinstance(item, dict) for item in downloaded_models)
|
||||||
|
):
|
||||||
|
self._send_json(400, {"error": "downloaded_models must be an array of objects"})
|
||||||
|
return
|
||||||
|
relay_addr = body.get("relay_addr") or None
|
||||||
cert_fingerprint = body.get("cert_fingerprint") or None
|
cert_fingerprint = body.get("cert_fingerprint") or None
|
||||||
peer_id = body.get("peer_id") or None
|
peer_id = body.get("peer_id") or None
|
||||||
|
|
||||||
@@ -2842,10 +2938,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
quantization=quantization,
|
quantization=quantization,
|
||||||
managed_assignment=managed_assignment or not explicit_shard,
|
managed_assignment=managed_assignment or not explicit_shard,
|
||||||
tracker_mode=tracker_mode,
|
tracker_mode=tracker_mode,
|
||||||
hf_repo=hf_repo,
|
hf_repo=hf_repo,
|
||||||
num_layers=num_layers,
|
num_layers=num_layers,
|
||||||
model_metadata=model_metadata,
|
model_metadata=model_metadata,
|
||||||
relay_addr=relay_addr,
|
downloaded_models=downloaded_models,
|
||||||
|
relay_addr=relay_addr,
|
||||||
cert_fingerprint=cert_fingerprint,
|
cert_fingerprint=cert_fingerprint,
|
||||||
peer_id=peer_id,
|
peer_id=peer_id,
|
||||||
)
|
)
|
||||||
@@ -2856,22 +2953,37 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
eid for eid, e in server.registry.items()
|
eid for eid, e in server.registry.items()
|
||||||
if e.endpoint == entry.endpoint.rstrip("/")
|
if e.endpoint == entry.endpoint.rstrip("/")
|
||||||
]
|
]
|
||||||
for eid in stale_ids:
|
for eid in stale_ids:
|
||||||
old = server.registry.pop(eid)
|
old = server.registry.pop(eid)
|
||||||
print(
|
_tracker_log(
|
||||||
f"[tracker] node re-registered: replaced {eid} with {node_id}"
|
server,
|
||||||
f" {old.endpoint}",
|
"info",
|
||||||
flush=True,
|
"node re-registered",
|
||||||
)
|
old_node_id=eid,
|
||||||
server.registry[node_id] = entry
|
node_id=node_id,
|
||||||
|
endpoint=old.endpoint,
|
||||||
|
)
|
||||||
|
server.registry[node_id] = entry
|
||||||
if entry.managed_assignment:
|
if entry.managed_assignment:
|
||||||
if entry.hf_repo:
|
if entry.hf_repo:
|
||||||
_rebalance_hf_model_locked(server, entry.hf_repo)
|
_rebalance_hf_model_locked(server, entry.hf_repo)
|
||||||
else:
|
else:
|
||||||
_rebalance_model_locked(server, model)
|
_rebalance_model_locked(server, model)
|
||||||
assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
|
assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
|
||||||
if assignment_directive is not None:
|
if assignment_directive is not None:
|
||||||
entry.pending_directives.clear()
|
entry.pending_directives.clear()
|
||||||
|
|
||||||
|
_tracker_log(
|
||||||
|
server,
|
||||||
|
"info",
|
||||||
|
"node registered",
|
||||||
|
node_id=node_id,
|
||||||
|
endpoint=entry.endpoint,
|
||||||
|
model=entry.model,
|
||||||
|
hf_repo=entry.hf_repo,
|
||||||
|
shard=f"{entry.shard_start}-{entry.shard_end}",
|
||||||
|
tracker_mode=entry.tracker_mode,
|
||||||
|
)
|
||||||
|
|
||||||
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
|
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
|
||||||
repo_info = f" [{hf_repo}]" if hf_repo else ""
|
repo_info = f" [{hf_repo}]" if hf_repo else ""
|
||||||
@@ -2910,11 +3022,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
with server.lock:
|
with server.lock:
|
||||||
self._purge_expired_nodes()
|
self._purge_expired_nodes()
|
||||||
entry = server.registry.get(node_id)
|
entry = server.registry.get(node_id)
|
||||||
if entry is None:
|
if entry is None:
|
||||||
self._send_json(404, {"error": "node not found"})
|
_tracker_log(server, "warn", "heartbeat for unknown node", node_id=node_id)
|
||||||
return
|
self._send_json(404, {"error": "node not found"})
|
||||||
|
return
|
||||||
entry.last_heartbeat = time.monotonic()
|
entry.last_heartbeat = time.monotonic()
|
||||||
entry.heartbeats_received += 1
|
entry.heartbeats_received += 1
|
||||||
# P2P metadata
|
# P2P metadata
|
||||||
@@ -3023,7 +3136,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
self._send_json(200, server.billing.snapshot())
|
self._send_json(200, server.billing.snapshot())
|
||||||
|
|
||||||
def _handle_dashboard(self):
|
def _handle_dashboard(self):
|
||||||
"""Serve the read-only web dashboard (US-035). Any tracker in the
|
"""Serve the read-only web dashboard (US-035). Any tracker in the
|
||||||
mesh — leader or follower — serves it from its replicated state."""
|
mesh — leader or follower — serves it from its replicated state."""
|
||||||
try:
|
try:
|
||||||
@@ -3038,10 +3151,16 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
self.end_headers()
|
self.end_headers()
|
||||||
try:
|
try:
|
||||||
self.wfile.write(body)
|
self.wfile.write(body)
|
||||||
except BrokenPipeError:
|
except BrokenPipeError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def _handle_registry_wallets(self):
|
def _handle_console(self):
|
||||||
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
|
with server.console_lock:
|
||||||
|
events = [dict(event) for event in server.console_events]
|
||||||
|
self._send_json(200, {"events": events})
|
||||||
|
|
||||||
|
def _handle_registry_wallets(self):
|
||||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||||
if not self._require_role("admin"):
|
if not self._require_role("admin"):
|
||||||
return
|
return
|
||||||
@@ -4194,8 +4313,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"model": e.model,
|
"model": e.model,
|
||||||
"hf_repo": e.hf_repo,
|
"hf_repo": e.hf_repo,
|
||||||
"num_layers": e.num_layers,
|
"num_layers": e.num_layers,
|
||||||
"model_metadata": dict(e.model_metadata),
|
"model_metadata": dict(e.model_metadata),
|
||||||
"shard_checksum": e.shard_checksum,
|
"downloaded_models": [dict(item) for item in e.downloaded_models],
|
||||||
|
"shard_checksum": e.shard_checksum,
|
||||||
"score": e.score,
|
"score": e.score,
|
||||||
}
|
}
|
||||||
for e, start in route_with_start
|
for e, start in route_with_start
|
||||||
@@ -4259,8 +4379,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
|||||||
"wallet_address": e.wallet_address,
|
"wallet_address": e.wallet_address,
|
||||||
"shard_start": e.shard_start,
|
"shard_start": e.shard_start,
|
||||||
"shard_end": e.shard_end,
|
"shard_end": e.shard_end,
|
||||||
"model": e.model,
|
"model": e.model,
|
||||||
"shard_checksum": e.shard_checksum,
|
"downloaded_models": [dict(item) for item in e.downloaded_models],
|
||||||
|
"shard_checksum": e.shard_checksum,
|
||||||
"score": e.score,
|
"score": e.score,
|
||||||
}
|
}
|
||||||
for e in route
|
for e in route
|
||||||
@@ -4678,10 +4799,15 @@ class TrackerServer:
|
|||||||
wallet_address=payload.get("wallet_address"),
|
wallet_address=payload.get("wallet_address"),
|
||||||
score=float(payload.get("score", 1.0)),
|
score=float(payload.get("score", 1.0)),
|
||||||
tracker_mode=bool(payload.get("tracker_mode", False)),
|
tracker_mode=bool(payload.get("tracker_mode", False)),
|
||||||
hf_repo=payload.get("hf_repo"),
|
hf_repo=payload.get("hf_repo"),
|
||||||
num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
|
num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
|
||||||
model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
|
model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
|
||||||
)
|
downloaded_models=(
|
||||||
|
payload.get("downloaded_models")
|
||||||
|
if isinstance(payload.get("downloaded_models"), list)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
)
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._registry[node_id] = entry
|
self._registry[node_id] = entry
|
||||||
|
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ PANELS = [
|
|||||||
"Tracker hive", "Nodes & coverage", "Client balances",
|
"Tracker hive", "Nodes & coverage", "Client balances",
|
||||||
"Node pending payouts", "Settlement history",
|
"Node pending payouts", "Settlement history",
|
||||||
"Strikes / bans / forfeitures", "Model usage", "Node throughput",
|
"Strikes / bans / forfeitures", "Model usage", "Node throughput",
|
||||||
|
"Console output",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -64,3 +65,29 @@ def test_registry_wallets_endpoint():
|
|||||||
assert data["wallets"]["wallet-a"]["banned"] is False
|
assert data["wallets"]["wallet-a"]["banned"] is False
|
||||||
finally:
|
finally:
|
||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_console_endpoint_exposes_tracker_events():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
port = tracker.start()
|
||||||
|
try:
|
||||||
|
body = json.dumps({
|
||||||
|
"endpoint": "http://127.0.0.1:9001",
|
||||||
|
"model": "stub-model",
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 3,
|
||||||
|
"hardware_profile": {},
|
||||||
|
}).encode()
|
||||||
|
req = urllib.request.Request(
|
||||||
|
f"http://127.0.0.1:{port}/v1/nodes/register",
|
||||||
|
data=body,
|
||||||
|
headers={"Content-Type": "application/json"},
|
||||||
|
method="POST",
|
||||||
|
)
|
||||||
|
urllib.request.urlopen(req).read()
|
||||||
|
|
||||||
|
data = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/console").read())
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert any(event["message"] == "node registered" for event in data["events"])
|
||||||
|
|||||||
@@ -399,7 +399,7 @@ def test_download_shard_uses_huggingface_when_repo_is_assigned(tmp_path, monkeyp
|
|||||||
progress=False,
|
progress=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert shard_dir == tmp_path / "tiny-llama" / "layers_0-3"
|
assert shard_dir == tmp_path / "tiny-llama"
|
||||||
assert calls == [{
|
assert calls == [{
|
||||||
"repo_id": "org/tiny-llama-shards",
|
"repo_id": "org/tiny-llama-shards",
|
||||||
"cache_dir": str(tmp_path),
|
"cache_dir": str(tmp_path),
|
||||||
@@ -407,6 +407,54 @@ def test_download_shard_uses_huggingface_when_repo_is_assigned(tmp_path, monkeyp
|
|||||||
}]
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_shard_reuses_model_cache_for_narrower_layer_range(
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A wider cached shard satisfies a later narrower assignment for the same model."""
|
||||||
|
cache_dir = tmp_path / "cache"
|
||||||
|
model_dir = cache_dir / "tiny-llama"
|
||||||
|
model_dir.mkdir(parents=True)
|
||||||
|
(model_dir / "config.json").write_bytes(b"{}")
|
||||||
|
(model_dir / "model-00001-of-00002.safetensors").write_bytes(b"a" * 3)
|
||||||
|
(model_dir / "model-00002-of-00002.safetensors").write_bytes(b"b" * 5)
|
||||||
|
|
||||||
|
def unexpected_urlopen(*args, **kwargs):
|
||||||
|
raise AssertionError("cached files should avoid tracker download")
|
||||||
|
|
||||||
|
def unexpected_snapshot_download(*args, **kwargs):
|
||||||
|
raise AssertionError("cached files should avoid HuggingFace download")
|
||||||
|
|
||||||
|
monkeypatch.setattr(urllib.request, "urlopen", unexpected_urlopen)
|
||||||
|
monkeypatch.setitem(
|
||||||
|
sys.modules,
|
||||||
|
"huggingface_hub",
|
||||||
|
types.SimpleNamespace(snapshot_download=unexpected_snapshot_download),
|
||||||
|
)
|
||||||
|
|
||||||
|
shard_dir = download_shard(
|
||||||
|
"tiny-llama",
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
cache_dir=cache_dir,
|
||||||
|
hf_repo="org/tiny-llama-shards",
|
||||||
|
model_sources=[{
|
||||||
|
"type": "tracker",
|
||||||
|
"url": "http://tracker/v1/model-files/download?model=tiny-llama",
|
||||||
|
"files": ["config.json", "model-00001-of-00002.safetensors"],
|
||||||
|
"file_sizes": {
|
||||||
|
"config.json": 2,
|
||||||
|
"model-00001-of-00002.safetensors": 3,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
peers=[{"endpoint": "http://peer", "checksum": "unused"}],
|
||||||
|
progress=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert shard_dir == model_dir
|
||||||
|
assert (model_dir / "model-00002-of-00002.safetensors").read_bytes() == b"b" * 5
|
||||||
|
|
||||||
|
|
||||||
def test_download_shard_prefers_tracker_model_source_over_huggingface(
|
def test_download_shard_prefers_tracker_model_source_over_huggingface(
|
||||||
tmp_path,
|
tmp_path,
|
||||||
monkeypatch,
|
monkeypatch,
|
||||||
@@ -1329,12 +1377,7 @@ def test_full_startup_sequence(tmp_path):
|
|||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
def test_second_node_downloads_same_shard_from_peer_without_huggingface(
|
def test_preset_model_startup_starts_heartbeat(tmp_path, monkeypatch):
|
||||||
tmp_path,
|
|
||||||
monkeypatch,
|
|
||||||
capsys,
|
|
||||||
):
|
|
||||||
"""Node A downloads from HF stub; node B downloads same assignment from node A."""
|
|
||||||
import meshnet_node.startup as startup_mod
|
import meshnet_node.startup as startup_mod
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
@@ -1342,6 +1385,53 @@ def test_second_node_downloads_same_shard_from_peer_without_huggingface(
|
|||||||
"detect_hardware",
|
"detect_hardware",
|
||||||
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
|
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
|
||||||
)
|
)
|
||||||
|
heartbeat_calls = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
startup_mod,
|
||||||
|
"_start_heartbeat",
|
||||||
|
lambda *args, **kwargs: heartbeat_calls.append((args, kwargs)),
|
||||||
|
)
|
||||||
|
|
||||||
|
tracker = TrackerServer(model_presets={"stub-model": {"layers_start": 0, "layers_end": 15}})
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
tracker_url = f"http://127.0.0.1:{tracker_port}"
|
||||||
|
try:
|
||||||
|
node = run_startup(
|
||||||
|
tracker_url=tracker_url,
|
||||||
|
model="stub-model",
|
||||||
|
wallet_path=tmp_path / "wallet.json",
|
||||||
|
cache_dir=tmp_path / "shards",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
assert len(heartbeat_calls) == 1
|
||||||
|
args, kwargs = heartbeat_calls[0]
|
||||||
|
assert args[0] == tracker_url
|
||||||
|
assert args[2]["model"] == "stub-model"
|
||||||
|
assert kwargs["node_ref"] is node
|
||||||
|
finally:
|
||||||
|
node.stop()
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_model_startup_registers_downloaded_inventory_without_checksum(
|
||||||
|
tmp_path,
|
||||||
|
monkeypatch,
|
||||||
|
capsys,
|
||||||
|
):
|
||||||
|
"""Real model folders are reported as inventory without hashing their contents."""
|
||||||
|
import meshnet_node.startup as startup_mod
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
startup_mod,
|
||||||
|
"detect_hardware",
|
||||||
|
lambda: {"device": "cpu", "gpu_name": None, "vram_mb": 0},
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
startup_mod,
|
||||||
|
"compute_shard_checksum",
|
||||||
|
lambda _path: (_ for _ in ()).throw(AssertionError("real model startup must not hash model files")),
|
||||||
|
)
|
||||||
hf_calls = []
|
hf_calls = []
|
||||||
|
|
||||||
def fake_snapshot_download(repo_id, cache_dir, local_dir):
|
def fake_snapshot_download(repo_id, cache_dir, local_dir):
|
||||||
@@ -1365,36 +1455,75 @@ def test_second_node_downloads_same_shard_from_peer_without_huggingface(
|
|||||||
})
|
})
|
||||||
tracker_port = tracker.start()
|
tracker_port = tracker.start()
|
||||||
tracker_url = f"http://127.0.0.1:{tracker_port}"
|
tracker_url = f"http://127.0.0.1:{tracker_port}"
|
||||||
nodes = []
|
|
||||||
try:
|
try:
|
||||||
node_a = run_startup(
|
node = run_startup(
|
||||||
tracker_url=tracker_url,
|
tracker_url=tracker_url,
|
||||||
model="tiny-llama",
|
model="tiny-llama",
|
||||||
wallet_path=tmp_path / "wallet-a.json",
|
wallet_path=tmp_path / "wallet.json",
|
||||||
cache_dir=tmp_path / "node-a-shards",
|
cache_dir=tmp_path / "node-shards",
|
||||||
)
|
)
|
||||||
nodes.append(node_a)
|
try:
|
||||||
assert len(hf_calls) == 1
|
assert len(hf_calls) == 1
|
||||||
|
assert (tmp_path / "node-shards" / "tiny-llama" / "weights.json").exists()
|
||||||
node_b = run_startup(
|
output = capsys.readouterr().out
|
||||||
tracker_url=tracker_url,
|
assert "Cached at:" in output
|
||||||
model="tiny-llama",
|
network_map = _get_json(f"{tracker_url}/v1/network/map")
|
||||||
wallet_path=tmp_path / "wallet-b.json",
|
registered = network_map["nodes"][0]
|
||||||
cache_dir=tmp_path / "node-b-shards",
|
assert registered["downloaded_models"] == [{
|
||||||
)
|
"model": "tiny-llama",
|
||||||
nodes.append(node_b)
|
"shard_start": 0,
|
||||||
|
"shard_end": 15,
|
||||||
assert len(hf_calls) == 1
|
"path": str(tmp_path / "node-shards" / "tiny-llama"),
|
||||||
assert (tmp_path / "node-b-shards" / "tiny-llama" / "layers_0-15" / "weights.json").exists()
|
"file_count": 1,
|
||||||
output = capsys.readouterr().out
|
"total_bytes": (tmp_path / "node-shards" / "tiny-llama" / "weights.json").stat().st_size,
|
||||||
assert "download source: HuggingFace" in output
|
"hf_repo": "org/tiny-llama-shards",
|
||||||
assert "download source: peer" in output
|
}]
|
||||||
finally:
|
finally:
|
||||||
for node in reversed(nodes):
|
|
||||||
node.stop()
|
node.stop()
|
||||||
|
finally:
|
||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_downloaded_model_inventory_reports_local_model_percentage(tmp_path):
|
||||||
|
import meshnet_node.startup as startup_mod
|
||||||
|
|
||||||
|
model_dir = tmp_path / "models" / "tiny-llama"
|
||||||
|
model_dir.mkdir(parents=True)
|
||||||
|
(model_dir / "config.json").write_bytes(b"{}")
|
||||||
|
(model_dir / "weights-a.safetensors").write_bytes(b"a" * 3)
|
||||||
|
|
||||||
|
inventory = startup_mod._downloaded_model_inventory(
|
||||||
|
"tiny-llama",
|
||||||
|
0,
|
||||||
|
1,
|
||||||
|
model_dir,
|
||||||
|
hf_repo="org/tiny-llama",
|
||||||
|
model_sources=[{
|
||||||
|
"full_files": ["config.json", "weights-a.safetensors", "weights-b.safetensors"],
|
||||||
|
"file_sizes": {
|
||||||
|
"config.json": 2,
|
||||||
|
"weights-a.safetensors": 3,
|
||||||
|
"weights-b.safetensors": 5,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert inventory == [{
|
||||||
|
"model": "tiny-llama",
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 1,
|
||||||
|
"path": str(model_dir),
|
||||||
|
"file_count": 2,
|
||||||
|
"total_bytes": 5,
|
||||||
|
"hf_repo": "org/tiny-llama",
|
||||||
|
"expected_file_count": 3,
|
||||||
|
"local_expected_file_count": 2,
|
||||||
|
"expected_bytes": 10,
|
||||||
|
"local_expected_bytes": 5,
|
||||||
|
"local_model_percentage": 50.0,
|
||||||
|
}]
|
||||||
|
|
||||||
|
|
||||||
def test_network_assign_gap_found_field():
|
def test_network_assign_gap_found_field():
|
||||||
"""network/assign sets gap_found=True when a real gap exists, False when fully covered."""
|
"""network/assign sets gap_found=True when a real gap exists, False when fully covered."""
|
||||||
import json as _json
|
import json as _json
|
||||||
@@ -1587,6 +1716,30 @@ def test_startup_cpu_fallback(tmp_path, monkeypatch):
|
|||||||
# --------------------------------------------------- layer detection (US: composite configs)
|
# --------------------------------------------------- layer detection (US: composite configs)
|
||||||
|
|
||||||
|
|
||||||
|
def test_detect_num_layers_prefers_flattened_local_model_config(tmp_path, monkeypatch):
|
||||||
|
import meshnet_node.startup as startup_mod
|
||||||
|
|
||||||
|
model_dir = tmp_path / "Qwen3.6-35B-A3B"
|
||||||
|
model_dir.mkdir()
|
||||||
|
(model_dir / "config.json").write_text("{}")
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class AutoConfigStub:
|
||||||
|
@staticmethod
|
||||||
|
def from_pretrained(model_id, cache_dir=None):
|
||||||
|
calls.append({"model_id": model_id, "cache_dir": cache_dir})
|
||||||
|
return types.SimpleNamespace(num_hidden_layers=37)
|
||||||
|
|
||||||
|
monkeypatch.setitem(
|
||||||
|
sys.modules,
|
||||||
|
"transformers",
|
||||||
|
types.SimpleNamespace(AutoConfig=AutoConfigStub),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert startup_mod._detect_num_layers("unsloth/Qwen3.6-35B-A3B", cache_dir=tmp_path) == 37
|
||||||
|
assert calls == [{"model_id": str(model_dir), "cache_dir": None}]
|
||||||
|
|
||||||
|
|
||||||
def test_layers_from_config_top_level():
|
def test_layers_from_config_top_level():
|
||||||
from meshnet_node.model_catalog import layers_from_config
|
from meshnet_node.model_catalog import layers_from_config
|
||||||
|
|
||||||
|
|||||||
@@ -75,6 +75,13 @@ def test_tracker_exposes_registered_model_metadata():
|
|||||||
"activated_parameters": "32B",
|
"activated_parameters": "32B",
|
||||||
"context_length": 256000,
|
"context_length": 256000,
|
||||||
},
|
},
|
||||||
|
"downloaded_models": [{
|
||||||
|
"model": "Kimi-K2.7-Code",
|
||||||
|
"hf_repo": "unsloth/Kimi-K2.7-Code",
|
||||||
|
"shard_start": 0,
|
||||||
|
"shard_end": 60,
|
||||||
|
"local_model_percentage": 100.0,
|
||||||
|
}],
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -91,6 +98,7 @@ def test_tracker_exposes_registered_model_metadata():
|
|||||||
registered = network_map["nodes"][0]
|
registered = network_map["nodes"][0]
|
||||||
assert registered["num_layers"] == 61
|
assert registered["num_layers"] == 61
|
||||||
assert registered["model_metadata"]["context_length"] == 256000
|
assert registered["model_metadata"]["context_length"] == 256000
|
||||||
|
assert registered["downloaded_models"][0]["local_model_percentage"] == 100.0
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_lists_recommended_kimi_before_nodes_register():
|
def test_tracker_lists_recommended_kimi_before_nodes_register():
|
||||||
@@ -975,6 +983,30 @@ def test_tracker_route_rejects_non_extending_overlap():
|
|||||||
tracker.stop()
|
tracker.stop()
|
||||||
|
|
||||||
|
|
||||||
|
def test_tracker_console_records_model_not_available():
|
||||||
|
tracker = TrackerServer()
|
||||||
|
tracker_port = tracker.start()
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
_post_json(
|
||||||
|
f"http://127.0.0.1:{tracker_port}/v1/chat/completions",
|
||||||
|
{"model": "missing-model", "messages": [{"role": "user", "content": "hi"}]},
|
||||||
|
)
|
||||||
|
raise AssertionError("Expected 503")
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
assert exc.code == 503
|
||||||
|
|
||||||
|
console = _get_json(f"http://127.0.0.1:{tracker_port}/v1/console")
|
||||||
|
finally:
|
||||||
|
tracker.stop()
|
||||||
|
|
||||||
|
assert any(
|
||||||
|
event["message"] == "no nodes available for model"
|
||||||
|
and event["fields"]["model"] == "missing-model"
|
||||||
|
for event in console["events"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_tracker_registration_rejects_invalid_payload():
|
def test_tracker_registration_rejects_invalid_payload():
|
||||||
"""Registration errors return a defined JSON 400 response."""
|
"""Registration errors return a defined JSON 400 response."""
|
||||||
tracker = TrackerServer()
|
tracker = TrackerServer()
|
||||||
|
|||||||
Reference in New Issue
Block a user