6208 lines
250 KiB
Python
6208 lines
250 KiB
Python
"""Tracker HTTP server — node registry and route selection for the inference mesh.
|
||
|
||
HTTP API contract:
|
||
- POST /v1/nodes/register
|
||
Request JSON: {
|
||
"endpoint": "http://host:port", "shard_start": int, "shard_end": int,
|
||
"model": str optional, "shard_checksum": str optional,
|
||
"hardware_profile": object, "wallet_address": str optional,
|
||
"score": number optional
|
||
}
|
||
Response 200: {"node_id": str}
|
||
Response 400: {"error": str}
|
||
- POST /v1/nodes/{node_id}/heartbeat
|
||
Response 200: {}
|
||
Response 404: {"error": "node not found"}
|
||
- GET /v1/route?model=<preset>
|
||
Response 200: {"route": ["http://node-a", "http://node-b"]}
|
||
Response 400/404/503: {"error": str}
|
||
- GET /v1/routes?model=<preset>&redundancy=<n>
|
||
Response 200: {"routes": [{"route": [...], "nodes": [...]}]}
|
||
Response 400/404/503: {"error": str}
|
||
- GET /v1/routing?model=<name> (ADR-0021 learned route table)
|
||
Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}}
|
||
"""
|
||
|
||
import http.cookies
|
||
import http.server
|
||
import hashlib
|
||
import itertools
|
||
import json
|
||
import os
|
||
import random
|
||
import select
|
||
import socketserver
|
||
import sqlite3
|
||
import tarfile
|
||
import threading
|
||
import time
|
||
import urllib.parse
|
||
import urllib.request
|
||
import uuid
|
||
from collections import deque
|
||
from dataclasses import dataclass, field
|
||
from importlib.resources import files
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
|
||
from .auth import is_validator_token, sign_hive_request, verify_hive_request
|
||
from .wallet_proof import binding_message, verify_wallet_signature
|
||
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
|
||
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
|
||
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
|
||
from .gossip import NodeGossip
|
||
from .logging_setup import tracker_logger
|
||
from .routing_stats import (
|
||
RouteCandidate,
|
||
RouteStatsStore,
|
||
RoutingConfig,
|
||
choose_route,
|
||
route_signature,
|
||
route_table,
|
||
)
|
||
from .model_files import files_for_layer_range, snapshot_dir_for_repo
|
||
from .raft import RaftNode
|
||
|
||
|
||
_CONSOLE_LIMIT = 300
|
||
_PROXY_PROGRESS_LOG_INTERVAL = 5.0
|
||
_SESSION_COOKIE_NAME = "meshnet_session"
|
||
|
||
|
||
def _preset_price_keys(name: str, preset: dict) -> set[str]:
|
||
"""All model strings a client may bill under for one preset.
|
||
|
||
``BillingLedger.price_for`` is keyed by the raw ``model`` string in the
|
||
request, so the preset price must be registered under the preset name,
|
||
its ``hf_repo``, and every alias — otherwise ``unsloth/Qwen…`` style
|
||
requests silently fall back to the default rate.
|
||
"""
|
||
keys = {name}
|
||
hf_repo = preset.get("hf_repo")
|
||
if isinstance(hf_repo, str) and hf_repo:
|
||
keys.add(hf_repo)
|
||
for alias in preset.get("aliases") or []:
|
||
if isinstance(alias, str) and alias:
|
||
keys.add(alias)
|
||
return keys
|
||
|
||
|
||
def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
|
||
"""Return wss://host/ws when url is a public HTTPS tracker origin."""
|
||
if not url:
|
||
return None
|
||
parsed = urllib.parse.urlparse(url)
|
||
if parsed.scheme != "https":
|
||
return None
|
||
host = parsed.hostname
|
||
if not host or host in ("127.0.0.1", "localhost"):
|
||
return None
|
||
return f"wss://{parsed.netloc}/ws"
|
||
|
||
|
||
def _load_model_presets_from_data() -> dict[str, dict]:
|
||
"""Load recommended model presets from package JSON data."""
|
||
try:
|
||
raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
|
||
data = json.loads(raw)
|
||
except Exception:
|
||
return {}
|
||
models = data.get("models", {})
|
||
if not isinstance(models, dict):
|
||
return {}
|
||
return {
|
||
str(name): preset
|
||
for name, preset in models.items()
|
||
if isinstance(preset, dict)
|
||
}
|
||
|
||
|
||
DEFAULT_MODEL_PRESETS: dict[str, dict] = {
|
||
"stub-model": {
|
||
"layers_start": 0,
|
||
"layers_end": 31,
|
||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
|
||
},
|
||
"openai-community/gpt2": {
|
||
"layers_start": 0,
|
||
"layers_end": 11,
|
||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
|
||
},
|
||
**_load_model_presets_from_data(),
|
||
}
|
||
|
||
def _clone_model_presets(presets: dict[str, dict]) -> dict[str, dict]:
|
||
"""Shallow-copy each preset dict so a runtime mutation (e.g. issue 23's
|
||
dynamic pricing refresh writing hf_last_price_per_1k/hf_last_updated)
|
||
never leaks into the shared module-level DEFAULT_MODEL_PRESETS and from
|
||
there into other TrackerServer instances in the same process."""
|
||
return {name: dict(preset) for name, preset in presets.items()}
|
||
|
||
|
||
DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
|
||
DEFAULT_RAM_BYTES = 16 * 1024 * 1024 * 1024
|
||
|
||
|
||
def _snapshot_regular_files(snapshot_dir: Path) -> list[str]:
|
||
return sorted(
|
||
path.relative_to(snapshot_dir).as_posix()
|
||
for path in snapshot_dir.rglob("*")
|
||
if path.is_file()
|
||
)
|
||
DEFAULT_QUANTIZATIONS = ["bfloat16"]
|
||
DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
|
||
# US-039/US-040 — single source of truth for the credit defaults (referenced by
|
||
# TrackerServer, _TrackerHTTPServer, and the CLI). Alpha runs devnet-friendly;
|
||
# flip both to 0 before any deployment holding a mainnet treasury.
|
||
DEFAULT_CALLER_CREDIT_USDT = 1.0
|
||
DEFAULT_DEVNET_TOPUP_USDT = 1.0
|
||
|
||
|
||
def _endpoint_key(url: str) -> str:
|
||
"""Normalize http(s) endpoints for host:port comparison."""
|
||
parsed = urllib.parse.urlparse(url.rstrip("/"))
|
||
host = (parsed.hostname or "").lower()
|
||
if not host:
|
||
return url.rstrip("/").lower()
|
||
port = parsed.port
|
||
if port is None:
|
||
port = 443 if parsed.scheme == "https" else 80
|
||
return f"{host}:{port}"
|
||
|
||
|
||
def _model_aliases(model: str | None) -> set[str]:
|
||
"""Return stable lookup aliases for a model repo or display name."""
|
||
if not model:
|
||
return set()
|
||
normalized = model.strip()
|
||
if not normalized:
|
||
return set()
|
||
aliases = {normalized}
|
||
short = normalized.rsplit("/", 1)[-1]
|
||
aliases.add(short)
|
||
lowered = short.lower()
|
||
aliases.add(lowered)
|
||
if lowered.endswith("-instruct"):
|
||
aliases.add(lowered.removesuffix("-instruct"))
|
||
return aliases
|
||
|
||
|
||
def _preset_aliases(name: str, preset: dict | None) -> set[str]:
|
||
aliases = _model_aliases(name)
|
||
if not preset:
|
||
return aliases
|
||
hf_repo = preset.get("hf_repo")
|
||
if isinstance(hf_repo, str):
|
||
aliases |= _model_aliases(hf_repo)
|
||
for alias in preset.get("aliases", []) or []:
|
||
if isinstance(alias, str):
|
||
aliases |= _model_aliases(alias)
|
||
return aliases
|
||
|
||
|
||
def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
|
||
requested = _model_aliases(model)
|
||
for name, preset in model_presets.items():
|
||
if requested & _preset_aliases(name, preset):
|
||
return name, preset
|
||
return None, None
|
||
|
||
|
||
def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
|
||
requested = _model_aliases(model)
|
||
if not requested:
|
||
return False
|
||
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
|
||
|
||
|
||
def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
|
||
requested = _preset_aliases(name, preset)
|
||
return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
|
||
|
||
|
||
class _RollingCounter:
|
||
"""Circular-bucket request counter.
|
||
|
||
Tracks events in `num_buckets` slots, each covering `bucket_seconds` of wall time.
|
||
Old buckets are silently discarded when their epoch expires.
|
||
"""
|
||
|
||
def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
|
||
self._num = num_buckets
|
||
self._bsec = bucket_seconds
|
||
self._counts: list[int] = [0] * num_buckets
|
||
self._epochs: list[int] = [-1] * num_buckets # which epoch each slot holds
|
||
|
||
def _epoch(self, now: float) -> int:
|
||
return int(now) // self._bsec
|
||
|
||
def record(self, now: float | None = None) -> None:
|
||
t = now if now is not None else time.time()
|
||
ep = self._epoch(t)
|
||
idx = ep % self._num
|
||
if self._epochs[idx] != ep:
|
||
self._counts[idx] = 0
|
||
self._epochs[idx] = ep
|
||
self._counts[idx] += 1
|
||
|
||
def rpm(self, now: float | None = None) -> float:
|
||
t = now if now is not None else time.time()
|
||
cutoff = self._epoch(t) - self._num # epochs <= this are stale
|
||
total = sum(self._counts[i] for i in range(self._num) if self._epochs[i] > cutoff)
|
||
window_minutes = (self._num * self._bsec) / 60.0
|
||
return total / window_minutes if window_minutes > 0 else 0.0
|
||
|
||
def buckets(self) -> list[tuple[int, int]]:
|
||
return [(self._epochs[i], self._counts[i]) for i in range(self._num)]
|
||
|
||
def restore_buckets(self, data: list[tuple[int, int]]) -> None:
|
||
for i, (ep, cnt) in enumerate(data):
|
||
if i < self._num:
|
||
self._epochs[i] = ep
|
||
self._counts[i] = cnt
|
||
|
||
|
||
class _RollingThroughput:
|
||
"""Circular buckets for observed tokens/sec.
|
||
|
||
Each bucket stores total output tokens and total elapsed seconds. TPS for a
|
||
window is the ratio across non-stale buckets, which avoids over-weighting
|
||
small fast requests.
|
||
"""
|
||
|
||
def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
|
||
self._num = num_buckets
|
||
self._bsec = bucket_seconds
|
||
self._tokens: list[int] = [0] * num_buckets
|
||
self._seconds: list[float] = [0.0] * num_buckets
|
||
self._samples: list[int] = [0] * num_buckets
|
||
self._epochs: list[int] = [-1] * num_buckets
|
||
|
||
def _epoch(self, now: float) -> int:
|
||
return int(now) // self._bsec
|
||
|
||
def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
|
||
if total_tokens <= 0 or elapsed_seconds <= 0:
|
||
return
|
||
t = now if now is not None else time.time()
|
||
ep = self._epoch(t)
|
||
idx = ep % self._num
|
||
if self._epochs[idx] != ep:
|
||
self._tokens[idx] = 0
|
||
self._seconds[idx] = 0.0
|
||
self._samples[idx] = 0
|
||
self._epochs[idx] = ep
|
||
self._tokens[idx] += int(total_tokens)
|
||
self._seconds[idx] += float(elapsed_seconds)
|
||
self._samples[idx] += 1
|
||
|
||
def stats(self, now: float | None = None) -> dict:
|
||
t = now if now is not None else time.time()
|
||
cutoff = self._epoch(t) - self._num
|
||
tokens = 0
|
||
seconds = 0.0
|
||
samples = 0
|
||
for i in range(self._num):
|
||
if self._epochs[i] > cutoff:
|
||
tokens += self._tokens[i]
|
||
seconds += self._seconds[i]
|
||
samples += self._samples[i]
|
||
return {
|
||
"tokens_per_sec": round(tokens / seconds, 4) if seconds > 0 else None,
|
||
"tokens": tokens,
|
||
"seconds": round(seconds, 6),
|
||
"sample_count": samples,
|
||
}
|
||
|
||
def buckets(self) -> list[tuple[int, int, float, int]]:
|
||
return [
|
||
(self._epochs[i], self._tokens[i], self._seconds[i], self._samples[i])
|
||
for i in range(self._num)
|
||
]
|
||
|
||
def restore_buckets(self, data: list[tuple[int, int, float, int]]) -> None:
|
||
for i, (ep, tokens, seconds, samples) in enumerate(data):
|
||
if i < self._num:
|
||
self._epochs[i] = ep
|
||
self._tokens[i] = tokens
|
||
self._seconds[i] = seconds
|
||
self._samples[i] = samples
|
||
|
||
|
||
class _ModelStats:
|
||
"""Three rolling windows for one model: last hour, last day, last month."""
|
||
|
||
def __init__(self) -> None:
|
||
self.per_minute = _RollingCounter(60, 60) # 60 × 1-min buckets = 1 hour
|
||
self.per_hour = _RollingCounter(24, 3600) # 24 × 1-hr buckets = 1 day
|
||
self.per_day = _RollingCounter(30, 86400) # 30 × 1-day buckets = ~1 month
|
||
|
||
def record(self, now: float | None = None) -> None:
|
||
t = now if now is not None else time.time()
|
||
self.per_minute.record(t)
|
||
self.per_hour.record(t)
|
||
self.per_day.record(t)
|
||
|
||
def rpms(self, now: float | None = None) -> dict:
|
||
t = now if now is not None else time.time()
|
||
return {
|
||
"rpm_last_hour": round(self.per_minute.rpm(t), 4),
|
||
"rpm_last_day": round(self.per_hour.rpm(t), 4),
|
||
"rpm_last_month": round(self.per_day.rpm(t), 4),
|
||
}
|
||
|
||
|
||
class _NodeModelThroughput:
|
||
"""Observed throughput windows for one node/model pair."""
|
||
|
||
def __init__(self) -> None:
|
||
self.per_minute = _RollingThroughput(60, 60)
|
||
self.per_hour = _RollingThroughput(24, 3600)
|
||
self.per_day = _RollingThroughput(30, 86400)
|
||
|
||
def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
|
||
t = now if now is not None else time.time()
|
||
self.per_minute.record(total_tokens, elapsed_seconds, t)
|
||
self.per_hour.record(total_tokens, elapsed_seconds, t)
|
||
self.per_day.record(total_tokens, elapsed_seconds, t)
|
||
|
||
def stats(self, now: float | None = None) -> dict:
|
||
hour = self.per_minute.stats(now)
|
||
day = self.per_hour.stats(now)
|
||
month = self.per_day.stats(now)
|
||
return {
|
||
"tokens_per_sec_last_hour": hour["tokens_per_sec"],
|
||
"tokens_per_sec_last_day": day["tokens_per_sec"],
|
||
"tokens_per_sec_last_month": month["tokens_per_sec"],
|
||
"sample_count_last_hour": hour["sample_count"],
|
||
"tokens_last_hour": hour["tokens"],
|
||
"seconds_last_hour": hour["seconds"],
|
||
}
|
||
|
||
|
||
class _StatsCollector:
|
||
"""Thread-safe model request stats with SQLite persistence and peer slice merging."""
|
||
|
||
SAVE_INTERVAL = 60.0 # seconds between DB saves
|
||
|
||
def __init__(self, db_path: str | None = None) -> None:
|
||
self._lock = threading.Lock()
|
||
self._local: dict[str, _ModelStats] = {}
|
||
self._node_model: dict[tuple[str, str], _NodeModelThroughput] = {}
|
||
self._peer_rpms: dict[str, dict[str, dict]] = {} # tracker_url -> model -> rpms
|
||
self._db_path = db_path
|
||
if db_path:
|
||
self._init_db()
|
||
self._load_from_db()
|
||
|
||
# ---------- public API ----------
|
||
|
||
def record_request(self, model: str, now: float | None = None) -> None:
|
||
t = now if now is not None else time.time()
|
||
with self._lock:
|
||
if model not in self._local:
|
||
self._local[model] = _ModelStats()
|
||
self._local[model].record(t)
|
||
|
||
def record_node_throughput(
|
||
self,
|
||
node_id: str,
|
||
model: str,
|
||
total_tokens: int,
|
||
elapsed_seconds: float,
|
||
now: float | None = None,
|
||
) -> None:
|
||
if not node_id or not model or total_tokens <= 0 or elapsed_seconds <= 0:
|
||
return
|
||
t = now if now is not None else time.time()
|
||
with self._lock:
|
||
key = (node_id, model)
|
||
if key not in self._node_model:
|
||
self._node_model[key] = _NodeModelThroughput()
|
||
self._node_model[key].record(total_tokens, elapsed_seconds, t)
|
||
|
||
def get_local_rpms(self, now: float | None = None) -> dict[str, dict]:
|
||
t = now if now is not None else time.time()
|
||
with self._lock:
|
||
return {m: s.rpms(t) for m, s in self._local.items()}
|
||
|
||
def get_node_model_stats(self, node_id: str, model: str, now: float | None = None) -> dict:
|
||
t = now if now is not None else time.time()
|
||
empty = {
|
||
"tokens_per_sec_last_hour": None,
|
||
"tokens_per_sec_last_day": None,
|
||
"tokens_per_sec_last_month": None,
|
||
"sample_count_last_hour": 0,
|
||
"tokens_last_hour": 0,
|
||
"seconds_last_hour": 0.0,
|
||
}
|
||
with self._lock:
|
||
stat = self._node_model.get((node_id, model))
|
||
return stat.stats(t) if stat is not None else dict(empty)
|
||
|
||
def get_node_throughput_stats(self, now: float | None = None) -> dict[str, dict]:
|
||
t = now if now is not None else time.time()
|
||
with self._lock:
|
||
result: dict[str, dict] = {}
|
||
for (node_id, model), stat in self._node_model.items():
|
||
result.setdefault(node_id, {"models": {}})["models"][model] = stat.stats(t)
|
||
return result
|
||
|
||
def merge_peer_rpms(self, tracker_url: str, rpms: dict[str, dict]) -> None:
|
||
with self._lock:
|
||
self._peer_rpms[tracker_url] = dict(rpms)
|
||
|
||
def get_combined_stats(self, now: float | None = None) -> dict:
|
||
t = now if now is not None else time.time()
|
||
with self._lock:
|
||
combined: dict[str, dict] = {}
|
||
for model, ms in self._local.items():
|
||
combined[model] = ms.rpms(t)
|
||
for _url, peer in self._peer_rpms.items():
|
||
for model, rpms in peer.items():
|
||
if model not in combined:
|
||
combined[model] = {"rpm_last_hour": 0.0, "rpm_last_day": 0.0, "rpm_last_month": 0.0}
|
||
for key in ("rpm_last_hour", "rpm_last_day", "rpm_last_month"):
|
||
combined[model][key] = round(combined[model].get(key, 0.0) + rpms.get(key, 0.0), 4)
|
||
return combined
|
||
|
||
# ---------- persistence ----------
|
||
|
||
def _init_db(self) -> None:
|
||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||
con.execute(
|
||
"CREATE TABLE IF NOT EXISTS model_rpm_buckets "
|
||
"(model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, count INTEGER, "
|
||
"PRIMARY KEY (model, window, bucket_idx))"
|
||
)
|
||
con.execute(
|
||
"CREATE TABLE IF NOT EXISTS node_model_tps_buckets "
|
||
"(node_id TEXT, model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, "
|
||
"tokens INTEGER, seconds REAL, samples INTEGER, "
|
||
"PRIMARY KEY (node_id, model, window, bucket_idx))"
|
||
)
|
||
con.commit()
|
||
con.close()
|
||
|
||
def save_to_db(self) -> None:
|
||
if not self._db_path:
|
||
return
|
||
with self._lock:
|
||
rows = []
|
||
for model, ms in self._local.items():
|
||
for window_name, counter in (
|
||
("hour", ms.per_minute),
|
||
("day", ms.per_hour),
|
||
("month", ms.per_day),
|
||
):
|
||
for idx, (ep, cnt) in enumerate(counter.buckets()):
|
||
if ep >= 0:
|
||
rows.append((model, window_name, idx, ep, cnt))
|
||
tps_rows = []
|
||
for (node_id, model), stat in self._node_model.items():
|
||
for window_name, counter in (
|
||
("hour", stat.per_minute),
|
||
("day", stat.per_hour),
|
||
("month", stat.per_day),
|
||
):
|
||
for idx, (ep, tokens, seconds, samples) in enumerate(counter.buckets()):
|
||
if ep >= 0:
|
||
tps_rows.append((node_id, model, window_name, idx, ep, tokens, seconds, samples))
|
||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||
con.executemany(
|
||
"INSERT OR REPLACE INTO model_rpm_buckets VALUES (?,?,?,?,?)", rows
|
||
)
|
||
con.executemany(
|
||
"INSERT OR REPLACE INTO node_model_tps_buckets VALUES (?,?,?,?,?,?,?,?)",
|
||
tps_rows,
|
||
)
|
||
con.commit()
|
||
con.close()
|
||
|
||
def _load_from_db(self) -> None:
|
||
con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
|
||
rows = con.execute("SELECT model, window, bucket_idx, bucket_epoch, count FROM model_rpm_buckets").fetchall()
|
||
tps_rows = con.execute(
|
||
"SELECT node_id, model, window, bucket_idx, bucket_epoch, tokens, seconds, samples "
|
||
"FROM node_model_tps_buckets"
|
||
).fetchall()
|
||
con.close()
|
||
grouped: dict[str, dict[str, list[tuple[int, int]]]] = {}
|
||
for model, window, idx, ep, cnt in rows:
|
||
grouped.setdefault(model, {}).setdefault(window, []).append((idx, ep, cnt))
|
||
for model, windows in grouped.items():
|
||
ms = _ModelStats()
|
||
for window_name, entries in windows.items():
|
||
counter = {"hour": ms.per_minute, "day": ms.per_hour, "month": ms.per_day}.get(window_name)
|
||
if counter is None:
|
||
continue
|
||
data = [(0, -1)] * counter._num
|
||
for idx, ep, cnt in entries:
|
||
if 0 <= idx < counter._num:
|
||
data[idx] = (ep, cnt)
|
||
counter.restore_buckets(data)
|
||
self._local[model] = ms
|
||
tps_grouped: dict[tuple[str, str], dict[str, list[tuple[int, int, int, float, int]]]] = {}
|
||
for node_id, model, window, idx, ep, tokens, seconds, samples in tps_rows:
|
||
tps_grouped.setdefault((node_id, model), {}).setdefault(window, []).append(
|
||
(idx, ep, tokens, seconds, samples)
|
||
)
|
||
for key, windows in tps_grouped.items():
|
||
stat = _NodeModelThroughput()
|
||
for window_name, entries in windows.items():
|
||
counter = {"hour": stat.per_minute, "day": stat.per_hour, "month": stat.per_day}.get(window_name)
|
||
if counter is None:
|
||
continue
|
||
data = [(0, 0, 0.0, 0)] * counter._num
|
||
for idx, ep, tokens, seconds, samples in entries:
|
||
if 0 <= idx < counter._num:
|
||
data[idx] = (ep, int(tokens), float(seconds), int(samples))
|
||
counter.restore_buckets(data)
|
||
self._node_model[key] = stat
|
||
|
||
|
||
class _NodeEntry:
|
||
__slots__ = (
|
||
"node_id", "endpoint", "shard_start", "shard_end",
|
||
"model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "downloaded_models", "hardware_profile", "wallet_address",
|
||
"score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
|
||
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
|
||
"model_tokens_per_sec",
|
||
"pending_directives", "last_heartbeat", "tracker_mode",
|
||
"relay_addr", "cert_fingerprint", "peer_id",
|
||
# heartbeat stats (reported by node, cumulative)
|
||
"total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
|
||
"current_requests",
|
||
"status", # "ready" | "loading"
|
||
"heartbeats_expected", "heartbeats_received",
|
||
# dynamic reassignment queued by the tracker
|
||
"pending_new_assignment",
|
||
)
|
||
|
||
def __init__(
|
||
self,
|
||
node_id: str,
|
||
endpoint: str,
|
||
shard_start: int | None,
|
||
shard_end: int | None,
|
||
model: str | None,
|
||
shard_checksum: str | None,
|
||
hardware_profile: dict,
|
||
wallet_address: str | None,
|
||
score: float,
|
||
vram_bytes: int = DEFAULT_VRAM_BYTES,
|
||
ram_bytes: int = DEFAULT_RAM_BYTES,
|
||
quantizations: list[str] | None = None,
|
||
max_loaded_shards: int = 1,
|
||
benchmark_tokens_per_sec: float = DEFAULT_BENCHMARK_TOKENS_PER_SEC,
|
||
quantization: str | None = None,
|
||
managed_assignment: bool = False,
|
||
tracker_mode: bool = False,
|
||
hf_repo: str | None = None,
|
||
num_layers: int | None = None,
|
||
model_metadata: dict | None = None,
|
||
downloaded_models: list[dict] | None = None,
|
||
relay_addr: str | None = None,
|
||
cert_fingerprint: str | None = None,
|
||
peer_id: str | None = None,
|
||
) -> None:
|
||
self.node_id = node_id
|
||
self.endpoint = endpoint
|
||
self.shard_start = shard_start
|
||
self.shard_end = shard_end
|
||
self.model = model
|
||
self.shard_checksum = shard_checksum
|
||
self.hardware_profile = hardware_profile
|
||
self.wallet_address = wallet_address
|
||
self.score = score
|
||
self.vram_bytes = vram_bytes
|
||
self.ram_bytes = ram_bytes
|
||
self.quantizations = quantizations or list(DEFAULT_QUANTIZATIONS)
|
||
self.max_loaded_shards = max_loaded_shards
|
||
self.benchmark_tokens_per_sec = benchmark_tokens_per_sec
|
||
self.quantization = quantization
|
||
self.managed_assignment = managed_assignment
|
||
self.model_tokens_per_sec: dict[str, float] = {}
|
||
self.tracker_mode = tracker_mode
|
||
self.hf_repo = hf_repo
|
||
self.num_layers = num_layers
|
||
self.model_metadata = dict(model_metadata or {})
|
||
self.downloaded_models = [dict(item) for item in (downloaded_models or []) if isinstance(item, dict)]
|
||
self.relay_addr = relay_addr
|
||
self.cert_fingerprint = cert_fingerprint
|
||
self.peer_id = peer_id
|
||
self.pending_directives: list[dict] = []
|
||
self.last_heartbeat: float = time.monotonic()
|
||
self.total_requests: int = 0
|
||
self.failed_requests: int = 0
|
||
self.queue_depth: int = 0
|
||
self.proxy_inflight: int = 0
|
||
self.current_requests: list[dict] = []
|
||
self.uptime_seconds: float = 0.0
|
||
self.status: str = "ready"
|
||
self.heartbeats_expected: int = 0
|
||
self.heartbeats_received: int = 0
|
||
self.pending_new_assignment: dict | None = None
|
||
|
||
|
||
def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
|
||
"""Effective tokens/s accounting for current queue depth."""
|
||
observed = None
|
||
if model:
|
||
observed = node.model_tokens_per_sec.get(model)
|
||
if observed is None:
|
||
for alias in _model_aliases(model):
|
||
observed = node.model_tokens_per_sec.get(alias)
|
||
if observed is not None:
|
||
break
|
||
base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
|
||
return base / (_effective_queue_depth(node) + 1)
|
||
|
||
|
||
def _effective_queue_depth(node: "_NodeEntry") -> int:
|
||
"""Best current load estimate: heartbeat queue or tracker-routed in-flight."""
|
||
return max(node.queue_depth, node.proxy_inflight)
|
||
|
||
|
||
_CURRENT_REQUEST_FIELDS = frozenset({
|
||
"request_id", "model", "kind", "tokens", "tokens_per_sec",
|
||
"elapsed_seconds", "routing_complete",
|
||
})
|
||
|
||
|
||
def _normalize_current_requests(items: object, *, limit: int = 32) -> list[dict]:
|
||
"""Sanitize node-reported in-flight request snapshots from heartbeats."""
|
||
if not isinstance(items, list):
|
||
return []
|
||
out: list[dict] = []
|
||
for item in items[:limit]:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
request_id = item.get("request_id")
|
||
if not request_id:
|
||
continue
|
||
rec: dict = {"request_id": str(request_id)}
|
||
for key in _CURRENT_REQUEST_FIELDS:
|
||
if key == "request_id" or key not in item:
|
||
continue
|
||
value = item[key]
|
||
if key in {"tokens"}:
|
||
rec[key] = int(value)
|
||
elif key in {"tokens_per_sec", "elapsed_seconds"}:
|
||
rec[key] = float(value)
|
||
elif key == "routing_complete":
|
||
rec[key] = bool(value)
|
||
else:
|
||
rec[key] = str(value)
|
||
out.append(rec)
|
||
return out
|
||
|
||
|
||
def _record_proxy_inflight(
|
||
server: "_TrackerHTTPServer",
|
||
nodes: list["_NodeEntry"],
|
||
delta: int,
|
||
) -> None:
|
||
with server.lock:
|
||
for node in nodes:
|
||
node.proxy_inflight = max(0, node.proxy_inflight + delta)
|
||
|
||
|
||
def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
|
||
"""ADR-0018 §6: veteran, good-standing nodes route more -- earnings scale
|
||
with tenure/reputation. Maps reputation [0, 1] to a [0.5, 1.0] routing
|
||
weight so a damaged-but-not-banned wallet loses traffic gradually rather
|
||
than being cut off outright (banning is handled separately)."""
|
||
if contracts is None or not node.wallet_address:
|
||
return 1.0
|
||
reputation = contracts.registry.get_wallet(node.wallet_address).reputation
|
||
return 0.5 + 0.5 * reputation
|
||
|
||
|
||
def _select_route(
|
||
nodes: list[_NodeEntry],
|
||
required_start: int,
|
||
required_end: int,
|
||
model: str | None = None,
|
||
contracts: Any | None = None,
|
||
) -> tuple[list[_NodeEntry], str]:
|
||
"""Greedy interval-cover biased toward fast, lightly-loaded, reputable nodes.
|
||
|
||
Among nodes that equally advance coverage, prefer the one with higher
|
||
effective throughput: observed per-model tokens/sec / (queue_depth + 1),
|
||
falling back to startup benchmark_tokens_per_sec until observations exist,
|
||
weighted by the node's reputation multiplier (see `_reputation_multiplier`).
|
||
Tiebreak: higher shard_end (fewer hops).
|
||
"""
|
||
candidates = sorted(
|
||
[node for node in nodes if node.shard_start is not None and node.shard_end is not None],
|
||
key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
|
||
)
|
||
route: list[_NodeEntry] = []
|
||
covered_up_to = required_start - 1
|
||
|
||
def _routing_score(node: "_NodeEntry") -> float:
|
||
return _effective_throughput(node, model) * _reputation_multiplier(node, contracts)
|
||
|
||
while covered_up_to < required_end:
|
||
best: _NodeEntry | None = None
|
||
for node in candidates:
|
||
if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
|
||
if best is None:
|
||
best = node
|
||
elif node.shard_end > best.shard_end:
|
||
best = node
|
||
elif node.shard_end == best.shard_end and _routing_score(node) > _routing_score(best):
|
||
best = node
|
||
if best is None:
|
||
missing = covered_up_to + 1
|
||
return [], f"no route available: no registered node covers layer {missing}"
|
||
route.append(best)
|
||
covered_up_to = best.shard_end
|
||
candidates = [n for n in candidates if n is not best]
|
||
|
||
return route, ""
|
||
|
||
|
||
def _enumerate_routes(
|
||
nodes: list["_NodeEntry"],
|
||
required_start: int,
|
||
required_end: int,
|
||
model: str | None = None,
|
||
contracts: Any | None = None,
|
||
max_candidates: int = 8,
|
||
) -> list["RouteCandidate"]:
|
||
"""Enumerate viable route candidates for bandit selection (ADR-0021).
|
||
|
||
One candidate per distinct head (a node that can embed the prompt, i.e.
|
||
covers `required_start` from layer 0 of the range), each greedily completed
|
||
with the longest-advancing hops. The route's prior throughput estimate is
|
||
its bottleneck hop's queue-adjusted effective throughput — used only until
|
||
observed route samples exist.
|
||
"""
|
||
sharded = [
|
||
n for n in nodes
|
||
if n.shard_start is not None and n.shard_end is not None
|
||
]
|
||
# Heads must start the pipeline at the first required layer (they tokenize
|
||
# and embed the prompt — same condition as tracker_mode registration).
|
||
heads = [n for n in sharded if n.shard_start == required_start]
|
||
candidates: dict[str, RouteCandidate] = {}
|
||
for head in heads:
|
||
route = [head]
|
||
covered_up_to = head.shard_end
|
||
pool = [n for n in sharded if n is not head]
|
||
while covered_up_to < required_end:
|
||
best = None
|
||
for n in pool:
|
||
if n.shard_start <= covered_up_to + 1 and n.shard_end > covered_up_to:
|
||
if best is None or n.shard_end > best.shard_end or (
|
||
n.shard_end == best.shard_end
|
||
and _effective_throughput(n, model) * _reputation_multiplier(n, contracts)
|
||
> _effective_throughput(best, model) * _reputation_multiplier(best, contracts)
|
||
):
|
||
best = n
|
||
if best is None:
|
||
route = []
|
||
break
|
||
route.append(best)
|
||
covered_up_to = best.shard_end
|
||
pool = [n for n in pool if n is not best]
|
||
if not route:
|
||
continue
|
||
signature = route_signature(model or "?", route)
|
||
if signature in candidates:
|
||
continue
|
||
prior = min(
|
||
_effective_throughput(n, model) * _reputation_multiplier(n, contracts)
|
||
for n in route
|
||
)
|
||
candidates[signature] = RouteCandidate(nodes=route, signature=signature, prior_tps=prior)
|
||
ranked = sorted(candidates.values(), key=lambda c: -c.prior_tps)
|
||
return ranked[:max_candidates]
|
||
|
||
|
||
def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
|
||
return [
|
||
{
|
||
"node_id": node.node_id,
|
||
"model": node.model,
|
||
"hf_repo": node.hf_repo,
|
||
"endpoint": node.endpoint,
|
||
"shard": f"{node.shard_start}-{node.shard_end}",
|
||
"num_layers": node.num_layers,
|
||
"queue_depth": _effective_queue_depth(node),
|
||
"heartbeat_queue_depth": node.queue_depth,
|
||
"proxy_inflight": node.proxy_inflight,
|
||
"current_requests": list(node.current_requests),
|
||
}
|
||
for node in nodes
|
||
]
|
||
|
||
|
||
def _coverage_percentage(
|
||
nodes: list[_NodeEntry],
|
||
required_start: int,
|
||
required_end: int,
|
||
) -> float:
|
||
required_layers = required_end - required_start + 1
|
||
if required_layers <= 0:
|
||
return 0.0
|
||
|
||
intervals = sorted(
|
||
(
|
||
(max(required_start, node.shard_start), min(required_end, node.shard_end))
|
||
for node in nodes
|
||
if node.shard_start is not None
|
||
and node.shard_end is not None
|
||
if node.shard_end >= required_start and node.shard_start <= required_end
|
||
),
|
||
key=lambda interval: interval[0],
|
||
)
|
||
covered = 0
|
||
covered_until = required_start - 1
|
||
for start, end in intervals:
|
||
if end <= covered_until:
|
||
continue
|
||
next_start = max(start, covered_until + 1)
|
||
if next_start > end:
|
||
continue
|
||
covered += end - next_start + 1
|
||
covered_until = end
|
||
return round((covered / required_layers) * 100, 2)
|
||
|
||
|
||
def _served_model_copies(
|
||
nodes: list[_NodeEntry],
|
||
required_start: int,
|
||
required_end: int,
|
||
) -> float:
|
||
required_layers = required_end - required_start + 1
|
||
if required_layers <= 0:
|
||
return 0.0
|
||
|
||
layer_counts = []
|
||
for layer in range(required_start, required_end + 1):
|
||
count = 0
|
||
for node in nodes:
|
||
if node.shard_start is None or node.shard_end is None:
|
||
continue
|
||
if node.shard_start <= layer <= node.shard_end:
|
||
count += 1
|
||
layer_counts.append(count)
|
||
|
||
if not layer_counts:
|
||
return 0.0
|
||
|
||
complete_copies = min(layer_counts)
|
||
residual_layers = sum(1 for count in layer_counts if count > complete_copies)
|
||
return round(complete_copies + (residual_layers / required_layers), 2)
|
||
|
||
|
||
def _model_health_summary(
|
||
server: "_TrackerHTTPServer",
|
||
model: str | None,
|
||
hf_repo: str | None = None,
|
||
) -> dict:
|
||
lookup = hf_repo or model
|
||
if not lookup:
|
||
return {}
|
||
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, lookup)
|
||
if preset is not None:
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
model_nodes = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
]
|
||
model_id = resolved_name or lookup
|
||
else:
|
||
model_nodes = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_model(node, lookup)
|
||
and node.shard_start is not None
|
||
and node.shard_end is not None
|
||
and node.num_layers is not None
|
||
]
|
||
if not model_nodes:
|
||
return {
|
||
"model": lookup,
|
||
"served_model_copies": 0.0,
|
||
"coverage_percentage": 0.0,
|
||
"node_count": 0,
|
||
}
|
||
required_start = 0
|
||
required_end = max(node.num_layers for node in model_nodes if node.num_layers is not None) - 1
|
||
model_id = hf_repo or model or lookup
|
||
|
||
return {
|
||
"model": model_id,
|
||
"required_start": required_start,
|
||
"required_end": required_end,
|
||
"served_model_copies": _served_model_copies(model_nodes, required_start, required_end),
|
||
"coverage_percentage": _coverage_percentage(model_nodes, required_start, required_end),
|
||
"node_count": len(model_nodes),
|
||
}
|
||
|
||
|
||
def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
|
||
start = int(preset.get("layers_start", 0))
|
||
if "layers_end" in preset:
|
||
return start, int(preset["layers_end"])
|
||
return start, start + int(preset["total_layers"]) - 1
|
||
|
||
|
||
def _preset_bytes_per_layer(preset: dict) -> dict[str, int]:
|
||
raw = preset.get("bytes_per_layer", preset.get("bytes_per_layer_at_quant", {}))
|
||
if isinstance(raw, dict) and raw:
|
||
return {str(quant): int(value) for quant, value in raw.items()}
|
||
return {"bfloat16": 30 * 1024 * 1024}
|
||
|
||
|
||
def _node_quantization(node: _NodeEntry, preset: dict) -> str:
|
||
bytes_per_layer = _preset_bytes_per_layer(preset)
|
||
if node.quantization in bytes_per_layer:
|
||
return node.quantization
|
||
for quantization in node.quantizations:
|
||
if quantization in bytes_per_layer:
|
||
return quantization
|
||
return next(iter(bytes_per_layer))
|
||
|
||
|
||
def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
|
||
"""Return the memory pool used for shard-capacity planning."""
|
||
if node.vram_bytes > 0:
|
||
return node.vram_bytes, "vram"
|
||
if node.ram_bytes > 0:
|
||
return node.ram_bytes, "ram"
|
||
return DEFAULT_RAM_BYTES, "ram-default"
|
||
|
||
|
||
def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
|
||
bytes_per_layer = _preset_bytes_per_layer(preset)
|
||
quantization = _node_quantization(node, preset)
|
||
layer_bytes = bytes_per_layer[quantization]
|
||
if layer_bytes <= 0:
|
||
return 0
|
||
memory_budget_bytes, _ = _node_memory_budget_bytes(node)
|
||
return int((memory_budget_bytes * 0.8) // layer_bytes)
|
||
|
||
|
||
def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
|
||
"""Operator-facing capacity fields for inspection endpoints."""
|
||
memory_budget_bytes, memory_budget_source = _node_memory_budget_bytes(node)
|
||
summary = {
|
||
"vram_bytes": node.vram_bytes,
|
||
"ram_bytes": node.ram_bytes,
|
||
"memory_budget_bytes": memory_budget_bytes,
|
||
"memory_budget_source": memory_budget_source,
|
||
"max_loaded_shards": node.max_loaded_shards,
|
||
"quantizations": list(node.quantizations),
|
||
"quantization": node.quantization,
|
||
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
|
||
"effective_throughput": round(_effective_throughput(node), 4),
|
||
}
|
||
if preset is not None:
|
||
summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
|
||
return summary
|
||
|
||
|
||
def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
|
||
budget, _source = _node_memory_budget_bytes(node)
|
||
if preset is None:
|
||
return int(budget * 0.8)
|
||
return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))
|
||
|
||
|
||
def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
|
||
total_vram = sum(max(0, node.vram_bytes) for node in nodes)
|
||
total_ram = sum(max(0, node.ram_bytes) for node in nodes)
|
||
total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
|
||
effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
|
||
return {
|
||
"node_count": len(nodes),
|
||
"total_vram_bytes": total_vram,
|
||
"total_ram_bytes": total_ram,
|
||
"total_memory_budget_bytes": total_budget,
|
||
"effective_assignable_memory_bytes": effective_budget,
|
||
"total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
|
||
"total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
|
||
}
|
||
|
||
|
||
def _repo_demand_rpm(
|
||
server: "_TrackerHTTPServer",
|
||
repo: str,
|
||
local_rpms: dict[str, dict],
|
||
) -> float:
|
||
"""Look up recent request rate by hf_repo or preset alias."""
|
||
if repo in local_rpms:
|
||
return float(local_rpms[repo].get("rpm_last_hour", 0.0) or 0.0)
|
||
resolved_name, _preset = _resolve_model_preset(server.model_presets, repo)
|
||
if resolved_name and resolved_name in local_rpms:
|
||
return float(local_rpms[resolved_name].get("rpm_last_hour", 0.0) or 0.0)
|
||
return 0.0
|
||
|
||
|
||
def _preset_for_node(server: "_TrackerHTTPServer", node: _NodeEntry) -> dict | None:
|
||
if node.hf_repo:
|
||
_resolved, preset = _resolve_model_preset(server.model_presets, node.hf_repo)
|
||
if preset is not None:
|
||
return preset
|
||
if node.model:
|
||
preset = server.model_presets.get(node.model)
|
||
if preset is not None:
|
||
return preset
|
||
if node.hf_repo and node.num_layers:
|
||
return _hf_rebalance_preset([node])
|
||
return None
|
||
|
||
|
||
def _assignment_memory_bytes(node: _NodeEntry, preset: dict | None) -> int:
|
||
if preset is None or node.shard_start is None or node.shard_end is None:
|
||
return 0
|
||
layers = node.shard_end - node.shard_start + 1
|
||
if layers <= 0:
|
||
return 0
|
||
quantization = _node_quantization(node, preset)
|
||
layer_bytes = _preset_bytes_per_layer(preset).get(quantization)
|
||
if not layer_bytes:
|
||
layer_bytes = next(iter(_preset_bytes_per_layer(preset).values()))
|
||
return layers * int(layer_bytes)
|
||
|
||
|
||
def _endpoint_memory_pool(
|
||
server: "_TrackerHTTPServer",
|
||
nodes: list[_NodeEntry],
|
||
) -> dict:
|
||
"""Per-host RAM/VRAM budget, usage, spare capacity, and loaded assignments."""
|
||
if not nodes:
|
||
return {}
|
||
host = nodes[0]
|
||
budget_bytes, budget_source = _node_memory_budget_bytes(host)
|
||
reserve_bytes = int(budget_bytes * 0.8)
|
||
loaded: list[dict] = []
|
||
used_bytes = 0
|
||
for node in nodes:
|
||
preset = _preset_for_node(server, node)
|
||
assignment_bytes = _assignment_memory_bytes(node, preset)
|
||
used_bytes += assignment_bytes
|
||
loaded.append({
|
||
"node_id": node.node_id,
|
||
"model": node.model,
|
||
"hf_repo": node.hf_repo,
|
||
"shard_start": node.shard_start,
|
||
"shard_end": node.shard_end,
|
||
"memory_bytes": assignment_bytes,
|
||
"status": node.status,
|
||
})
|
||
max_slots = max(node.max_loaded_shards for node in nodes)
|
||
loaded_slots = len([node for node in nodes if node.shard_start is not None])
|
||
spare_bytes = max(0, reserve_bytes - used_bytes)
|
||
return {
|
||
"endpoint": host.endpoint,
|
||
"memory_budget_bytes": budget_bytes,
|
||
"memory_budget_source": budget_source,
|
||
"memory_reserve_bytes": reserve_bytes,
|
||
"memory_used_bytes": used_bytes,
|
||
"memory_spare_bytes": spare_bytes,
|
||
"loaded_slots": loaded_slots,
|
||
"max_loaded_shards": max_slots,
|
||
"spare_slots": max(0, max_slots - loaded_slots),
|
||
"loaded": loaded,
|
||
}
|
||
|
||
|
||
def _memory_pool_map(server: "_TrackerHTTPServer") -> dict:
|
||
"""Aggregate and per-endpoint view of assignable RAM across the hive."""
|
||
from collections import defaultdict
|
||
|
||
groups: dict[str, list[_NodeEntry]] = defaultdict(list)
|
||
for node in server.registry.values():
|
||
groups[node.endpoint.rstrip("/")].append(node)
|
||
|
||
hosts = [_endpoint_memory_pool(server, group) for group in groups.values()]
|
||
hosts.sort(key=lambda item: (-item.get("memory_spare_bytes", 0), item.get("endpoint", "")))
|
||
total_budget = sum(item["memory_budget_bytes"] for item in hosts)
|
||
total_used = sum(item["memory_used_bytes"] for item in hosts)
|
||
total_spare = sum(item["memory_spare_bytes"] for item in hosts)
|
||
total_slots = sum(item["max_loaded_shards"] for item in hosts)
|
||
loaded_slots = sum(item["loaded_slots"] for item in hosts)
|
||
return {
|
||
"total_memory_budget_bytes": total_budget,
|
||
"total_memory_used_bytes": total_used,
|
||
"total_memory_spare_bytes": total_spare,
|
||
"total_loaded_slots": loaded_slots,
|
||
"total_max_loaded_shards": total_slots,
|
||
"total_spare_slots": max(0, total_slots - loaded_slots),
|
||
"hosts": hosts,
|
||
}
|
||
|
||
|
||
def _add_shard_directive(
|
||
node: _NodeEntry,
|
||
model: str,
|
||
start: int,
|
||
end: int,
|
||
quantization: str,
|
||
*,
|
||
model_sources: list[dict] | None = None,
|
||
) -> dict:
|
||
directive = {
|
||
"action": "ADD_SHARD",
|
||
"model": model,
|
||
"start_layer": start,
|
||
"end_layer": end,
|
||
"shard_start": start,
|
||
"shard_end": end,
|
||
"quantization": quantization,
|
||
}
|
||
if model_sources:
|
||
directive["model_sources"] = model_sources
|
||
return directive
|
||
|
||
|
||
def _model_demand_and_supply(
|
||
server: "_TrackerHTTPServer",
|
||
model_key: str,
|
||
preset: dict,
|
||
) -> dict:
|
||
resolved_name, _ = _resolve_model_preset(server.model_presets, model_key)
|
||
lookup = preset.get("hf_repo") or resolved_name or model_key
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
model_nodes = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name or model_key, preset) # type: ignore[arg-type]
|
||
]
|
||
health = _model_health_summary(server, model_key, preset.get("hf_repo"))
|
||
local_rpms = server.stats.get_local_rpms() if server.stats is not None else {}
|
||
demand_rpm = _repo_demand_rpm(server, str(lookup), local_rpms)
|
||
return {
|
||
"model": resolved_name or model_key,
|
||
"hf_repo": preset.get("hf_repo"),
|
||
"preset": preset,
|
||
"demand_rpm": demand_rpm,
|
||
"served_model_copies": float(health.get("served_model_copies") or 0.0),
|
||
"coverage_percentage": float(health.get("coverage_percentage") or 0.0),
|
||
"required_start": required_start,
|
||
"required_end": required_end,
|
||
"model_nodes": model_nodes,
|
||
}
|
||
|
||
|
||
def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
|
||
"""Use spare host RAM/slots to add copies of high-demand models."""
|
||
pool = _memory_pool_map(server)
|
||
candidates: list[tuple[float, str, dict]] = []
|
||
for model_key, preset in server.model_presets.items():
|
||
if not isinstance(preset, dict) or not preset.get("hf_repo"):
|
||
continue
|
||
info = _model_demand_and_supply(server, model_key, preset)
|
||
demand = info["demand_rpm"]
|
||
copies = info["served_model_copies"]
|
||
coverage = info["coverage_percentage"]
|
||
if coverage < 100.0:
|
||
continue
|
||
target_copies = max(1.0, demand / 5.0)
|
||
if copies >= target_copies:
|
||
continue
|
||
candidates.append((demand, model_key, info))
|
||
|
||
if not candidates:
|
||
return
|
||
|
||
candidates.sort(key=lambda item: (-item[0], item[2]["served_model_copies"]))
|
||
for _demand, model_key, info in candidates:
|
||
preset = info["preset"]
|
||
hf_repo = str(info["hf_repo"])
|
||
required_start = info["required_start"]
|
||
required_end = info["required_end"]
|
||
total_layers = required_end - required_start + 1
|
||
layer_bytes = next(iter(_preset_bytes_per_layer(preset).values()))
|
||
full_copy_bytes = total_layers * layer_bytes
|
||
|
||
for host in pool["hosts"]:
|
||
if host["spare_slots"] <= 0 or host["memory_spare_bytes"] < full_copy_bytes:
|
||
continue
|
||
host_nodes = [
|
||
server.registry[node["node_id"]]
|
||
for node in host["loaded"]
|
||
if node["node_id"] in server.registry
|
||
]
|
||
if not host_nodes:
|
||
continue
|
||
if any((n.hf_repo or n.model) == hf_repo for n in host_nodes):
|
||
continue
|
||
anchor = max(host_nodes, key=lambda n: n.benchmark_tokens_per_sec)
|
||
if anchor.status != "ready" or anchor.pending_new_assignment is not None:
|
||
continue
|
||
capacity = min(_node_layer_capacity(anchor, preset), total_layers)
|
||
if capacity <= 0:
|
||
continue
|
||
quantization = _node_quantization(anchor, preset)
|
||
shard_end = min(required_end, required_start + capacity - 1)
|
||
assignment = _add_shard_directive(
|
||
anchor,
|
||
hf_repo,
|
||
required_start,
|
||
shard_end,
|
||
quantization,
|
||
)
|
||
anchor.pending_new_assignment = assignment
|
||
anchor.pending_directives.append(assignment)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"scale demanded model copy",
|
||
endpoint=anchor.endpoint,
|
||
node_id=anchor.node_id,
|
||
model=model_key,
|
||
hf_repo=hf_repo,
|
||
shard=f"{required_start}-{shard_end}",
|
||
demand_rpm=round(info["demand_rpm"], 4),
|
||
served_model_copies=info["served_model_copies"],
|
||
target_copies=round(max(1.0, info["demand_rpm"] / 5.0), 2),
|
||
host_spare_bytes=host["memory_spare_bytes"],
|
||
)
|
||
break
|
||
|
||
|
||
def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
|
||
if preset is None:
|
||
return {"recommended": False}
|
||
pool = _pool_summary(nodes, preset)
|
||
required = int(preset.get("required_model_bytes", 0) or 0)
|
||
deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
|
||
missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
|
||
return {
|
||
"recommended": bool(preset.get("recommended", False)),
|
||
"status": preset.get("deployment_status", "available"),
|
||
"required_model_bytes": required or None,
|
||
"download_size_bytes": preset.get("download_size_bytes"),
|
||
"native_quantization": preset.get("native_quantization"),
|
||
"pool": pool,
|
||
"deployable": deployable,
|
||
"missing_effective_memory_bytes": missing,
|
||
}
|
||
|
||
|
||
def _max_layers_for_memory(
|
||
memory_mb: int,
|
||
total_layers: int,
|
||
preset: dict | None = None,
|
||
*,
|
||
device: str | None = None,
|
||
) -> int:
|
||
if total_layers <= 0:
|
||
return 0
|
||
if memory_mb <= 0:
|
||
return max(1, total_layers // 2)
|
||
memory_bytes = memory_mb * 1024 * 1024
|
||
bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
|
||
safety_fraction = 0.55 if device == "cpu" else 0.8
|
||
return min(
|
||
total_layers,
|
||
max(1, int((memory_bytes * safety_fraction) // bytes_per_layer)),
|
||
)
|
||
|
||
|
||
def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
|
||
metadata: dict = {}
|
||
for node in nodes:
|
||
if node.model_metadata:
|
||
metadata.update(node.model_metadata)
|
||
if "num_layers" not in metadata:
|
||
layers = [node.num_layers for node in nodes if node.num_layers is not None]
|
||
if layers:
|
||
metadata["num_layers"] = max(layers)
|
||
return metadata
|
||
|
||
|
||
def _coverage_map(
|
||
nodes: list[_NodeEntry],
|
||
required_start: int,
|
||
required_end: int,
|
||
) -> list[dict]:
|
||
layer_counts = []
|
||
for layer in range(required_start, required_end + 1):
|
||
count = 0
|
||
for node in nodes:
|
||
if node.shard_start is None or node.shard_end is None:
|
||
continue
|
||
if node.shard_start <= layer <= node.shard_end:
|
||
count += 1
|
||
layer_counts.append((layer, count))
|
||
|
||
coverage: list[dict] = []
|
||
for layer, count in layer_counts:
|
||
if coverage and coverage[-1]["node_count"] == count and coverage[-1]["end_layer"] == layer - 1:
|
||
coverage[-1]["end_layer"] = layer
|
||
else:
|
||
coverage.append({"start_layer": layer, "end_layer": layer, "node_count": count})
|
||
return coverage
|
||
|
||
|
||
def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
|
||
"""Per-node health detail for the availability map."""
|
||
age = time.monotonic() - node.last_heartbeat
|
||
alive = age <= heartbeat_timeout
|
||
hb_expected = max(1, round(node.uptime_seconds / 20.0)) # assume ~20s interval
|
||
hb_rate = round(min(1.0, node.heartbeats_received / hb_expected), 4) if node.heartbeats_received else 0.0
|
||
total = node.total_requests
|
||
inf_rate = round((total - node.failed_requests) / total, 4) if total > 0 else 1.0
|
||
return {
|
||
"node_id": node.node_id,
|
||
"endpoint": node.endpoint,
|
||
"alive": alive,
|
||
"last_seen_seconds_ago": round(age, 1),
|
||
"status": node.status,
|
||
"queue_depth": _effective_queue_depth(node),
|
||
"heartbeat_queue_depth": node.queue_depth,
|
||
"proxy_inflight": node.proxy_inflight,
|
||
"current_requests": list(node.current_requests),
|
||
"total_requests": node.total_requests,
|
||
"heartbeat_success_rate": hb_rate,
|
||
"inference_success_rate": inf_rate,
|
||
"capacity": _node_capacity_summary(node),
|
||
}
|
||
|
||
|
||
def _coverage_map_detailed(
|
||
nodes: list["_NodeEntry"],
|
||
required_start: int,
|
||
required_end: int,
|
||
heartbeat_timeout: float,
|
||
) -> list[dict]:
|
||
"""Like _coverage_map but with per-node identity and health in each band.
|
||
|
||
Includes all nodes (alive and stale) so operators can see both coverage
|
||
holes and which dead nodes used to fill them.
|
||
"""
|
||
now = time.monotonic()
|
||
|
||
def covers(node: "_NodeEntry", layer: int) -> bool:
|
||
return (
|
||
node.shard_start is not None
|
||
and node.shard_end is not None
|
||
and node.shard_start <= layer <= node.shard_end
|
||
)
|
||
|
||
# Build per-layer list of nodes (with alive flag)
|
||
layer_nodes: list[list[tuple["_NodeEntry", bool]]] = []
|
||
for layer in range(required_start, required_end + 1):
|
||
ns = [
|
||
(n, (now - n.last_heartbeat) <= heartbeat_timeout)
|
||
for n in nodes
|
||
if covers(n, layer)
|
||
]
|
||
layer_nodes.append(ns)
|
||
|
||
# Merge consecutive layers with identical node-set (same node_ids, same alive)
|
||
coverage: list[dict] = []
|
||
for i, layer_idx in enumerate(range(required_start, required_end + 1)):
|
||
ns = layer_nodes[i]
|
||
node_ids = [n.node_id for n, _ in ns]
|
||
alive_flags = [a for _, a in ns]
|
||
# Compare with last band
|
||
if (
|
||
coverage
|
||
and coverage[-1]["end_layer"] == layer_idx - 1
|
||
and [nd["node_id"] for nd in coverage[-1]["nodes"]] == node_ids
|
||
and [nd["alive"] for nd in coverage[-1]["nodes"]] == alive_flags
|
||
):
|
||
coverage[-1]["end_layer"] = layer_idx
|
||
else:
|
||
coverage.append({
|
||
"start_layer": layer_idx,
|
||
"end_layer": layer_idx,
|
||
"node_count": len(ns),
|
||
"nodes": [_node_health(n, heartbeat_timeout) for n, _ in ns],
|
||
})
|
||
return coverage
|
||
|
||
|
||
def _coverage_gaps(coverage: list[dict]) -> list[tuple[int, int]]:
|
||
return [
|
||
(segment["start_layer"], segment["end_layer"])
|
||
for segment in coverage
|
||
if segment["node_count"] == 0
|
||
]
|
||
|
||
|
||
def _unassigned_managed_nodes(nodes: list["_NodeEntry"]) -> list["_NodeEntry"]:
|
||
return [
|
||
node for node in nodes
|
||
if node.managed_assignment
|
||
and (node.shard_start is None or node.shard_end is None)
|
||
]
|
||
|
||
|
||
def _emit_shard_change_directives(
|
||
node: "_NodeEntry",
|
||
model: str,
|
||
previous_range: tuple[int | None, int | None, str | None],
|
||
preset: dict,
|
||
) -> None:
|
||
"""Queue DROP/LOAD directives when a managed node's shard assignment changes."""
|
||
previous_start, previous_end, previous_quantization = previous_range
|
||
current_range = (node.shard_start, node.shard_end, node.quantization)
|
||
if node.shard_start is None or node.shard_end is None or current_range == previous_range:
|
||
return
|
||
if previous_start is not None and previous_end is not None:
|
||
node.pending_directives.append(
|
||
_drop_directive(
|
||
node,
|
||
model,
|
||
previous_start,
|
||
previous_end,
|
||
previous_quantization or _node_quantization(node, preset),
|
||
)
|
||
)
|
||
node.pending_directives.append(
|
||
_load_directive(
|
||
node,
|
||
model,
|
||
node.shard_start,
|
||
node.shard_end,
|
||
node.quantization or _node_quantization(node, preset),
|
||
)
|
||
)
|
||
|
||
|
||
def _assign_redundant_managed_nodes(
|
||
managed_nodes: list["_NodeEntry"],
|
||
model: str,
|
||
preset: dict,
|
||
required_start: int,
|
||
required_end: int,
|
||
) -> None:
|
||
"""Give newly joined managed nodes their own copy without reshuffling incumbents."""
|
||
total_layers = required_end - required_start + 1
|
||
for node in sorted(
|
||
managed_nodes,
|
||
key=lambda entry: (
|
||
-entry.benchmark_tokens_per_sec,
|
||
-_node_layer_capacity(entry, preset),
|
||
entry.node_id,
|
||
),
|
||
):
|
||
if node.shard_start is not None and node.shard_end is not None:
|
||
continue
|
||
capacity = min(_node_layer_capacity(node, preset), total_layers)
|
||
if capacity <= 0:
|
||
continue
|
||
previous_range = (node.shard_start, node.shard_end, node.quantization)
|
||
quantization = _node_quantization(node, preset)
|
||
node.quantization = quantization
|
||
node.shard_start = required_start
|
||
node.shard_end = min(required_end, required_start + capacity - 1)
|
||
_emit_shard_change_directives(node, model, previous_range, preset)
|
||
|
||
|
||
def _relay_http_request_frames(
|
||
relay_addr: str,
|
||
path: str,
|
||
body: bytes,
|
||
headers: dict[str, str],
|
||
timeout: float = 310.0,
|
||
idle_timeout: float = 120.0,
|
||
*,
|
||
cancel_event: threading.Event | None = None,
|
||
ws_holder: list[Any] | None = None,
|
||
# Quoted: threading.Lock is a factory function (not a class) before
|
||
# Python 3.13, so an unquoted `| None` union crashes at import time.
|
||
ws_lock: "threading.Lock | None" = None,
|
||
):
|
||
"""Send an HTTP-shaped request through a relay RPC WebSocket, yielding
|
||
response frames until a terminal one (US-036).
|
||
|
||
A frame with ``stream: true`` is part of a chunked SSE response ending with
|
||
``done: true``; a frame without ``stream`` is a complete single response.
|
||
Yields nothing when the relay is unreachable; stops silently on idle or
|
||
overall timeout (the caller bills whatever was observed).
|
||
"""
|
||
try:
|
||
import websockets.sync.client as wsc # type: ignore[import]
|
||
except Exception:
|
||
return
|
||
request_id = str(uuid.uuid4())
|
||
deadline = time.monotonic() + timeout
|
||
try:
|
||
with wsc.connect(relay_addr, open_timeout=10, close_timeout=5) as ws:
|
||
if ws_holder is not None:
|
||
if ws_lock is not None:
|
||
with ws_lock:
|
||
ws_holder.clear()
|
||
ws_holder.append(ws)
|
||
else:
|
||
ws_holder.clear()
|
||
ws_holder.append(ws)
|
||
ws.send(json.dumps({
|
||
"request_id": request_id,
|
||
"method": "POST",
|
||
"path": path,
|
||
"headers": headers,
|
||
"body": body.decode(errors="replace"),
|
||
}))
|
||
while True:
|
||
if cancel_event is not None and cancel_event.is_set():
|
||
return
|
||
remaining = deadline - time.monotonic()
|
||
if remaining <= 0:
|
||
return
|
||
raw = ws.recv(timeout=min(idle_timeout, remaining))
|
||
frame = json.loads(raw)
|
||
if frame.get("request_id") not in {None, request_id}:
|
||
continue
|
||
yield frame
|
||
if not frame.get("stream") or frame.get("done"):
|
||
return
|
||
except Exception:
|
||
return
|
||
|
||
|
||
def _relay_http_request(
|
||
relay_addr: str,
|
||
path: str,
|
||
body: bytes,
|
||
headers: dict[str, str],
|
||
timeout: float = 310.0,
|
||
) -> dict | None:
|
||
"""Send an HTTP-shaped request through a relay and buffer the response.
|
||
|
||
Streamed frame sequences are collapsed into one response dict whose body is
|
||
the concatenated SSE text — used by non-chat callers and kept for
|
||
backward compatibility; the chat proxy streams frames directly.
|
||
"""
|
||
frames = _relay_http_request_frames(relay_addr, path, body, headers, timeout=timeout)
|
||
first = next(frames, None)
|
||
if first is None:
|
||
return None
|
||
if not first.get("stream"):
|
||
return first
|
||
chunks = [first.get("chunk") or ""]
|
||
for frame in frames:
|
||
chunks.append(frame.get("chunk") or "")
|
||
return {
|
||
"request_id": first.get("request_id"),
|
||
"status": first.get("status", 200),
|
||
"headers": first.get("headers") or {},
|
||
"body": "".join(chunks),
|
||
}
|
||
|
||
|
||
def _usage_split(payload: dict) -> dict | None:
|
||
"""Parse a usage block into {"prompt", "completion", "total"} (ints or None)."""
|
||
usage = payload.get("usage")
|
||
if not isinstance(usage, dict):
|
||
return None
|
||
|
||
def _num(key: str) -> int | None:
|
||
value = usage.get(key)
|
||
return int(value) if isinstance(value, (int, float)) else None
|
||
|
||
return {
|
||
"prompt": _num("prompt_tokens"),
|
||
"completion": _num("completion_tokens"),
|
||
"total": _usage_total_tokens(payload),
|
||
}
|
||
|
||
|
||
def _stream_line_tokens(line: bytes) -> tuple[int, dict | None]:
|
||
"""Token accounting for one SSE line: (observed output delta, usage split or None)."""
|
||
if not line.startswith(b"data:"):
|
||
return 0, None
|
||
payload = line[5:].strip()
|
||
if not payload or payload == b"[DONE]":
|
||
return 0, None
|
||
try:
|
||
chunk_payload = json.loads(payload)
|
||
except json.JSONDecodeError:
|
||
return 1, None
|
||
return _observed_stream_tokens(chunk_payload), _usage_split(chunk_payload)
|
||
|
||
|
||
def _stream_billable_split(
|
||
observed_output: int, usage: dict | None, request_body: dict
|
||
) -> tuple[int, int]:
|
||
"""(input_tokens, output_tokens) for a streamed response (US-045).
|
||
|
||
Output: observed deltas, capped by reported completion count when the
|
||
stream carried a usage chunk. Input: reported prompt count, else the
|
||
prompt estimate from the request body.
|
||
"""
|
||
prompt = (usage or {}).get("prompt")
|
||
completion = (usage or {}).get("completion")
|
||
total = (usage or {}).get("total")
|
||
if prompt is None:
|
||
prompt = _estimate_prompt_tokens(request_body) or 0
|
||
if completion is None and total is not None:
|
||
completion = max(0, total - prompt)
|
||
return max(0, prompt), _billable_stream_tokens(observed_output, completion)
|
||
|
||
|
||
def _billable_non_stream_split(payload: dict, request_body: dict) -> tuple[int, int]:
|
||
"""(input_tokens, output_tokens) for a buffered response (US-045).
|
||
|
||
Prefers the response usage block; falls back to content estimates.
|
||
Completion stays capped by the request's max-tokens bound, as before.
|
||
"""
|
||
usage = _usage_split(payload)
|
||
prompt_estimate = _estimate_prompt_tokens(request_body) or 0
|
||
prompt = (usage or {}).get("prompt")
|
||
completion = (usage or {}).get("completion")
|
||
if prompt is None:
|
||
prompt = prompt_estimate
|
||
if completion is None:
|
||
total = (usage or {}).get("total")
|
||
if total is not None:
|
||
completion = max(0, total - prompt)
|
||
else:
|
||
completion = _observed_non_stream_completion_tokens(payload)
|
||
limit = _requested_completion_token_limit(request_body)
|
||
if limit is not None and completion > limit:
|
||
completion = min(completion, limit)
|
||
prompt = max(prompt, prompt_estimate)
|
||
return max(0, prompt), max(0, completion)
|
||
|
||
|
||
def _find_pinned_route(
|
||
nodes: list[_NodeEntry],
|
||
required_start: int,
|
||
required_end: int,
|
||
hop_count: int,
|
||
) -> list[_NodeEntry] | None:
|
||
"""First combination of exactly ``hop_count`` distinct nodes covering the
|
||
layer range, where every node extends coverage (US-030 benchmark routes)."""
|
||
for combo in itertools.permutations(nodes, hop_count):
|
||
covered = required_start - 1
|
||
valid = True
|
||
for candidate in combo:
|
||
if candidate.shard_start is None or candidate.shard_end is None:
|
||
valid = False
|
||
break
|
||
if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
|
||
valid = False
|
||
break
|
||
covered = candidate.shard_end
|
||
if valid and covered >= required_end:
|
||
return list(combo)
|
||
return None
|
||
|
||
|
||
def _nodes_and_bounds_for_model(
|
||
server: "_TrackerHTTPServer",
|
||
model: str,
|
||
) -> tuple[list[_NodeEntry], int, int] | None:
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
if preset is not None:
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
return [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
], required_start, required_end
|
||
|
||
nodes = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_model(node, model)
|
||
and node.shard_start is not None
|
||
and node.shard_end is not None
|
||
and node.num_layers is not None
|
||
]
|
||
if not nodes:
|
||
return None
|
||
return nodes, 0, max(node.num_layers for node in nodes) - 1
|
||
|
||
|
||
def _fetch_toploc_commitment(
|
||
node: _NodeEntry,
|
||
*,
|
||
session_id: str,
|
||
model: str,
|
||
messages: list[dict],
|
||
) -> dict | None:
|
||
"""Fetch a node's own on-demand TOPLOC boundary commitment (ADR-0018 §3),
|
||
same protocol as `ValidatorProcess._fetch_hop_commitment`."""
|
||
endpoint = node.endpoint
|
||
if not isinstance(endpoint, str) or not endpoint:
|
||
return None
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{endpoint.rstrip('/')}/v1/audit/toploc/commitment",
|
||
data=json.dumps({
|
||
"session_id": session_id,
|
||
"model": model,
|
||
"messages": messages,
|
||
"shard_start": node.shard_start,
|
||
"shard_end": node.shard_end,
|
||
}).encode(),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=5.0) as resp:
|
||
response = json.loads(resp.read())
|
||
except (OSError, ValueError, json.JSONDecodeError):
|
||
return None
|
||
proof = response.get("toploc_proof") or response.get("activation_proof")
|
||
token_ids = response.get("claimed_token_ids") or response.get("output_token_ids")
|
||
if not isinstance(proof, dict):
|
||
return None
|
||
if not isinstance(token_ids, list) or not all(isinstance(t, int) for t in token_ids):
|
||
return None
|
||
return {"toploc_proof": proof, "claimed_token_ids": token_ids}
|
||
|
||
|
||
def _fetch_toploc_reference_activations(
|
||
reference_node_url: str,
|
||
*,
|
||
model: str,
|
||
messages: list[dict],
|
||
claimed_token_ids: list[int],
|
||
claim: Any,
|
||
) -> list | None:
|
||
"""Teacher-force the claimed tokens on the reference node (same contract
|
||
as `ValidatorProcess._run_teacher_forced_prefill` / validator README's
|
||
"TOPLOC audit contract")."""
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{reference_node_url.rstrip('/')}/v1/audit/toploc",
|
||
data=json.dumps({
|
||
"model": model,
|
||
"messages": messages,
|
||
"claimed_token_ids": claimed_token_ids,
|
||
"dtype": claim.dtype,
|
||
"quantization": claim.quantization,
|
||
"decode_batching_size": claim.decode_batching_size,
|
||
"topk": claim.topk,
|
||
"skip_prefill": claim.skip_prefill,
|
||
}).encode(),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=300.0) as resp:
|
||
response = json.loads(resp.read())
|
||
except (OSError, ValueError, json.JSONDecodeError):
|
||
return None
|
||
activations = response.get("activations")
|
||
if not isinstance(activations, list):
|
||
return None
|
||
return activations
|
||
|
||
|
||
def _load_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
|
||
return {
|
||
"action": "LOAD_SHARD",
|
||
"model": model,
|
||
"start_layer": start,
|
||
"end_layer": end,
|
||
"shard_start": start,
|
||
"shard_end": end,
|
||
"quantization": quantization,
|
||
}
|
||
|
||
|
||
def _drop_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
|
||
return {
|
||
"action": "DROP_SHARD",
|
||
"model": model,
|
||
"start_layer": start,
|
||
"end_layer": end,
|
||
"shard_start": start,
|
||
"shard_end": end,
|
||
"quantization": quantization,
|
||
}
|
||
|
||
|
||
def _route_stats_keys(server: "_TrackerHTTPServer", entry: "_NodeEntry") -> list[str]:
|
||
"""All stats keys a node's routes may be recorded under (model, repo, resolved preset)."""
|
||
keys = {entry.model, entry.hf_repo}
|
||
resolved, _ = _resolve_model_preset(server.model_presets, entry.hf_repo or entry.model)
|
||
keys.add(resolved)
|
||
return [key for key in keys if key]
|
||
|
||
|
||
def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
|
||
now = time.monotonic()
|
||
expired_ids = [
|
||
node_id for node_id, entry in server.registry.items()
|
||
if (now - entry.last_heartbeat) > server.heartbeat_timeout
|
||
]
|
||
expired_entries: list[tuple[str, _NodeEntry]] = []
|
||
for node_id in expired_ids:
|
||
entry = server.registry.pop(node_id)
|
||
expired_entries.append((node_id, entry))
|
||
if expired_ids:
|
||
_rebalance_all_locked(server)
|
||
server.route_stats.bump_epoch(
|
||
key
|
||
for _, entry in expired_entries
|
||
for key in _route_stats_keys(server, entry)
|
||
)
|
||
for node_id, entry in expired_entries:
|
||
_tracker_log(
|
||
server,
|
||
"warn",
|
||
"node expired",
|
||
node_id=node_id,
|
||
endpoint=entry.endpoint,
|
||
model=entry.model,
|
||
hf_repo=entry.hf_repo,
|
||
shard=f"{entry.shard_start}-{entry.shard_end}",
|
||
heartbeat_timeout_seconds=server.heartbeat_timeout,
|
||
model_health=_model_health_summary(server, entry.model, entry.hf_repo),
|
||
)
|
||
return expired_ids
|
||
|
||
|
||
def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
if preset is None:
|
||
return
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
total_layers = required_end - required_start + 1
|
||
model_nodes = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
]
|
||
managed_nodes = [node for node in model_nodes if node.managed_assignment]
|
||
if not managed_nodes:
|
||
return
|
||
|
||
coverage = _coverage_map(model_nodes, required_start, required_end)
|
||
gaps = _coverage_gaps(coverage)
|
||
unassigned = _unassigned_managed_nodes(managed_nodes)
|
||
if not gaps and not unassigned:
|
||
return
|
||
if not gaps and unassigned:
|
||
_assign_redundant_managed_nodes(
|
||
unassigned, model, preset, required_start, required_end,
|
||
)
|
||
return
|
||
|
||
previous_ranges = {
|
||
node.node_id: (node.shard_start, node.shard_end, node.quantization)
|
||
for node in managed_nodes
|
||
}
|
||
for node in managed_nodes:
|
||
node.shard_start = None
|
||
node.shard_end = None
|
||
|
||
managed_nodes.sort(
|
||
key=lambda node: (
|
||
-node.benchmark_tokens_per_sec,
|
||
-_node_layer_capacity(node, preset),
|
||
node.node_id,
|
||
)
|
||
)
|
||
base_nodes = [node for node in model_nodes if not node.managed_assignment]
|
||
base_coverage = _coverage_map(base_nodes, required_start, required_end)
|
||
gaps = _coverage_gaps(base_coverage)
|
||
|
||
eligible_nodes = [
|
||
node for node in managed_nodes
|
||
if _node_layer_capacity(node, preset) > 0
|
||
]
|
||
node_index = 0
|
||
for gap_start, gap_end in gaps:
|
||
cursor = gap_start
|
||
while cursor <= gap_end and node_index < len(eligible_nodes):
|
||
node = eligible_nodes[node_index]
|
||
remaining_layers = gap_end - cursor + 1
|
||
remaining_nodes_after = len(eligible_nodes) - node_index - 1
|
||
capacity = min(
|
||
_node_layer_capacity(node, preset),
|
||
total_layers,
|
||
max(1, remaining_layers - remaining_nodes_after),
|
||
)
|
||
if capacity <= 0:
|
||
node_index += 1
|
||
continue
|
||
quantization = _node_quantization(node, preset)
|
||
node.quantization = quantization
|
||
node.shard_start = cursor
|
||
node.shard_end = min(gap_end, cursor + capacity - 1)
|
||
cursor = node.shard_end + 1
|
||
node_index += 1
|
||
|
||
for node in managed_nodes:
|
||
_emit_shard_change_directives(
|
||
node,
|
||
model,
|
||
previous_ranges[node.node_id],
|
||
preset,
|
||
)
|
||
|
||
|
||
def _hf_rebalance_preset(nodes: list[_NodeEntry]) -> dict:
|
||
total_layers = max(node.num_layers or 0 for node in nodes)
|
||
return {
|
||
"layers_start": 0,
|
||
"layers_end": total_layers - 1,
|
||
"bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
|
||
}
|
||
|
||
|
||
def _rebalance_hf_model_locked(server: "_TrackerHTTPServer", hf_repo: str) -> None:
|
||
model_nodes = [
|
||
node for node in server.registry.values()
|
||
if node.hf_repo == hf_repo
|
||
and node.num_layers is not None
|
||
]
|
||
managed_nodes = [node for node in model_nodes if node.managed_assignment]
|
||
if not model_nodes or not managed_nodes:
|
||
return
|
||
|
||
preset = _hf_rebalance_preset(model_nodes)
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
total_layers = required_end - required_start + 1
|
||
if total_layers <= 0:
|
||
return
|
||
|
||
coverage = _coverage_map(model_nodes, required_start, required_end)
|
||
gaps = _coverage_gaps(coverage)
|
||
unassigned = _unassigned_managed_nodes(managed_nodes)
|
||
if not gaps and not unassigned:
|
||
return
|
||
if not gaps and unassigned:
|
||
_assign_redundant_managed_nodes(
|
||
unassigned, hf_repo, preset, required_start, required_end,
|
||
)
|
||
return
|
||
|
||
previous_ranges = {
|
||
node.node_id: (node.shard_start, node.shard_end, node.quantization)
|
||
for node in managed_nodes
|
||
}
|
||
for node in managed_nodes:
|
||
node.shard_start = None
|
||
node.shard_end = None
|
||
|
||
managed_nodes.sort(
|
||
key=lambda node: (
|
||
-node.benchmark_tokens_per_sec,
|
||
-_node_layer_capacity(node, preset),
|
||
node.node_id,
|
||
)
|
||
)
|
||
base_nodes = [node for node in model_nodes if not node.managed_assignment]
|
||
base_coverage = _coverage_map(base_nodes, required_start, required_end)
|
||
gaps = _coverage_gaps(base_coverage)
|
||
|
||
eligible_nodes = [
|
||
node for node in managed_nodes
|
||
if _node_layer_capacity(node, preset) > 0
|
||
]
|
||
node_index = 0
|
||
for gap_start, gap_end in gaps:
|
||
cursor = gap_start
|
||
while cursor <= gap_end and node_index < len(eligible_nodes):
|
||
node = eligible_nodes[node_index]
|
||
remaining_layers = gap_end - cursor + 1
|
||
remaining_nodes_after = len(eligible_nodes) - node_index - 1
|
||
capacity = min(
|
||
_node_layer_capacity(node, preset),
|
||
total_layers,
|
||
max(1, remaining_layers - remaining_nodes_after),
|
||
)
|
||
if capacity <= 0:
|
||
node_index += 1
|
||
continue
|
||
quantization = _node_quantization(node, preset)
|
||
node.quantization = quantization
|
||
node.shard_start = cursor
|
||
node.shard_end = min(gap_end, cursor + capacity - 1)
|
||
cursor = node.shard_end + 1
|
||
node_index += 1
|
||
|
||
for node in managed_nodes:
|
||
_emit_shard_change_directives(
|
||
node,
|
||
hf_repo,
|
||
previous_ranges[node.node_id],
|
||
preset,
|
||
)
|
||
|
||
|
||
def _rebalance_all_locked(server: "_TrackerHTTPServer") -> None:
|
||
for model in list(server.model_presets):
|
||
_rebalance_model_locked(server, model)
|
||
for hf_repo in sorted({node.hf_repo for node in server.registry.values() if node.hf_repo}):
|
||
_rebalance_hf_model_locked(server, hf_repo)
|
||
|
||
|
||
def _api_key_from_headers(headers) -> str | None:
|
||
auth = headers.get("Authorization")
|
||
if not auth:
|
||
return None
|
||
if auth.lower().startswith("bearer "):
|
||
return auth.split(" ", 1)[1].strip() or None
|
||
return auth.strip() or None
|
||
|
||
|
||
def _session_token_from_headers(headers) -> str | None:
|
||
token = _api_key_from_headers(headers)
|
||
if token:
|
||
return token
|
||
cookie_header = headers.get("Cookie")
|
||
if not cookie_header:
|
||
return None
|
||
cookie = http.cookies.SimpleCookie()
|
||
try:
|
||
cookie.load(cookie_header)
|
||
except http.cookies.CookieError:
|
||
return None
|
||
morsel = cookie.get(_SESSION_COOKIE_NAME)
|
||
if morsel is None:
|
||
return None
|
||
return morsel.value.strip() or None
|
||
|
||
|
||
def _session_cookie_header(token: str | None) -> str:
|
||
cookie = http.cookies.SimpleCookie()
|
||
cookie[_SESSION_COOKIE_NAME] = token or ""
|
||
morsel = cookie[_SESSION_COOKIE_NAME]
|
||
morsel["path"] = "/"
|
||
morsel["httponly"] = True
|
||
morsel["samesite"] = "Lax"
|
||
if token:
|
||
morsel["max-age"] = str(int(7 * 86400))
|
||
else:
|
||
morsel["max-age"] = "0"
|
||
return morsel.OutputString()
|
||
|
||
|
||
def _usage_total_tokens(payload: dict) -> int | None:
|
||
usage = payload.get("usage")
|
||
if not isinstance(usage, dict):
|
||
return None
|
||
total = usage.get("total_tokens")
|
||
if isinstance(total, (int, float)):
|
||
return int(total)
|
||
prompt = usage.get("prompt_tokens")
|
||
completion = usage.get("completion_tokens")
|
||
if isinstance(prompt, (int, float)) or isinstance(completion, (int, float)):
|
||
return int(prompt or 0) + int(completion or 0)
|
||
return None
|
||
|
||
|
||
def _estimate_text_tokens(value: Any) -> int | None:
|
||
if isinstance(value, str):
|
||
text = value.strip()
|
||
if not text:
|
||
return 0
|
||
return len(text.split())
|
||
if isinstance(value, list):
|
||
total = 0
|
||
found = False
|
||
for item in value:
|
||
if isinstance(item, str):
|
||
estimated = _estimate_text_tokens(item)
|
||
elif isinstance(item, dict):
|
||
estimated = _estimate_text_tokens(item.get("text"))
|
||
else:
|
||
estimated = None
|
||
if estimated is not None:
|
||
total += estimated
|
||
found = True
|
||
return total if found else None
|
||
return None
|
||
|
||
|
||
def _estimate_prompt_tokens(body: dict) -> int | None:
|
||
messages = body.get("messages")
|
||
if isinstance(messages, list):
|
||
total = 0
|
||
found = False
|
||
for message in messages:
|
||
if not isinstance(message, dict):
|
||
continue
|
||
estimated = _estimate_text_tokens(message.get("content"))
|
||
if estimated is not None:
|
||
total += estimated
|
||
found = True
|
||
return total if found else None
|
||
prompt = body.get("prompt")
|
||
return _estimate_text_tokens(prompt)
|
||
|
||
|
||
def _requested_completion_token_limit(body: dict) -> int | None:
|
||
for field in ("max_completion_tokens", "max_tokens"):
|
||
value = body.get(field)
|
||
if isinstance(value, bool):
|
||
return None
|
||
if isinstance(value, (int, float)):
|
||
return max(0, int(value))
|
||
return None
|
||
|
||
|
||
def _request_total_token_upper_bound(body: dict) -> int | None:
|
||
completion_limit = _requested_completion_token_limit(body)
|
||
if completion_limit is None:
|
||
return None
|
||
prompt_estimate = _estimate_prompt_tokens(body)
|
||
return completion_limit + (prompt_estimate or 0)
|
||
|
||
|
||
def _billable_non_stream_tokens(payload: dict, request_body: dict) -> int:
|
||
reported = _usage_total_tokens(payload)
|
||
upper_bound = _request_total_token_upper_bound(request_body)
|
||
if reported is None:
|
||
completion_estimate = _observed_non_stream_completion_tokens(payload)
|
||
if completion_estimate > 0:
|
||
billable = completion_estimate + (_estimate_prompt_tokens(request_body) or 0)
|
||
elif upper_bound is not None:
|
||
billable = upper_bound
|
||
else:
|
||
return 0
|
||
return min(billable, upper_bound) if upper_bound is not None else billable
|
||
billable = max(0, reported)
|
||
if upper_bound is not None:
|
||
billable = min(billable, upper_bound)
|
||
return billable
|
||
|
||
|
||
def _observed_non_stream_completion_tokens(payload: dict) -> int:
|
||
choices = payload.get("choices")
|
||
if not isinstance(choices, list):
|
||
return 0
|
||
observed = 0
|
||
for choice in choices:
|
||
if not isinstance(choice, dict):
|
||
continue
|
||
message = choice.get("message")
|
||
estimated: int | None = None
|
||
if isinstance(message, dict):
|
||
estimated = _estimate_text_tokens(message.get("content"))
|
||
if estimated is None:
|
||
estimated = _estimate_text_tokens(choice.get("text"))
|
||
if estimated is not None:
|
||
observed += estimated
|
||
elif choice:
|
||
observed += 1
|
||
return observed
|
||
|
||
|
||
def _observed_output_from_non_stream_payload(payload: dict) -> str:
|
||
choices = payload.get("choices")
|
||
if not isinstance(choices, list) or not choices:
|
||
return ""
|
||
first = choices[0]
|
||
if not isinstance(first, dict):
|
||
return ""
|
||
message = first.get("message")
|
||
if isinstance(message, dict) and isinstance(message.get("content"), str):
|
||
return message["content"]
|
||
text = first.get("text")
|
||
return text if isinstance(text, str) else ""
|
||
|
||
|
||
def _observed_stream_tokens(payload: dict) -> int:
|
||
choices = payload.get("choices")
|
||
if not isinstance(choices, list):
|
||
return 0
|
||
observed = 0
|
||
for choice in choices:
|
||
if not isinstance(choice, dict):
|
||
continue
|
||
delta = choice.get("delta")
|
||
if isinstance(delta, dict):
|
||
estimated = _estimate_text_tokens(delta.get("content"))
|
||
if estimated:
|
||
observed += estimated
|
||
continue
|
||
if choice:
|
||
observed += 1
|
||
return observed
|
||
|
||
|
||
def _billable_stream_tokens(observed_tokens: int, reported_tokens: int | None) -> int:
|
||
observed = max(0, observed_tokens)
|
||
if reported_tokens is None:
|
||
return observed
|
||
return min(observed, max(0, reported_tokens))
|
||
|
||
|
||
def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
|
||
if contracts is None or not wallet_address:
|
||
return None
|
||
if contracts.registry.get_wallet(wallet_address).banned:
|
||
return "wallet is banned"
|
||
return None
|
||
|
||
|
||
def _tracker_log(
|
||
server: "_TrackerHTTPServer",
|
||
level: str,
|
||
message: str,
|
||
*,
|
||
stdout: bool = True,
|
||
update_console_key: str | None = None,
|
||
**fields: Any,
|
||
) -> None:
|
||
log_level = {
|
||
"debug": 10,
|
||
"info": 20,
|
||
"warn": 30,
|
||
"warning": 30,
|
||
"error": 40,
|
||
}.get(level.lower(), 20)
|
||
event = {
|
||
"ts": time.time(),
|
||
"level": level,
|
||
"message": message,
|
||
"fields": {
|
||
key: value
|
||
for key, value in fields.items()
|
||
if value is not None
|
||
},
|
||
}
|
||
with server.console_lock:
|
||
if update_console_key is not None:
|
||
updated = False
|
||
for existing in reversed(server.console_events):
|
||
if (
|
||
existing.get("message") == message
|
||
and existing.get("fields", {}).get("request_id") == update_console_key
|
||
):
|
||
existing["ts"] = event["ts"]
|
||
existing["fields"] = event["fields"]
|
||
updated = True
|
||
break
|
||
if not updated:
|
||
server.console_events.append(event)
|
||
else:
|
||
server.console_events.append(event)
|
||
extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
|
||
suffix = f" {extras}" if extras else ""
|
||
tracker_logger().log(log_level, f"{message}{suffix}")
|
||
if stdout:
|
||
print(f"[tracker] {level}: {message}{suffix}", flush=True)
|
||
|
||
|
||
@dataclass
|
||
class _ActiveProxyContext:
|
||
request_id: str
|
||
cancel_event: threading.Event = field(default_factory=threading.Event)
|
||
upstream: Any | None = None
|
||
upstream_lock: threading.Lock = field(default_factory=threading.Lock)
|
||
relay_ws: Any | None = None
|
||
relay_ws_lock: threading.Lock = field(default_factory=threading.Lock)
|
||
|
||
|
||
def _register_active_proxy(server: "_TrackerHTTPServer", request_id: str) -> _ActiveProxyContext:
|
||
ctx = _ActiveProxyContext(request_id=request_id)
|
||
with server.active_proxies_lock:
|
||
server.active_proxies[request_id] = ctx
|
||
return ctx
|
||
|
||
|
||
def _unregister_active_proxy(server: "_TrackerHTTPServer", request_id: str) -> None:
|
||
with server.active_proxies_lock:
|
||
server.active_proxies.pop(request_id, None)
|
||
|
||
|
||
def _request_proxy_cancel(server: "_TrackerHTTPServer", request_id: str) -> bool:
|
||
with server.active_proxies_lock:
|
||
ctx = server.active_proxies.get(request_id)
|
||
if ctx is None:
|
||
return False
|
||
ctx.cancel_event.set()
|
||
|
||
def _close_resources() -> None:
|
||
with ctx.upstream_lock:
|
||
upstream = ctx.upstream
|
||
if upstream is not None:
|
||
try:
|
||
upstream.close()
|
||
except Exception:
|
||
pass
|
||
with ctx.relay_ws_lock:
|
||
relay_ws = ctx.relay_ws
|
||
if relay_ws is not None:
|
||
try:
|
||
relay_ws.close()
|
||
except Exception:
|
||
pass
|
||
|
||
threading.Thread(target=_close_resources, daemon=True).start()
|
||
return True
|
||
|
||
|
||
def _upstream_socket(upstream: Any) -> Any | None:
|
||
fp = getattr(upstream, "fp", None)
|
||
raw = getattr(fp, "raw", None) if fp is not None else None
|
||
return getattr(raw, "_sock", None) if raw is not None else None
|
||
|
||
|
||
def _set_upstream_read_timeout(upstream: Any, timeout: float | None) -> None:
|
||
sock = _upstream_socket(upstream)
|
||
if sock is not None:
|
||
sock.settimeout(timeout)
|
||
|
||
|
||
def _clear_proxy_progress_log_state(server: "_TrackerHTTPServer", request_id: str) -> None:
|
||
state = getattr(server, "_proxy_progress_log_state", None)
|
||
if state is not None:
|
||
state.pop(request_id, None)
|
||
|
||
|
||
def _tracker_log_proxy_progress(
|
||
server: "_TrackerHTTPServer",
|
||
*,
|
||
request_id: str,
|
||
model: str,
|
||
route_model: str,
|
||
tokens: int,
|
||
started: float,
|
||
route_nodes: list["_NodeEntry"],
|
||
stream: bool = True,
|
||
relay: bool = False,
|
||
) -> None:
|
||
elapsed = time.monotonic() - started
|
||
effective_elapsed = max(elapsed, 1e-6)
|
||
now = time.monotonic()
|
||
state = getattr(server, "_proxy_progress_log_state", None)
|
||
if state is None:
|
||
state = {}
|
||
server._proxy_progress_log_state = state
|
||
last_stdout = state.get(request_id)
|
||
stdout = last_stdout is None or (now - last_stdout) >= _PROXY_PROGRESS_LOG_INTERVAL
|
||
if stdout:
|
||
state[request_id] = now
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy progress",
|
||
stdout=stdout,
|
||
update_console_key=request_id,
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
stream=stream,
|
||
relay=relay or None,
|
||
tokens=tokens,
|
||
elapsed_seconds=round(elapsed, 4),
|
||
tokens_per_sec=round(tokens / effective_elapsed, 4) if tokens > 0 else 0.0,
|
||
route=_node_route_summary(route_nodes),
|
||
)
|
||
|
||
|
||
def _node_id_for_registration(
|
||
endpoint: str,
|
||
model: str,
|
||
wallet_address: str | None,
|
||
shard_start: int | None,
|
||
shard_end: int | None,
|
||
hf_repo: str | None,
|
||
) -> str:
|
||
wallet_prefix = wallet_address[:8] if wallet_address else "anon"
|
||
stable_key = "|".join([
|
||
wallet_address or "",
|
||
endpoint.rstrip("/"),
|
||
model,
|
||
hf_repo or "",
|
||
"" if shard_start is None else str(shard_start),
|
||
"" if shard_end is None else str(shard_end),
|
||
])
|
||
digest = hashlib.sha256(stable_key.encode()).hexdigest()[:12]
|
||
return f"{wallet_prefix}-{digest}"
|
||
|
||
|
||
class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
|
||
daemon_threads = True
|
||
|
||
def __init__(
|
||
self,
|
||
addr: tuple,
|
||
handler,
|
||
registry: dict,
|
||
lock: threading.Lock,
|
||
heartbeat_timeout: float,
|
||
model_presets: dict,
|
||
contracts: Any | None,
|
||
minimum_stake: int,
|
||
relay_url: str | None = None,
|
||
raft: "RaftNode | None" = None,
|
||
gossip: "NodeGossip | None" = None,
|
||
stats: "_StatsCollector | None" = None,
|
||
billing: "BillingLedger | None" = None,
|
||
accounts: "AccountStore | None" = None,
|
||
benchmark_results_path: str | None = None,
|
||
validator_service_token: str | None = None,
|
||
hive_secret: str | None = None,
|
||
max_charge_per_request: float | None = None,
|
||
starting_credit: float = DEFAULT_CALLER_CREDIT_USDT,
|
||
devnet_topup_amount: float = DEFAULT_DEVNET_TOPUP_USDT,
|
||
toploc_calibration: "ToplocCalibrationStore | None" = None,
|
||
toploc_reference_node_url: str | None = None,
|
||
toploc_calibration_gate_min_hardware_profiles: int = 1,
|
||
toploc_backend: Any | None = None,
|
||
hf_pricing_log: "HfPricingLog | None" = None,
|
||
models_dir: Path | None = None,
|
||
route_stats: "RouteStatsStore | None" = None,
|
||
) -> None:
|
||
super().__init__(addr, handler)
|
||
self.registry = registry
|
||
self.lock = lock
|
||
self.heartbeat_timeout = heartbeat_timeout
|
||
self.model_presets = model_presets
|
||
self.contracts = contracts
|
||
self.minimum_stake = minimum_stake
|
||
self.relay_url = relay_url.rstrip("/") if relay_url else None
|
||
self.raft = raft
|
||
self.gossip = gossip
|
||
self.stats: _StatsCollector | None = stats
|
||
self.billing: BillingLedger | None = billing
|
||
self.accounts: AccountStore | None = accounts
|
||
self.benchmark_results_path = benchmark_results_path or os.path.join(
|
||
os.getcwd(), "benchmark_results.json"
|
||
)
|
||
self.benchmark_lock = threading.Lock()
|
||
self.validator_service_token = validator_service_token
|
||
self.hive_secret = hive_secret
|
||
self.max_charge_per_request = max_charge_per_request
|
||
self.starting_credit = starting_credit
|
||
self.devnet_topup_amount = devnet_topup_amount
|
||
self.toploc_calibration: ToplocCalibrationStore | None = toploc_calibration
|
||
self.toploc_reference_node_url = (
|
||
toploc_reference_node_url.rstrip("/") if toploc_reference_node_url else None
|
||
)
|
||
self.toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
|
||
self.toploc_backend = toploc_backend
|
||
self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
|
||
self.models_dir = models_dir
|
||
self.console_events = deque(maxlen=_CONSOLE_LIMIT)
|
||
self.console_lock = threading.Lock()
|
||
self.active_proxies: dict[str, _ActiveProxyContext] = {}
|
||
self.active_proxies_lock = threading.Lock()
|
||
self.route_stats: RouteStatsStore = route_stats or RouteStatsStore()
|
||
self.route_rng = random.Random()
|
||
|
||
|
||
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||
def log_message(self, fmt, *args): # noqa: suppress request logs in tests
|
||
pass
|
||
|
||
def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
|
||
body = json.dumps(data).encode()
|
||
self.send_response(status)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
for name, value in (headers or {}).items():
|
||
self.send_header(name, value)
|
||
self.end_headers()
|
||
try:
|
||
self.wfile.write(body)
|
||
except BrokenPipeError:
|
||
pass
|
||
|
||
# ---- unified auth boundary (ADR-0017) ----
|
||
|
||
def _resolve_identity(self) -> tuple[str | None, dict | None]:
|
||
"""Resolve the caller to (role, account).
|
||
|
||
Roles: "validator" (service token), "admin"/"user" (session token).
|
||
Client API keys resolve to no privileged role — they authorize
|
||
inference and wallet binding only, never operator endpoints.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
token = _session_token_from_headers(self.headers)
|
||
if not token:
|
||
return None, None
|
||
if is_validator_token(token, server.validator_service_token):
|
||
return "validator", None
|
||
if server.accounts is not None:
|
||
account = server.accounts.session_account(token)
|
||
if account is not None:
|
||
return account.get("role", "user"), account
|
||
return None, None
|
||
|
||
def _require_role(self, *allowed: str) -> bool:
|
||
"""Gate a privileged handler; sends 401/403 and returns False on failure.
|
||
|
||
401 when no credential was presented; 403 when a credential was
|
||
presented but does not resolve to an allowed role — this covers
|
||
client API keys and garbage bearer strings on operator endpoints.
|
||
"""
|
||
role, _account = self._resolve_identity()
|
||
if role in allowed:
|
||
return True
|
||
if _api_key_from_headers(self.headers) is None:
|
||
self._send_json(401, {"error": "authentication required (admin session or service token)"})
|
||
else:
|
||
self._send_json(403, {"error": "this endpoint requires an admin session or service token"})
|
||
return False
|
||
|
||
def _read_hive_authenticated_body(self) -> dict | None:
|
||
"""Read + verify a hive gossip body (HMAC per ADR-0017 §3).
|
||
|
||
Fails closed: without a configured --hive-secret no gossip is
|
||
accepted. Sends the error response itself and returns None on failure.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
length = int(self.headers.get("Content-Length", 0))
|
||
raw = self.rfile.read(length) if length else b"{}"
|
||
if not verify_hive_request(server.hive_secret, self.headers, raw):
|
||
self._send_json(401, {"error": "valid hive signature required"})
|
||
return None
|
||
try:
|
||
body = json.loads(raw or b"{}")
|
||
except json.JSONDecodeError:
|
||
self._send_json(400, {"error": "invalid JSON body"})
|
||
return None
|
||
if not isinstance(body, dict):
|
||
self._send_json(400, {"error": "JSON body must be an object"})
|
||
return None
|
||
return body
|
||
|
||
def _read_json_body(self) -> dict | None:
|
||
length = int(self.headers.get("Content-Length", 0))
|
||
try:
|
||
body = json.loads(self.rfile.read(length) or b"{}")
|
||
except json.JSONDecodeError:
|
||
self._send_json(400, {"error": "invalid JSON body"})
|
||
return None
|
||
if not isinstance(body, dict):
|
||
self._send_json(400, {"error": "JSON body must be an object"})
|
||
return None
|
||
return body
|
||
|
||
def _purge_expired_nodes(self) -> None:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
_purge_expired_nodes_locked(server)
|
||
|
||
def do_POST(self):
|
||
if self.path == "/v1/chat/completions":
|
||
self._handle_proxy_chat()
|
||
return
|
||
if self.path == "/v1/nodes/register":
|
||
self._handle_register()
|
||
return
|
||
if self.path == "/v1/raft/vote":
|
||
self._handle_raft_vote()
|
||
return
|
||
if self.path == "/v1/raft/append":
|
||
self._handle_raft_append()
|
||
return
|
||
if self.path == "/v1/gossip":
|
||
self._handle_gossip()
|
||
return
|
||
if self.path == "/v1/stats/gossip":
|
||
self._handle_stats_gossip()
|
||
return
|
||
if self.path == "/v1/billing/gossip":
|
||
self._handle_billing_gossip()
|
||
return
|
||
if self.path == "/v1/billing/forfeit":
|
||
self._handle_billing_forfeit()
|
||
return
|
||
if self.path == "/v1/auth/register":
|
||
self._handle_auth_register()
|
||
return
|
||
if self.path == "/v1/auth/login":
|
||
self._handle_auth_login()
|
||
return
|
||
if self.path == "/v1/auth/logout":
|
||
self._handle_auth_logout()
|
||
return
|
||
if self.path == "/v1/account/keys":
|
||
self._handle_account_key_create()
|
||
return
|
||
if self.path == "/v1/account/keys/revoke":
|
||
self._handle_account_key_revoke()
|
||
return
|
||
if self.path == "/v1/account/topup":
|
||
self._handle_account_topup()
|
||
return
|
||
if self.path == "/v1/accounts/gossip":
|
||
self._handle_accounts_gossip()
|
||
return
|
||
if self.path == "/v1/registry/gossip":
|
||
self._handle_registry_gossip()
|
||
return
|
||
if self.path == "/v1/benchmark/hop-penalty":
|
||
self._handle_benchmark_hop_penalty()
|
||
return
|
||
if self.path == "/v1/calibration/toploc/run":
|
||
self._handle_toploc_calibration_run()
|
||
return
|
||
if self.path == "/v1/wallet/register":
|
||
self._handle_wallet_register()
|
||
return
|
||
parts = self.path.split("/")
|
||
# /v1/nodes/<node_id>/heartbeat -> ['', 'v1', 'nodes', '<id>', 'heartbeat']
|
||
if len(parts) == 5 and parts[1] == "v1" and parts[2] == "nodes" and parts[4] == "heartbeat":
|
||
self._handle_heartbeat(parts[3])
|
||
return
|
||
# /v1/proxy/requests/<request_id>/cancel
|
||
if (
|
||
len(parts) == 6
|
||
and parts[1] == "v1"
|
||
and parts[2] == "proxy"
|
||
and parts[3] == "requests"
|
||
and parts[5] == "cancel"
|
||
and parts[4]
|
||
):
|
||
self._handle_proxy_request_cancel(urllib.parse.unquote(parts[4]))
|
||
return
|
||
self.send_response(404)
|
||
self.end_headers()
|
||
|
||
def do_GET(self):
|
||
parsed = urllib.parse.urlparse(self.path)
|
||
if parsed.path == "/v1/route":
|
||
self._handle_route(parsed)
|
||
elif parsed.path == "/v1/routes":
|
||
self._handle_routes(parsed)
|
||
elif parsed.path == "/v1/routing":
|
||
self._handle_routing(parsed)
|
||
elif parsed.path == "/v1/nodes/assign":
|
||
self._handle_assign(parsed)
|
||
elif parsed.path == "/v1/network/assign":
|
||
self._handle_network_assign(parsed)
|
||
elif parsed.path == "/v1/network/map":
|
||
self._handle_network_map()
|
||
elif parsed.path == "/v1/models":
|
||
self._handle_models()
|
||
elif parsed.path == "/v1/model-files/download":
|
||
self._handle_model_files_download(parsed)
|
||
elif parsed.path.startswith("/v1/coverage/"):
|
||
model = urllib.parse.unquote(parsed.path.removeprefix("/v1/coverage/"))
|
||
self._handle_coverage(model)
|
||
elif parsed.path.startswith("/v1/tracker-nodes/"):
|
||
model = urllib.parse.unquote(parsed.path.removeprefix("/v1/tracker-nodes/"))
|
||
self._handle_tracker_nodes(model)
|
||
elif parsed.path.startswith("/v1/head-workers/"):
|
||
model = urllib.parse.unquote(parsed.path.removeprefix("/v1/head-workers/"))
|
||
self._handle_tracker_nodes(model)
|
||
elif parsed.path == "/v1/raft/status":
|
||
self._handle_raft_status()
|
||
elif parsed.path == "/v1/stats":
|
||
self._handle_stats()
|
||
elif parsed.path == "/v1/console":
|
||
self._handle_console()
|
||
elif parsed.path == "/v1/billing/summary":
|
||
self._handle_billing_summary()
|
||
elif parsed.path == "/v1/billing/settlements":
|
||
self._handle_billing_settlements()
|
||
elif parsed.path == "/v1/account":
|
||
self._handle_account_me()
|
||
elif parsed.path == "/v1/admin/accounts":
|
||
self._handle_admin_accounts()
|
||
elif parsed.path == "/v1/benchmark/results":
|
||
self._handle_benchmark_results()
|
||
elif parsed.path == "/v1/calibration/toploc/results":
|
||
self._handle_toploc_calibration_results()
|
||
elif parsed.path == "/v1/pricing/hf/history":
|
||
self._handle_hf_pricing_history(parsed)
|
||
elif parsed.path == "/v1/registry/wallets":
|
||
self._handle_registry_wallets()
|
||
elif parsed.path in ("/dashboard", "/dashboard/"):
|
||
self._handle_dashboard()
|
||
elif parsed.path == "/v1/health":
|
||
self._send_json(200, {"status": "ok"})
|
||
else:
|
||
self.send_response(404)
|
||
self.end_headers()
|
||
|
||
def _handle_models(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
created = int(time.time())
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
alive = list(server.registry.values())
|
||
if server.contracts is not None:
|
||
alive = [
|
||
node for node in alive
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
data = []
|
||
seen_ids: set[str] = set()
|
||
for name, preset in server.model_presets.items():
|
||
model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
|
||
if not model_nodes and not preset.get("recommended"):
|
||
continue
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
coverage = _coverage_percentage(
|
||
model_nodes,
|
||
required_start,
|
||
required_end,
|
||
)
|
||
aliases = [name]
|
||
hf_repo = preset.get("hf_repo")
|
||
if hf_repo and hf_repo not in aliases:
|
||
aliases.append(hf_repo)
|
||
for alias in preset.get("aliases", []) or []:
|
||
if isinstance(alias, str) and alias not in aliases:
|
||
aliases.append(alias)
|
||
data.append({
|
||
"id": name,
|
||
"object": "model",
|
||
"created": created,
|
||
"owned_by": "meshnet",
|
||
"name": name,
|
||
"hf_repo": hf_repo,
|
||
"aliases": aliases,
|
||
"metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
|
||
"recommended": bool(preset.get("recommended", False)),
|
||
"deployment": _deployment_summary(alive, preset),
|
||
"shard_coverage_percentage": coverage,
|
||
})
|
||
seen_ids.add(name)
|
||
if hf_repo:
|
||
seen_ids.add(hf_repo)
|
||
|
||
hf_model_ids = sorted({
|
||
node.hf_repo or node.model
|
||
for node in alive
|
||
if node.model is not None
|
||
and node.model not in server.model_presets
|
||
and node.shard_start is not None
|
||
and node.shard_end is not None
|
||
and node.num_layers is not None
|
||
})
|
||
for model_id in hf_model_ids:
|
||
if model_id is None or model_id in seen_ids:
|
||
continue
|
||
model_nodes = [
|
||
node for node in alive
|
||
if node.shard_start is not None
|
||
and node.shard_end is not None
|
||
and node.num_layers is not None
|
||
and (node.hf_repo == model_id or (node.hf_repo is None and node.model == model_id))
|
||
]
|
||
if not model_nodes:
|
||
continue
|
||
short_names = sorted({node.model for node in model_nodes if node.model})
|
||
aliases = [model_id, *[name for name in short_names if name != model_id]]
|
||
required_start = 0
|
||
required_end = max(node.num_layers for node in model_nodes) - 1
|
||
data.append({
|
||
"id": model_id,
|
||
"object": "model",
|
||
"created": created,
|
||
"owned_by": "meshnet",
|
||
"name": short_names[0] if short_names else model_id,
|
||
"hf_repo": model_id if any(node.hf_repo == model_id for node in model_nodes) else None,
|
||
"aliases": aliases,
|
||
"metadata": _model_metadata_from_nodes(model_nodes),
|
||
"shard_coverage_percentage": _coverage_percentage(
|
||
model_nodes,
|
||
required_start,
|
||
required_end,
|
||
),
|
||
})
|
||
seen_ids.add(model_id)
|
||
self._send_json(200, {"object": "list", "data": data})
|
||
|
||
def _handle_coverage(self, model: str):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
# Do NOT purge before coverage — dead nodes are included with alive=false
|
||
# so operators can see what was covering each layer band before failure.
|
||
with server.lock:
|
||
resolved = _nodes_and_bounds_for_model(server, model)
|
||
if resolved is None:
|
||
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
|
||
return
|
||
all_nodes, required_start, required_end = resolved
|
||
if server.contracts is not None:
|
||
all_nodes = [
|
||
node for node in all_nodes
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
coverage = _coverage_map_detailed(all_nodes, required_start, required_end, server.heartbeat_timeout)
|
||
self._send_json(200, {"model": model, "coverage": coverage})
|
||
|
||
def _handle_tracker_nodes(self, model: str):
|
||
"""Return head workers: worker nodes that can start inference for a model.
|
||
|
||
The historical endpoint name is /v1/tracker-nodes, but these are not
|
||
tracker processes and they are the only machines that load model shards.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
if preset is None:
|
||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||
return
|
||
required_start, _ = _preset_layer_bounds(preset)
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
alive = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
]
|
||
if server.contracts is not None:
|
||
alive = [
|
||
node for node in alive
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
tracker_nodes = [
|
||
node for node in alive
|
||
if node.shard_start is not None
|
||
and node.shard_start == required_start
|
||
and node.tracker_mode
|
||
]
|
||
self._send_json(200, {
|
||
"model": resolved_name,
|
||
"head_workers": [
|
||
{
|
||
"node_id": node.node_id,
|
||
"endpoint": node.endpoint,
|
||
"relay_addr": node.relay_addr,
|
||
"peer_id": node.peer_id,
|
||
}
|
||
for node in tracker_nodes
|
||
],
|
||
"tracker_nodes": [
|
||
{
|
||
"node_id": node.node_id,
|
||
"endpoint": node.endpoint,
|
||
"relay_addr": node.relay_addr,
|
||
"peer_id": node.peer_id,
|
||
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
|
||
}
|
||
for node in tracker_nodes
|
||
],
|
||
})
|
||
|
||
def _handle_network_map(self) -> None:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
nodes = list(server.registry.values())
|
||
memory_pool = _memory_pool_map(server)
|
||
|
||
def capacity_for(node: _NodeEntry) -> dict:
|
||
preset = None
|
||
if node.model:
|
||
preset = server.model_presets.get(node.model)
|
||
if preset is None and node.hf_repo and node.num_layers:
|
||
preset = _hf_rebalance_preset([node])
|
||
return _node_capacity_summary(node, preset)
|
||
|
||
def throughput_for(node: _NodeEntry) -> dict:
|
||
if server.stats is None:
|
||
return {}
|
||
models = [m for m in (node.hf_repo, node.model) if m]
|
||
result = {}
|
||
for model in models:
|
||
result[model] = server.stats.get_node_model_stats(node.node_id, model)
|
||
return result
|
||
|
||
def model_supply_for(node: _NodeEntry) -> dict:
|
||
return _model_health_summary(server, node.model, node.hf_repo)
|
||
|
||
self._send_json(200, {
|
||
"relay_url": server.relay_url,
|
||
"pool": _pool_summary(nodes),
|
||
"memory_pool": memory_pool,
|
||
"recommended_models": [
|
||
{
|
||
"id": name,
|
||
"hf_repo": preset.get("hf_repo"),
|
||
"aliases": list(preset.get("aliases", []) or []),
|
||
"metadata": dict(preset.get("metadata") or {}),
|
||
"deployment": _deployment_summary(nodes, preset),
|
||
}
|
||
for name, preset in server.model_presets.items()
|
||
if preset.get("recommended")
|
||
],
|
||
"nodes": [
|
||
{
|
||
"node_id": node.node_id,
|
||
"endpoint": node.endpoint,
|
||
"relay_addr": node.relay_addr,
|
||
"peer_id": node.peer_id,
|
||
"model": node.model,
|
||
"hf_repo": node.hf_repo,
|
||
"num_layers": node.num_layers,
|
||
"model_metadata": dict(node.model_metadata),
|
||
"downloaded_models": [dict(item) for item in node.downloaded_models],
|
||
"shard_start": node.shard_start,
|
||
"shard_end": node.shard_end,
|
||
"tracker_mode": node.tracker_mode,
|
||
"last_heartbeat": node.last_heartbeat,
|
||
"capacity": capacity_for(node),
|
||
"model_supply": model_supply_for(node),
|
||
"throughput": throughput_for(node),
|
||
"stats": _node_health(node, server.heartbeat_timeout),
|
||
}
|
||
for node in nodes
|
||
],
|
||
})
|
||
|
||
# ---------------------------------------------------------------- OpenAI proxy
|
||
|
||
def _handle_proxy_chat(self) -> None:
|
||
"""Proxy POST /v1/chat/completions to a tracker-mode (first-shard) node.
|
||
|
||
Picks a live tracker-mode node for the requested model using round-robin,
|
||
then forwards the request verbatim and relays the response (including
|
||
streaming SSE chunks) back to the caller.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
|
||
length = int(self.headers.get("Content-Length", 0))
|
||
raw_body = self.rfile.read(length) if length else b"{}"
|
||
|
||
try:
|
||
body = json.loads(raw_body)
|
||
except json.JSONDecodeError:
|
||
self._send_json(400, {"error": {"message": "invalid JSON", "type": "invalid_request_error", "code": "invalid_request"}})
|
||
return
|
||
|
||
model: str = body.get("model", "")
|
||
is_stream: bool = bool(body.get("stream", False))
|
||
|
||
if model and server.stats is not None:
|
||
server.stats.record_request(model)
|
||
|
||
# Billing gate (ADR-0015): reject before any routing — no free work.
|
||
api_key = _api_key_from_headers(self.headers)
|
||
if server.billing is not None:
|
||
if api_key is None:
|
||
self._send_json(401, {"error": {
|
||
"message": "missing API key: send Authorization: Bearer <key>",
|
||
"type": "invalid_request_error",
|
||
"code": "missing_api_key",
|
||
}})
|
||
return
|
||
if server.accounts is not None and server.accounts.is_key_revoked(api_key):
|
||
self._send_json(401, {"error": {
|
||
"message": "API key has been revoked",
|
||
"type": "invalid_request_error",
|
||
"code": "invalid_api_key",
|
||
}})
|
||
return
|
||
# US-039: with accounts enabled, only real account keys may spend —
|
||
# arbitrary bearer strings must never become billable clients.
|
||
if server.accounts is not None and not server.accounts.is_active_key(api_key):
|
||
self._send_json(401, {"error": {
|
||
"message": "unknown API key: create one at /dashboard (register, then + new key)",
|
||
"type": "invalid_request_error",
|
||
"code": "invalid_api_key",
|
||
}})
|
||
return
|
||
if not server.billing.has_funds(api_key):
|
||
self._send_json(402, {"error": {
|
||
"message": "insufficient balance: deposit USDT to continue",
|
||
"type": "insufficient_quota",
|
||
"code": "insufficient_balance",
|
||
}})
|
||
return
|
||
if server.max_charge_per_request is not None:
|
||
token_limit = _requested_completion_token_limit(body)
|
||
if token_limit is None:
|
||
self._send_json(400, {"error": {
|
||
"message": (
|
||
"max_charge_per_request is enabled; include max_tokens "
|
||
"or max_completion_tokens so this request can be capped"
|
||
),
|
||
"type": "invalid_request_error",
|
||
"code": "missing_token_limit",
|
||
}})
|
||
return
|
||
in_rate, out_rate = server.billing.prices_for(model)
|
||
prompt_estimate = _estimate_prompt_tokens(body) or 0
|
||
estimated_charge = (in_rate * prompt_estimate + out_rate * token_limit) / 1000.0
|
||
if estimated_charge > server.max_charge_per_request:
|
||
self._send_json(402, {"error": {
|
||
"message": (
|
||
f"request exceeds max_charge_per_request "
|
||
f"({estimated_charge:.6f} USDT estimated > "
|
||
f"{server.max_charge_per_request:.6f} USDT cap)"
|
||
),
|
||
"type": "insufficient_quota",
|
||
"code": "spend_cap_exceeded",
|
||
}})
|
||
return
|
||
|
||
# US-030: optional pinned route — "route": [node_id, ...] uses those
|
||
# nodes in order instead of auto-selection. Absent field: unchanged.
|
||
pinned_ids = body.get("route")
|
||
pinned_nodes: list[_NodeEntry] | None = None
|
||
if pinned_ids is not None:
|
||
if (
|
||
not isinstance(pinned_ids, list)
|
||
or not pinned_ids
|
||
or not all(isinstance(nid, str) and nid for nid in pinned_ids)
|
||
):
|
||
self._send_json(400, {"error": {
|
||
"message": "route must be a non-empty list of node id strings",
|
||
"type": "invalid_request_error",
|
||
"code": "invalid_route",
|
||
}})
|
||
return
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
missing = [nid for nid in pinned_ids if nid not in server.registry]
|
||
if missing:
|
||
self._send_json(400, {"error": {
|
||
"message": f"unknown node ids in route: {missing}",
|
||
"type": "invalid_request_error",
|
||
"code": "unknown_route_nodes",
|
||
}})
|
||
return
|
||
pinned_nodes = [server.registry[nid] for nid in pinned_ids]
|
||
|
||
if pinned_nodes is not None:
|
||
node = pinned_nodes[0]
|
||
else:
|
||
# Find a live tracker-mode node for this model
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
candidates = [
|
||
n for n in server.registry.values()
|
||
if n.tracker_mode and _node_matches_model(n, model)
|
||
]
|
||
|
||
if not candidates:
|
||
# Fall back: any node serving shard_start=0 for this model
|
||
with server.lock:
|
||
candidates = [
|
||
n for n in server.registry.values()
|
||
if n.shard_start == 0 and _node_matches_model(n, model)
|
||
]
|
||
|
||
if not candidates:
|
||
with server.lock:
|
||
registered = [
|
||
{
|
||
"node_id": n.node_id,
|
||
"model": n.model,
|
||
"hf_repo": n.hf_repo,
|
||
"shard": f"{n.shard_start}-{n.shard_end}",
|
||
"tracker_mode": n.tracker_mode,
|
||
}
|
||
for n in server.registry.values()
|
||
]
|
||
_tracker_log(
|
||
server,
|
||
"warn",
|
||
"no nodes available for model",
|
||
model=model,
|
||
registered_nodes=registered,
|
||
)
|
||
self._send_json(503, {"error": {
|
||
"message": f"no nodes available for model {model!r}",
|
||
"type": "service_unavailable",
|
||
"code": "model_not_available",
|
||
}})
|
||
return
|
||
|
||
node = max(candidates, key=lambda n: _effective_throughput(n, model))
|
||
target_url = f"{node.endpoint}/v1/chat/completions"
|
||
request_id = str(body.get("id") or f"req-{time.time_ns():x}")
|
||
body["id"] = request_id
|
||
raw_body = json.dumps(body).encode()
|
||
|
||
# Pre-resolve the downstream route so the first-shard node skips its own
|
||
# tracker query. We already hold the full registry picture — no need for
|
||
# a second round-trip.
|
||
route_model = node.hf_repo or node.model or model
|
||
with server.lock:
|
||
resolved_route_model, route_preset = _resolve_model_preset(server.model_presets, route_model)
|
||
if route_preset is not None:
|
||
route_model = resolved_route_model or route_model
|
||
preset = route_preset
|
||
rs, re = _preset_layer_bounds(preset)
|
||
all_nodes: list = [
|
||
n for n in server.registry.values()
|
||
if _node_matches_preset(n, route_model, preset)
|
||
and n.shard_start is not None
|
||
and n.shard_end is not None
|
||
]
|
||
else:
|
||
all_nodes = [
|
||
n for n in server.registry.values()
|
||
if _node_matches_model(n, route_model)
|
||
and n.shard_start is not None and n.num_layers is not None
|
||
]
|
||
rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
|
||
if pinned_nodes is not None:
|
||
route_nodes = pinned_nodes
|
||
routing_decision = {"mode": "pinned"}
|
||
else:
|
||
# ADR-0021: enumerate viable routes and pick one bandit-style —
|
||
# ε-scout among unproven routes, otherwise weighted ∝ observed tps^α.
|
||
route_candidates = _enumerate_routes(
|
||
all_nodes, rs, re,
|
||
model=route_model,
|
||
contracts=server.contracts,
|
||
max_candidates=server.route_stats.config.max_candidate_routes,
|
||
)
|
||
picked, routing_decision = choose_route(
|
||
route_candidates, server.route_stats, route_model, rng=server.route_rng,
|
||
)
|
||
if picked is not None:
|
||
route_nodes = picked.nodes
|
||
else:
|
||
# No head-anchored candidate — legacy greedy cover as fallback
|
||
# (also produces the layer-gap error message).
|
||
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
|
||
routing_decision = {"mode": "greedy-fallback"}
|
||
if route_error:
|
||
_tracker_log(
|
||
server,
|
||
"warn",
|
||
"route unavailable",
|
||
model=model,
|
||
route_model=route_model,
|
||
error=route_error,
|
||
candidate_count=len(all_nodes),
|
||
candidates=_node_route_summary(all_nodes),
|
||
)
|
||
self._send_json(503, {"error": {
|
||
"message": route_error,
|
||
"type": "service_unavailable",
|
||
"code": "route_not_available",
|
||
}})
|
||
return
|
||
# The proxy target must be the route's own head: an independently
|
||
# chosen fastest node may not be part of the planned route, which
|
||
# previously injected downstream hops with wrong start layers
|
||
# (ADR-0020 mixed-topology flaw).
|
||
if route_nodes:
|
||
node = route_nodes[0]
|
||
target_url = f"{node.endpoint}/v1/chat/completions"
|
||
# Compute start_layer for each hop: each node begins where the previous ended + 1.
|
||
# This allows overlapping shard registrations without double-computation.
|
||
covered_up_to = rs - 1
|
||
route_hops: list[dict] = []
|
||
node_work: list[tuple[str | None, int]] = []
|
||
for rn in route_nodes:
|
||
hop: dict = {"endpoint": rn.endpoint, "start_layer": covered_up_to + 1}
|
||
if rn.relay_addr:
|
||
hop["relay_addr"] = rn.relay_addr
|
||
route_hops.append(hop)
|
||
effective_end = rn.shard_end if rn.shard_end is not None else covered_up_to
|
||
node_work.append((rn.wallet_address, max(0, effective_end - covered_up_to)))
|
||
covered_up_to = effective_end
|
||
# Strip the first-shard node we're about to proxy to — it's already handling the request.
|
||
downstream_hops = [
|
||
h for h in route_hops
|
||
if _endpoint_key(h["endpoint"]) != _endpoint_key(node.endpoint)
|
||
]
|
||
downstream_urls = json.dumps(downstream_hops)
|
||
route_debug = " -> ".join(
|
||
f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
|
||
for n in route_nodes
|
||
)
|
||
inflight_nodes = route_nodes or [node]
|
||
inflight_recorded = True
|
||
_record_proxy_inflight(server, inflight_nodes, 1)
|
||
|
||
def finish_proxy_inflight() -> None:
|
||
nonlocal inflight_recorded
|
||
if inflight_recorded:
|
||
_record_proxy_inflight(server, inflight_nodes, -1)
|
||
inflight_recorded = False
|
||
_unregister_active_proxy(server, request_id)
|
||
|
||
proxy_ctx = _register_active_proxy(server, request_id)
|
||
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy route selected",
|
||
request_id=request_id,
|
||
model=model,
|
||
head_node_id=node.node_id,
|
||
head_endpoint=node.endpoint,
|
||
downstream=downstream_urls,
|
||
route=route_debug or "<empty>",
|
||
routing=routing_decision,
|
||
nodes=_node_route_summary(route_nodes),
|
||
)
|
||
|
||
req = urllib.request.Request(
|
||
target_url,
|
||
data=raw_body,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
"X-Meshnet-Route": downstream_urls,
|
||
"X-Meshnet-Request-Id": request_id,
|
||
},
|
||
method="POST",
|
||
)
|
||
# Copy Authorization header from client if present
|
||
auth = self.headers.get("Authorization")
|
||
if auth:
|
||
req.add_header("Authorization", auth)
|
||
|
||
relay_headers = {
|
||
"Content-Type": "application/json",
|
||
"X-Meshnet-Route": downstream_urls,
|
||
"X-Meshnet-Request-Id": request_id,
|
||
**({"Authorization": auth} if auth else {}),
|
||
}
|
||
|
||
if node.relay_addr:
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy via relay",
|
||
request_id=request_id,
|
||
relay_addr=node.relay_addr,
|
||
direct_endpoint=target_url,
|
||
)
|
||
started = time.monotonic()
|
||
relay_ws_holder: list[Any] = []
|
||
frames = _relay_http_request_frames(
|
||
node.relay_addr,
|
||
path="/v1/chat/completions",
|
||
body=raw_body,
|
||
headers=relay_headers,
|
||
cancel_event=proxy_ctx.cancel_event,
|
||
ws_holder=relay_ws_holder,
|
||
ws_lock=proxy_ctx.relay_ws_lock,
|
||
)
|
||
first = next(frames, None)
|
||
with proxy_ctx.relay_ws_lock:
|
||
proxy_ctx.relay_ws = relay_ws_holder[0] if relay_ws_holder else None
|
||
if proxy_ctx.cancel_event.is_set():
|
||
if self._finalize_proxy_cancel(
|
||
proxy_ctx=proxy_ctx,
|
||
server=server,
|
||
request_id=request_id,
|
||
started=started,
|
||
model=model,
|
||
route_model=route_model,
|
||
route_nodes=route_nodes,
|
||
api_key=api_key,
|
||
node_work=node_work,
|
||
body=body,
|
||
finish_proxy_inflight=finish_proxy_inflight,
|
||
):
|
||
return
|
||
if first is not None and first.get("stream"):
|
||
# Streamed response (US-036): forward SSE chunks as they arrive
|
||
# and run the same token accounting as the direct stream path.
|
||
self._stream_relayed_frames(
|
||
first, frames, started,
|
||
model, route_model, route_nodes, api_key, node_work,
|
||
request_body=body,
|
||
request_id=request_id,
|
||
proxy_ctx=proxy_ctx,
|
||
finish_proxy_inflight=finish_proxy_inflight,
|
||
)
|
||
finish_proxy_inflight()
|
||
return
|
||
if first is not None:
|
||
elapsed = time.monotonic() - started
|
||
self._send_relayed_response(first)
|
||
if int(first.get("status", 503)) < 400:
|
||
body_text = first.get("body") or ""
|
||
try:
|
||
in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body)
|
||
except (json.JSONDecodeError, TypeError):
|
||
in_tokens, out_tokens = 0, 0
|
||
tokens = in_tokens + out_tokens
|
||
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy complete",
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
status=int(first.get("status", 503)),
|
||
tokens=tokens,
|
||
elapsed_seconds=round(elapsed, 4),
|
||
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||
route=_node_route_summary(route_nodes),
|
||
)
|
||
self._bill_completed(
|
||
api_key, model, tokens, node_work,
|
||
input_tokens=in_tokens, output_tokens=out_tokens,
|
||
)
|
||
finish_proxy_inflight()
|
||
return
|
||
_tracker_log(
|
||
server,
|
||
"warn",
|
||
"relay proxy failed, trying direct",
|
||
request_id=request_id,
|
||
relay_addr=node.relay_addr,
|
||
direct_endpoint=target_url,
|
||
)
|
||
|
||
try:
|
||
started = time.monotonic()
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy connecting",
|
||
request_id=request_id,
|
||
target_url=target_url,
|
||
stream=is_stream or None,
|
||
)
|
||
upstream_result: list[Any] = []
|
||
connect_errors: list[BaseException] = []
|
||
|
||
def _connect_upstream() -> None:
|
||
try:
|
||
upstream_result.append(urllib.request.urlopen(req, timeout=300.0))
|
||
except BaseException as exc:
|
||
connect_errors.append(exc)
|
||
|
||
connect_thread = threading.Thread(target=_connect_upstream, daemon=True)
|
||
connect_thread.start()
|
||
while connect_thread.is_alive():
|
||
if proxy_ctx.cancel_event.is_set():
|
||
connect_thread.join(timeout=310.0)
|
||
if upstream_result:
|
||
try:
|
||
upstream_result[0].close()
|
||
except Exception:
|
||
pass
|
||
if self._finalize_proxy_cancel(
|
||
proxy_ctx=proxy_ctx,
|
||
server=server,
|
||
request_id=request_id,
|
||
started=started,
|
||
model=model,
|
||
route_model=route_model,
|
||
route_nodes=route_nodes,
|
||
api_key=api_key,
|
||
node_work=node_work,
|
||
body=body,
|
||
finish_proxy_inflight=finish_proxy_inflight,
|
||
):
|
||
return
|
||
connect_thread.join(0.2)
|
||
|
||
if proxy_ctx.cancel_event.is_set():
|
||
if upstream_result:
|
||
try:
|
||
upstream_result[0].close()
|
||
except Exception:
|
||
pass
|
||
if self._finalize_proxy_cancel(
|
||
proxy_ctx=proxy_ctx,
|
||
server=server,
|
||
request_id=request_id,
|
||
started=started,
|
||
model=model,
|
||
route_model=route_model,
|
||
route_nodes=route_nodes,
|
||
api_key=api_key,
|
||
node_work=node_work,
|
||
body=body,
|
||
finish_proxy_inflight=finish_proxy_inflight,
|
||
):
|
||
return
|
||
|
||
if connect_errors:
|
||
raise connect_errors[0]
|
||
|
||
upstream = upstream_result[0]
|
||
with proxy_ctx.upstream_lock:
|
||
proxy_ctx.upstream = upstream
|
||
upstream_sock = _upstream_socket(upstream)
|
||
if upstream_sock is not None:
|
||
_set_upstream_read_timeout(upstream, None)
|
||
else:
|
||
_set_upstream_read_timeout(upstream, 0.5)
|
||
_tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
|
||
except urllib.error.HTTPError as exc:
|
||
# Relay error status + body from node
|
||
err_body = exc.read()
|
||
self.send_response(exc.code)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(err_body)))
|
||
self.end_headers()
|
||
try:
|
||
self.wfile.write(err_body)
|
||
except BrokenPipeError:
|
||
pass
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
finish_proxy_inflight()
|
||
return
|
||
except Exception as exc:
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
if node.relay_addr:
|
||
_tracker_log(
|
||
server,
|
||
"error",
|
||
"direct proxy failed after relay",
|
||
request_id=request_id,
|
||
target_url=target_url,
|
||
relay_addr=node.relay_addr,
|
||
error=repr(exc),
|
||
)
|
||
else:
|
||
_tracker_log(
|
||
server,
|
||
"error",
|
||
"proxy failed",
|
||
request_id=request_id,
|
||
target_url=target_url,
|
||
error=repr(exc),
|
||
)
|
||
self._send_json(503, {"error": {
|
||
"message": f"upstream node unreachable: {exc}",
|
||
"type": "service_unavailable",
|
||
"code": "upstream_error",
|
||
}})
|
||
finish_proxy_inflight()
|
||
return
|
||
|
||
with upstream:
|
||
content_type = upstream.headers.get("Content-Type", "application/json")
|
||
if is_stream or "text/event-stream" in content_type:
|
||
# Relay SSE stream chunk-by-chunk
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
|
||
self.send_header("Cache-Control", "no-cache")
|
||
self.send_header("X-Accel-Buffering", "no")
|
||
self.end_headers()
|
||
stream_usage: dict | None = None
|
||
observed_stream_tokens = 0
|
||
client_gone = False
|
||
try:
|
||
while True:
|
||
if proxy_ctx.cancel_event.is_set():
|
||
break
|
||
if upstream_sock is not None:
|
||
readable, _, _ = select.select([upstream_sock], [], [], 0.5)
|
||
if not readable:
|
||
continue
|
||
try:
|
||
line = upstream.readline()
|
||
except TimeoutError:
|
||
continue
|
||
if not line:
|
||
if proxy_ctx.cancel_event.is_set():
|
||
break
|
||
break
|
||
if not client_gone:
|
||
try:
|
||
self.wfile.write(line)
|
||
self.wfile.flush()
|
||
except (BrokenPipeError, ConnectionResetError):
|
||
# Keep draining upstream so completed node work is still billed.
|
||
client_gone = True
|
||
observed, usage = _stream_line_tokens(line)
|
||
observed_stream_tokens += observed
|
||
if observed:
|
||
_tracker_log_proxy_progress(
|
||
server,
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
tokens=observed_stream_tokens,
|
||
started=started,
|
||
route_nodes=route_nodes,
|
||
)
|
||
if usage is not None:
|
||
stream_usage = usage
|
||
except (BrokenPipeError, ConnectionResetError):
|
||
client_gone = True
|
||
if self._finalize_proxy_cancel(
|
||
proxy_ctx=proxy_ctx,
|
||
server=server,
|
||
request_id=request_id,
|
||
started=started,
|
||
model=model,
|
||
route_model=route_model,
|
||
route_nodes=route_nodes,
|
||
api_key=api_key,
|
||
node_work=node_work,
|
||
body=body,
|
||
finish_proxy_inflight=finish_proxy_inflight,
|
||
observed_stream_tokens=observed_stream_tokens,
|
||
stream_usage=stream_usage,
|
||
):
|
||
return
|
||
elapsed = time.monotonic() - started
|
||
# Bill even on client disconnect — the nodes did the work.
|
||
# Observed stream chunks are authoritative for the upper bound;
|
||
# upstream usage may only lower that count.
|
||
in_tokens, out_tokens = _stream_billable_split(
|
||
observed_stream_tokens, stream_usage, body
|
||
)
|
||
self._record_observed_throughput(
|
||
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
|
||
)
|
||
tokens = in_tokens + out_tokens
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy complete",
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
status=200,
|
||
stream=True,
|
||
client_disconnected=client_gone,
|
||
tokens=tokens,
|
||
elapsed_seconds=round(elapsed, 4),
|
||
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||
route=_node_route_summary(route_nodes),
|
||
)
|
||
self._bill_completed(
|
||
api_key, model, tokens, node_work,
|
||
input_tokens=in_tokens, output_tokens=out_tokens,
|
||
)
|
||
else:
|
||
# Non-streaming: buffer and relay
|
||
resp_body = upstream.read()
|
||
elapsed = time.monotonic() - started
|
||
observed_output = ""
|
||
try:
|
||
response_payload = json.loads(resp_body)
|
||
in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
|
||
observed_output = _observed_output_from_non_stream_payload(response_payload)
|
||
except json.JSONDecodeError:
|
||
in_tokens, out_tokens = 0, 0
|
||
tokens = in_tokens + out_tokens
|
||
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy complete",
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
target_url=target_url,
|
||
status=200,
|
||
bytes=len(resp_body),
|
||
tokens=tokens,
|
||
elapsed_seconds=round(elapsed, 4),
|
||
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||
route=_node_route_summary(route_nodes),
|
||
)
|
||
self._record_validation_event(request_id, model, body, observed_output, route_nodes)
|
||
self._bill_completed(
|
||
api_key, model, tokens, node_work,
|
||
input_tokens=in_tokens, output_tokens=out_tokens,
|
||
)
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", content_type)
|
||
self.send_header("Content-Length", str(len(resp_body)))
|
||
self.end_headers()
|
||
try:
|
||
self.wfile.write(resp_body)
|
||
except (BrokenPipeError, ConnectionResetError):
|
||
pass
|
||
finish_proxy_inflight()
|
||
|
||
def _record_observed_throughput(
|
||
self,
|
||
requested_model: str,
|
||
route_model: str,
|
||
total_tokens: int,
|
||
elapsed_seconds: float,
|
||
route_nodes: list[_NodeEntry],
|
||
) -> None:
|
||
"""Record observed route TPS for participating nodes.
|
||
|
||
The tracker sees end-to-end request duration, not per-hop timings, so
|
||
each hop gets the same route-level observation for now. Per-hop telemetry
|
||
can refine this later without changing the external stats shape.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if route_model and route_nodes:
|
||
# Route-level bandit sample (ADR-0021); the store itself rejects
|
||
# near-empty completions that would poison the arm.
|
||
server.route_stats.record_sample(
|
||
route_model,
|
||
route_signature(route_model, route_nodes),
|
||
total_tokens,
|
||
elapsed_seconds,
|
||
)
|
||
if server.stats is None or total_tokens <= 0:
|
||
return
|
||
elapsed_seconds = max(elapsed_seconds, 1e-6)
|
||
models = [m for m in (requested_model, route_model) if m]
|
||
if len(models) == 2 and models[0] == models[1]:
|
||
models = [models[0]]
|
||
for node in route_nodes:
|
||
for model in models:
|
||
server.stats.record_node_throughput(
|
||
node.node_id,
|
||
model,
|
||
total_tokens=total_tokens,
|
||
elapsed_seconds=elapsed_seconds,
|
||
)
|
||
stats = server.stats.get_node_model_stats(node.node_id, model)
|
||
observed = stats.get("tokens_per_sec_last_hour")
|
||
if observed is not None:
|
||
node.model_tokens_per_sec[model] = float(observed)
|
||
|
||
def _record_validation_event(
|
||
self,
|
||
session_id: str,
|
||
model: str,
|
||
request_body: dict,
|
||
observed_output: str,
|
||
route_nodes: list[_NodeEntry],
|
||
) -> None:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
validation = getattr(server.contracts, "validation", None) if server.contracts is not None else None
|
||
if validation is None:
|
||
return
|
||
messages = request_body.get("messages")
|
||
if not isinstance(messages, list):
|
||
messages = []
|
||
event_nodes = [
|
||
{
|
||
"node_id": node.node_id,
|
||
"endpoint": node.endpoint,
|
||
"wallet_address": node.wallet_address,
|
||
"shard_start": node.shard_start,
|
||
"shard_end": node.shard_end,
|
||
}
|
||
for node in route_nodes
|
||
]
|
||
try:
|
||
validation.record_completed_inference(
|
||
session_id=session_id,
|
||
model=model,
|
||
messages=[dict(message) for message in messages if isinstance(message, dict)],
|
||
observed_output=observed_output,
|
||
route_nodes=event_nodes,
|
||
)
|
||
except Exception as exc:
|
||
print(f"[tracker] validation event recording failed for {session_id}: {exc}", flush=True)
|
||
|
||
def _bill_completed(
|
||
self,
|
||
api_key: str | None,
|
||
model: str,
|
||
total_tokens: int,
|
||
node_work: list[tuple[str | None, int]],
|
||
*,
|
||
input_tokens: int | None = None,
|
||
output_tokens: int | None = None,
|
||
) -> None:
|
||
"""Charge a completed request against the billing ledger (ADR-0015).
|
||
|
||
With ``input_tokens``/``output_tokens`` the ledger bills each side at
|
||
its own rate (US-045); ``total_tokens`` alone falls back to the
|
||
blended rate.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.billing is None or api_key is None:
|
||
return
|
||
# Probationary period (issue 08): a wallet's first N jobs earn nothing —
|
||
# its share stays in the protocol cut. Job counts live in the registry
|
||
# contract, so this only applies when the tracker runs with contracts.
|
||
if server.contracts is not None:
|
||
adjusted: list[tuple[str | None, int]] = []
|
||
for wallet, work in node_work:
|
||
if wallet:
|
||
in_probation = server.contracts.registry.probationary_jobs_remaining(wallet) > 0
|
||
server.contracts.registry.record_completed_job(wallet)
|
||
if in_probation:
|
||
wallet = None
|
||
adjusted.append((wallet, work))
|
||
node_work = adjusted
|
||
try:
|
||
event = server.billing.charge_request(
|
||
api_key, model, total_tokens, node_work,
|
||
input_tokens=input_tokens, output_tokens=output_tokens,
|
||
)
|
||
print(
|
||
f"[tracker] billed api_key=…{api_key[-6:]}: model={model!r} "
|
||
f"tokens={event['total_tokens']} "
|
||
f"(in={event.get('input_tokens', '?')} out={event.get('output_tokens', '?')}) "
|
||
f"cost={event['cost']:.6f} USDT shares={event['shares']}",
|
||
flush=True,
|
||
)
|
||
except Exception as exc:
|
||
print(f"[tracker] billing failed for model={model!r}: {exc}", flush=True)
|
||
|
||
def _stream_relayed_frames(
|
||
self,
|
||
first: dict,
|
||
frames,
|
||
started: float,
|
||
model: str,
|
||
route_model: str,
|
||
route_nodes: list,
|
||
api_key: str | None,
|
||
node_work: list,
|
||
request_body: dict,
|
||
request_id: str,
|
||
*,
|
||
proxy_ctx: _ActiveProxyContext | None = None,
|
||
finish_proxy_inflight: Any = None,
|
||
) -> None:
|
||
"""Forward a streamed relay response (US-036) to the client as SSE,
|
||
billing with the same accounting as the direct stream path."""
|
||
headers = first.get("headers") if isinstance(first.get("headers"), dict) else {}
|
||
self.send_response(int(first.get("status", 200)))
|
||
self.send_header("Content-Type", headers.get("Content-Type", "text/event-stream; charset=utf-8"))
|
||
self.send_header("Cache-Control", "no-cache")
|
||
self.send_header("X-Accel-Buffering", "no")
|
||
self.end_headers()
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
stream_usage: dict | None = None
|
||
observed_stream_tokens = 0
|
||
client_gone = False
|
||
for frame in itertools.chain([first], frames):
|
||
if proxy_ctx is not None and proxy_ctx.cancel_event.is_set():
|
||
break
|
||
chunk = frame.get("chunk") or ""
|
||
if not chunk:
|
||
continue
|
||
data = chunk.encode()
|
||
if not client_gone:
|
||
try:
|
||
self.wfile.write(data)
|
||
self.wfile.flush()
|
||
except BrokenPipeError:
|
||
# Keep draining frames — the nodes did the work; bill it.
|
||
client_gone = True
|
||
for line in data.splitlines():
|
||
observed, usage = _stream_line_tokens(line)
|
||
observed_stream_tokens += observed
|
||
if observed:
|
||
_tracker_log_proxy_progress(
|
||
server,
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
tokens=observed_stream_tokens,
|
||
started=started,
|
||
route_nodes=route_nodes,
|
||
relay=True,
|
||
)
|
||
if usage is not None:
|
||
stream_usage = usage
|
||
if (
|
||
proxy_ctx is not None
|
||
and finish_proxy_inflight is not None
|
||
and self._finalize_proxy_cancel(
|
||
proxy_ctx=proxy_ctx,
|
||
server=server,
|
||
request_id=request_id,
|
||
started=started,
|
||
model=model,
|
||
route_model=route_model,
|
||
route_nodes=route_nodes,
|
||
api_key=api_key,
|
||
node_work=node_work,
|
||
body=request_body,
|
||
finish_proxy_inflight=finish_proxy_inflight,
|
||
observed_stream_tokens=observed_stream_tokens,
|
||
stream_usage=stream_usage,
|
||
)
|
||
):
|
||
return
|
||
elapsed = time.monotonic() - started
|
||
in_tokens, out_tokens = _stream_billable_split(
|
||
observed_stream_tokens, stream_usage, request_body
|
||
)
|
||
self._record_observed_throughput(
|
||
model, route_model, in_tokens + out_tokens, elapsed, route_nodes
|
||
)
|
||
tokens = in_tokens + out_tokens
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy complete",
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
status=200,
|
||
stream=True,
|
||
client_disconnected=client_gone,
|
||
relay=True,
|
||
tokens=tokens,
|
||
elapsed_seconds=round(elapsed, 4),
|
||
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
|
||
route=_node_route_summary(route_nodes),
|
||
)
|
||
self._bill_completed(
|
||
api_key, model, tokens, node_work,
|
||
input_tokens=in_tokens, output_tokens=out_tokens,
|
||
)
|
||
|
||
def _send_relayed_response(self, response: dict) -> None:
|
||
status = int(response.get("status", 503))
|
||
headers = response.get("headers") if isinstance(response.get("headers"), dict) else {}
|
||
body_text = response.get("body") or ""
|
||
body = body_text.encode() if isinstance(body_text, str) else bytes(body_text)
|
||
self.send_response(status)
|
||
self.send_header("Content-Type", headers.get("Content-Type", "application/json"))
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
try:
|
||
self.wfile.write(body)
|
||
except BrokenPipeError:
|
||
pass
|
||
|
||
def _handle_register(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
|
||
# --- Raft cluster mode: forward to leader or propose via Raft ---
|
||
if server.raft is not None:
|
||
if not server.raft.is_leader:
|
||
leader = server.raft.leader()
|
||
if leader is None:
|
||
self._send_json(503, {"error": "no leader elected — retry in a moment"})
|
||
return
|
||
# Proxy to leader
|
||
try:
|
||
data = json.dumps(body).encode()
|
||
req = urllib.request.Request(
|
||
f"{leader}/v1/nodes/register",
|
||
data=data,
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=5.0) as r:
|
||
resp_body = r.read()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(resp_body)))
|
||
self.end_headers()
|
||
self.wfile.write(resp_body)
|
||
except Exception as exc:
|
||
self._send_json(503, {"error": f"leader proxy failed: {exc}"})
|
||
return
|
||
# Leader path: fall through to normal registration, then replicate via Raft.
|
||
# We let the registration run first (to generate node_id), then asynchronously
|
||
# propose to the Raft log so followers can replicate the entry.
|
||
# Raft proposal happens after the response is sent (fire-and-forget replication).
|
||
|
||
endpoint = body.get("endpoint")
|
||
if not isinstance(endpoint, str) or not endpoint:
|
||
self._send_json(400, {"error": "endpoint is required"})
|
||
return
|
||
parsed_endpoint = urllib.parse.urlparse(endpoint)
|
||
if parsed_endpoint.scheme not in {"http", "https"} or not parsed_endpoint.netloc:
|
||
self._send_json(400, {"error": "endpoint must be an http(s) URL"})
|
||
return
|
||
|
||
shard_start: int | None
|
||
shard_end: int | None
|
||
explicit_shard = "shard_start" in body or "shard_end" in body
|
||
if explicit_shard:
|
||
try:
|
||
shard_start = int(body["shard_start"])
|
||
shard_end = int(body["shard_end"])
|
||
except (KeyError, TypeError, ValueError):
|
||
self._send_json(400, {"error": "shard_start and shard_end must be numeric"})
|
||
return
|
||
if shard_start < 0 or shard_end < 0 or shard_start > shard_end:
|
||
self._send_json(400, {"error": "shard range must be non-negative and ordered"})
|
||
return
|
||
else:
|
||
shard_start = None
|
||
shard_end = None
|
||
try:
|
||
score = float(body.get("score", 1.0))
|
||
except (TypeError, ValueError):
|
||
self._send_json(400, {"error": "score must be numeric"})
|
||
return
|
||
|
||
hardware_profile = body.get("hardware_profile", {})
|
||
if not isinstance(hardware_profile, dict):
|
||
self._send_json(400, {"error": "hardware_profile must be an object"})
|
||
return
|
||
model = body.get("model")
|
||
if model is None:
|
||
model = "stub-model"
|
||
if not isinstance(model, str):
|
||
self._send_json(400, {"error": "model must be a string"})
|
||
return
|
||
shard_checksum = body.get("shard_checksum")
|
||
if shard_checksum is not None and not isinstance(shard_checksum, str):
|
||
self._send_json(400, {"error": "shard_checksum must be a string"})
|
||
return
|
||
try:
|
||
vram_bytes = int(body.get("vram_bytes", DEFAULT_VRAM_BYTES))
|
||
ram_bytes = int(body.get("ram_bytes", DEFAULT_RAM_BYTES))
|
||
max_loaded_shards = int(body.get("max_loaded_shards", 1))
|
||
benchmark_tokens_per_sec = float(
|
||
body.get("benchmark_tokens_per_sec", DEFAULT_BENCHMARK_TOKENS_PER_SEC)
|
||
)
|
||
except (TypeError, ValueError):
|
||
self._send_json(400, {"error": "vram_bytes, ram_bytes, max_loaded_shards, and benchmark_tokens_per_sec must be numeric"})
|
||
return
|
||
if vram_bytes < 0 or ram_bytes < 0 or max_loaded_shards < 1 or benchmark_tokens_per_sec <= 0:
|
||
self._send_json(400, {"error": "capability values must be positive"})
|
||
return
|
||
quantizations_body = body.get("quantizations", DEFAULT_QUANTIZATIONS)
|
||
if not (
|
||
isinstance(quantizations_body, list)
|
||
and quantizations_body
|
||
and all(isinstance(item, str) and item for item in quantizations_body)
|
||
):
|
||
self._send_json(400, {"error": "quantizations must be a non-empty string array"})
|
||
return
|
||
quantizations = list(quantizations_body)
|
||
quantization = body.get("quantization")
|
||
if quantization is not None and not isinstance(quantization, str):
|
||
self._send_json(400, {"error": "quantization must be a string"})
|
||
return
|
||
wallet_address = body.get("wallet_address")
|
||
if wallet_address is not None and not isinstance(wallet_address, str):
|
||
self._send_json(400, {"error": "wallet_address must be a string"})
|
||
return
|
||
ban_error = _registration_ban_error(server.contracts, wallet_address)
|
||
if ban_error:
|
||
self._send_json(403, {"error": ban_error})
|
||
return
|
||
|
||
tracker_mode = bool(body.get("tracker_mode", False))
|
||
managed_assignment = bool(body.get("managed_assignment", False))
|
||
hf_repo = body.get("hf_repo")
|
||
if hf_repo is not None and not isinstance(hf_repo, str):
|
||
self._send_json(400, {"error": "hf_repo must be a string"})
|
||
return
|
||
num_layers_body = body.get("num_layers")
|
||
num_layers: int | None = None
|
||
if num_layers_body is not None:
|
||
try:
|
||
num_layers = int(num_layers_body)
|
||
except (TypeError, ValueError):
|
||
self._send_json(400, {"error": "num_layers must be an integer"})
|
||
return
|
||
model_metadata = body.get("model_metadata", {})
|
||
if model_metadata is None:
|
||
model_metadata = {}
|
||
if not isinstance(model_metadata, dict):
|
||
self._send_json(400, {"error": "model_metadata must be an object"})
|
||
return
|
||
downloaded_models = body.get("downloaded_models", [])
|
||
if downloaded_models is None:
|
||
downloaded_models = []
|
||
if not (
|
||
isinstance(downloaded_models, list)
|
||
and all(isinstance(item, dict) for item in downloaded_models)
|
||
):
|
||
self._send_json(400, {"error": "downloaded_models must be an array of objects"})
|
||
return
|
||
relay_addr = body.get("relay_addr") or None
|
||
cert_fingerprint = body.get("cert_fingerprint") or None
|
||
peer_id = body.get("peer_id") or None
|
||
|
||
node_id = _node_id_for_registration(
|
||
endpoint,
|
||
model,
|
||
wallet_address,
|
||
shard_start,
|
||
shard_end,
|
||
hf_repo,
|
||
)
|
||
entry = _NodeEntry(
|
||
node_id=node_id,
|
||
endpoint=endpoint.rstrip("/"),
|
||
shard_start=shard_start,
|
||
shard_end=shard_end,
|
||
model=model,
|
||
shard_checksum=shard_checksum,
|
||
hardware_profile=hardware_profile,
|
||
wallet_address=wallet_address,
|
||
score=score,
|
||
vram_bytes=vram_bytes,
|
||
ram_bytes=ram_bytes,
|
||
quantizations=quantizations,
|
||
max_loaded_shards=max_loaded_shards,
|
||
benchmark_tokens_per_sec=benchmark_tokens_per_sec,
|
||
quantization=quantization,
|
||
managed_assignment=managed_assignment or not explicit_shard,
|
||
tracker_mode=tracker_mode,
|
||
hf_repo=hf_repo,
|
||
num_layers=num_layers,
|
||
model_metadata=model_metadata,
|
||
downloaded_models=downloaded_models,
|
||
relay_addr=relay_addr,
|
||
cert_fingerprint=cert_fingerprint,
|
||
peer_id=peer_id,
|
||
)
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
# Dedup: replace the same node id or the same endpoint+model assignment.
|
||
endpoint_key = entry.endpoint.rstrip("/")
|
||
model_key = entry.hf_repo or entry.model
|
||
stale_ids = [
|
||
eid for eid, e in server.registry.items()
|
||
if eid == node_id
|
||
or (
|
||
e.endpoint.rstrip("/") == endpoint_key
|
||
and (e.hf_repo or e.model) == model_key
|
||
)
|
||
]
|
||
stale_entries: list[tuple[str, _NodeEntry]] = []
|
||
for eid in stale_ids:
|
||
old = server.registry.pop(eid)
|
||
stale_entries.append((eid, old))
|
||
is_topology_change = node_id not in stale_ids or any(
|
||
(old.shard_start, old.shard_end) != (entry.shard_start, entry.shard_end)
|
||
for eid, old in stale_entries
|
||
if eid == node_id
|
||
)
|
||
server.registry[node_id] = entry
|
||
if is_topology_change:
|
||
server.route_stats.bump_epoch(_route_stats_keys(server, entry))
|
||
if entry.managed_assignment and not explicit_shard:
|
||
if entry.hf_repo:
|
||
_rebalance_hf_model_locked(server, entry.hf_repo)
|
||
else:
|
||
_rebalance_model_locked(server, model)
|
||
assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
|
||
if assignment_directive is not None:
|
||
entry.pending_directives.clear()
|
||
|
||
model_health = _model_health_summary(server, entry.model, entry.hf_repo)
|
||
for eid, old in stale_entries:
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"node re-registered",
|
||
old_node_id=eid,
|
||
node_id=node_id,
|
||
endpoint=old.endpoint,
|
||
old_model=old.model,
|
||
old_hf_repo=old.hf_repo,
|
||
old_model_health=_model_health_summary(server, old.model, old.hf_repo),
|
||
model_health=model_health,
|
||
)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"node registered",
|
||
node_id=node_id,
|
||
endpoint=entry.endpoint,
|
||
model=entry.model,
|
||
hf_repo=entry.hf_repo,
|
||
shard=f"{entry.shard_start}-{entry.shard_end}",
|
||
tracker_mode=entry.tracker_mode,
|
||
model_health=model_health,
|
||
)
|
||
|
||
shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
|
||
repo_info = f" [{hf_repo}]" if hf_repo else ""
|
||
budget_bytes, budget_source = _node_memory_budget_bytes(entry)
|
||
budget_gb = budget_bytes / (1024 ** 3)
|
||
print(
|
||
f"[tracker] node registered: {node_id} {endpoint} {model}{repo_info} {shard_info} "
|
||
f"capacity={budget_gb:.1f}GB {budget_source} slots={max_loaded_shards}",
|
||
flush=True,
|
||
)
|
||
|
||
payload = {"node_id": node_id}
|
||
if assignment_directive is not None:
|
||
payload["directive"] = assignment_directive
|
||
self._send_json(200, payload)
|
||
|
||
# Raft replication: leader proposes this registration to followers so their
|
||
# registries stay in sync. Fire-and-forget (async) — the client already
|
||
# got its node_id; replication happens in the background.
|
||
if server.raft is not None and server.raft.is_leader:
|
||
raft_payload = dict(body)
|
||
raft_payload["node_id"] = node_id # include the generated ID
|
||
threading.Thread(
|
||
target=server.raft.propose,
|
||
args=("register", raft_payload),
|
||
daemon=True,
|
||
).start()
|
||
|
||
def _handle_heartbeat(self, node_id: str):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body: dict = {}
|
||
content_length = int(self.headers.get("Content-Length", 0))
|
||
if content_length > 0:
|
||
try:
|
||
body = json.loads(self.rfile.read(content_length))
|
||
except Exception:
|
||
pass
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
entry = server.registry.get(node_id)
|
||
if entry is None:
|
||
_tracker_log(server, "warn", "heartbeat for unknown node", node_id=node_id)
|
||
self._send_json(404, {"error": "node not found"})
|
||
return
|
||
entry.last_heartbeat = time.monotonic()
|
||
entry.heartbeats_received += 1
|
||
# P2P metadata
|
||
if body.get("relay_addr"):
|
||
entry.relay_addr = body["relay_addr"]
|
||
if body.get("cert_fingerprint"):
|
||
entry.cert_fingerprint = body["cert_fingerprint"]
|
||
if body.get("peer_id"):
|
||
entry.peer_id = body["peer_id"]
|
||
# Node stats (cumulative — node always sends totals, tracker replaces)
|
||
if "total_requests" in body:
|
||
entry.total_requests = int(body["total_requests"])
|
||
if "failed_requests" in body:
|
||
entry.failed_requests = int(body["failed_requests"])
|
||
if "queue_depth" in body:
|
||
entry.queue_depth = int(body["queue_depth"])
|
||
if "current_requests" in body:
|
||
entry.current_requests = _normalize_current_requests(body["current_requests"])
|
||
if "uptime_seconds" in body:
|
||
entry.uptime_seconds = float(body["uptime_seconds"])
|
||
if "status" in body and body["status"] in ("ready", "loading"):
|
||
entry.status = body["status"]
|
||
directives = list(entry.pending_directives)
|
||
entry.pending_directives.clear()
|
||
new_assignment = entry.pending_new_assignment
|
||
if new_assignment is not None:
|
||
entry.pending_new_assignment = None
|
||
if server.gossip is not None:
|
||
server.gossip.record(node_id)
|
||
resp: dict = {}
|
||
if directives:
|
||
resp["directives"] = directives
|
||
if new_assignment is not None:
|
||
resp["new_assignment"] = new_assignment
|
||
self._send_json(200, resp)
|
||
|
||
# ---------------------------------------------------------------- Raft handlers
|
||
|
||
def _handle_raft_vote(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
if server.raft is None:
|
||
self._send_json(503, {"error": "raft not enabled"})
|
||
return
|
||
result = server.raft.handle_request_vote(body)
|
||
self._send_json(200, result)
|
||
|
||
def _handle_raft_append(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
if server.raft is None:
|
||
self._send_json(503, {"error": "raft not enabled"})
|
||
return
|
||
result = server.raft.handle_append_entries(body)
|
||
self._send_json(200, result)
|
||
|
||
def _handle_raft_status(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.raft is None:
|
||
self._send_json(200, {"role": "standalone", "term": 0, "leader": None})
|
||
return
|
||
self._send_json(200, server.raft.status())
|
||
|
||
def _handle_gossip(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
if server.gossip is not None and isinstance(body, dict):
|
||
server.gossip.merge({k: float(v) for k, v in body.items()})
|
||
self._send_json(200, {})
|
||
|
||
def _handle_stats(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.stats is None:
|
||
self._send_json(200, {"models": {}, "nodes": {}})
|
||
return
|
||
self._send_json(200, {
|
||
"models": server.stats.get_combined_stats(),
|
||
"nodes": server.stats.get_node_throughput_stats(),
|
||
})
|
||
|
||
def _handle_stats_gossip(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_hive_authenticated_body()
|
||
if body is None:
|
||
return
|
||
tracker_url = body.get("tracker_url", "")
|
||
rpms = body.get("stats", {})
|
||
if server.stats is not None and tracker_url and isinstance(rpms, dict):
|
||
server.stats.merge_peer_rpms(tracker_url, rpms)
|
||
self._send_json(200, {})
|
||
|
||
def _handle_billing_summary(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin"):
|
||
return
|
||
if server.billing is None:
|
||
self._send_json(404, {"error": "billing is not enabled on this tracker"})
|
||
return
|
||
self._send_json(200, server.billing.snapshot())
|
||
|
||
def _handle_dashboard(self):
|
||
"""Serve the read-only web dashboard (US-035). Any tracker in the
|
||
mesh — leader or follower — serves it from its replicated state."""
|
||
try:
|
||
html = files("meshnet_tracker").joinpath("dashboard.html").read_text()
|
||
except (FileNotFoundError, OSError):
|
||
self._send_json(404, {"error": "dashboard asset missing"})
|
||
return
|
||
body = html.encode()
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||
self.send_header("Content-Length", str(len(body)))
|
||
self.end_headers()
|
||
try:
|
||
self.wfile.write(body)
|
||
except BrokenPipeError:
|
||
pass
|
||
|
||
def _handle_console(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
with server.console_lock:
|
||
events = [dict(event) for event in server.console_events]
|
||
self._send_json(200, {"events": events})
|
||
|
||
def _handle_proxy_request_cancel(self, request_id: str) -> None:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.accounts is not None and not self._require_role("admin"):
|
||
return
|
||
if not _request_proxy_cancel(server, request_id):
|
||
self._send_json(404, {"error": f"no active proxy for request {request_id!r}"})
|
||
return
|
||
self._send_json(200, {"status": "canceled", "request_id": request_id})
|
||
|
||
def _finalize_proxy_cancel(
|
||
self,
|
||
*,
|
||
proxy_ctx: _ActiveProxyContext,
|
||
server: "_TrackerHTTPServer",
|
||
request_id: str,
|
||
started: float,
|
||
model: str,
|
||
route_model: str,
|
||
route_nodes: list,
|
||
api_key: str | None,
|
||
node_work: list,
|
||
body: dict,
|
||
finish_proxy_inflight: Any,
|
||
observed_stream_tokens: int = 0,
|
||
stream_usage: dict | None = None,
|
||
) -> bool:
|
||
if not proxy_ctx.cancel_event.is_set():
|
||
return False
|
||
elapsed = time.monotonic() - started
|
||
_clear_proxy_progress_log_state(server, request_id)
|
||
tokens = observed_stream_tokens
|
||
if observed_stream_tokens > 0:
|
||
in_tokens, out_tokens = _stream_billable_split(
|
||
observed_stream_tokens, stream_usage, body,
|
||
)
|
||
tokens = in_tokens + out_tokens
|
||
self._record_observed_throughput(
|
||
model, route_model, tokens, elapsed, route_nodes,
|
||
)
|
||
_tracker_log(
|
||
server,
|
||
"info",
|
||
"proxy canceled",
|
||
request_id=request_id,
|
||
model=model,
|
||
route_model=route_model,
|
||
tokens=tokens,
|
||
elapsed_seconds=round(elapsed, 4),
|
||
tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 and tokens > 0 else 0.0,
|
||
route=_node_route_summary(route_nodes),
|
||
)
|
||
if observed_stream_tokens > 0:
|
||
in_tokens, out_tokens = _stream_billable_split(
|
||
observed_stream_tokens, stream_usage, body,
|
||
)
|
||
self._bill_completed(
|
||
api_key, model, in_tokens + out_tokens, node_work,
|
||
input_tokens=in_tokens, output_tokens=out_tokens,
|
||
)
|
||
finish_proxy_inflight()
|
||
return True
|
||
|
||
def _handle_registry_wallets(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin"):
|
||
return
|
||
if server.contracts is None:
|
||
self._send_json(200, {"wallets": {}})
|
||
return
|
||
wallets = server.contracts.registry.list_wallets()
|
||
self._send_json(200, {"wallets": {
|
||
wallet: {
|
||
"stake_balance": state.stake_balance,
|
||
"strike_count": state.strike_count,
|
||
"banned": state.banned,
|
||
"completed_jobs": state.completed_job_count,
|
||
"reputation": state.reputation,
|
||
"last_audit_ts": state.last_audit_ts,
|
||
"probationary_jobs_remaining": (
|
||
server.contracts.registry.probationary_jobs_remaining(wallet)
|
||
),
|
||
}
|
||
for wallet, state in wallets.items()
|
||
}})
|
||
|
||
def _handle_billing_settlements(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin"):
|
||
return
|
||
if server.billing is None:
|
||
self._send_json(404, {"error": "billing is not enabled on this tracker"})
|
||
return
|
||
self._send_json(200, {"settlements": server.billing.settlement_history()})
|
||
|
||
def _handle_billing_gossip(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_hive_authenticated_body()
|
||
if body is None:
|
||
return
|
||
if server.billing is None:
|
||
self._send_json(200, {"applied": 0})
|
||
return
|
||
events = body.get("events")
|
||
if not isinstance(events, list):
|
||
self._send_json(400, {"error": "events must be a list"})
|
||
return
|
||
applied = server.billing.apply_events([e for e in events if isinstance(e, dict)])
|
||
self._send_json(200, {"applied": applied})
|
||
|
||
def _handle_billing_forfeit(self):
|
||
"""Privileged: forfeit a node's pending balance + record a strike (US-034).
|
||
|
||
ADR-0017 §4: validator service token or admin session only. Client
|
||
API keys — valid or not — are rejected.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("validator", "admin"):
|
||
return
|
||
if server.billing is None:
|
||
self._send_json(404, {"error": "billing is not enabled on this tracker"})
|
||
return
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
wallet = body.get("wallet")
|
||
if not wallet or not isinstance(wallet, str):
|
||
self._send_json(400, {"error": "wallet is required"})
|
||
return
|
||
reason = body.get("reason") or "fraud"
|
||
event = server.billing.forfeit_pending(wallet, reason=str(reason))
|
||
strike_state: dict = {}
|
||
if server.contracts is not None:
|
||
server.contracts.registry.record_strike(wallet)
|
||
wallet_state = server.contracts.registry.get_wallet(wallet)
|
||
strike_state = {
|
||
"strike_count": wallet_state.strike_count,
|
||
"banned": wallet_state.banned,
|
||
}
|
||
print(
|
||
f"[tracker] forfeited pending balance of {wallet}: "
|
||
f"{event['amount']:.6f} USDT ({reason}) strikes={strike_state.get('strike_count', 'n/a')}",
|
||
flush=True,
|
||
)
|
||
self._send_json(200, {"forfeited": event["amount"], **strike_state})
|
||
|
||
# ---- user accounts (registration, login, API keys) ----
|
||
|
||
def _session_account(self) -> dict | None:
|
||
"""Resolve the session token in the Authorization header, or None."""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.accounts is None:
|
||
return None
|
||
return server.accounts.session_account(_session_token_from_headers(self.headers))
|
||
|
||
def _require_accounts(self) -> "AccountStore | None":
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.accounts is None:
|
||
self._send_json(404, {"error": "accounts are not enabled on this tracker"})
|
||
return None
|
||
return server.accounts
|
||
|
||
def _handle_auth_register(self):
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
try:
|
||
account = accounts.register(
|
||
email=body.get("email"),
|
||
wallet=body.get("wallet"),
|
||
password=str(body.get("password") or ""),
|
||
)
|
||
except ValueError as exc:
|
||
self._send_json(400, {"error": str(exc)})
|
||
return
|
||
token = accounts.create_session(account["account_id"])
|
||
api_key = accounts.create_api_key(account["account_id"])
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.billing is not None:
|
||
# US-039: registration creates the account's first key, so Caller
|
||
# Credit lands here; the account-derived event id keeps later
|
||
# grants (e.g. via /v1/account/keys) no-ops.
|
||
server.billing.grant_caller_credit(
|
||
api_key, account["account_id"], server.starting_credit
|
||
)
|
||
server.billing.ensure_client(api_key)
|
||
print(
|
||
f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
|
||
f"role={account['role']}", flush=True,
|
||
)
|
||
self._send_json(
|
||
200,
|
||
{"account": account, "session_token": token, "api_key": api_key},
|
||
headers={"Set-Cookie": _session_cookie_header(token)},
|
||
)
|
||
|
||
def _handle_auth_login(self):
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
identifier = body.get("identifier") or body.get("email") or body.get("wallet") or ""
|
||
account = accounts.verify_login(str(identifier), str(body.get("password") or ""))
|
||
if account is None:
|
||
self._send_json(401, {"error": "invalid credentials"})
|
||
return
|
||
token = accounts.create_session(account["account_id"])
|
||
self._send_json(
|
||
200,
|
||
{"account": account, "session_token": token},
|
||
headers={"Set-Cookie": _session_cookie_header(token)},
|
||
)
|
||
|
||
def _handle_auth_logout(self):
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
accounts.destroy_session(_session_token_from_headers(self.headers))
|
||
self._send_json(200, {"ok": True}, headers={"Set-Cookie": _session_cookie_header(None)})
|
||
|
||
def _handle_account_me(self):
|
||
"""Balance, usage, and API keys for the logged-in account."""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if self._require_accounts() is None:
|
||
return
|
||
account = self._session_account()
|
||
if account is None:
|
||
self._send_json(401, {"error": "login required"})
|
||
return
|
||
keys = server.accounts.keys_for(account["account_id"]) # type: ignore[union-attr]
|
||
balances = {}
|
||
usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": [], "records": []}
|
||
if server.billing is not None:
|
||
balances = {key: server.billing.get_client_balance(key) for key in keys}
|
||
usage = server.billing.usage_for(keys)
|
||
self._send_json(200, {
|
||
"account": account,
|
||
"api_keys": keys,
|
||
"balances": balances,
|
||
"total_balance": sum(balances.values()),
|
||
"usage": usage,
|
||
"topup_amount": server.devnet_topup_amount if server.billing is not None else 0.0,
|
||
})
|
||
|
||
def _handle_account_key_create(self):
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
account = self._session_account()
|
||
if account is None:
|
||
self._send_json(401, {"error": "login required"})
|
||
return
|
||
api_key = accounts.create_api_key(account["account_id"])
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
credited = False
|
||
if server.billing is not None:
|
||
# US-039: Caller Credit lands with the account's first key; the
|
||
# account-derived event id keeps it once-per-account forever.
|
||
credited = server.billing.grant_caller_credit(
|
||
api_key, account["account_id"], server.starting_credit
|
||
)
|
||
server.billing.ensure_client(api_key)
|
||
self._send_json(200, {"api_key": api_key, "caller_credit_granted": credited})
|
||
|
||
def _handle_account_topup(self):
|
||
"""Devnet faucet (US-040): credit the configured amount to one of the
|
||
logged-in account's keys. 404 unless the operator enabled it."""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
if server.billing is None or server.devnet_topup_amount <= 0:
|
||
self._send_json(404, {"error": "top-up is not enabled on this tracker"})
|
||
return
|
||
account = self._session_account()
|
||
if account is None:
|
||
self._send_json(401, {"error": "login required"})
|
||
return
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
api_key = body.get("api_key")
|
||
if not api_key or not isinstance(api_key, str):
|
||
self._send_json(400, {"error": "api_key is required"})
|
||
return
|
||
if accounts.owner_of_key(api_key) != account["account_id"]:
|
||
self._send_json(403, {"error": "key not found on this account"})
|
||
return
|
||
balance = server.billing.credit_client(
|
||
api_key, server.devnet_topup_amount, note="devnet-topup"
|
||
)
|
||
self._send_json(200, {
|
||
"api_key": api_key,
|
||
"credited": server.devnet_topup_amount,
|
||
"balance": balance,
|
||
})
|
||
|
||
def _handle_account_key_revoke(self):
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
account = self._session_account()
|
||
if account is None:
|
||
self._send_json(401, {"error": "login required"})
|
||
return
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
api_key = body.get("api_key")
|
||
if not api_key or not isinstance(api_key, str):
|
||
self._send_json(400, {"error": "api_key is required"})
|
||
return
|
||
if not accounts.revoke_api_key(account["account_id"], api_key):
|
||
self._send_json(404, {"error": "key not found on this account"})
|
||
return
|
||
self._send_json(200, {"revoked": api_key})
|
||
|
||
def _handle_admin_accounts(self):
|
||
"""Admin-only: all accounts with their keys and balances."""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
accounts = self._require_accounts()
|
||
if accounts is None:
|
||
return
|
||
account = self._session_account()
|
||
if account is None:
|
||
self._send_json(401, {"error": "login required"})
|
||
return
|
||
if account["role"] != "admin":
|
||
self._send_json(403, {"error": "admin role required"})
|
||
return
|
||
listing = accounts.list_accounts()
|
||
if server.billing is not None:
|
||
for entry in listing:
|
||
entry["balances"] = {
|
||
key: server.billing.get_client_balance(key)
|
||
for key in entry.get("api_keys", [])
|
||
}
|
||
self._send_json(200, {"accounts": listing})
|
||
|
||
def _handle_accounts_gossip(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_hive_authenticated_body()
|
||
if body is None:
|
||
return
|
||
if server.accounts is None:
|
||
self._send_json(200, {"applied": 0})
|
||
return
|
||
events = body.get("events")
|
||
if not isinstance(events, list):
|
||
self._send_json(400, {"error": "events must be a list"})
|
||
return
|
||
applied = server.accounts.apply_events([e for e in events if isinstance(e, dict)])
|
||
self._send_json(200, {"applied": applied})
|
||
|
||
def _handle_registry_gossip(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
body = self._read_hive_authenticated_body()
|
||
if body is None:
|
||
return
|
||
registry_log = getattr(server.contracts, "registry_log", None)
|
||
if registry_log is None:
|
||
self._send_json(200, {"applied": 0})
|
||
return
|
||
events = body.get("events")
|
||
if not isinstance(events, list):
|
||
self._send_json(400, {"error": "events must be a list"})
|
||
return
|
||
applied = registry_log.apply_events([e for e in events if isinstance(e, dict)])
|
||
self._send_json(200, {"applied": applied})
|
||
|
||
def _handle_wallet_register(self):
|
||
"""Bind a client wallet pubkey to an API key (US-032, C6).
|
||
|
||
Deposits from that wallet into the treasury are then credited to the
|
||
API key's ledger balance by the deposit watcher.
|
||
|
||
Requires a signature proving ownership of the wallet's private key —
|
||
the caller signs ``binding_message(api_key, wallet)`` with the
|
||
wallet's ed25519 key. An admin session may force-rebind a wallet
|
||
already bound to a different API key (documented override; no
|
||
signed-release flow is implemented).
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.billing is None:
|
||
self._send_json(404, {"error": "billing is not enabled on this tracker"})
|
||
return
|
||
role, _account = self._resolve_identity()
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
wallet = body.get("wallet")
|
||
if not wallet or not isinstance(wallet, str):
|
||
self._send_json(400, {"error": "wallet is required"})
|
||
return
|
||
|
||
if role == "admin":
|
||
api_key = body.get("api_key")
|
||
if not api_key or not isinstance(api_key, str):
|
||
self._send_json(400, {"error": "api_key is required for admin override"})
|
||
return
|
||
event = server.billing.bind_wallet(api_key, wallet, admin_override=True)
|
||
else:
|
||
api_key = _api_key_from_headers(self.headers)
|
||
if not api_key:
|
||
self._send_json(401, {"error": "Authorization header required"})
|
||
return
|
||
signature = body.get("signature")
|
||
if not signature or not isinstance(signature, str):
|
||
self._send_json(400, {"error": "signature is required to prove wallet ownership"})
|
||
return
|
||
if not verify_wallet_signature(wallet, binding_message(api_key, wallet), signature):
|
||
self._send_json(401, {"error": "invalid wallet signature"})
|
||
return
|
||
event = server.billing.bind_wallet(api_key, wallet)
|
||
|
||
if event.get("rejected"):
|
||
self._send_json(409, {"error": "wallet already bound to a different API key"})
|
||
return
|
||
print(f"[tracker] wallet bound: {wallet} -> api_key …{api_key[-6:]}", flush=True)
|
||
self._send_json(200, {"wallet": wallet, "bound": True})
|
||
|
||
def _handle_benchmark_hop_penalty(self):
|
||
"""Privileged: run the same prompt through 1/2/3-node pinned routes (US-030).
|
||
|
||
Data collection only — the routing algorithm is unchanged. Per-hop
|
||
latency is derived incrementally: the k-node route's final hop penalty
|
||
is its total minus the (k-1)-node route's total.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin", "validator"):
|
||
return
|
||
auth = self.headers.get("Authorization")
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
model = body.get("model", "")
|
||
if not model:
|
||
self._send_json(400, {"error": "model is required"})
|
||
return
|
||
prompt = body.get("prompt") or "Benchmark: say OK."
|
||
max_new_tokens = int(body.get("max_new_tokens", 64))
|
||
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
resolved = _nodes_and_bounds_for_model(server, model)
|
||
if resolved is None or not resolved[0]:
|
||
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
|
||
return
|
||
all_nodes, rs, re = resolved
|
||
|
||
self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||
route_results: list[dict] = []
|
||
prev_total_ms: float | None = None
|
||
prev_per_hop: list[float] = []
|
||
for hop_count in (1, 2, 3):
|
||
combo = _find_pinned_route(all_nodes, rs, re, hop_count)
|
||
if combo is None:
|
||
continue # insufficient coverage for this hop count — skip
|
||
request_body = json.dumps({
|
||
"model": model,
|
||
"messages": [{"role": "user", "content": prompt}],
|
||
"max_tokens": max_new_tokens,
|
||
"route": [n.node_id for n in combo],
|
||
}).encode()
|
||
req = urllib.request.Request(
|
||
f"{self_url}/v1/chat/completions",
|
||
data=request_body,
|
||
headers={"Content-Type": "application/json", "Authorization": auth},
|
||
method="POST",
|
||
)
|
||
started = time.monotonic()
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=300.0) as resp:
|
||
payload = json.loads(resp.read())
|
||
except Exception as exc:
|
||
route_results.append({
|
||
"route": [n.node_id for n in combo],
|
||
"error": str(exc),
|
||
})
|
||
continue
|
||
total_ms = (time.monotonic() - started) * 1000.0
|
||
if prev_total_ms is not None and len(prev_per_hop) == hop_count - 1:
|
||
per_hop_ms = prev_per_hop + [max(0.0, total_ms - prev_total_ms)]
|
||
else:
|
||
per_hop_ms = [total_ms / hop_count] * hop_count
|
||
prev_total_ms = total_ms
|
||
prev_per_hop = per_hop_ms
|
||
route_results.append({
|
||
"route": [n.node_id for n in combo],
|
||
"total_ms": total_ms,
|
||
"per_hop_ms": per_hop_ms,
|
||
"tokens_generated": _usage_total_tokens(payload) or 0,
|
||
})
|
||
|
||
record = {
|
||
"timestamp": time.time(),
|
||
"model": model,
|
||
"prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
|
||
"routes": route_results,
|
||
}
|
||
with server.benchmark_lock:
|
||
existing: list = []
|
||
if os.path.exists(server.benchmark_results_path):
|
||
try:
|
||
with open(server.benchmark_results_path, encoding="utf-8") as fh:
|
||
existing = json.load(fh)
|
||
except (json.JSONDecodeError, OSError):
|
||
existing = []
|
||
if not isinstance(existing, list):
|
||
existing = []
|
||
existing.append(record)
|
||
with open(server.benchmark_results_path, "w", encoding="utf-8") as fh:
|
||
json.dump(existing, fh, indent=2)
|
||
self._send_json(200, record)
|
||
|
||
def _handle_benchmark_results(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin", "validator"):
|
||
return
|
||
results: list = []
|
||
with server.benchmark_lock:
|
||
if os.path.exists(server.benchmark_results_path):
|
||
try:
|
||
with open(server.benchmark_results_path, encoding="utf-8") as fh:
|
||
results = json.load(fh)
|
||
except (json.JSONDecodeError, OSError):
|
||
results = []
|
||
self._send_json(200, {"results": results if isinstance(results, list) else []})
|
||
|
||
def _handle_toploc_calibration_run(self):
|
||
"""Privileged: honest-noise TOPLOC calibration dispatch (issue 21).
|
||
|
||
Fans the same fixed prompt through every currently registered node
|
||
that can solo-serve the full model (one pinned-route hop, mirroring
|
||
`_handle_benchmark_hop_penalty`'s dispatch pattern), then verifies
|
||
each node's own on-demand TOPLOC commitment against a teacher-forced
|
||
replay on the reference node — same audit contract the validator
|
||
uses (`packages/validator/README.md` "TOPLOC audit contract"). Each
|
||
node's raw divergence (not just pass/fail) is recorded into the
|
||
calibration corpus, keyed by wallet + GPU model + dtype, so
|
||
thresholds can eventually be derived instead of guessed.
|
||
|
||
Nodes that only hold a partial shard (need a multi-hop route) are
|
||
skipped for this pass — solo dispatch isolates one node's hardware
|
||
noise without a route composition confound — and nodes that don't
|
||
answer the on-demand commitment fetch (endpoint down, or node-side
|
||
TOPLOC serving not yet wired) are skipped and reported, not treated
|
||
as a pass or a fail.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin", "validator"):
|
||
return
|
||
if server.toploc_calibration is None:
|
||
self._send_json(503, {"error": "toploc calibration store is not enabled on this tracker"})
|
||
return
|
||
if not server.toploc_reference_node_url:
|
||
self._send_json(503, {"error": "toploc_reference_node_url is not configured on this tracker"})
|
||
return
|
||
auth = self.headers.get("Authorization")
|
||
body = self._read_json_body()
|
||
if body is None:
|
||
return
|
||
model = body.get("model", "")
|
||
if not model:
|
||
self._send_json(400, {"error": "model is required"})
|
||
return
|
||
prompt = body.get("prompt") or "Calibration: say OK."
|
||
max_new_tokens = int(body.get("max_new_tokens", 32))
|
||
seed = body.get("seed", 0)
|
||
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
resolved = _nodes_and_bounds_for_model(server, model)
|
||
if resolved is None or not resolved[0]:
|
||
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
|
||
return
|
||
all_nodes, rs, re = resolved
|
||
if server.contracts is not None:
|
||
all_nodes = [
|
||
node for node in all_nodes
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
solo_nodes = [
|
||
node for node in all_nodes
|
||
if node.shard_start is not None and node.shard_end is not None
|
||
and node.shard_start <= rs and node.shard_end >= re
|
||
]
|
||
|
||
self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
|
||
messages = [{"role": "user", "content": prompt}]
|
||
node_results: list[dict] = []
|
||
for node in solo_nodes:
|
||
request_id = f"cal-{uuid.uuid4().hex}"
|
||
request_body = json.dumps({
|
||
"id": request_id,
|
||
"model": model,
|
||
"messages": messages,
|
||
"max_tokens": max_new_tokens,
|
||
"temperature": 0,
|
||
"seed": seed,
|
||
"route": [node.node_id],
|
||
}).encode()
|
||
req = urllib.request.Request(
|
||
f"{self_url}/v1/chat/completions",
|
||
data=request_body,
|
||
headers={"Content-Type": "application/json", "Authorization": auth},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=300.0) as resp:
|
||
json.loads(resp.read())
|
||
except Exception as exc:
|
||
node_results.append({"node_id": node.node_id, "wallet_address": node.wallet_address, "error": str(exc)})
|
||
continue
|
||
|
||
outcome = self._verify_node_toploc_calibration(
|
||
server, node, request_id=request_id, model=model, messages=messages,
|
||
)
|
||
node_results.append(outcome)
|
||
|
||
skipped_partial_shard = [
|
||
node.node_id for node in all_nodes if node not in solo_nodes
|
||
]
|
||
record = {
|
||
"timestamp": time.time(),
|
||
"model": model,
|
||
"prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
|
||
"nodes": node_results,
|
||
"skipped_partial_shard_node_ids": skipped_partial_shard,
|
||
"gate_status": server.toploc_calibration.gate_status(
|
||
min_hardware_profiles=self._toploc_calibration_gate_min_hardware_profiles(),
|
||
),
|
||
}
|
||
self._send_json(200, record)
|
||
|
||
def _toploc_calibration_gate_min_hardware_profiles(self) -> int:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
return server.toploc_calibration_gate_min_hardware_profiles
|
||
|
||
def _verify_node_toploc_calibration(
|
||
self,
|
||
server: "_TrackerHTTPServer",
|
||
node: "_NodeEntry",
|
||
*,
|
||
request_id: str,
|
||
model: str,
|
||
messages: list[dict],
|
||
) -> dict:
|
||
"""One node's calibration outcome: fetch its on-demand commitment,
|
||
teacher-force the claimed tokens on the reference node, verify, and
|
||
persist the raw divergence into the corpus."""
|
||
from meshnet_validator.audit import ToplocProofClaim, verify_activation_proofs_detailed
|
||
|
||
gpu_model = (node.hardware_profile or {}).get("gpu_name") or (node.hardware_profile or {}).get("device") or "unknown"
|
||
dtype = node.quantization or "unknown"
|
||
base_result = {
|
||
"node_id": node.node_id,
|
||
"wallet_address": node.wallet_address,
|
||
"gpu_model": gpu_model,
|
||
"dtype": dtype,
|
||
}
|
||
commitment = _fetch_toploc_commitment(
|
||
node, session_id=request_id, model=model, messages=messages,
|
||
)
|
||
if commitment is None:
|
||
return {**base_result, "skipped": "no on-demand toploc commitment available"}
|
||
|
||
try:
|
||
claim = ToplocProofClaim.from_mapping(commitment["toploc_proof"])
|
||
except (KeyError, TypeError, ValueError):
|
||
return {**base_result, "skipped": "malformed toploc commitment"}
|
||
|
||
reference_activations = _fetch_toploc_reference_activations(
|
||
server.toploc_reference_node_url,
|
||
model=model,
|
||
messages=messages,
|
||
claimed_token_ids=commitment["claimed_token_ids"],
|
||
claim=claim,
|
||
)
|
||
if reference_activations is None:
|
||
return {**base_result, "skipped": "reference node teacher-forced replay failed"}
|
||
|
||
result = verify_activation_proofs_detailed(reference_activations, claim, backend=server.toploc_backend)
|
||
if node.wallet_address:
|
||
server.toploc_calibration.record_run(
|
||
node_wallet=node.wallet_address,
|
||
gpu_model=gpu_model,
|
||
dtype=dtype,
|
||
model=model,
|
||
passed=result.passed,
|
||
exp_intersections=result.exp_intersections,
|
||
mant_err_mean=result.mant_err_mean,
|
||
mant_err_median=result.mant_err_median,
|
||
)
|
||
return {
|
||
**base_result,
|
||
"passed": result.passed,
|
||
"exp_intersections": result.exp_intersections,
|
||
"mant_err_mean": result.mant_err_mean,
|
||
"mant_err_median": result.mant_err_median,
|
||
"chunk_count": result.chunk_count,
|
||
}
|
||
|
||
def _handle_toploc_calibration_results(self):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin", "validator"):
|
||
return
|
||
if server.toploc_calibration is None:
|
||
self._send_json(503, {"error": "toploc calibration store is not enabled on this tracker"})
|
||
return
|
||
min_profiles = self._toploc_calibration_gate_min_hardware_profiles()
|
||
self._send_json(200, {
|
||
"runs": server.toploc_calibration.runs(),
|
||
"envelope": server.toploc_calibration.envelope(),
|
||
"gate_status": server.toploc_calibration.gate_status(min_hardware_profiles=min_profiles),
|
||
})
|
||
|
||
def _handle_hf_pricing_history(self, parsed: urllib.parse.ParseResult):
|
||
"""Dispute-auditability log for the dynamic HF-benchmarked pricing (issue 23)."""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if not self._require_role("admin", "validator"):
|
||
return
|
||
if server.hf_pricing_log is None:
|
||
self._send_json(503, {"error": "hf pricing log is not enabled on this tracker"})
|
||
return
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
model = params.get("model", [None])[0]
|
||
self._send_json(200, {"changes": server.hf_pricing_log.history(model)})
|
||
|
||
def _handle_assign(self, parsed: urllib.parse.ParseResult):
|
||
"""Return an optimal shard assignment for a node given its hardware profile.
|
||
|
||
Query params:
|
||
model — model preset name (default: first preset)
|
||
device — "cuda" | "cpu"
|
||
vram_mb — integer VRAM in MB (0 for CPU)
|
||
ram_mb — integer system RAM in MB, used when vram_mb=0
|
||
|
||
The greedy strategy: find the first gap in current layer coverage
|
||
and assign it. If no gap exists, assign the full model range so the
|
||
node provides redundant coverage.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
|
||
model_list = params.get("model")
|
||
if not model_list:
|
||
model = next(iter(server.model_presets), None)
|
||
if model is None:
|
||
self._send_json(503, {"error": "no model presets configured"})
|
||
return
|
||
else:
|
||
model = model_list[0]
|
||
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
if preset is None:
|
||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||
return
|
||
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
alive = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
]
|
||
if server.contracts is not None:
|
||
alive = [
|
||
node for node in alive
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
|
||
device = params.get("device", ["cpu"])[0]
|
||
try:
|
||
vram_mb = int(params.get("vram_mb", ["0"])[0])
|
||
except ValueError:
|
||
vram_mb = 0
|
||
try:
|
||
ram_mb = int(params.get("ram_mb", ["0"])[0])
|
||
except ValueError:
|
||
ram_mb = 0
|
||
max_layers = required_end - required_start + 1
|
||
# VRAM only bounds the shard for CUDA nodes; a CPU node may still report
|
||
# a detected-but-unusable GPU, and must be sized by system RAM.
|
||
memory_mb = vram_mb if (device == "cuda" and vram_mb > 0) else ram_mb
|
||
if memory_mb > 0:
|
||
layer_bytes = _preset_bytes_per_layer(preset).get("bfloat16", 30 * 1024 * 1024)
|
||
max_layers = min(max_layers, max(1, int(((memory_mb * 1024 * 1024) * 0.8) // layer_bytes)))
|
||
elif device != "cuda" or vram_mb < 8192:
|
||
max_layers = min(max_layers, 16)
|
||
|
||
# Collect covered intervals sorted by start layer.
|
||
covered = sorted(
|
||
[
|
||
(n.shard_start, n.shard_end)
|
||
for n in alive
|
||
if n.shard_start is not None and n.shard_end is not None
|
||
],
|
||
key=lambda t: t[0],
|
||
)
|
||
|
||
# Walk from required_start to find the first uncovered layer.
|
||
gap_start = required_start
|
||
for s, e in covered:
|
||
if s <= gap_start:
|
||
gap_start = max(gap_start, e + 1)
|
||
else:
|
||
break # gap found before this node
|
||
|
||
if gap_start > required_end:
|
||
# Full coverage exists — assign the full range for redundancy.
|
||
shard_start = required_start
|
||
shard_end = min(required_end, shard_start + max_layers - 1)
|
||
else:
|
||
shard_start = gap_start
|
||
shard_end = min(required_end, shard_start + max_layers - 1)
|
||
|
||
peers = [
|
||
{"endpoint": node.endpoint, "checksum": node.shard_checksum}
|
||
for node in alive
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
and node.shard_start == shard_start
|
||
and node.shard_end == shard_end
|
||
and node.shard_checksum
|
||
]
|
||
|
||
self._send_json(200, {
|
||
"shard_start": shard_start,
|
||
"shard_end": shard_end,
|
||
"model": resolved_name,
|
||
"model_layers_end": required_end,
|
||
"peers": peers,
|
||
"bytes_per_layer": _preset_bytes_per_layer(preset),
|
||
"model_sources": self._model_sources(
|
||
resolved_name,
|
||
preset,
|
||
shard_start,
|
||
shard_end,
|
||
),
|
||
**({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
|
||
})
|
||
|
||
def _model_sources(self, model: str, preset: dict, shard_start: int, shard_end: int) -> list[dict]:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
hf_repo = preset.get("hf_repo")
|
||
if not server.models_dir or not isinstance(hf_repo, str) or not hf_repo:
|
||
return []
|
||
snapshot_dir = snapshot_dir_for_repo(server.models_dir, hf_repo)
|
||
if snapshot_dir is None:
|
||
return []
|
||
files = files_for_layer_range(snapshot_dir, shard_start, shard_end)
|
||
if not files:
|
||
return []
|
||
host = self.headers.get("Host") or f"{self.server.server_address[0]}:{self.server.server_address[1]}"
|
||
base_url = f"http://{host}"
|
||
query = urllib.parse.urlencode({
|
||
"model": model,
|
||
"shard_start": shard_start,
|
||
"shard_end": shard_end,
|
||
})
|
||
full_query = urllib.parse.urlencode({"model": model, "full": 1})
|
||
full_files = _snapshot_regular_files(snapshot_dir)
|
||
file_sizes: dict[str, int] = {}
|
||
for rel in set(files) | set(full_files):
|
||
try:
|
||
file_sizes[rel] = (snapshot_dir / rel).resolve().stat().st_size
|
||
except OSError:
|
||
continue
|
||
return [{
|
||
"type": "tracker",
|
||
"endpoint": base_url,
|
||
"url": f"{base_url}/v1/model-files/download?{query}",
|
||
"full_url": f"{base_url}/v1/model-files/download?{full_query}",
|
||
"files": files,
|
||
"full_files": full_files,
|
||
"file_sizes": file_sizes,
|
||
}]
|
||
|
||
def _handle_model_files_download(self, parsed: urllib.parse.ParseResult) -> None:
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
if server.models_dir is None:
|
||
self._send_json(404, {"error": "tracker model-file source is not enabled"})
|
||
return
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
model = params.get("model", [""])[0]
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
if preset is None:
|
||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||
return
|
||
hf_repo = preset.get("hf_repo")
|
||
if not isinstance(hf_repo, str) or not hf_repo:
|
||
self._send_json(404, {"error": f"model preset has no hf_repo: {resolved_name!r}"})
|
||
return
|
||
snapshot_dir = snapshot_dir_for_repo(server.models_dir, hf_repo)
|
||
if snapshot_dir is None:
|
||
self._send_json(404, {"error": f"local snapshot not found for {hf_repo}"})
|
||
return
|
||
full_download = params.get("full", ["0"])[0] in {"1", "true", "yes"}
|
||
if full_download:
|
||
rel_files = _snapshot_regular_files(snapshot_dir)
|
||
else:
|
||
try:
|
||
shard_start = int(params.get("shard_start", [""])[0])
|
||
shard_end = int(params.get("shard_end", [""])[0])
|
||
except ValueError:
|
||
self._send_json(400, {"error": "shard_start and shard_end must be integers"})
|
||
return
|
||
rel_files = files_for_layer_range(snapshot_dir, shard_start, shard_end)
|
||
if not rel_files:
|
||
self._send_json(404, {"error": "no local files matched the assigned shard"})
|
||
return
|
||
|
||
# Single-file mode: ?file=<rel> streams one file with Content-Length so
|
||
# clients can verify completeness and retry per file instead of
|
||
# restarting one giant tar stream after a network hiccup.
|
||
single = params.get("file", [""])[0]
|
||
if single:
|
||
if single not in set(rel_files):
|
||
self._send_json(404, {"error": f"file not in requested shard: {single!r}"})
|
||
return
|
||
file_path = snapshot_dir / single
|
||
try:
|
||
# resolve() dereferences HF snapshot symlinks into blobs/.
|
||
size = file_path.resolve().stat().st_size
|
||
except OSError:
|
||
self._send_json(404, {"error": f"file unreadable: {single!r}"})
|
||
return
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/octet-stream")
|
||
self.send_header("Content-Length", str(size))
|
||
self.send_header("X-Meshnet-Model-Source", "tracker")
|
||
self.end_headers()
|
||
try:
|
||
with file_path.open("rb") as f:
|
||
while True:
|
||
chunk = f.read(1024 * 1024)
|
||
if not chunk:
|
||
break
|
||
self.wfile.write(chunk)
|
||
except (BrokenPipeError, ConnectionResetError):
|
||
print(
|
||
f"model-file download aborted by client "
|
||
f"({self.client_address[0]}, model={resolved_name}, file={single})",
|
||
flush=True,
|
||
)
|
||
return
|
||
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/x-tar")
|
||
self.send_header("X-Meshnet-Model-Source", "tracker")
|
||
self.end_headers()
|
||
try:
|
||
# dereference: HF cache snapshots are symlinks into blobs/ — ship contents.
|
||
with tarfile.open(fileobj=self.wfile, mode="w|", dereference=True) as archive:
|
||
for rel in rel_files:
|
||
archive.add(snapshot_dir / rel, arcname=rel)
|
||
except (BrokenPipeError, ConnectionResetError):
|
||
print(
|
||
f"model-file download aborted by client "
|
||
f"({self.client_address[0]}, model={resolved_name})",
|
||
flush=True,
|
||
)
|
||
|
||
def _handle_network_assign(self, parsed: urllib.parse.ParseResult):
|
||
"""Assign a new node to fill the biggest uncovered shard gap across HF-model nodes.
|
||
|
||
Query params:
|
||
vram_mb — integer VRAM in MB (0 = CPU-only node)
|
||
ram_mb — integer system RAM in MB, used when vram_mb=0
|
||
device — "cuda" | "cpu"
|
||
hf_repo — optional; if set, restrict search to this repo only
|
||
|
||
Returns:
|
||
{hf_repo, shard_start, shard_end, num_layers, gap_found}
|
||
gap_found=true means a real uncovered gap was assigned; false means redundancy.
|
||
"""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
try:
|
||
vram_mb = int(params.get("vram_mb", ["0"])[0])
|
||
except ValueError:
|
||
vram_mb = 0
|
||
try:
|
||
ram_mb = int(params.get("ram_mb", ["0"])[0])
|
||
except ValueError:
|
||
ram_mb = 0
|
||
device = params.get("device", ["cpu"])[0]
|
||
filter_repo = params.get("hf_repo", [None])[0] # optional repo filter
|
||
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
all_nodes = list(server.registry.values())
|
||
|
||
# Collect only nodes that registered a real HF model (have hf_repo + shard bounds).
|
||
hf_nodes = [
|
||
n for n in all_nodes
|
||
if n.hf_repo
|
||
and n.shard_start is not None
|
||
and n.shard_end is not None
|
||
and n.num_layers is not None
|
||
and (filter_repo is None or n.hf_repo == filter_repo)
|
||
]
|
||
|
||
if not hf_nodes:
|
||
# The caller is not registered yet — count its memory toward
|
||
# deployability, or the first node can never bootstrap the network.
|
||
caller = _NodeEntry(
|
||
node_id="assign-candidate",
|
||
endpoint="",
|
||
shard_start=None,
|
||
shard_end=None,
|
||
model=None,
|
||
shard_checksum=None,
|
||
hardware_profile={},
|
||
wallet_address=None,
|
||
score=1.0,
|
||
vram_bytes=vram_mb * 1024 * 1024,
|
||
ram_bytes=ram_mb * 1024 * 1024,
|
||
)
|
||
pool_with_caller = all_nodes + [caller]
|
||
resolved_name = None
|
||
preset = None
|
||
if filter_repo:
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
|
||
else:
|
||
deployable = [
|
||
(name, preset)
|
||
for name, preset in server.model_presets.items()
|
||
if preset.get("recommended") and _deployment_summary(pool_with_caller, preset)["deployable"]
|
||
]
|
||
if deployable:
|
||
resolved_name, preset = deployable[0]
|
||
if preset is not None and preset.get("hf_repo"):
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
total_l = required_end - required_start + 1
|
||
memory_mb = vram_mb if (device == "cuda" and vram_mb > 0) else ram_mb
|
||
max_layers = _max_layers_for_memory(memory_mb, total_l, preset, device=device)
|
||
shard_start = required_start
|
||
shard_end = min(required_end, shard_start + max_layers - 1)
|
||
self._send_json(200, {
|
||
"hf_repo": preset["hf_repo"],
|
||
"model": resolved_name,
|
||
"shard_start": shard_start,
|
||
"shard_end": shard_end,
|
||
"num_layers": total_l,
|
||
"gap_found": True,
|
||
"price_per_token": 0.0,
|
||
"bytes_per_layer": _preset_bytes_per_layer(preset),
|
||
"deployment": _deployment_summary(pool_with_caller, preset),
|
||
"model_sources": self._model_sources(
|
||
str(resolved_name),
|
||
preset,
|
||
shard_start,
|
||
shard_end,
|
||
),
|
||
})
|
||
return
|
||
|
||
msg = (
|
||
f"no HF-model nodes registered for {filter_repo!r}"
|
||
if filter_repo
|
||
else "no HF-model nodes registered; cannot assign shards"
|
||
)
|
||
self._send_json(503, {"error": msg})
|
||
return
|
||
|
||
# Group by hf_repo; pick the one with the largest total_layers and biggest gap.
|
||
from collections import defaultdict
|
||
repo_groups: dict = defaultdict(list)
|
||
repo_layers: dict = {}
|
||
for n in hf_nodes:
|
||
repo_groups[n.hf_repo].append(n)
|
||
# Use the largest num_layers seen for this repo.
|
||
if n.hf_repo not in repo_layers or n.num_layers > repo_layers[n.hf_repo]:
|
||
repo_layers[n.hf_repo] = n.num_layers
|
||
|
||
# Smart scoring: demand_rpm × coverage_deficit
|
||
# coverage_deficit = uncovered_layers / total_layers (0 = fully covered)
|
||
# demand_rpm comes from the stats collector; defaults to 0.0 when no traffic yet.
|
||
demand_rpms: dict[str, float] = {}
|
||
if server.stats is not None:
|
||
local_rpms = server.stats.get_local_rpms()
|
||
for repo in repo_groups:
|
||
demand_rpms[repo] = _repo_demand_rpm(server, repo, local_rpms)
|
||
|
||
best_repo = None
|
||
best_score = -1.0
|
||
best_gap_size = -1
|
||
best_gap_start = 0
|
||
best_num_layers = 0
|
||
|
||
for repo, nodes in repo_groups.items():
|
||
total = repo_layers[repo]
|
||
served_copies = _served_model_copies(nodes, 0, total - 1)
|
||
covered = sorted(
|
||
[(n.shard_start, n.shard_end) for n in nodes],
|
||
key=lambda t: t[0],
|
||
)
|
||
# Walk from 0 to find first uncovered layer.
|
||
gap_start = 0
|
||
for s, e in covered:
|
||
if s <= gap_start:
|
||
gap_start = max(gap_start, e + 1)
|
||
else:
|
||
break
|
||
gap_size = max(0, (total - 1) - gap_start + 1)
|
||
coverage_deficit = gap_size / max(total, 1)
|
||
demand = demand_rpms.get(repo, 0.0)
|
||
# Prefer demanded models already being served; coverage gaps still dominate.
|
||
score = (demand + 1.0) * (coverage_deficit + 0.01) + served_copies * 0.01
|
||
if score > best_score or (score == best_score and gap_size > best_gap_size):
|
||
best_score = score
|
||
best_gap_size = gap_size
|
||
best_gap_start = gap_start
|
||
best_repo = repo
|
||
best_num_layers = total
|
||
|
||
gap_found = best_gap_size > 0
|
||
if not gap_found:
|
||
# All shards covered — add redundancy on the most demanded/served model.
|
||
best_repo = max(
|
||
repo_groups,
|
||
key=lambda repo: (
|
||
demand_rpms.get(repo, 0.0),
|
||
_served_model_copies(repo_groups[repo], 0, repo_layers[repo] - 1),
|
||
len(repo_groups[repo]),
|
||
),
|
||
)
|
||
best_gap_start = 0
|
||
best_num_layers = repo_layers[best_repo]
|
||
|
||
# Capacity: use the same 80%-of-memory rule as registered node planning.
|
||
total_l = best_num_layers
|
||
memory_mb = vram_mb if (device == "cuda" and vram_mb > 0) else ram_mb
|
||
resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
|
||
if memory_mb > 0:
|
||
max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset, device=device)
|
||
elif device == "cuda" and vram_mb >= 8192:
|
||
max_layers = total_l
|
||
else:
|
||
max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset, device=device)
|
||
|
||
shard_start = best_gap_start
|
||
shard_end = min(total_l - 1, shard_start + max_layers - 1)
|
||
|
||
self._send_json(200, {
|
||
"hf_repo": best_repo,
|
||
"model": resolved_name,
|
||
"shard_start": shard_start,
|
||
"shard_end": shard_end,
|
||
"num_layers": total_l,
|
||
"gap_found": gap_found,
|
||
"price_per_token": 0.0,
|
||
"bytes_per_layer": _preset_bytes_per_layer(best_preset) if best_preset is not None else None,
|
||
"model_sources": self._model_sources(
|
||
str(resolved_name or best_repo),
|
||
best_preset or {"hf_repo": best_repo},
|
||
shard_start,
|
||
shard_end,
|
||
),
|
||
})
|
||
|
||
def _handle_route(self, parsed: urllib.parse.ParseResult):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
model_list = params.get("model")
|
||
if not model_list:
|
||
self._send_json(400, {"error": "missing 'model' query parameter"})
|
||
return
|
||
|
||
model = model_list[0]
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
if preset is not None:
|
||
# Preset-based routing (stub-model system).
|
||
alive = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
]
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
else:
|
||
# HF model routing: match by hf_repo (full) or model short name.
|
||
alive = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_model(node, model)
|
||
and node.shard_start is not None
|
||
and node.shard_end is not None
|
||
and node.num_layers is not None
|
||
]
|
||
if not alive:
|
||
self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
|
||
return
|
||
required_start = 0
|
||
required_end = max(n.num_layers for n in alive) - 1 # type: ignore[type-var]
|
||
|
||
if server.contracts is not None:
|
||
alive = [
|
||
node for node in alive
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
|
||
route_model = resolved_name or model
|
||
candidates = _enumerate_routes(
|
||
alive,
|
||
required_start,
|
||
required_end,
|
||
model=route_model,
|
||
contracts=server.contracts,
|
||
max_candidates=server.route_stats.config.max_candidate_routes,
|
||
)
|
||
if candidates:
|
||
# Prefer a distributed multi-hop route when available. Greedy
|
||
# _select_route alone would pick a single full-shard node and omit
|
||
# partial head shards that share the same port on another host.
|
||
route = max(candidates, key=lambda cand: (len(cand.nodes), cand.prior_tps)).nodes
|
||
error = ""
|
||
else:
|
||
route, error = _select_route(
|
||
alive, required_start, required_end, contracts=server.contracts,
|
||
)
|
||
if error:
|
||
_tracker_log(
|
||
server,
|
||
"warn",
|
||
"route unavailable",
|
||
model=model,
|
||
route_model=resolved_name or model,
|
||
error=error,
|
||
candidate_count=len(alive),
|
||
candidates=_node_route_summary(alive),
|
||
)
|
||
self._send_json(503, {"error": error})
|
||
return
|
||
|
||
covered_up_to = required_start - 1
|
||
route_with_start: list[tuple] = []
|
||
for rn in route:
|
||
route_with_start.append((rn, covered_up_to + 1))
|
||
covered_up_to = rn.shard_end if rn.shard_end is not None else covered_up_to
|
||
|
||
self._send_json(200, {
|
||
"route": [e.endpoint for e, _ in route_with_start],
|
||
"nodes": [
|
||
{
|
||
"node_id": e.node_id,
|
||
"endpoint": e.endpoint,
|
||
"start_layer": start,
|
||
"relay_addr": e.relay_addr,
|
||
"peer_id": e.peer_id,
|
||
"wallet_address": e.wallet_address,
|
||
"shard_start": e.shard_start,
|
||
"shard_end": e.shard_end,
|
||
"model": e.model,
|
||
"hf_repo": e.hf_repo,
|
||
"num_layers": e.num_layers,
|
||
"model_metadata": dict(e.model_metadata),
|
||
"downloaded_models": [dict(item) for item in e.downloaded_models],
|
||
"shard_checksum": e.shard_checksum,
|
||
"score": e.score,
|
||
}
|
||
for e, start in route_with_start
|
||
],
|
||
})
|
||
|
||
def _handle_routing(self, parsed: urllib.parse.ParseResult):
|
||
"""Learned route table (ADR-0021): per-model candidates with observed
|
||
tps, coefficient vs the best proven route, and expected traffic share."""
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
only_model = (params.get("model") or [None])[0]
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
alive = list(server.registry.values())
|
||
model_keys: dict[str, list] = {}
|
||
for node in alive:
|
||
if node.shard_start is None or node.shard_end is None:
|
||
continue
|
||
key = node.hf_repo or node.model
|
||
if not key:
|
||
continue
|
||
model_keys.setdefault(key, []).append(node)
|
||
cfg = server.route_stats.config
|
||
out: dict[str, dict] = {}
|
||
for key, nodes in model_keys.items():
|
||
if only_model and key != only_model and not any(
|
||
key == alias for alias in _model_aliases(only_model)
|
||
):
|
||
continue
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, key)
|
||
# Stats are recorded under the proxy's resolved route_model —
|
||
# use the same key here or lookups always miss.
|
||
stats_key = resolved_name or key
|
||
if preset is not None:
|
||
rs, re = _preset_layer_bounds(preset)
|
||
else:
|
||
layer_counts = [n.num_layers for n in nodes if n.num_layers is not None]
|
||
if not layer_counts:
|
||
continue
|
||
rs, re = 0, max(layer_counts) - 1
|
||
candidates = _enumerate_routes(
|
||
nodes, rs, re,
|
||
model=stats_key,
|
||
contracts=server.contracts,
|
||
max_candidates=cfg.max_candidate_routes,
|
||
)
|
||
out[stats_key] = {
|
||
"epoch": server.route_stats.epoch(stats_key),
|
||
"routes": route_table(candidates, server.route_stats, stats_key),
|
||
}
|
||
self._send_json(200, {
|
||
"config": {
|
||
"explore_share": cfg.explore_share,
|
||
"weight_alpha": cfg.weight_alpha,
|
||
"stats_half_life_seconds": cfg.stats_half_life_seconds,
|
||
"min_sample_tokens": cfg.min_sample_tokens,
|
||
},
|
||
"models": out,
|
||
})
|
||
|
||
def _handle_routes(self, parsed: urllib.parse.ParseResult):
|
||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||
params = urllib.parse.parse_qs(parsed.query)
|
||
model_list = params.get("model")
|
||
if not model_list:
|
||
self._send_json(400, {"error": "missing 'model' query parameter"})
|
||
return
|
||
|
||
try:
|
||
redundancy = int(params.get("redundancy", ["1"])[0])
|
||
except ValueError:
|
||
self._send_json(400, {"error": "redundancy must be an integer"})
|
||
return
|
||
if redundancy < 1:
|
||
self._send_json(400, {"error": "redundancy must be at least 1"})
|
||
return
|
||
|
||
model = model_list[0]
|
||
resolved_name, preset = _resolve_model_preset(server.model_presets, model)
|
||
if preset is None:
|
||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||
return
|
||
|
||
required_start, required_end = _preset_layer_bounds(preset)
|
||
|
||
with server.lock:
|
||
self._purge_expired_nodes()
|
||
candidates = [
|
||
node for node in server.registry.values()
|
||
if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
|
||
]
|
||
if server.contracts is not None:
|
||
candidates = [
|
||
node for node in candidates
|
||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||
]
|
||
|
||
routes = []
|
||
remaining = list(candidates)
|
||
for _ in range(redundancy):
|
||
route, error = _select_route(remaining, required_start, required_end, contracts=server.contracts)
|
||
if error:
|
||
self._send_json(503, {"error": error})
|
||
return
|
||
route_endpoints = {node.endpoint for node in route}
|
||
remaining = [node for node in remaining if node.endpoint not in route_endpoints]
|
||
routes.append({
|
||
"route": [e.endpoint for e in route],
|
||
"nodes": [
|
||
{
|
||
"node_id": e.node_id,
|
||
"endpoint": e.endpoint,
|
||
"relay_addr": e.relay_addr,
|
||
"peer_id": e.peer_id,
|
||
"wallet_address": e.wallet_address,
|
||
"shard_start": e.shard_start,
|
||
"shard_end": e.shard_end,
|
||
"model": e.model,
|
||
"downloaded_models": [dict(item) for item in e.downloaded_models],
|
||
"shard_checksum": e.shard_checksum,
|
||
"score": e.score,
|
||
}
|
||
for e in route
|
||
],
|
||
})
|
||
|
||
self._send_json(200, {"routes": routes})
|
||
|
||
|
||
class TrackerServer:
|
||
"""HTTP tracker that manages node registration and resolves inference routes.
|
||
|
||
Nodes register via POST /v1/nodes/register and keep themselves alive with
|
||
POST /v1/nodes/<id>/heartbeat. The gateway queries GET /v1/route?model=<name>
|
||
to obtain an ordered list of node endpoints whose shards cover all layers.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
host: str = "127.0.0.1",
|
||
port: int = 0,
|
||
heartbeat_timeout: float = 90.0,
|
||
rebalance_interval: float = 30.0,
|
||
model_presets: dict | None = None,
|
||
contracts: Any | None = None,
|
||
minimum_stake: int = 0,
|
||
cluster_peers: list[str] | None = None,
|
||
cluster_self_url: str | None = None,
|
||
stats_db: str | None = None,
|
||
relay_url: str | None = None,
|
||
billing: BillingLedger | None = None,
|
||
enable_billing: bool = False,
|
||
billing_db: str | None = None,
|
||
accounts: AccountStore | None = None,
|
||
accounts_db: str | None = None,
|
||
benchmark_results_path: str | None = None,
|
||
treasury: Any | None = None,
|
||
deposit_poll_interval: float = 15.0,
|
||
settle_period: float = 86400.0,
|
||
payout_threshold: float = 5.0,
|
||
payout_dust_floor: float = 0.01,
|
||
settlement_check_interval: float | None = None,
|
||
validator_service_token: str | None = None,
|
||
hive_secret: str | None = None,
|
||
max_charge_per_request: float | None = None,
|
||
starting_credit: float = DEFAULT_CALLER_CREDIT_USDT,
|
||
devnet_topup_amount: float = DEFAULT_DEVNET_TOPUP_USDT,
|
||
toploc_calibration: ToplocCalibrationStore | None = None,
|
||
toploc_calibration_db: str | None = None,
|
||
toploc_reference_node_url: str | None = None,
|
||
toploc_calibration_gate_min_hardware_profiles: int = 1,
|
||
toploc_backend: Any | None = None,
|
||
enable_hf_pricing: bool = False,
|
||
hf_pricing_log: HfPricingLog | None = None,
|
||
hf_pricing_log_db: str | None = None,
|
||
hf_pricing_refresh_interval: float = 86400.0,
|
||
hf_pricing_fetch_html: Any | None = None,
|
||
models_dir: str | Path | None = None,
|
||
routing_config: RoutingConfig | None = None,
|
||
) -> None:
|
||
self._host = host
|
||
self._requested_port = port
|
||
self._heartbeat_timeout = heartbeat_timeout
|
||
self._rebalance_interval = rebalance_interval
|
||
self._model_presets: dict = (
|
||
model_presets if model_presets is not None else _clone_model_presets(DEFAULT_MODEL_PRESETS)
|
||
)
|
||
self._contracts = contracts
|
||
self._minimum_stake = minimum_stake
|
||
self._cluster_peers: list[str] = list(cluster_peers) if cluster_peers else []
|
||
self._cluster_self_url: str | None = cluster_self_url
|
||
self._relay_url = relay_url
|
||
self._registry: dict[str, _NodeEntry] = {}
|
||
self._lock = threading.Lock()
|
||
self._server: _TrackerHTTPServer | None = None
|
||
self._thread: threading.Thread | None = None
|
||
self._rebalance_stop = threading.Event()
|
||
self._rebalance_thread: threading.Thread | None = None
|
||
self._raft: RaftNode | None = None
|
||
self._gossip: NodeGossip | None = None
|
||
self._stats: _StatsCollector | None = _StatsCollector(db_path=stats_db) if stats_db else _StatsCollector()
|
||
self._stats_stop = threading.Event()
|
||
self._stats_thread: threading.Thread | None = None
|
||
if billing is None:
|
||
db_path = billing_db
|
||
if db_path is None and enable_billing:
|
||
db_path = DEFAULT_BILLING_DB_PATH
|
||
if db_path:
|
||
preset_prices: dict[str, tuple[float, float]] = {}
|
||
for name, preset in self._model_presets.items():
|
||
if not isinstance(preset, dict):
|
||
continue
|
||
base = preset.get("price_per_1k_tokens")
|
||
input_rate = preset.get("input_price_per_1k_tokens", base)
|
||
output_rate = preset.get("output_price_per_1k_tokens", base)
|
||
if input_rate is None or output_rate is None:
|
||
continue
|
||
for key in _preset_price_keys(name, preset):
|
||
preset_prices[key] = (float(input_rate), float(output_rate))
|
||
billing = BillingLedger(db_path=db_path, prices=preset_prices)
|
||
self._billing: BillingLedger | None = billing
|
||
self._billing_gossip_cursor = 0
|
||
if accounts is None:
|
||
accounts_path = accounts_db
|
||
if accounts_path is None and enable_billing:
|
||
accounts_path = DEFAULT_ACCOUNTS_DB_PATH
|
||
if accounts_path:
|
||
accounts = AccountStore(db_path=accounts_path)
|
||
self._accounts: AccountStore | None = accounts
|
||
self._accounts_gossip_cursor = 0
|
||
self._registry_gossip_cursor = 0
|
||
self._benchmark_results_path = benchmark_results_path
|
||
self._treasury = treasury
|
||
self._deposit_poll_interval = deposit_poll_interval
|
||
self._deposit_stop = threading.Event()
|
||
self._deposit_thread: threading.Thread | None = None
|
||
self._settle_period = settle_period
|
||
self._payout_threshold = payout_threshold
|
||
self._payout_dust_floor = payout_dust_floor
|
||
self._settlement_check_interval = settlement_check_interval or max(
|
||
1.0, min(settle_period / 4.0, 15.0)
|
||
)
|
||
self._settlement_stop = threading.Event()
|
||
self._settlement_thread: threading.Thread | None = None
|
||
if max_charge_per_request is not None and max_charge_per_request <= 0.0:
|
||
raise ValueError("max_charge_per_request must be positive")
|
||
self._max_charge_per_request = max_charge_per_request
|
||
self._starting_credit = max(0.0, starting_credit)
|
||
self._devnet_topup_amount = max(0.0, devnet_topup_amount)
|
||
self._validator_service_token = (
|
||
validator_service_token
|
||
if validator_service_token is not None
|
||
else os.environ.get("MESHNET_VALIDATOR_SERVICE_TOKEN") or None
|
||
)
|
||
self._hive_secret = (
|
||
hive_secret
|
||
if hive_secret is not None
|
||
else os.environ.get("MESHNET_HIVE_SECRET") or None
|
||
)
|
||
if toploc_calibration is None and toploc_calibration_db:
|
||
toploc_calibration = ToplocCalibrationStore(db_path=toploc_calibration_db)
|
||
self._toploc_calibration: ToplocCalibrationStore | None = toploc_calibration
|
||
self._toploc_reference_node_url = toploc_reference_node_url
|
||
self._toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
|
||
self._toploc_backend = toploc_backend
|
||
if hf_pricing_log is None and (enable_hf_pricing or hf_pricing_log_db):
|
||
hf_pricing_log = HfPricingLog(db_path=hf_pricing_log_db or DEFAULT_HF_PRICING_LOG_DB_PATH)
|
||
self._hf_pricing_log: HfPricingLog | None = hf_pricing_log
|
||
self._enable_hf_pricing = enable_hf_pricing
|
||
self._hf_pricing_refresh_interval = hf_pricing_refresh_interval
|
||
self._hf_pricing_fetch_html = hf_pricing_fetch_html
|
||
# MESHNET_DOWNLOAD_DIR is the node-side model store; on a box running
|
||
# both tracker and node it doubles as the tracker's snapshot source.
|
||
raw_models_dir = (
|
||
models_dir
|
||
if models_dir is not None
|
||
else os.environ.get("MESHNET_MODELS_DIR") or os.environ.get("MESHNET_DOWNLOAD_DIR")
|
||
)
|
||
self._models_dir = Path(raw_models_dir).expanduser() if raw_models_dir else None
|
||
self._hf_pricing_stop = threading.Event()
|
||
self._hf_pricing_thread: threading.Thread | None = None
|
||
if routing_config is None:
|
||
routing_config = RoutingConfig(
|
||
explore_share=float(os.environ.get("MESHNET_ROUTE_EXPLORE_SHARE", RoutingConfig.explore_share)),
|
||
weight_alpha=float(os.environ.get("MESHNET_ROUTE_WEIGHT_ALPHA", RoutingConfig.weight_alpha)),
|
||
stats_half_life_seconds=float(
|
||
os.environ.get("MESHNET_ROUTE_STATS_HALF_LIFE", RoutingConfig.stats_half_life_seconds)
|
||
),
|
||
min_sample_tokens=int(
|
||
os.environ.get("MESHNET_ROUTE_MIN_SAMPLE_TOKENS", RoutingConfig.min_sample_tokens)
|
||
),
|
||
)
|
||
self._route_stats = RouteStatsStore(routing_config)
|
||
self.port: int | None = None
|
||
|
||
def start(self) -> int:
|
||
if self._server is not None:
|
||
raise RuntimeError("TrackerServer is already running")
|
||
|
||
effective_relay_url = (
|
||
self._relay_url
|
||
or derive_relay_url_from_public_tracker_url(self._cluster_self_url)
|
||
)
|
||
|
||
# Start HTTP server first so we know our port
|
||
self._server = _TrackerHTTPServer(
|
||
(self._host, self._requested_port),
|
||
_TrackerHandler,
|
||
self._registry,
|
||
self._lock,
|
||
self._heartbeat_timeout,
|
||
self._model_presets,
|
||
self._contracts,
|
||
self._minimum_stake,
|
||
relay_url=effective_relay_url,
|
||
stats=self._stats,
|
||
billing=self._billing,
|
||
accounts=self._accounts,
|
||
benchmark_results_path=self._benchmark_results_path,
|
||
validator_service_token=self._validator_service_token,
|
||
hive_secret=self._hive_secret,
|
||
max_charge_per_request=self._max_charge_per_request,
|
||
starting_credit=self._starting_credit,
|
||
devnet_topup_amount=self._devnet_topup_amount,
|
||
toploc_calibration=self._toploc_calibration,
|
||
toploc_reference_node_url=self._toploc_reference_node_url,
|
||
toploc_calibration_gate_min_hardware_profiles=self._toploc_calibration_gate_min_hardware_profiles,
|
||
toploc_backend=self._toploc_backend,
|
||
hf_pricing_log=self._hf_pricing_log,
|
||
models_dir=self._models_dir,
|
||
route_stats=self._route_stats,
|
||
)
|
||
self.port = self._server.server_address[1]
|
||
|
||
# Start Raft + gossip if cluster peers are configured
|
||
if self._cluster_peers:
|
||
self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
|
||
self._raft = RaftNode(
|
||
self_url=self_url,
|
||
peers=self._cluster_peers,
|
||
apply_fn=self._raft_apply,
|
||
)
|
||
self._gossip = NodeGossip(peers=self._cluster_peers)
|
||
self._server.raft = self._raft
|
||
self._server.gossip = self._gossip
|
||
self._raft.start()
|
||
self._gossip.start()
|
||
|
||
self._rebalance_stop.clear()
|
||
self._stats_stop.clear()
|
||
self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
|
||
self._thread.start()
|
||
self._rebalance_thread = threading.Thread(target=self._rebalance_loop, daemon=True)
|
||
self._rebalance_thread.start()
|
||
self._stats_thread = threading.Thread(target=self._stats_loop, daemon=True)
|
||
self._stats_thread.start()
|
||
if self._treasury is not None and self._billing is not None:
|
||
self._deposit_stop.clear()
|
||
self._deposit_thread = threading.Thread(target=self._deposit_loop, daemon=True)
|
||
self._deposit_thread.start()
|
||
self._settlement_stop.clear()
|
||
self._settlement_thread = threading.Thread(target=self._settlement_loop, daemon=True)
|
||
self._settlement_thread.start()
|
||
if self._enable_hf_pricing and self._billing is not None:
|
||
self._hf_pricing_stop.clear()
|
||
self._hf_pricing_thread = threading.Thread(target=self._hf_pricing_loop, daemon=True)
|
||
self._hf_pricing_thread.start()
|
||
return self.port
|
||
|
||
def _settlement_loop(self) -> None:
|
||
"""Leader-only on-chain settlement (US-033, ADR-0015).
|
||
|
||
Pay a node when pending ≥ threshold OR its pending age ≥ max period,
|
||
with a dust floor. Pending is debited (payout events, replicated)
|
||
before the transaction is sent; unconfirmed batches are resent with
|
||
the same settlement id, so retries never double-pay.
|
||
"""
|
||
billing = self._billing
|
||
treasury = self._treasury
|
||
assert billing is not None and treasury is not None
|
||
while not self._settlement_stop.wait(self._settlement_check_interval):
|
||
if self._raft is not None and not self._raft.is_leader:
|
||
continue # followers replicate the ledger but never sign
|
||
# resend batches whose transaction never confirmed
|
||
for settlement_id, payouts in billing.unconfirmed_settlements().items():
|
||
self._send_settlement(settlement_id, payouts)
|
||
banned: set[str] = set()
|
||
if self._server is not None and self._server.contracts is not None:
|
||
registry = self._server.contracts.registry
|
||
banned = {
|
||
wallet for wallet, _ in billing.payables(
|
||
threshold=0.0, max_period=0.0, dust_floor=0.0,
|
||
)
|
||
if registry.get_wallet(wallet).banned
|
||
}
|
||
due = billing.payables(
|
||
threshold=self._payout_threshold,
|
||
max_period=self._settle_period,
|
||
dust_floor=self._payout_dust_floor,
|
||
exclude=banned,
|
||
)
|
||
if not due:
|
||
continue
|
||
settlement_id = uuid.uuid4().hex
|
||
registry = (
|
||
self._server.contracts.registry
|
||
if self._server is not None and self._server.contracts is not None
|
||
else None
|
||
)
|
||
settled: list[tuple[str, float]] = []
|
||
for wallet, amount in due:
|
||
# ADR-0015: a fraud forfeiture landing between the payables()
|
||
# snapshot above and this debit must never be paid out on top
|
||
# of — recheck the ban and let settle_node_payout clamp to
|
||
# whatever is still actually pending.
|
||
if registry is not None and registry.get_wallet(wallet).banned:
|
||
continue
|
||
event = billing.settle_node_payout(wallet, amount, reference=settlement_id)
|
||
if event["amount"] > 0:
|
||
settled.append((wallet, event["amount"]))
|
||
if settled:
|
||
self._send_settlement(settlement_id, settled)
|
||
|
||
def _send_settlement(self, settlement_id: str, payouts: list) -> None:
|
||
billing = self._billing
|
||
treasury = self._treasury
|
||
assert billing is not None and treasury is not None
|
||
try:
|
||
signature = treasury.send_payouts([(w, a) for w, a in payouts])
|
||
except Exception as exc:
|
||
print(
|
||
f"[tracker] settlement {settlement_id} failed (will retry): {exc}",
|
||
flush=True,
|
||
)
|
||
return
|
||
billing.confirm_settlement(settlement_id, signature)
|
||
total = sum(a for _, a in payouts)
|
||
print(
|
||
f"[tracker] settled {settlement_id}: {len(payouts)} payout(s), "
|
||
f"{total:.6f} USDT total (tx {signature})",
|
||
flush=True,
|
||
)
|
||
|
||
def _deposit_loop(self) -> None:
|
||
"""Poll the treasury token account and credit confirmed deposits (US-032)."""
|
||
billing = self._billing
|
||
treasury = self._treasury
|
||
assert billing is not None and treasury is not None
|
||
while not self._deposit_stop.wait(self._deposit_poll_interval):
|
||
try:
|
||
deposits = treasury.list_new_deposits(
|
||
lambda sig: billing.has_event(f"deposit-{sig}")
|
||
)
|
||
except Exception as exc:
|
||
print(f"[tracker] deposit poll failed: {exc}", flush=True)
|
||
continue
|
||
for deposit in deposits:
|
||
api_key = billing.api_key_for_wallet(deposit.sender_wallet)
|
||
if api_key is None:
|
||
print(
|
||
f"[tracker] unattributed deposit {deposit.signature}: "
|
||
f"{deposit.amount_usdt} USDT from unbound wallet "
|
||
f"{deposit.sender_wallet} — register via /v1/wallet/register",
|
||
flush=True,
|
||
)
|
||
continue
|
||
credited = billing.credit_deposit(
|
||
api_key, deposit.amount_usdt, deposit.signature
|
||
)
|
||
if credited is not None:
|
||
print(
|
||
f"[tracker] deposit credited: {deposit.amount_usdt} USDT "
|
||
f"-> api_key …{api_key[-6:]} (tx {deposit.signature})",
|
||
flush=True,
|
||
)
|
||
|
||
def _hf_pricing_loop(self) -> None:
|
||
"""Daily dynamic pricing refresh benchmarked against HF inference rates (issue 23).
|
||
|
||
For every preset with a curated, human-verified ``hf_aliases`` list,
|
||
fetch current HF marketplace pricing and set the client price to 80%
|
||
of the cheapest matching alias. Presets with no (or an empty)
|
||
``hf_aliases`` are left entirely alone — they keep the static
|
||
default price. Any single preset's fetch/parse failure is logged and
|
||
skipped; it never raises into this loop or the request path.
|
||
"""
|
||
billing = self._billing
|
||
assert billing is not None
|
||
while not self._hf_pricing_stop.wait(self._hf_pricing_refresh_interval):
|
||
for name, preset in list(self._model_presets.items()):
|
||
if not isinstance(preset, dict) or not preset.get("hf_aliases"):
|
||
continue
|
||
try:
|
||
result = refresh_preset_price(
|
||
model_name=name,
|
||
preset=preset,
|
||
current_price=billing.price_for(name),
|
||
fetch_html=self._hf_pricing_fetch_html,
|
||
)
|
||
except Exception as exc:
|
||
print(f"[tracker] hf pricing refresh failed for {name!r}: {exc}", flush=True)
|
||
continue
|
||
if result is None:
|
||
continue
|
||
new_input = result.get("new_input_price_per_1k", result["new_price_per_1k"])
|
||
new_output = result.get("new_output_price_per_1k", result["new_price_per_1k"])
|
||
for key in _preset_price_keys(name, preset):
|
||
billing.set_prices(key, new_input, new_output)
|
||
preset["hf_last_price_per_1k"] = result["new_price_per_1k"]
|
||
preset["hf_last_updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||
if self._hf_pricing_log is not None:
|
||
self._hf_pricing_log.record_change(
|
||
model=name,
|
||
old_price_per_1k=result["old_price_per_1k"],
|
||
new_price_per_1k=result["new_price_per_1k"],
|
||
source_repo_id=result["source_repo_id"],
|
||
source_provider=result["source_provider"],
|
||
)
|
||
print(
|
||
f"[tracker] hf pricing: {name} {result['old_price_per_1k']:.6f} -> "
|
||
f"{result['new_price_per_1k']:.6f} USDT/1k tokens "
|
||
f"(80% of {result['source_repo_id']}::{result['source_provider']})",
|
||
flush=True,
|
||
)
|
||
|
||
def _raft_apply(self, command: str, payload: dict) -> None:
|
||
"""Called by RaftNode when a log entry is committed — replicate to local registry."""
|
||
if command != "register":
|
||
return
|
||
# Re-apply the registration to our local registry (follower path).
|
||
# On the leader this is a no-op since it already registered locally.
|
||
node_id = payload.get("node_id")
|
||
if not node_id or node_id in self._registry:
|
||
return # already present (leader case) or malformed
|
||
endpoint = payload.get("endpoint", "")
|
||
try:
|
||
shard_start = int(payload["shard_start"]) if payload.get("shard_start") is not None else None
|
||
shard_end = int(payload["shard_end"]) if payload.get("shard_end") is not None else None
|
||
except (TypeError, ValueError):
|
||
return
|
||
entry = _NodeEntry(
|
||
node_id=node_id,
|
||
endpoint=endpoint.rstrip("/"),
|
||
shard_start=shard_start,
|
||
shard_end=shard_end,
|
||
model=payload.get("model", "stub-model"),
|
||
shard_checksum=payload.get("shard_checksum"),
|
||
hardware_profile=payload.get("hardware_profile", {}),
|
||
wallet_address=payload.get("wallet_address"),
|
||
score=float(payload.get("score", 1.0)),
|
||
tracker_mode=bool(payload.get("tracker_mode", False)),
|
||
hf_repo=payload.get("hf_repo"),
|
||
num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
|
||
model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
|
||
downloaded_models=(
|
||
payload.get("downloaded_models")
|
||
if isinstance(payload.get("downloaded_models"), list)
|
||
else None
|
||
),
|
||
)
|
||
with self._lock:
|
||
self._registry[node_id] = entry
|
||
|
||
def _rebalance_loop(self) -> None:
|
||
while not self._rebalance_stop.wait(self._rebalance_interval):
|
||
server = self._server
|
||
if server is None:
|
||
return
|
||
with self._lock:
|
||
_purge_expired_nodes_locked(server)
|
||
_rebalance_all_locked(server)
|
||
_scale_demanded_models_locked(server)
|
||
|
||
def _push_to_peers(self, path: str, body: bytes) -> bool:
|
||
"""POST a hive-signed payload to every cluster peer; True if all succeeded."""
|
||
headers = {"Content-Type": "application/json"}
|
||
if self._hive_secret:
|
||
headers.update(sign_hive_request(self._hive_secret, body))
|
||
delivered_all = True
|
||
for peer in self._cluster_peers:
|
||
try:
|
||
req = urllib.request.Request(
|
||
f"{peer}{path}", data=body, headers=headers, method="POST",
|
||
)
|
||
with urllib.request.urlopen(req, timeout=2.0) as r:
|
||
r.read()
|
||
except Exception:
|
||
delivered_all = False
|
||
return delivered_all
|
||
|
||
def _stats_loop(self) -> None:
|
||
"""Periodically save stats/billing to DB and push local slices to cluster peers."""
|
||
while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
|
||
if self._stats is not None:
|
||
self._stats.save_to_db()
|
||
if self._billing is not None:
|
||
self._billing.save_to_db()
|
||
if self._accounts is not None:
|
||
self._accounts.save_to_db()
|
||
registry_log = getattr(self._contracts, "registry_log", None)
|
||
if registry_log is not None:
|
||
registry_log.save_to_db()
|
||
if self._cluster_peers and not self._hive_secret:
|
||
print(
|
||
"[tracker] WARNING: cluster peers configured without --hive-secret — "
|
||
"gossip pushes will be rejected (ADR-0017)",
|
||
flush=True,
|
||
)
|
||
if self._stats is not None and self._cluster_peers:
|
||
self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
|
||
local_rpms = self._stats.get_local_rpms()
|
||
body = json.dumps({"tracker_url": self_url, "stats": local_rpms}).encode()
|
||
self._push_to_peers("/v1/stats/gossip", body)
|
||
if self._billing is not None and self._cluster_peers:
|
||
events, cursor = self._billing.events_since(self._billing_gossip_cursor)
|
||
if events:
|
||
body = json.dumps({"events": events}).encode()
|
||
# Only advance past events every peer has seen; unreachable
|
||
# peers get the full backlog on the next tick (idempotent
|
||
# via event-id dedupe on the receiving side).
|
||
if self._push_to_peers("/v1/billing/gossip", body):
|
||
self._billing_gossip_cursor = cursor
|
||
if self._accounts is not None and self._cluster_peers:
|
||
events, cursor = self._accounts.events_since(self._accounts_gossip_cursor)
|
||
if events:
|
||
body = json.dumps({"events": events}).encode()
|
||
if self._push_to_peers("/v1/accounts/gossip", body):
|
||
self._accounts_gossip_cursor = cursor
|
||
registry_log = getattr(self._contracts, "registry_log", None)
|
||
if registry_log is not None and self._cluster_peers:
|
||
events, cursor = registry_log.events_since(self._registry_gossip_cursor)
|
||
if events:
|
||
body = json.dumps({"events": events}).encode()
|
||
if self._push_to_peers("/v1/registry/gossip", body):
|
||
self._registry_gossip_cursor = cursor
|
||
|
||
def stop(self) -> None:
|
||
if self._raft is not None:
|
||
self._raft.stop()
|
||
if self._gossip is not None:
|
||
self._gossip.stop()
|
||
if self._server is None:
|
||
return
|
||
self._rebalance_stop.set()
|
||
self._stats_stop.set()
|
||
self._deposit_stop.set()
|
||
self._settlement_stop.set()
|
||
self._hf_pricing_stop.set()
|
||
if self._stats is not None:
|
||
self._stats.save_to_db()
|
||
if self._billing is not None:
|
||
self._billing.save_to_db()
|
||
if self._accounts is not None:
|
||
self._accounts.save_to_db()
|
||
registry_log = getattr(self._contracts, "registry_log", None)
|
||
if registry_log is not None:
|
||
registry_log.save_to_db()
|
||
self._server.shutdown()
|
||
self._server.server_close()
|
||
if self._thread is not None:
|
||
self._thread.join(timeout=1)
|
||
if self._rebalance_thread is not None:
|
||
self._rebalance_thread.join(timeout=1)
|
||
if self._stats_thread is not None:
|
||
self._stats_thread.join(timeout=1)
|
||
if self._deposit_thread is not None:
|
||
self._deposit_thread.join(timeout=1)
|
||
self._deposit_thread = None
|
||
if self._settlement_thread is not None:
|
||
self._settlement_thread.join(timeout=1)
|
||
self._settlement_thread = None
|
||
if self._hf_pricing_thread is not None:
|
||
self._hf_pricing_thread.join(timeout=1)
|
||
self._hf_pricing_thread = None
|
||
self._server = None
|
||
self._thread = None
|
||
self._rebalance_thread = None
|
||
self._stats_thread = None
|
||
self._raft = None
|
||
self._gossip = None
|
||
self.port = None
|