diff --git a/.scratch/alpha-hardening/issues/01-c1-gossip-auth.md b/.scratch/alpha-hardening/issues/01-c1-gossip-auth.md
index 2b9ebc9..52dc0e6 100644
--- a/.scratch/alpha-hardening/issues/01-c1-gossip-auth.md
+++ b/.scratch/alpha-hardening/issues/01-c1-gossip-auth.md
@@ -1,4 +1,4 @@
-Status: ready-for-agent
+Status: done
# 01 — C1: Authenticate hive gossip endpoints
diff --git a/.scratch/alpha-hardening/issues/02-a2-unified-auth-boundary.md b/.scratch/alpha-hardening/issues/02-a2-unified-auth-boundary.md
index 4ca1c28..039594a 100644
--- a/.scratch/alpha-hardening/issues/02-a2-unified-auth-boundary.md
+++ b/.scratch/alpha-hardening/issues/02-a2-unified-auth-boundary.md
@@ -1,4 +1,4 @@
-Status: ready-for-agent
+Status: done
# 02 — A2: Unified auth boundary for privileged and financial reads
diff --git a/.scratch/alpha-hardening/issues/20-validator-service-token.md b/.scratch/alpha-hardening/issues/20-validator-service-token.md
index a0ec7dd..ea6de6d 100644
--- a/.scratch/alpha-hardening/issues/20-validator-service-token.md
+++ b/.scratch/alpha-hardening/issues/20-validator-service-token.md
@@ -1,4 +1,4 @@
-Status: ready-for-agent
+Status: done
# 20 — Validator service token for `/v1/billing/forfeit`
diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html
index 874cc35..63377cd 100644
--- a/packages/tracker/meshnet_tracker/dashboard.html
+++ b/packages/tracker/meshnet_tracker/dashboard.html
@@ -1,383 +1,383 @@
-
-
-
-
-
-meshnet tracker
-
-
-
-
-
-
-
-
-
-
- Node pending payouts
loading…
- Settlement history
loading…
- Strikes / bans / forfeitures
loading…
- Model usage (RPM)
loading…
-
-
-
-
-
+
+
+
+
+
+meshnet tracker
+
+
+
+
+
+
+
+
+
+
+ Node pending payouts
loading…
+ Settlement history
loading…
+ Strikes / bans / forfeitures
loading…
+ Model usage (RPM)
loading…
+
+
+
+
+
diff --git a/packages/tracker/meshnet_tracker/server.py b/packages/tracker/meshnet_tracker/server.py
index 9c32d1d..d1a97e1 100644
--- a/packages/tracker/meshnet_tracker/server.py
+++ b/packages/tracker/meshnet_tracker/server.py
@@ -1,3624 +1,3624 @@
-"""Tracker HTTP server — node registry and route selection for the inference mesh.
-
-HTTP API contract:
-- POST /v1/nodes/register
- Request JSON: {
- "endpoint": "http://host:port", "shard_start": int, "shard_end": int,
- "model": str optional, "shard_checksum": str optional,
- "hardware_profile": object, "wallet_address": str optional,
- "score": number optional
- }
- Response 200: {"node_id": str}
- Response 400: {"error": str}
-- POST /v1/nodes/{node_id}/heartbeat
- Response 200: {}
- Response 404: {"error": "node not found"}
-- GET /v1/route?model=
- Response 200: {"route": ["http://node-a", "http://node-b"]}
- Response 400/404/503: {"error": str}
-- GET /v1/routes?model=&redundancy=
- Response 200: {"routes": [{"route": [...], "nodes": [...]}]}
- Response 400/404/503: {"error": str}
-"""
-
-import http.server
-import hashlib
-import itertools
-import json
-import os
-import socketserver
-import sqlite3
-import threading
-import time
-import urllib.parse
-import urllib.request
-import uuid
-from importlib.resources import files
-from typing import Any
-
-from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
-from .auth import is_validator_token, sign_hive_request, verify_hive_request
-from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
-from .gossip import NodeGossip
-from .raft import RaftNode
-
-
-def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
- """Return wss://host/ws when url is a public HTTPS tracker origin."""
- if not url:
- return None
- parsed = urllib.parse.urlparse(url)
- if parsed.scheme != "https":
- return None
- host = parsed.hostname
- if not host or host in ("127.0.0.1", "localhost"):
- return None
- return f"wss://{parsed.netloc}/ws"
-
-
-def _load_model_presets_from_data() -> dict[str, dict]:
- """Load recommended model presets from package JSON data."""
- try:
- raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
- data = json.loads(raw)
- except Exception:
- return {}
- models = data.get("models", {})
- if not isinstance(models, dict):
- return {}
- return {
- str(name): preset
- for name, preset in models.items()
- if isinstance(preset, dict)
- }
-
-
-DEFAULT_MODEL_PRESETS: dict[str, dict] = {
- "stub-model": {
- "layers_start": 0,
- "layers_end": 31,
- "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
- },
- "openai-community/gpt2": {
- "layers_start": 0,
- "layers_end": 11,
- "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
- },
- **_load_model_presets_from_data(),
-}
-
-DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
-DEFAULT_RAM_BYTES = 16 * 1024 * 1024 * 1024
-DEFAULT_QUANTIZATIONS = ["bfloat16"]
-DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
-
-
-def _model_aliases(model: str | None) -> set[str]:
- """Return stable lookup aliases for a model repo or display name."""
- if not model:
- return set()
- aliases = {model}
- short = model.rsplit("/", 1)[-1]
- aliases.add(short)
- lowered = short.lower()
- aliases.add(lowered)
- if lowered.endswith("-instruct"):
- aliases.add(lowered.removesuffix("-instruct"))
- return aliases
-
-
-def _preset_aliases(name: str, preset: dict | None) -> set[str]:
- aliases = _model_aliases(name)
- if not preset:
- return aliases
- hf_repo = preset.get("hf_repo")
- if isinstance(hf_repo, str):
- aliases |= _model_aliases(hf_repo)
- for alias in preset.get("aliases", []) or []:
- if isinstance(alias, str):
- aliases |= _model_aliases(alias)
- return aliases
-
-
-def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
- requested = _model_aliases(model)
- for name, preset in model_presets.items():
- if requested & _preset_aliases(name, preset):
- return name, preset
- return None, None
-
-
-def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
- requested = _model_aliases(model)
- if not requested:
- return False
- return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
-
-
-def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
- requested = _preset_aliases(name, preset)
- return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
-
-
-class _RollingCounter:
- """Circular-bucket request counter.
-
- Tracks events in `num_buckets` slots, each covering `bucket_seconds` of wall time.
- Old buckets are silently discarded when their epoch expires.
- """
-
- def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
- self._num = num_buckets
- self._bsec = bucket_seconds
- self._counts: list[int] = [0] * num_buckets
- self._epochs: list[int] = [-1] * num_buckets # which epoch each slot holds
-
- def _epoch(self, now: float) -> int:
- return int(now) // self._bsec
-
- def record(self, now: float | None = None) -> None:
- t = now if now is not None else time.time()
- ep = self._epoch(t)
- idx = ep % self._num
- if self._epochs[idx] != ep:
- self._counts[idx] = 0
- self._epochs[idx] = ep
- self._counts[idx] += 1
-
- def rpm(self, now: float | None = None) -> float:
- t = now if now is not None else time.time()
- cutoff = self._epoch(t) - self._num # epochs <= this are stale
- total = sum(self._counts[i] for i in range(self._num) if self._epochs[i] > cutoff)
- window_minutes = (self._num * self._bsec) / 60.0
- return total / window_minutes if window_minutes > 0 else 0.0
-
- def buckets(self) -> list[tuple[int, int]]:
- return [(self._epochs[i], self._counts[i]) for i in range(self._num)]
-
- def restore_buckets(self, data: list[tuple[int, int]]) -> None:
- for i, (ep, cnt) in enumerate(data):
- if i < self._num:
- self._epochs[i] = ep
- self._counts[i] = cnt
-
-
-class _RollingThroughput:
- """Circular buckets for observed tokens/sec.
-
- Each bucket stores total output tokens and total elapsed seconds. TPS for a
- window is the ratio across non-stale buckets, which avoids over-weighting
- small fast requests.
- """
-
- def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
- self._num = num_buckets
- self._bsec = bucket_seconds
- self._tokens: list[int] = [0] * num_buckets
- self._seconds: list[float] = [0.0] * num_buckets
- self._samples: list[int] = [0] * num_buckets
- self._epochs: list[int] = [-1] * num_buckets
-
- def _epoch(self, now: float) -> int:
- return int(now) // self._bsec
-
- def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
- if total_tokens <= 0 or elapsed_seconds <= 0:
- return
- t = now if now is not None else time.time()
- ep = self._epoch(t)
- idx = ep % self._num
- if self._epochs[idx] != ep:
- self._tokens[idx] = 0
- self._seconds[idx] = 0.0
- self._samples[idx] = 0
- self._epochs[idx] = ep
- self._tokens[idx] += int(total_tokens)
- self._seconds[idx] += float(elapsed_seconds)
- self._samples[idx] += 1
-
- def stats(self, now: float | None = None) -> dict:
- t = now if now is not None else time.time()
- cutoff = self._epoch(t) - self._num
- tokens = 0
- seconds = 0.0
- samples = 0
- for i in range(self._num):
- if self._epochs[i] > cutoff:
- tokens += self._tokens[i]
- seconds += self._seconds[i]
- samples += self._samples[i]
- return {
- "tokens_per_sec": round(tokens / seconds, 4) if seconds > 0 else None,
- "tokens": tokens,
- "seconds": round(seconds, 6),
- "sample_count": samples,
- }
-
- def buckets(self) -> list[tuple[int, int, float, int]]:
- return [
- (self._epochs[i], self._tokens[i], self._seconds[i], self._samples[i])
- for i in range(self._num)
- ]
-
- def restore_buckets(self, data: list[tuple[int, int, float, int]]) -> None:
- for i, (ep, tokens, seconds, samples) in enumerate(data):
- if i < self._num:
- self._epochs[i] = ep
- self._tokens[i] = tokens
- self._seconds[i] = seconds
- self._samples[i] = samples
-
-
-class _ModelStats:
- """Three rolling windows for one model: last hour, last day, last month."""
-
- def __init__(self) -> None:
- self.per_minute = _RollingCounter(60, 60) # 60 × 1-min buckets = 1 hour
- self.per_hour = _RollingCounter(24, 3600) # 24 × 1-hr buckets = 1 day
- self.per_day = _RollingCounter(30, 86400) # 30 × 1-day buckets = ~1 month
-
- def record(self, now: float | None = None) -> None:
- t = now if now is not None else time.time()
- self.per_minute.record(t)
- self.per_hour.record(t)
- self.per_day.record(t)
-
- def rpms(self, now: float | None = None) -> dict:
- t = now if now is not None else time.time()
- return {
- "rpm_last_hour": round(self.per_minute.rpm(t), 4),
- "rpm_last_day": round(self.per_hour.rpm(t), 4),
- "rpm_last_month": round(self.per_day.rpm(t), 4),
- }
-
-
-class _NodeModelThroughput:
- """Observed throughput windows for one node/model pair."""
-
- def __init__(self) -> None:
- self.per_minute = _RollingThroughput(60, 60)
- self.per_hour = _RollingThroughput(24, 3600)
- self.per_day = _RollingThroughput(30, 86400)
-
- def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
- t = now if now is not None else time.time()
- self.per_minute.record(total_tokens, elapsed_seconds, t)
- self.per_hour.record(total_tokens, elapsed_seconds, t)
- self.per_day.record(total_tokens, elapsed_seconds, t)
-
- def stats(self, now: float | None = None) -> dict:
- hour = self.per_minute.stats(now)
- day = self.per_hour.stats(now)
- month = self.per_day.stats(now)
- return {
- "tokens_per_sec_last_hour": hour["tokens_per_sec"],
- "tokens_per_sec_last_day": day["tokens_per_sec"],
- "tokens_per_sec_last_month": month["tokens_per_sec"],
- "sample_count_last_hour": hour["sample_count"],
- "tokens_last_hour": hour["tokens"],
- "seconds_last_hour": hour["seconds"],
- }
-
-
-class _StatsCollector:
- """Thread-safe model request stats with SQLite persistence and peer slice merging."""
-
- SAVE_INTERVAL = 60.0 # seconds between DB saves
-
- def __init__(self, db_path: str | None = None) -> None:
- self._lock = threading.Lock()
- self._local: dict[str, _ModelStats] = {}
- self._node_model: dict[tuple[str, str], _NodeModelThroughput] = {}
- self._peer_rpms: dict[str, dict[str, dict]] = {} # tracker_url -> model -> rpms
- self._db_path = db_path
- if db_path:
- self._init_db()
- self._load_from_db()
-
- # ---------- public API ----------
-
- def record_request(self, model: str, now: float | None = None) -> None:
- t = now if now is not None else time.time()
- with self._lock:
- if model not in self._local:
- self._local[model] = _ModelStats()
- self._local[model].record(t)
-
- def record_node_throughput(
- self,
- node_id: str,
- model: str,
- total_tokens: int,
- elapsed_seconds: float,
- now: float | None = None,
- ) -> None:
- if not node_id or not model or total_tokens <= 0 or elapsed_seconds <= 0:
- return
- t = now if now is not None else time.time()
- with self._lock:
- key = (node_id, model)
- if key not in self._node_model:
- self._node_model[key] = _NodeModelThroughput()
- self._node_model[key].record(total_tokens, elapsed_seconds, t)
-
- def get_local_rpms(self, now: float | None = None) -> dict[str, dict]:
- t = now if now is not None else time.time()
- with self._lock:
- return {m: s.rpms(t) for m, s in self._local.items()}
-
- def get_node_model_stats(self, node_id: str, model: str, now: float | None = None) -> dict:
- t = now if now is not None else time.time()
- empty = {
- "tokens_per_sec_last_hour": None,
- "tokens_per_sec_last_day": None,
- "tokens_per_sec_last_month": None,
- "sample_count_last_hour": 0,
- "tokens_last_hour": 0,
- "seconds_last_hour": 0.0,
- }
- with self._lock:
- stat = self._node_model.get((node_id, model))
- return stat.stats(t) if stat is not None else dict(empty)
-
- def get_node_throughput_stats(self, now: float | None = None) -> dict[str, dict]:
- t = now if now is not None else time.time()
- with self._lock:
- result: dict[str, dict] = {}
- for (node_id, model), stat in self._node_model.items():
- result.setdefault(node_id, {"models": {}})["models"][model] = stat.stats(t)
- return result
-
- def merge_peer_rpms(self, tracker_url: str, rpms: dict[str, dict]) -> None:
- with self._lock:
- self._peer_rpms[tracker_url] = dict(rpms)
-
- def get_combined_stats(self, now: float | None = None) -> dict:
- t = now if now is not None else time.time()
- with self._lock:
- combined: dict[str, dict] = {}
- for model, ms in self._local.items():
- combined[model] = ms.rpms(t)
- for _url, peer in self._peer_rpms.items():
- for model, rpms in peer.items():
- if model not in combined:
- combined[model] = {"rpm_last_hour": 0.0, "rpm_last_day": 0.0, "rpm_last_month": 0.0}
- for key in ("rpm_last_hour", "rpm_last_day", "rpm_last_month"):
- combined[model][key] = round(combined[model].get(key, 0.0) + rpms.get(key, 0.0), 4)
- return combined
-
- # ---------- persistence ----------
-
- def _init_db(self) -> None:
- con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
- con.execute(
- "CREATE TABLE IF NOT EXISTS model_rpm_buckets "
- "(model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, count INTEGER, "
- "PRIMARY KEY (model, window, bucket_idx))"
- )
- con.execute(
- "CREATE TABLE IF NOT EXISTS node_model_tps_buckets "
- "(node_id TEXT, model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, "
- "tokens INTEGER, seconds REAL, samples INTEGER, "
- "PRIMARY KEY (node_id, model, window, bucket_idx))"
- )
- con.commit()
- con.close()
-
- def save_to_db(self) -> None:
- if not self._db_path:
- return
- with self._lock:
- rows = []
- for model, ms in self._local.items():
- for window_name, counter in (
- ("hour", ms.per_minute),
- ("day", ms.per_hour),
- ("month", ms.per_day),
- ):
- for idx, (ep, cnt) in enumerate(counter.buckets()):
- if ep >= 0:
- rows.append((model, window_name, idx, ep, cnt))
- tps_rows = []
- for (node_id, model), stat in self._node_model.items():
- for window_name, counter in (
- ("hour", stat.per_minute),
- ("day", stat.per_hour),
- ("month", stat.per_day),
- ):
- for idx, (ep, tokens, seconds, samples) in enumerate(counter.buckets()):
- if ep >= 0:
- tps_rows.append((node_id, model, window_name, idx, ep, tokens, seconds, samples))
- con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
- con.executemany(
- "INSERT OR REPLACE INTO model_rpm_buckets VALUES (?,?,?,?,?)", rows
- )
- con.executemany(
- "INSERT OR REPLACE INTO node_model_tps_buckets VALUES (?,?,?,?,?,?,?,?)",
- tps_rows,
- )
- con.commit()
- con.close()
-
- def _load_from_db(self) -> None:
- con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
- rows = con.execute("SELECT model, window, bucket_idx, bucket_epoch, count FROM model_rpm_buckets").fetchall()
- tps_rows = con.execute(
- "SELECT node_id, model, window, bucket_idx, bucket_epoch, tokens, seconds, samples "
- "FROM node_model_tps_buckets"
- ).fetchall()
- con.close()
- grouped: dict[str, dict[str, list[tuple[int, int]]]] = {}
- for model, window, idx, ep, cnt in rows:
- grouped.setdefault(model, {}).setdefault(window, []).append((idx, ep, cnt))
- for model, windows in grouped.items():
- ms = _ModelStats()
- for window_name, entries in windows.items():
- counter = {"hour": ms.per_minute, "day": ms.per_hour, "month": ms.per_day}.get(window_name)
- if counter is None:
- continue
- data = [(0, -1)] * counter._num
- for idx, ep, cnt in entries:
- if 0 <= idx < counter._num:
- data[idx] = (ep, cnt)
- counter.restore_buckets(data)
- self._local[model] = ms
- tps_grouped: dict[tuple[str, str], dict[str, list[tuple[int, int, int, float, int]]]] = {}
- for node_id, model, window, idx, ep, tokens, seconds, samples in tps_rows:
- tps_grouped.setdefault((node_id, model), {}).setdefault(window, []).append(
- (idx, ep, tokens, seconds, samples)
- )
- for key, windows in tps_grouped.items():
- stat = _NodeModelThroughput()
- for window_name, entries in windows.items():
- counter = {"hour": stat.per_minute, "day": stat.per_hour, "month": stat.per_day}.get(window_name)
- if counter is None:
- continue
- data = [(0, 0, 0.0, 0)] * counter._num
- for idx, ep, tokens, seconds, samples in entries:
- if 0 <= idx < counter._num:
- data[idx] = (ep, int(tokens), float(seconds), int(samples))
- counter.restore_buckets(data)
- self._node_model[key] = stat
-
-
-class _NodeEntry:
- __slots__ = (
- "node_id", "endpoint", "shard_start", "shard_end",
- "model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "hardware_profile", "wallet_address",
- "score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
- "benchmark_tokens_per_sec", "quantization", "managed_assignment",
- "model_tokens_per_sec",
- "pending_directives", "last_heartbeat", "tracker_mode",
- "relay_addr", "cert_fingerprint", "peer_id",
- # heartbeat stats (reported by node, cumulative)
- "total_requests", "failed_requests", "queue_depth", "uptime_seconds",
- "status", # "ready" | "loading"
- "heartbeats_expected", "heartbeats_received",
- # dynamic reassignment queued by the tracker
- "pending_new_assignment",
- )
-
- def __init__(
- self,
- node_id: str,
- endpoint: str,
- shard_start: int | None,
- shard_end: int | None,
- model: str | None,
- shard_checksum: str | None,
- hardware_profile: dict,
- wallet_address: str | None,
- score: float,
- vram_bytes: int = DEFAULT_VRAM_BYTES,
- ram_bytes: int = DEFAULT_RAM_BYTES,
- quantizations: list[str] | None = None,
- max_loaded_shards: int = 1,
- benchmark_tokens_per_sec: float = DEFAULT_BENCHMARK_TOKENS_PER_SEC,
- quantization: str | None = None,
- managed_assignment: bool = False,
- tracker_mode: bool = False,
- hf_repo: str | None = None,
- num_layers: int | None = None,
- model_metadata: dict | None = None,
- relay_addr: str | None = None,
- cert_fingerprint: str | None = None,
- peer_id: str | None = None,
- ) -> None:
- self.node_id = node_id
- self.endpoint = endpoint
- self.shard_start = shard_start
- self.shard_end = shard_end
- self.model = model
- self.shard_checksum = shard_checksum
- self.hardware_profile = hardware_profile
- self.wallet_address = wallet_address
- self.score = score
- self.vram_bytes = vram_bytes
- self.ram_bytes = ram_bytes
- self.quantizations = quantizations or list(DEFAULT_QUANTIZATIONS)
- self.max_loaded_shards = max_loaded_shards
- self.benchmark_tokens_per_sec = benchmark_tokens_per_sec
- self.quantization = quantization
- self.managed_assignment = managed_assignment
- self.model_tokens_per_sec: dict[str, float] = {}
- self.tracker_mode = tracker_mode
- self.hf_repo = hf_repo
- self.num_layers = num_layers
- self.model_metadata = dict(model_metadata or {})
- self.relay_addr = relay_addr
- self.cert_fingerprint = cert_fingerprint
- self.peer_id = peer_id
- self.pending_directives: list[dict] = []
- self.last_heartbeat: float = time.monotonic()
- self.total_requests: int = 0
- self.failed_requests: int = 0
- self.queue_depth: int = 0
- self.uptime_seconds: float = 0.0
- self.status: str = "ready"
- self.heartbeats_expected: int = 0
- self.heartbeats_received: int = 0
- self.pending_new_assignment: dict | None = None
-
-
-def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
- """Effective tokens/s accounting for current queue depth."""
- observed = None
- if model:
- observed = node.model_tokens_per_sec.get(model)
- if observed is None:
- for alias in _model_aliases(model):
- observed = node.model_tokens_per_sec.get(alias)
- if observed is not None:
- break
- base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
- return base / (node.queue_depth + 1)
-
-
-def _select_route(
- nodes: list[_NodeEntry],
- required_start: int,
- required_end: int,
- model: str | None = None,
-) -> tuple[list[_NodeEntry], str]:
- """Greedy interval-cover biased toward fast, lightly-loaded nodes.
-
- Among nodes that equally advance coverage, prefer the one with higher
- effective throughput: observed per-model tokens/sec / (queue_depth + 1),
- falling back to startup benchmark_tokens_per_sec until observations exist.
- Tiebreak: higher shard_end (fewer hops).
- """
- candidates = sorted(
- [node for node in nodes if node.shard_start is not None and node.shard_end is not None],
- key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
- )
- route: list[_NodeEntry] = []
- covered_up_to = required_start - 1
-
- while covered_up_to < required_end:
- best: _NodeEntry | None = None
- for node in candidates:
- if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
- if best is None:
- best = node
- elif node.shard_end > best.shard_end:
- best = node
- elif node.shard_end == best.shard_end and _effective_throughput(node, model) > _effective_throughput(best, model):
- best = node
- if best is None:
- missing = covered_up_to + 1
- return [], f"no route available: no registered node covers layer {missing}"
- route.append(best)
- covered_up_to = best.shard_end
- candidates = [n for n in candidates if n is not best]
-
- return route, ""
-
-
-def _coverage_percentage(
- nodes: list[_NodeEntry],
- required_start: int,
- required_end: int,
-) -> float:
- required_layers = required_end - required_start + 1
- if required_layers <= 0:
- return 0.0
-
- intervals = sorted(
- (
- (max(required_start, node.shard_start), min(required_end, node.shard_end))
- for node in nodes
- if node.shard_start is not None
- and node.shard_end is not None
- if node.shard_end >= required_start and node.shard_start <= required_end
- ),
- key=lambda interval: interval[0],
- )
- covered = 0
- covered_until = required_start - 1
- for start, end in intervals:
- if end <= covered_until:
- continue
- next_start = max(start, covered_until + 1)
- if next_start > end:
- continue
- covered += end - next_start + 1
- covered_until = end
- return round((covered / required_layers) * 100, 2)
-
-
-def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
- start = int(preset.get("layers_start", 0))
- if "layers_end" in preset:
- return start, int(preset["layers_end"])
- return start, start + int(preset["total_layers"]) - 1
-
-
-def _preset_bytes_per_layer(preset: dict) -> dict[str, int]:
- raw = preset.get("bytes_per_layer", preset.get("bytes_per_layer_at_quant", {}))
- if isinstance(raw, dict) and raw:
- return {str(quant): int(value) for quant, value in raw.items()}
- return {"bfloat16": 30 * 1024 * 1024}
-
-
-def _node_quantization(node: _NodeEntry, preset: dict) -> str:
- bytes_per_layer = _preset_bytes_per_layer(preset)
- if node.quantization in bytes_per_layer:
- return node.quantization
- for quantization in node.quantizations:
- if quantization in bytes_per_layer:
- return quantization
- return next(iter(bytes_per_layer))
-
-
-def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
- """Return the memory pool used for shard-capacity planning."""
- if node.vram_bytes > 0:
- return node.vram_bytes, "vram"
- if node.ram_bytes > 0:
- return node.ram_bytes, "ram"
- return DEFAULT_RAM_BYTES, "ram-default"
-
-
-def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
- bytes_per_layer = _preset_bytes_per_layer(preset)
- quantization = _node_quantization(node, preset)
- layer_bytes = bytes_per_layer[quantization]
- if layer_bytes <= 0:
- return 0
- memory_budget_bytes, _ = _node_memory_budget_bytes(node)
- return int((memory_budget_bytes * 0.8) // layer_bytes)
-
-
-def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
- """Operator-facing capacity fields for inspection endpoints."""
- memory_budget_bytes, memory_budget_source = _node_memory_budget_bytes(node)
- summary = {
- "vram_bytes": node.vram_bytes,
- "ram_bytes": node.ram_bytes,
- "memory_budget_bytes": memory_budget_bytes,
- "memory_budget_source": memory_budget_source,
- "max_loaded_shards": node.max_loaded_shards,
- "quantizations": list(node.quantizations),
- "quantization": node.quantization,
- "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
- "effective_throughput": round(_effective_throughput(node), 4),
- }
- if preset is not None:
- summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
- return summary
-
-
-def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
- budget, _source = _node_memory_budget_bytes(node)
- if preset is None:
- return int(budget * 0.8)
- return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))
-
-
-def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
- total_vram = sum(max(0, node.vram_bytes) for node in nodes)
- total_ram = sum(max(0, node.ram_bytes) for node in nodes)
- total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
- effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
- return {
- "node_count": len(nodes),
- "total_vram_bytes": total_vram,
- "total_ram_bytes": total_ram,
- "total_memory_budget_bytes": total_budget,
- "effective_assignable_memory_bytes": effective_budget,
- "total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
- "total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
- }
-
-
-def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
- if preset is None:
- return {"recommended": False}
- pool = _pool_summary(nodes, preset)
- required = int(preset.get("required_model_bytes", 0) or 0)
- deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
- missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
- return {
- "recommended": bool(preset.get("recommended", False)),
- "status": preset.get("deployment_status", "available"),
- "required_model_bytes": required or None,
- "download_size_bytes": preset.get("download_size_bytes"),
- "native_quantization": preset.get("native_quantization"),
- "pool": pool,
- "deployable": deployable,
- "missing_effective_memory_bytes": missing,
- }
-
-
-def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
- if total_layers <= 0:
- return 0
- if memory_mb <= 0:
- return max(1, total_layers // 2)
- bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
- return min(
- total_layers,
- max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
- )
-
-
-def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
- metadata: dict = {}
- for node in nodes:
- if node.model_metadata:
- metadata.update(node.model_metadata)
- if "num_layers" not in metadata:
- layers = [node.num_layers for node in nodes if node.num_layers is not None]
- if layers:
- metadata["num_layers"] = max(layers)
- return metadata
-
-
-def _coverage_map(
- nodes: list[_NodeEntry],
- required_start: int,
- required_end: int,
-) -> list[dict]:
- layer_counts = []
- for layer in range(required_start, required_end + 1):
- count = 0
- for node in nodes:
- if node.shard_start is None or node.shard_end is None:
- continue
- if node.shard_start <= layer <= node.shard_end:
- count += 1
- layer_counts.append((layer, count))
-
- coverage: list[dict] = []
- for layer, count in layer_counts:
- if coverage and coverage[-1]["node_count"] == count and coverage[-1]["end_layer"] == layer - 1:
- coverage[-1]["end_layer"] = layer
- else:
- coverage.append({"start_layer": layer, "end_layer": layer, "node_count": count})
- return coverage
-
-
-def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
- """Per-node health detail for the availability map."""
- age = time.monotonic() - node.last_heartbeat
- alive = age <= heartbeat_timeout
- hb_expected = max(1, round(node.uptime_seconds / 20.0)) # assume ~20s interval
- hb_rate = round(min(1.0, node.heartbeats_received / hb_expected), 4) if node.heartbeats_received else 0.0
- total = node.total_requests
- inf_rate = round((total - node.failed_requests) / total, 4) if total > 0 else 1.0
- return {
- "node_id": node.node_id,
- "endpoint": node.endpoint,
- "alive": alive,
- "last_seen_seconds_ago": round(age, 1),
- "status": node.status,
- "queue_depth": node.queue_depth,
- "total_requests": node.total_requests,
- "heartbeat_success_rate": hb_rate,
- "inference_success_rate": inf_rate,
- "capacity": _node_capacity_summary(node),
- }
-
-
-def _coverage_map_detailed(
- nodes: list["_NodeEntry"],
- required_start: int,
- required_end: int,
- heartbeat_timeout: float,
-) -> list[dict]:
- """Like _coverage_map but with per-node identity and health in each band.
-
- Includes all nodes (alive and stale) so operators can see both coverage
- holes and which dead nodes used to fill them.
- """
- now = time.monotonic()
-
- def covers(node: "_NodeEntry", layer: int) -> bool:
- return (
- node.shard_start is not None
- and node.shard_end is not None
- and node.shard_start <= layer <= node.shard_end
- )
-
- # Build per-layer list of nodes (with alive flag)
- layer_nodes: list[list[tuple["_NodeEntry", bool]]] = []
- for layer in range(required_start, required_end + 1):
- ns = [
- (n, (now - n.last_heartbeat) <= heartbeat_timeout)
- for n in nodes
- if covers(n, layer)
- ]
- layer_nodes.append(ns)
-
- # Merge consecutive layers with identical node-set (same node_ids, same alive)
- coverage: list[dict] = []
- for i, layer_idx in enumerate(range(required_start, required_end + 1)):
- ns = layer_nodes[i]
- node_ids = [n.node_id for n, _ in ns]
- alive_flags = [a for _, a in ns]
- # Compare with last band
- if (
- coverage
- and coverage[-1]["end_layer"] == layer_idx - 1
- and [nd["node_id"] for nd in coverage[-1]["nodes"]] == node_ids
- and [nd["alive"] for nd in coverage[-1]["nodes"]] == alive_flags
- ):
- coverage[-1]["end_layer"] = layer_idx
- else:
- coverage.append({
- "start_layer": layer_idx,
- "end_layer": layer_idx,
- "node_count": len(ns),
- "nodes": [_node_health(n, heartbeat_timeout) for n, _ in ns],
- })
- return coverage
-
-
-def _coverage_gaps(coverage: list[dict]) -> list[tuple[int, int]]:
- return [
- (segment["start_layer"], segment["end_layer"])
- for segment in coverage
- if segment["node_count"] == 0
- ]
-
-
-def _relay_http_request(
- relay_addr: str,
- path: str,
- body: bytes,
- headers: dict[str, str],
- timeout: float = 310.0,
-) -> dict | None:
- """Send an HTTP-shaped request through a relay RPC WebSocket."""
- try:
- import websockets.sync.client as wsc # type: ignore[import]
-
- request_id = str(uuid.uuid4())
- with wsc.connect(relay_addr, open_timeout=10, close_timeout=5) as ws:
- ws.send(json.dumps({
- "request_id": request_id,
- "method": "POST",
- "path": path,
- "headers": headers,
- "body": body.decode(errors="replace"),
- }))
- raw = ws.recv(timeout=timeout)
- response = json.loads(raw)
- if response.get("request_id") not in {None, request_id}:
- return None
- return response
- except Exception:
- return None
-
-
-def _find_pinned_route(
- nodes: list[_NodeEntry],
- required_start: int,
- required_end: int,
- hop_count: int,
-) -> list[_NodeEntry] | None:
- """First combination of exactly ``hop_count`` distinct nodes covering the
- layer range, where every node extends coverage (US-030 benchmark routes)."""
- for combo in itertools.permutations(nodes, hop_count):
- covered = required_start - 1
- valid = True
- for candidate in combo:
- if candidate.shard_start is None or candidate.shard_end is None:
- valid = False
- break
- if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
- valid = False
- break
- covered = candidate.shard_end
- if valid and covered >= required_end:
- return list(combo)
- return None
-
-
-def _nodes_and_bounds_for_model(
- server: "_TrackerHTTPServer",
- model: str,
-) -> tuple[list[_NodeEntry], int, int] | None:
- resolved_name, preset = _resolve_model_preset(server.model_presets, model)
- if preset is not None:
- required_start, required_end = _preset_layer_bounds(preset)
- return [
- node for node in server.registry.values()
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- ], required_start, required_end
-
- nodes = [
- node for node in server.registry.values()
- if _node_matches_model(node, model)
- and node.shard_start is not None
- and node.shard_end is not None
- and node.num_layers is not None
- ]
- if not nodes:
- return None
- return nodes, 0, max(node.num_layers for node in nodes) - 1
-
-
-def _load_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
- return {
- "action": "LOAD_SHARD",
- "model": model,
- "start_layer": start,
- "end_layer": end,
- "shard_start": start,
- "shard_end": end,
- "quantization": quantization,
- }
-
-
-def _drop_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
- return {
- "action": "DROP_SHARD",
- "model": model,
- "start_layer": start,
- "end_layer": end,
- "shard_start": start,
- "shard_end": end,
- "quantization": quantization,
- }
-
-
-def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
- now = time.monotonic()
- expired_ids = [
- node_id for node_id, entry in server.registry.items()
- if (now - entry.last_heartbeat) > server.heartbeat_timeout
- ]
- for node_id in expired_ids:
- entry = server.registry.pop(node_id)
- print(
- f"[tracker] node expired: {node_id} {entry.endpoint} "
- f"(no heartbeat for >{server.heartbeat_timeout:.0f}s)",
- flush=True,
- )
- if expired_ids:
- _rebalance_all_locked(server)
- return expired_ids
-
-
-def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
- resolved_name, preset = _resolve_model_preset(server.model_presets, model)
- if preset is None:
- return
- required_start, required_end = _preset_layer_bounds(preset)
- total_layers = required_end - required_start + 1
- model_nodes = [
- node for node in server.registry.values()
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- ]
- managed_nodes = [node for node in model_nodes if node.managed_assignment]
- if not managed_nodes:
- return
-
- previous_ranges = {
- node.node_id: (node.shard_start, node.shard_end, node.quantization)
- for node in managed_nodes
- }
- for node in managed_nodes:
- node.shard_start = None
- node.shard_end = None
-
- managed_nodes.sort(
- key=lambda node: (
- -node.benchmark_tokens_per_sec,
- -_node_layer_capacity(node, preset),
- node.node_id,
- )
- )
- base_nodes = [node for node in model_nodes if not node.managed_assignment]
- coverage = _coverage_map(base_nodes, required_start, required_end)
- gaps = _coverage_gaps(coverage)
- if not gaps:
- gaps = [(required_start, required_end)]
-
- eligible_nodes = [
- node for node in managed_nodes
- if _node_layer_capacity(node, preset) > 0
- ]
- node_index = 0
- for gap_start, gap_end in gaps:
- cursor = gap_start
- while cursor <= gap_end and node_index < len(eligible_nodes):
- node = eligible_nodes[node_index]
- remaining_layers = gap_end - cursor + 1
- remaining_nodes_after = len(eligible_nodes) - node_index - 1
- capacity = min(
- _node_layer_capacity(node, preset),
- total_layers,
- max(1, remaining_layers - remaining_nodes_after),
- )
- if capacity <= 0:
- node_index += 1
- continue
- quantization = _node_quantization(node, preset)
- node.quantization = quantization
- node.shard_start = cursor
- node.shard_end = min(gap_end, cursor + capacity - 1)
- cursor = node.shard_end + 1
- node_index += 1
-
- for node in managed_nodes:
- previous_start, previous_end, previous_quantization = previous_ranges[node.node_id]
- current_range = (node.shard_start, node.shard_end, node.quantization)
- if node.shard_start is None or node.shard_end is None or current_range == previous_ranges[node.node_id]:
- continue
- if previous_start is not None and previous_end is not None:
- node.pending_directives.append(
- _drop_directive(
- node,
- model,
- previous_start,
- previous_end,
- previous_quantization or _node_quantization(node, preset),
- )
- )
- node.pending_directives.append(
- _load_directive(
- node,
- model,
- node.shard_start,
- node.shard_end,
- node.quantization or _node_quantization(node, preset),
- )
- )
-
-
-def _hf_rebalance_preset(nodes: list[_NodeEntry]) -> dict:
- total_layers = max(node.num_layers or 0 for node in nodes)
- return {
- "layers_start": 0,
- "layers_end": total_layers - 1,
- "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
- }
-
-
-def _rebalance_hf_model_locked(server: "_TrackerHTTPServer", hf_repo: str) -> None:
- model_nodes = [
- node for node in server.registry.values()
- if node.hf_repo == hf_repo
- and node.num_layers is not None
- ]
- managed_nodes = [node for node in model_nodes if node.managed_assignment]
- if not model_nodes or not managed_nodes:
- return
-
- preset = _hf_rebalance_preset(model_nodes)
- required_start, required_end = _preset_layer_bounds(preset)
- total_layers = required_end - required_start + 1
- if total_layers <= 0:
- return
-
- previous_ranges = {
- node.node_id: (node.shard_start, node.shard_end, node.quantization)
- for node in managed_nodes
- }
- for node in managed_nodes:
- node.shard_start = None
- node.shard_end = None
-
- managed_nodes.sort(
- key=lambda node: (
- -node.benchmark_tokens_per_sec,
- -_node_layer_capacity(node, preset),
- node.node_id,
- )
- )
- base_nodes = [node for node in model_nodes if not node.managed_assignment]
- coverage = _coverage_map(base_nodes, required_start, required_end)
- gaps = _coverage_gaps(coverage)
- if not gaps:
- gaps = [(required_start, required_end)]
-
- eligible_nodes = [
- node for node in managed_nodes
- if _node_layer_capacity(node, preset) > 0
- ]
- node_index = 0
- for gap_start, gap_end in gaps:
- cursor = gap_start
- while cursor <= gap_end and node_index < len(eligible_nodes):
- node = eligible_nodes[node_index]
- remaining_layers = gap_end - cursor + 1
- remaining_nodes_after = len(eligible_nodes) - node_index - 1
- capacity = min(
- _node_layer_capacity(node, preset),
- total_layers,
- max(1, remaining_layers - remaining_nodes_after),
- )
- if capacity <= 0:
- node_index += 1
- continue
- quantization = _node_quantization(node, preset)
- node.quantization = quantization
- node.shard_start = cursor
- node.shard_end = min(gap_end, cursor + capacity - 1)
- cursor = node.shard_end + 1
- node_index += 1
-
- for node in managed_nodes:
- previous_start, previous_end, previous_quantization = previous_ranges[node.node_id]
- current_range = (node.shard_start, node.shard_end, node.quantization)
- if node.shard_start is None or node.shard_end is None or current_range == previous_ranges[node.node_id]:
- continue
- if previous_start is not None and previous_end is not None:
- node.pending_directives.append(
- _drop_directive(
- node,
- hf_repo,
- previous_start,
- previous_end,
- previous_quantization or _node_quantization(node, preset),
- )
- )
- node.pending_directives.append(
- _load_directive(
- node,
- hf_repo,
- node.shard_start,
- node.shard_end,
- node.quantization or _node_quantization(node, preset),
- )
- )
-
-
-def _rebalance_all_locked(server: "_TrackerHTTPServer") -> None:
- for model in list(server.model_presets):
- _rebalance_model_locked(server, model)
- for hf_repo in sorted({node.hf_repo for node in server.registry.values() if node.hf_repo}):
- _rebalance_hf_model_locked(server, hf_repo)
-
-
-def _api_key_from_headers(headers) -> str | None:
- auth = headers.get("Authorization")
- if not auth:
- return None
- if auth.lower().startswith("bearer "):
- return auth.split(" ", 1)[1].strip() or None
- return auth.strip() or None
-
-
-def _usage_total_tokens(payload: dict) -> int | None:
- usage = payload.get("usage")
- if not isinstance(usage, dict):
- return None
- total = usage.get("total_tokens")
- if isinstance(total, (int, float)):
- return int(total)
- prompt = usage.get("prompt_tokens")
- completion = usage.get("completion_tokens")
- if isinstance(prompt, (int, float)) or isinstance(completion, (int, float)):
- return int(prompt or 0) + int(completion or 0)
- return None
-
-
-def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
- if contracts is None or not wallet_address:
- return None
- if contracts.registry.get_wallet(wallet_address).banned:
- return "wallet is banned"
- return None
-
-
-def _node_id_for_registration(
- endpoint: str,
- model: str,
- wallet_address: str | None,
- shard_start: int | None,
- shard_end: int | None,
- hf_repo: str | None,
-) -> str:
- wallet_prefix = wallet_address[:8] if wallet_address else "anon"
- stable_key = "|".join([
- wallet_address or "",
- endpoint.rstrip("/"),
- model,
- hf_repo or "",
- "" if shard_start is None else str(shard_start),
- "" if shard_end is None else str(shard_end),
- ])
- digest = hashlib.sha256(stable_key.encode()).hexdigest()[:12]
- return f"{wallet_prefix}-{digest}"
-
-
-class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
- daemon_threads = True
-
- def __init__(
- self,
- addr: tuple,
- handler,
- registry: dict,
- lock: threading.Lock,
- heartbeat_timeout: float,
- model_presets: dict,
- contracts: Any | None,
- minimum_stake: int,
- relay_url: str | None = None,
- raft: "RaftNode | None" = None,
- gossip: "NodeGossip | None" = None,
- stats: "_StatsCollector | None" = None,
- billing: "BillingLedger | None" = None,
- accounts: "AccountStore | None" = None,
- benchmark_results_path: str | None = None,
- validator_service_token: str | None = None,
- hive_secret: str | None = None,
- ) -> None:
- super().__init__(addr, handler)
- self.registry = registry
- self.lock = lock
- self.heartbeat_timeout = heartbeat_timeout
- self.model_presets = model_presets
- self.contracts = contracts
- self.minimum_stake = minimum_stake
- self.relay_url = relay_url.rstrip("/") if relay_url else None
- self.raft = raft
- self.gossip = gossip
- self.stats: _StatsCollector | None = stats
- self.billing: BillingLedger | None = billing
- self.accounts: AccountStore | None = accounts
- self.benchmark_results_path = benchmark_results_path or os.path.join(
- os.getcwd(), "benchmark_results.json"
- )
- self.benchmark_lock = threading.Lock()
- self.validator_service_token = validator_service_token
- self.hive_secret = hive_secret
-
-
-class _TrackerHandler(http.server.BaseHTTPRequestHandler):
- def log_message(self, fmt, *args): # noqa: suppress request logs in tests
- pass
-
- def _send_json(self, status: int, data: dict) -> None:
- body = json.dumps(data).encode()
- self.send_response(status)
- self.send_header("Content-Type", "application/json")
- self.send_header("Content-Length", str(len(body)))
- self.end_headers()
- try:
- self.wfile.write(body)
- except BrokenPipeError:
- pass
-
- # ---- unified auth boundary (ADR-0017) ----
-
- def _resolve_identity(self) -> tuple[str | None, dict | None]:
- """Resolve the caller to (role, account).
-
- Roles: "validator" (service token), "admin"/"user" (session token).
- Client API keys resolve to no privileged role — they authorize
- inference and wallet binding only, never operator endpoints.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- token = _api_key_from_headers(self.headers)
- if not token:
- return None, None
- if is_validator_token(token, server.validator_service_token):
- return "validator", None
- if server.accounts is not None:
- account = server.accounts.session_account(token)
- if account is not None:
- return account.get("role", "user"), account
- return None, None
-
- def _require_role(self, *allowed: str) -> bool:
- """Gate a privileged handler; sends 401/403 and returns False on failure.
-
- 401 when no credential was presented; 403 when a credential was
- presented but does not resolve to an allowed role — this covers
- client API keys and garbage bearer strings on operator endpoints.
- """
- role, _account = self._resolve_identity()
- if role in allowed:
- return True
- if _api_key_from_headers(self.headers) is None:
- self._send_json(401, {"error": "authentication required (admin session or service token)"})
- else:
- self._send_json(403, {"error": "this endpoint requires an admin session or service token"})
- return False
-
- def _read_hive_authenticated_body(self) -> dict | None:
- """Read + verify a hive gossip body (HMAC per ADR-0017 §3).
-
- Fails closed: without a configured --hive-secret no gossip is
- accepted. Sends the error response itself and returns None on failure.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- length = int(self.headers.get("Content-Length", 0))
- raw = self.rfile.read(length) if length else b"{}"
- if not verify_hive_request(server.hive_secret, self.headers, raw):
- self._send_json(401, {"error": "valid hive signature required"})
- return None
- try:
- body = json.loads(raw or b"{}")
- except json.JSONDecodeError:
- self._send_json(400, {"error": "invalid JSON body"})
- return None
- if not isinstance(body, dict):
- self._send_json(400, {"error": "JSON body must be an object"})
- return None
- return body
-
- def _read_json_body(self) -> dict | None:
- length = int(self.headers.get("Content-Length", 0))
- try:
- body = json.loads(self.rfile.read(length) or b"{}")
- except json.JSONDecodeError:
- self._send_json(400, {"error": "invalid JSON body"})
- return None
- if not isinstance(body, dict):
- self._send_json(400, {"error": "JSON body must be an object"})
- return None
- return body
-
- def _purge_expired_nodes(self) -> None:
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- _purge_expired_nodes_locked(server)
-
- def do_POST(self):
- if self.path == "/v1/chat/completions":
- self._handle_proxy_chat()
- return
- if self.path == "/v1/nodes/register":
- self._handle_register()
- return
- if self.path == "/v1/raft/vote":
- self._handle_raft_vote()
- return
- if self.path == "/v1/raft/append":
- self._handle_raft_append()
- return
- if self.path == "/v1/gossip":
- self._handle_gossip()
- return
- if self.path == "/v1/stats/gossip":
- self._handle_stats_gossip()
- return
- if self.path == "/v1/billing/gossip":
- self._handle_billing_gossip()
- return
- if self.path == "/v1/billing/forfeit":
- self._handle_billing_forfeit()
- return
- if self.path == "/v1/auth/register":
- self._handle_auth_register()
- return
- if self.path == "/v1/auth/login":
- self._handle_auth_login()
- return
- if self.path == "/v1/auth/logout":
- self._handle_auth_logout()
- return
- if self.path == "/v1/account/keys":
- self._handle_account_key_create()
- return
- if self.path == "/v1/account/keys/revoke":
- self._handle_account_key_revoke()
- return
- if self.path == "/v1/accounts/gossip":
- self._handle_accounts_gossip()
- return
- if self.path == "/v1/benchmark/hop-penalty":
- self._handle_benchmark_hop_penalty()
- return
- if self.path == "/v1/wallet/register":
- self._handle_wallet_register()
- return
- parts = self.path.split("/")
- # /v1/nodes//heartbeat -> ['', 'v1', 'nodes', '', 'heartbeat']
- if len(parts) == 5 and parts[1] == "v1" and parts[2] == "nodes" and parts[4] == "heartbeat":
- self._handle_heartbeat(parts[3])
- return
- self.send_response(404)
- self.end_headers()
-
- def do_GET(self):
- parsed = urllib.parse.urlparse(self.path)
- if parsed.path == "/v1/route":
- self._handle_route(parsed)
- elif parsed.path == "/v1/routes":
- self._handle_routes(parsed)
- elif parsed.path == "/v1/nodes/assign":
- self._handle_assign(parsed)
- elif parsed.path == "/v1/network/assign":
- self._handle_network_assign(parsed)
- elif parsed.path == "/v1/network/map":
- self._handle_network_map()
- elif parsed.path == "/v1/models":
- self._handle_models()
- elif parsed.path.startswith("/v1/coverage/"):
- model = urllib.parse.unquote(parsed.path.removeprefix("/v1/coverage/"))
- self._handle_coverage(model)
- elif parsed.path.startswith("/v1/tracker-nodes/"):
- model = urllib.parse.unquote(parsed.path.removeprefix("/v1/tracker-nodes/"))
- self._handle_tracker_nodes(model)
- elif parsed.path.startswith("/v1/head-workers/"):
- model = urllib.parse.unquote(parsed.path.removeprefix("/v1/head-workers/"))
- self._handle_tracker_nodes(model)
- elif parsed.path == "/v1/raft/status":
- self._handle_raft_status()
- elif parsed.path == "/v1/stats":
- self._handle_stats()
- elif parsed.path == "/v1/billing/summary":
- self._handle_billing_summary()
- elif parsed.path == "/v1/billing/settlements":
- self._handle_billing_settlements()
- elif parsed.path == "/v1/account":
- self._handle_account_me()
- elif parsed.path == "/v1/admin/accounts":
- self._handle_admin_accounts()
- elif parsed.path == "/v1/benchmark/results":
- self._handle_benchmark_results()
- elif parsed.path == "/v1/registry/wallets":
- self._handle_registry_wallets()
- elif parsed.path in ("/dashboard", "/dashboard/"):
- self._handle_dashboard()
- elif parsed.path == "/v1/health":
- self._send_json(200, {"status": "ok"})
- else:
- self.send_response(404)
- self.end_headers()
-
- def _handle_models(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- created = int(time.time())
- with server.lock:
- self._purge_expired_nodes()
- alive = list(server.registry.values())
- if server.contracts is not None:
- alive = [
- node for node in alive
- if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
- ]
- data = []
- seen_ids: set[str] = set()
- for name, preset in server.model_presets.items():
- model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
- if not model_nodes and not preset.get("recommended"):
- continue
- required_start, required_end = _preset_layer_bounds(preset)
- coverage = _coverage_percentage(
- model_nodes,
- required_start,
- required_end,
- )
- aliases = [name]
- hf_repo = preset.get("hf_repo")
- if hf_repo and hf_repo not in aliases:
- aliases.append(hf_repo)
- for alias in preset.get("aliases", []) or []:
- if isinstance(alias, str) and alias not in aliases:
- aliases.append(alias)
- data.append({
- "id": name,
- "object": "model",
- "created": created,
- "owned_by": "meshnet",
- "name": name,
- "hf_repo": hf_repo,
- "aliases": aliases,
- "metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
- "recommended": bool(preset.get("recommended", False)),
- "deployment": _deployment_summary(alive, preset),
- "shard_coverage_percentage": coverage,
- })
- seen_ids.add(name)
- if hf_repo:
- seen_ids.add(hf_repo)
-
- hf_model_ids = sorted({
- node.hf_repo or node.model
- for node in alive
- if node.model is not None
- and node.model not in server.model_presets
- and node.shard_start is not None
- and node.shard_end is not None
- and node.num_layers is not None
- })
- for model_id in hf_model_ids:
- if model_id is None or model_id in seen_ids:
- continue
- model_nodes = [
- node for node in alive
- if node.shard_start is not None
- and node.shard_end is not None
- and node.num_layers is not None
- and (node.hf_repo == model_id or (node.hf_repo is None and node.model == model_id))
- ]
- if not model_nodes:
- continue
- short_names = sorted({node.model for node in model_nodes if node.model})
- aliases = [model_id, *[name for name in short_names if name != model_id]]
- required_start = 0
- required_end = max(node.num_layers for node in model_nodes) - 1
- data.append({
- "id": model_id,
- "object": "model",
- "created": created,
- "owned_by": "meshnet",
- "name": short_names[0] if short_names else model_id,
- "hf_repo": model_id if any(node.hf_repo == model_id for node in model_nodes) else None,
- "aliases": aliases,
- "metadata": _model_metadata_from_nodes(model_nodes),
- "shard_coverage_percentage": _coverage_percentage(
- model_nodes,
- required_start,
- required_end,
- ),
- })
- seen_ids.add(model_id)
- self._send_json(200, {"object": "list", "data": data})
-
- def _handle_coverage(self, model: str):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- # Do NOT purge before coverage — dead nodes are included with alive=false
- # so operators can see what was covering each layer band before failure.
- with server.lock:
- resolved = _nodes_and_bounds_for_model(server, model)
- if resolved is None:
- self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
- return
- all_nodes, required_start, required_end = resolved
- if server.contracts is not None:
- all_nodes = [
- node for node in all_nodes
- if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
- ]
- coverage = _coverage_map_detailed(all_nodes, required_start, required_end, server.heartbeat_timeout)
- self._send_json(200, {"model": model, "coverage": coverage})
-
- def _handle_tracker_nodes(self, model: str):
- """Return head workers: worker nodes that can start inference for a model.
-
- The historical endpoint name is /v1/tracker-nodes, but these are not
- tracker processes and they are the only machines that load model shards.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- resolved_name, preset = _resolve_model_preset(server.model_presets, model)
- if preset is None:
- self._send_json(404, {"error": f"unknown model preset: {model!r}"})
- return
- required_start, _ = _preset_layer_bounds(preset)
- with server.lock:
- self._purge_expired_nodes()
- alive = [
- node for node in server.registry.values()
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- ]
- if server.contracts is not None:
- alive = [
- node for node in alive
- if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
- ]
- tracker_nodes = [
- node for node in alive
- if node.shard_start is not None
- and node.shard_start == required_start
- and node.tracker_mode
- ]
- self._send_json(200, {
- "model": resolved_name,
- "head_workers": [
- {
- "node_id": node.node_id,
- "endpoint": node.endpoint,
- "relay_addr": node.relay_addr,
- "peer_id": node.peer_id,
- }
- for node in tracker_nodes
- ],
- "tracker_nodes": [
- {
- "node_id": node.node_id,
- "endpoint": node.endpoint,
- "relay_addr": node.relay_addr,
- "peer_id": node.peer_id,
- "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
- }
- for node in tracker_nodes
- ],
- })
-
- def _handle_network_map(self) -> None:
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- with server.lock:
- self._purge_expired_nodes()
- nodes = list(server.registry.values())
-
- def capacity_for(node: _NodeEntry) -> dict:
- preset = None
- if node.model:
- preset = server.model_presets.get(node.model)
- if preset is None and node.hf_repo and node.num_layers:
- preset = _hf_rebalance_preset([node])
- return _node_capacity_summary(node, preset)
-
- def throughput_for(node: _NodeEntry) -> dict:
- if server.stats is None:
- return {}
- models = [m for m in (node.hf_repo, node.model) if m]
- result = {}
- for model in models:
- result[model] = server.stats.get_node_model_stats(node.node_id, model)
- return result
-
- self._send_json(200, {
- "relay_url": server.relay_url,
- "pool": _pool_summary(nodes),
- "recommended_models": [
- {
- "id": name,
- "hf_repo": preset.get("hf_repo"),
- "aliases": list(preset.get("aliases", []) or []),
- "metadata": dict(preset.get("metadata") or {}),
- "deployment": _deployment_summary(nodes, preset),
- }
- for name, preset in server.model_presets.items()
- if preset.get("recommended")
- ],
- "nodes": [
- {
- "node_id": node.node_id,
- "endpoint": node.endpoint,
- "relay_addr": node.relay_addr,
- "peer_id": node.peer_id,
- "model": node.model,
- "hf_repo": node.hf_repo,
- "num_layers": node.num_layers,
- "model_metadata": dict(node.model_metadata),
- "shard_start": node.shard_start,
- "shard_end": node.shard_end,
- "tracker_mode": node.tracker_mode,
- "last_heartbeat": node.last_heartbeat,
- "capacity": capacity_for(node),
- "throughput": throughput_for(node),
- "stats": _node_health(node, server.heartbeat_timeout),
- }
- for node in nodes
- ],
- })
-
- # ---------------------------------------------------------------- OpenAI proxy
-
- def _handle_proxy_chat(self) -> None:
- """Proxy POST /v1/chat/completions to a tracker-mode (first-shard) node.
-
- Picks a live tracker-mode node for the requested model using round-robin,
- then forwards the request verbatim and relays the response (including
- streaming SSE chunks) back to the caller.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
-
- length = int(self.headers.get("Content-Length", 0))
- raw_body = self.rfile.read(length) if length else b"{}"
-
- try:
- body = json.loads(raw_body)
- except json.JSONDecodeError:
- self._send_json(400, {"error": {"message": "invalid JSON", "type": "invalid_request_error", "code": "invalid_request"}})
- return
-
- model: str = body.get("model", "")
- is_stream: bool = bool(body.get("stream", False))
-
- if model and server.stats is not None:
- server.stats.record_request(model)
-
- # Billing gate (ADR-0015): reject before any routing — no free work.
- api_key = _api_key_from_headers(self.headers)
- if server.billing is not None:
- if api_key is None:
- self._send_json(401, {"error": {
- "message": "missing API key: send Authorization: Bearer ",
- "type": "invalid_request_error",
- "code": "missing_api_key",
- }})
- return
- if server.accounts is not None and server.accounts.is_key_revoked(api_key):
- self._send_json(401, {"error": {
- "message": "API key has been revoked",
- "type": "invalid_request_error",
- "code": "invalid_api_key",
- }})
- return
- if not server.billing.has_funds(api_key):
- self._send_json(402, {"error": {
- "message": "insufficient balance: deposit USDT to continue",
- "type": "insufficient_quota",
- "code": "insufficient_balance",
- }})
- return
-
- # US-030: optional pinned route — "route": [node_id, ...] uses those
- # nodes in order instead of auto-selection. Absent field: unchanged.
- pinned_ids = body.get("route")
- pinned_nodes: list[_NodeEntry] | None = None
- if pinned_ids is not None:
- if (
- not isinstance(pinned_ids, list)
- or not pinned_ids
- or not all(isinstance(nid, str) and nid for nid in pinned_ids)
- ):
- self._send_json(400, {"error": {
- "message": "route must be a non-empty list of node id strings",
- "type": "invalid_request_error",
- "code": "invalid_route",
- }})
- return
- with server.lock:
- self._purge_expired_nodes()
- missing = [nid for nid in pinned_ids if nid not in server.registry]
- if missing:
- self._send_json(400, {"error": {
- "message": f"unknown node ids in route: {missing}",
- "type": "invalid_request_error",
- "code": "unknown_route_nodes",
- }})
- return
- pinned_nodes = [server.registry[nid] for nid in pinned_ids]
-
- if pinned_nodes is not None:
- node = pinned_nodes[0]
- else:
- # Find a live tracker-mode node for this model
- with server.lock:
- self._purge_expired_nodes()
- candidates = [
- n for n in server.registry.values()
- if n.tracker_mode and _node_matches_model(n, model)
- ]
-
- if not candidates:
- # Fall back: any node serving shard_start=0 for this model
- with server.lock:
- candidates = [
- n for n in server.registry.values()
- if n.shard_start == 0 and _node_matches_model(n, model)
- ]
-
- if not candidates:
- self._send_json(503, {"error": {
- "message": f"no nodes available for model {model!r}",
- "type": "service_unavailable",
- "code": "model_not_available",
- }})
- return
-
- node = max(candidates, key=lambda n: _effective_throughput(n, model))
- target_url = f"{node.endpoint}/v1/chat/completions"
- request_id = str(body.get("id") or f"req-{time.time_ns():x}")
-
- # Pre-resolve the downstream route so the first-shard node skips its own
- # tracker query. We already hold the full registry picture — no need for
- # a second round-trip.
- route_model = node.hf_repo or node.model or model
- with server.lock:
- if route_model in server.model_presets:
- preset = server.model_presets[route_model]
- rs, re = _preset_layer_bounds(preset)
- all_nodes: list = [n for n in server.registry.values() if n.model == route_model]
- else:
- all_nodes = [
- n for n in server.registry.values()
- if _node_matches_model(n, route_model)
- and n.shard_start is not None and n.num_layers is not None
- ]
- rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
- if pinned_nodes is not None:
- route_nodes = pinned_nodes
- else:
- route_nodes, _ = _select_route(all_nodes, rs, re, model=route_model)
- # Compute start_layer for each hop: each node begins where the previous ended + 1.
- # This allows overlapping shard registrations without double-computation.
- covered_up_to = rs - 1
- route_hops: list[dict] = []
- node_work: list[tuple[str | None, int]] = []
- for rn in route_nodes:
- hop: dict = {"endpoint": rn.endpoint, "start_layer": covered_up_to + 1}
- if rn.relay_addr:
- hop["relay_addr"] = rn.relay_addr
- route_hops.append(hop)
- effective_end = rn.shard_end if rn.shard_end is not None else covered_up_to
- node_work.append((rn.wallet_address, max(0, effective_end - covered_up_to)))
- covered_up_to = effective_end
- # Strip the first-shard node we're about to proxy to — it's already handling the request.
- downstream_hops = [h for h in route_hops if h["endpoint"].rstrip("/") != node.endpoint.rstrip("/")]
- downstream_urls = json.dumps(downstream_hops)
- route_debug = " -> ".join(
- f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
- for n in route_nodes
- )
- print(
- f"[tracker] proxy route {request_id}: model={model!r} "
- f"head={node.node_id}@{node.endpoint} downstream={downstream_urls} "
- f"route={route_debug or ''}",
- flush=True,
- )
-
- req = urllib.request.Request(
- target_url,
- data=raw_body,
- headers={
- "Content-Type": "application/json",
- "X-Meshnet-Route": downstream_urls,
- },
- method="POST",
- )
- # Copy Authorization header from client if present
- auth = self.headers.get("Authorization")
- if auth:
- req.add_header("Authorization", auth)
-
- relay_headers = {
- "Content-Type": "application/json",
- "X-Meshnet-Route": downstream_urls,
- **({"Authorization": auth} if auth else {}),
- }
-
- if node.relay_addr:
- print(
- f"[tracker] proxy via relay {request_id}: {node.relay_addr} "
- f"(direct endpoint {target_url})",
- flush=True,
- )
- started = time.monotonic()
- relayed = _relay_http_request(
- node.relay_addr,
- path="/v1/chat/completions",
- body=raw_body,
- headers=relay_headers,
- )
- elapsed = time.monotonic() - started
- if relayed is not None:
- self._send_relayed_response(relayed)
- if int(relayed.get("status", 503)) < 400:
- body_text = relayed.get("body") or ""
- try:
- tokens = _usage_total_tokens(json.loads(body_text)) or 0
- except (json.JSONDecodeError, TypeError):
- tokens = 0
- self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
- self._bill_completed(api_key, model, tokens, node_work)
- return
- print(
- f"[tracker] relay proxy failed {request_id}: {node.relay_addr}; "
- f"trying direct {target_url}",
- flush=True,
- )
-
- try:
- started = time.monotonic()
- upstream = urllib.request.urlopen(req, timeout=300.0)
- print(f"[tracker] proxy connected {request_id}: {target_url}", flush=True)
- except urllib.error.HTTPError as exc:
- # Relay error status + body from node
- err_body = exc.read()
- self.send_response(exc.code)
- self.send_header("Content-Type", "application/json")
- self.send_header("Content-Length", str(len(err_body)))
- self.end_headers()
- try:
- self.wfile.write(err_body)
- except BrokenPipeError:
- pass
- return
- except Exception as exc:
- if node.relay_addr:
- print(
- f"[tracker] direct proxy failed {request_id}: {target_url}: {exc}; "
- f"relay already attempted for {node.relay_addr}",
- flush=True,
- )
- else:
- print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True)
- self._send_json(503, {"error": {
- "message": f"upstream node unreachable: {exc}",
- "type": "service_unavailable",
- "code": "upstream_error",
- }})
- return
-
- with upstream:
- content_type = upstream.headers.get("Content-Type", "application/json")
- if is_stream or "text/event-stream" in content_type:
- # Relay SSE stream chunk-by-chunk
- self.send_response(200)
- self.send_header("Content-Type", "text/event-stream; charset=utf-8")
- self.send_header("Cache-Control", "no-cache")
- self.end_headers()
- stream_tokens: int | None = None
- chunk_count = 0
- try:
- while True:
- line = upstream.readline()
- if not line:
- break
- self.wfile.write(line)
- self.wfile.flush()
- if line.startswith(b"data:"):
- payload = line[5:].strip()
- if payload and payload != b"[DONE]":
- chunk_count += 1
- if b'"usage"' in payload:
- try:
- found = _usage_total_tokens(json.loads(payload))
- if found:
- stream_tokens = found
- except json.JSONDecodeError:
- pass
- except BrokenPipeError:
- pass
- elapsed = time.monotonic() - started
- # Bill even on client disconnect — the nodes did the work.
- # Chunk count approximates generated tokens when the stream
- # carries no usage record.
- observed_tokens = stream_tokens if stream_tokens is not None else chunk_count
- self._record_observed_throughput(model, route_model, observed_tokens, elapsed, route_nodes)
- self._bill_completed(
- api_key, model,
- observed_tokens,
- node_work,
- )
- else:
- # Non-streaming: buffer and relay
- resp_body = upstream.read()
- elapsed = time.monotonic() - started
- print(
- f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
- flush=True,
- )
- self.send_response(200)
- self.send_header("Content-Type", content_type)
- self.send_header("Content-Length", str(len(resp_body)))
- self.end_headers()
- try:
- self.wfile.write(resp_body)
- except BrokenPipeError:
- pass
- try:
- tokens = _usage_total_tokens(json.loads(resp_body)) or 0
- except json.JSONDecodeError:
- tokens = 0
- self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
- self._bill_completed(api_key, model, tokens, node_work)
-
- def _record_observed_throughput(
- self,
- requested_model: str,
- route_model: str,
- total_tokens: int,
- elapsed_seconds: float,
- route_nodes: list[_NodeEntry],
- ) -> None:
- """Record observed route TPS for participating nodes.
-
- The tracker sees end-to-end request duration, not per-hop timings, so
- each hop gets the same route-level observation for now. Per-hop telemetry
- can refine this later without changing the external stats shape.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.stats is None or total_tokens <= 0 or elapsed_seconds <= 0:
- return
- models = [m for m in (requested_model, route_model) if m]
- if len(models) == 2 and models[0] == models[1]:
- models = [models[0]]
- for node in route_nodes:
- for model in models:
- server.stats.record_node_throughput(
- node.node_id,
- model,
- total_tokens=total_tokens,
- elapsed_seconds=elapsed_seconds,
- )
- stats = server.stats.get_node_model_stats(node.node_id, model)
- observed = stats.get("tokens_per_sec_last_hour")
- if observed is not None:
- node.model_tokens_per_sec[model] = float(observed)
-
- def _bill_completed(
- self,
- api_key: str | None,
- model: str,
- total_tokens: int,
- node_work: list[tuple[str | None, int]],
- ) -> None:
- """Charge a completed request against the billing ledger (ADR-0015)."""
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.billing is None or api_key is None:
- return
- # Probationary period (issue 08): a wallet's first N jobs earn nothing —
- # its share stays in the protocol cut. Job counts live in the registry
- # contract, so this only applies when the tracker runs with contracts.
- if server.contracts is not None:
- adjusted: list[tuple[str | None, int]] = []
- for wallet, work in node_work:
- if wallet:
- in_probation = server.contracts.registry.probationary_jobs_remaining(wallet) > 0
- server.contracts.registry.record_completed_job(wallet)
- if in_probation:
- wallet = None
- adjusted.append((wallet, work))
- node_work = adjusted
- try:
- event = server.billing.charge_request(api_key, model, total_tokens, node_work)
- print(
- f"[tracker] billed api_key=…{api_key[-6:]}: model={model!r} "
- f"tokens={total_tokens} cost={event['cost']:.6f} USDT "
- f"shares={event['shares']}",
- flush=True,
- )
- except Exception as exc:
- print(f"[tracker] billing failed for model={model!r}: {exc}", flush=True)
-
- def _send_relayed_response(self, response: dict) -> None:
- status = int(response.get("status", 503))
- headers = response.get("headers") if isinstance(response.get("headers"), dict) else {}
- body_text = response.get("body") or ""
- body = body_text.encode() if isinstance(body_text, str) else bytes(body_text)
- self.send_response(status)
- self.send_header("Content-Type", headers.get("Content-Type", "application/json"))
- self.send_header("Content-Length", str(len(body)))
- self.end_headers()
- try:
- self.wfile.write(body)
- except BrokenPipeError:
- pass
-
- def _handle_register(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_json_body()
- if body is None:
- return
-
- # --- Raft cluster mode: forward to leader or propose via Raft ---
- if server.raft is not None:
- if not server.raft.is_leader:
- leader = server.raft.leader()
- if leader is None:
- self._send_json(503, {"error": "no leader elected — retry in a moment"})
- return
- # Proxy to leader
- try:
- data = json.dumps(body).encode()
- req = urllib.request.Request(
- f"{leader}/v1/nodes/register",
- data=data,
- headers={"Content-Type": "application/json"},
- method="POST",
- )
- with urllib.request.urlopen(req, timeout=5.0) as r:
- resp_body = r.read()
- self.send_response(200)
- self.send_header("Content-Type", "application/json")
- self.send_header("Content-Length", str(len(resp_body)))
- self.end_headers()
- self.wfile.write(resp_body)
- except Exception as exc:
- self._send_json(503, {"error": f"leader proxy failed: {exc}"})
- return
- # Leader path: fall through to normal registration, then replicate via Raft.
- # We let the registration run first (to generate node_id), then asynchronously
- # propose to the Raft log so followers can replicate the entry.
- # Raft proposal happens after the response is sent (fire-and-forget replication).
-
- endpoint = body.get("endpoint")
- if not isinstance(endpoint, str) or not endpoint:
- self._send_json(400, {"error": "endpoint is required"})
- return
- parsed_endpoint = urllib.parse.urlparse(endpoint)
- if parsed_endpoint.scheme not in {"http", "https"} or not parsed_endpoint.netloc:
- self._send_json(400, {"error": "endpoint must be an http(s) URL"})
- return
-
- shard_start: int | None
- shard_end: int | None
- explicit_shard = "shard_start" in body or "shard_end" in body
- if explicit_shard:
- try:
- shard_start = int(body["shard_start"])
- shard_end = int(body["shard_end"])
- except (KeyError, TypeError, ValueError):
- self._send_json(400, {"error": "shard_start and shard_end must be numeric"})
- return
- if shard_start < 0 or shard_end < 0 or shard_start > shard_end:
- self._send_json(400, {"error": "shard range must be non-negative and ordered"})
- return
- else:
- shard_start = None
- shard_end = None
- try:
- score = float(body.get("score", 1.0))
- except (TypeError, ValueError):
- self._send_json(400, {"error": "score must be numeric"})
- return
-
- hardware_profile = body.get("hardware_profile", {})
- if not isinstance(hardware_profile, dict):
- self._send_json(400, {"error": "hardware_profile must be an object"})
- return
- model = body.get("model")
- if model is None:
- model = "stub-model"
- if not isinstance(model, str):
- self._send_json(400, {"error": "model must be a string"})
- return
- shard_checksum = body.get("shard_checksum")
- if shard_checksum is not None and not isinstance(shard_checksum, str):
- self._send_json(400, {"error": "shard_checksum must be a string"})
- return
- try:
- vram_bytes = int(body.get("vram_bytes", DEFAULT_VRAM_BYTES))
- ram_bytes = int(body.get("ram_bytes", DEFAULT_RAM_BYTES))
- max_loaded_shards = int(body.get("max_loaded_shards", 1))
- benchmark_tokens_per_sec = float(
- body.get("benchmark_tokens_per_sec", DEFAULT_BENCHMARK_TOKENS_PER_SEC)
- )
- except (TypeError, ValueError):
- self._send_json(400, {"error": "vram_bytes, ram_bytes, max_loaded_shards, and benchmark_tokens_per_sec must be numeric"})
- return
- if vram_bytes < 0 or ram_bytes < 0 or max_loaded_shards < 1 or benchmark_tokens_per_sec <= 0:
- self._send_json(400, {"error": "capability values must be positive"})
- return
- quantizations_body = body.get("quantizations", DEFAULT_QUANTIZATIONS)
- if not (
- isinstance(quantizations_body, list)
- and quantizations_body
- and all(isinstance(item, str) and item for item in quantizations_body)
- ):
- self._send_json(400, {"error": "quantizations must be a non-empty string array"})
- return
- quantizations = list(quantizations_body)
- quantization = body.get("quantization")
- if quantization is not None and not isinstance(quantization, str):
- self._send_json(400, {"error": "quantization must be a string"})
- return
- wallet_address = body.get("wallet_address")
- if wallet_address is not None and not isinstance(wallet_address, str):
- self._send_json(400, {"error": "wallet_address must be a string"})
- return
- ban_error = _registration_ban_error(server.contracts, wallet_address)
- if ban_error:
- self._send_json(403, {"error": ban_error})
- return
-
- tracker_mode = bool(body.get("tracker_mode", False))
- managed_assignment = bool(body.get("managed_assignment", False))
- hf_repo = body.get("hf_repo")
- if hf_repo is not None and not isinstance(hf_repo, str):
- self._send_json(400, {"error": "hf_repo must be a string"})
- return
- num_layers_body = body.get("num_layers")
- num_layers: int | None = None
- if num_layers_body is not None:
- try:
- num_layers = int(num_layers_body)
- except (TypeError, ValueError):
- self._send_json(400, {"error": "num_layers must be an integer"})
- return
- model_metadata = body.get("model_metadata", {})
- if model_metadata is None:
- model_metadata = {}
- if not isinstance(model_metadata, dict):
- self._send_json(400, {"error": "model_metadata must be an object"})
- return
- relay_addr = body.get("relay_addr") or None
- cert_fingerprint = body.get("cert_fingerprint") or None
- peer_id = body.get("peer_id") or None
-
- node_id = _node_id_for_registration(
- endpoint,
- model,
- wallet_address,
- shard_start,
- shard_end,
- hf_repo,
- )
- entry = _NodeEntry(
- node_id=node_id,
- endpoint=endpoint.rstrip("/"),
- shard_start=shard_start,
- shard_end=shard_end,
- model=model,
- shard_checksum=shard_checksum,
- hardware_profile=hardware_profile,
- wallet_address=wallet_address,
- score=score,
- vram_bytes=vram_bytes,
- ram_bytes=ram_bytes,
- quantizations=quantizations,
- max_loaded_shards=max_loaded_shards,
- benchmark_tokens_per_sec=benchmark_tokens_per_sec,
- quantization=quantization,
- managed_assignment=managed_assignment or not explicit_shard,
- tracker_mode=tracker_mode,
- hf_repo=hf_repo,
- num_layers=num_layers,
- model_metadata=model_metadata,
- relay_addr=relay_addr,
- cert_fingerprint=cert_fingerprint,
- peer_id=peer_id,
- )
- with server.lock:
- self._purge_expired_nodes()
- # Dedup: if this endpoint is already registered, remove the old entry first.
- stale_ids = [
- eid for eid, e in server.registry.items()
- if e.endpoint == entry.endpoint.rstrip("/")
- ]
- for eid in stale_ids:
- old = server.registry.pop(eid)
- print(
- f"[tracker] node re-registered: replaced {eid} with {node_id}"
- f" {old.endpoint}",
- flush=True,
- )
- server.registry[node_id] = entry
- if entry.managed_assignment:
- if entry.hf_repo:
- _rebalance_hf_model_locked(server, entry.hf_repo)
- else:
- _rebalance_model_locked(server, model)
- assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
- if assignment_directive is not None:
- entry.pending_directives.clear()
-
- shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
- repo_info = f" [{hf_repo}]" if hf_repo else ""
- budget_bytes, budget_source = _node_memory_budget_bytes(entry)
- budget_gb = budget_bytes / (1024 ** 3)
- print(
- f"[tracker] node registered: {node_id} {endpoint} {model}{repo_info} {shard_info} "
- f"capacity={budget_gb:.1f}GB {budget_source} slots={max_loaded_shards}",
- flush=True,
- )
-
- payload = {"node_id": node_id}
- if assignment_directive is not None:
- payload["directive"] = assignment_directive
- self._send_json(200, payload)
-
- # Raft replication: leader proposes this registration to followers so their
- # registries stay in sync. Fire-and-forget (async) — the client already
- # got its node_id; replication happens in the background.
- if server.raft is not None and server.raft.is_leader:
- raft_payload = dict(body)
- raft_payload["node_id"] = node_id # include the generated ID
- threading.Thread(
- target=server.raft.propose,
- args=("register", raft_payload),
- daemon=True,
- ).start()
-
- def _handle_heartbeat(self, node_id: str):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body: dict = {}
- content_length = int(self.headers.get("Content-Length", 0))
- if content_length > 0:
- try:
- body = json.loads(self.rfile.read(content_length))
- except Exception:
- pass
- with server.lock:
- self._purge_expired_nodes()
- entry = server.registry.get(node_id)
- if entry is None:
- self._send_json(404, {"error": "node not found"})
- return
- entry.last_heartbeat = time.monotonic()
- entry.heartbeats_received += 1
- # P2P metadata
- if body.get("relay_addr"):
- entry.relay_addr = body["relay_addr"]
- if body.get("cert_fingerprint"):
- entry.cert_fingerprint = body["cert_fingerprint"]
- if body.get("peer_id"):
- entry.peer_id = body["peer_id"]
- # Node stats (cumulative — node always sends totals, tracker replaces)
- if "total_requests" in body:
- entry.total_requests = int(body["total_requests"])
- if "failed_requests" in body:
- entry.failed_requests = int(body["failed_requests"])
- if "queue_depth" in body:
- entry.queue_depth = int(body["queue_depth"])
- if "uptime_seconds" in body:
- entry.uptime_seconds = float(body["uptime_seconds"])
- if "status" in body and body["status"] in ("ready", "loading"):
- entry.status = body["status"]
- if entry.hf_repo:
- _rebalance_hf_model_locked(server, entry.hf_repo)
- else:
- _rebalance_model_locked(server, entry.model or "stub-model")
- directives = list(entry.pending_directives)
- entry.pending_directives.clear()
- new_assignment = entry.pending_new_assignment
- if new_assignment is not None:
- entry.pending_new_assignment = None
- if server.gossip is not None:
- server.gossip.record(node_id)
- resp: dict = {}
- if directives:
- resp["directives"] = directives
- if new_assignment is not None:
- resp["new_assignment"] = new_assignment
- self._send_json(200, resp)
-
- # ---------------------------------------------------------------- Raft handlers
-
- def _handle_raft_vote(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_json_body()
- if body is None:
- return
- if server.raft is None:
- self._send_json(503, {"error": "raft not enabled"})
- return
- result = server.raft.handle_request_vote(body)
- self._send_json(200, result)
-
- def _handle_raft_append(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_json_body()
- if body is None:
- return
- if server.raft is None:
- self._send_json(503, {"error": "raft not enabled"})
- return
- result = server.raft.handle_append_entries(body)
- self._send_json(200, result)
-
- def _handle_raft_status(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.raft is None:
- self._send_json(200, {"role": "standalone", "term": 0, "leader": None})
- return
- self._send_json(200, server.raft.status())
-
- def _handle_gossip(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_json_body()
- if body is None:
- return
- if server.gossip is not None and isinstance(body, dict):
- server.gossip.merge({k: float(v) for k, v in body.items()})
- self._send_json(200, {})
-
- def _handle_stats(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.stats is None:
- self._send_json(200, {"models": {}, "nodes": {}})
- return
- self._send_json(200, {
- "models": server.stats.get_combined_stats(),
- "nodes": server.stats.get_node_throughput_stats(),
- })
-
- def _handle_stats_gossip(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_hive_authenticated_body()
- if body is None:
- return
- tracker_url = body.get("tracker_url", "")
- rpms = body.get("stats", {})
- if server.stats is not None and tracker_url and isinstance(rpms, dict):
- server.stats.merge_peer_rpms(tracker_url, rpms)
- self._send_json(200, {})
-
- def _handle_billing_summary(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if not self._require_role("admin"):
- return
- if server.billing is None:
- self._send_json(404, {"error": "billing is not enabled on this tracker"})
- return
- self._send_json(200, server.billing.snapshot())
-
- def _handle_dashboard(self):
- """Serve the read-only web dashboard (US-035). Any tracker in the
- mesh — leader or follower — serves it from its replicated state."""
- try:
- html = files("meshnet_tracker").joinpath("dashboard.html").read_text()
- except (FileNotFoundError, OSError):
- self._send_json(404, {"error": "dashboard asset missing"})
- return
- body = html.encode()
- self.send_response(200)
- self.send_header("Content-Type", "text/html; charset=utf-8")
- self.send_header("Content-Length", str(len(body)))
- self.end_headers()
- try:
- self.wfile.write(body)
- except BrokenPipeError:
- pass
-
- def _handle_registry_wallets(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if not self._require_role("admin"):
- return
- if server.contracts is None:
- self._send_json(200, {"wallets": {}})
- return
- wallets = server.contracts.registry.list_wallets()
- self._send_json(200, {"wallets": {
- wallet: {
- "stake_balance": state.stake_balance,
- "strike_count": state.strike_count,
- "banned": state.banned,
- "completed_jobs": state.completed_job_count,
- }
- for wallet, state in wallets.items()
- }})
-
- def _handle_billing_settlements(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if not self._require_role("admin"):
- return
- if server.billing is None:
- self._send_json(404, {"error": "billing is not enabled on this tracker"})
- return
- self._send_json(200, {"settlements": server.billing.settlement_history()})
-
- def _handle_billing_gossip(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_hive_authenticated_body()
- if body is None:
- return
- if server.billing is None:
- self._send_json(200, {"applied": 0})
- return
- events = body.get("events")
- if not isinstance(events, list):
- self._send_json(400, {"error": "events must be a list"})
- return
- applied = server.billing.apply_events([e for e in events if isinstance(e, dict)])
- self._send_json(200, {"applied": applied})
-
- def _handle_billing_forfeit(self):
- """Privileged: forfeit a node's pending balance + record a strike (US-034).
-
- ADR-0017 §4: validator service token or admin session only. Client
- API keys — valid or not — are rejected.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if not self._require_role("validator", "admin"):
- return
- if server.billing is None:
- self._send_json(404, {"error": "billing is not enabled on this tracker"})
- return
- body = self._read_json_body()
- if body is None:
- return
- wallet = body.get("wallet")
- if not wallet or not isinstance(wallet, str):
- self._send_json(400, {"error": "wallet is required"})
- return
- reason = body.get("reason") or "fraud"
- event = server.billing.forfeit_pending(wallet, reason=str(reason))
- strike_state: dict = {}
- if server.contracts is not None:
- server.contracts.registry.record_strike(wallet)
- wallet_state = server.contracts.registry.get_wallet(wallet)
- strike_state = {
- "strike_count": wallet_state.strike_count,
- "banned": wallet_state.banned,
- }
- print(
- f"[tracker] forfeited pending balance of {wallet}: "
- f"{event['amount']:.6f} USDT ({reason}) strikes={strike_state.get('strike_count', 'n/a')}",
- flush=True,
- )
- self._send_json(200, {"forfeited": event["amount"], **strike_state})
-
- # ---- user accounts (registration, login, API keys) ----
-
- def _session_account(self) -> dict | None:
- """Resolve the session token in the Authorization header, or None."""
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.accounts is None:
- return None
- return server.accounts.session_account(_api_key_from_headers(self.headers))
-
- def _require_accounts(self) -> "AccountStore | None":
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.accounts is None:
- self._send_json(404, {"error": "accounts are not enabled on this tracker"})
- return None
- return server.accounts
-
- def _handle_auth_register(self):
- accounts = self._require_accounts()
- if accounts is None:
- return
- body = self._read_json_body()
- if body is None:
- return
- try:
- account = accounts.register(
- email=body.get("email"),
- wallet=body.get("wallet"),
- password=str(body.get("password") or ""),
- )
- except ValueError as exc:
- self._send_json(400, {"error": str(exc)})
- return
- token = accounts.create_session(account["account_id"])
- api_key = accounts.create_api_key(account["account_id"])
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.billing is not None:
- server.billing.ensure_client(api_key) # grant Caller Credit up front
- print(
- f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
- f"role={account['role']}", flush=True,
- )
- self._send_json(200, {"account": account, "session_token": token, "api_key": api_key})
-
- def _handle_auth_login(self):
- accounts = self._require_accounts()
- if accounts is None:
- return
- body = self._read_json_body()
- if body is None:
- return
- identifier = body.get("identifier") or body.get("email") or body.get("wallet") or ""
- account = accounts.verify_login(str(identifier), str(body.get("password") or ""))
- if account is None:
- self._send_json(401, {"error": "invalid credentials"})
- return
- token = accounts.create_session(account["account_id"])
- self._send_json(200, {"account": account, "session_token": token})
-
- def _handle_auth_logout(self):
- accounts = self._require_accounts()
- if accounts is None:
- return
- accounts.destroy_session(_api_key_from_headers(self.headers))
- self._send_json(200, {"ok": True})
-
- def _handle_account_me(self):
- """Balance, usage, and API keys for the logged-in account."""
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if self._require_accounts() is None:
- return
- account = self._session_account()
- if account is None:
- self._send_json(401, {"error": "login required"})
- return
- keys = server.accounts.keys_for(account["account_id"]) # type: ignore[union-attr]
- balances = {}
- usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": []}
- if server.billing is not None:
- balances = {key: server.billing.get_client_balance(key) for key in keys}
- usage = server.billing.usage_for(keys)
- self._send_json(200, {
- "account": account,
- "api_keys": keys,
- "balances": balances,
- "total_balance": sum(balances.values()),
- "usage": usage,
- })
-
- def _handle_account_key_create(self):
- accounts = self._require_accounts()
- if accounts is None:
- return
- account = self._session_account()
- if account is None:
- self._send_json(401, {"error": "login required"})
- return
- api_key = accounts.create_api_key(account["account_id"])
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if server.billing is not None:
- server.billing.ensure_client(api_key)
- self._send_json(200, {"api_key": api_key})
-
- def _handle_account_key_revoke(self):
- accounts = self._require_accounts()
- if accounts is None:
- return
- account = self._session_account()
- if account is None:
- self._send_json(401, {"error": "login required"})
- return
- body = self._read_json_body()
- if body is None:
- return
- api_key = body.get("api_key")
- if not api_key or not isinstance(api_key, str):
- self._send_json(400, {"error": "api_key is required"})
- return
- if not accounts.revoke_api_key(account["account_id"], api_key):
- self._send_json(404, {"error": "key not found on this account"})
- return
- self._send_json(200, {"revoked": api_key})
-
- def _handle_admin_accounts(self):
- """Admin-only: all accounts with their keys and balances."""
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- accounts = self._require_accounts()
- if accounts is None:
- return
- account = self._session_account()
- if account is None:
- self._send_json(401, {"error": "login required"})
- return
- if account["role"] != "admin":
- self._send_json(403, {"error": "admin role required"})
- return
- listing = accounts.list_accounts()
- if server.billing is not None:
- for entry in listing:
- entry["balances"] = {
- key: server.billing.get_client_balance(key)
- for key in entry.get("api_keys", [])
- }
- self._send_json(200, {"accounts": listing})
-
- def _handle_accounts_gossip(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- body = self._read_hive_authenticated_body()
- if body is None:
- return
- if server.accounts is None:
- self._send_json(200, {"applied": 0})
- return
- events = body.get("events")
- if not isinstance(events, list):
- self._send_json(400, {"error": "events must be a list"})
- return
- applied = server.accounts.apply_events([e for e in events if isinstance(e, dict)])
- self._send_json(200, {"applied": applied})
-
- def _handle_wallet_register(self):
- """Bind the caller's client wallet pubkey to their API key (US-032).
-
- Deposits from that wallet into the treasury are then credited to the
- API key's ledger balance by the deposit watcher.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- api_key = _api_key_from_headers(self.headers)
- if not api_key:
- self._send_json(401, {"error": "Authorization header required"})
- return
- if server.billing is None:
- self._send_json(404, {"error": "billing is not enabled on this tracker"})
- return
- body = self._read_json_body()
- if body is None:
- return
- wallet = body.get("wallet")
- if not wallet or not isinstance(wallet, str):
- self._send_json(400, {"error": "wallet is required"})
- return
- server.billing.bind_wallet(api_key, wallet)
- print(f"[tracker] wallet bound: {wallet} -> api_key …{api_key[-6:]}", flush=True)
- self._send_json(200, {"wallet": wallet, "bound": True})
-
- def _handle_benchmark_hop_penalty(self):
- """Privileged: run the same prompt through 1/2/3-node pinned routes (US-030).
-
- Data collection only — the routing algorithm is unchanged. Per-hop
- latency is derived incrementally: the k-node route's final hop penalty
- is its total minus the (k-1)-node route's total.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if not self._require_role("admin", "validator"):
- return
- auth = self.headers.get("Authorization")
- body = self._read_json_body()
- if body is None:
- return
- model = body.get("model", "")
- if not model:
- self._send_json(400, {"error": "model is required"})
- return
- prompt = body.get("prompt") or "Benchmark: say OK."
- max_new_tokens = int(body.get("max_new_tokens", 64))
-
- with server.lock:
- self._purge_expired_nodes()
- resolved = _nodes_and_bounds_for_model(server, model)
- if resolved is None or not resolved[0]:
- self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
- return
- all_nodes, rs, re = resolved
-
- self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
- route_results: list[dict] = []
- prev_total_ms: float | None = None
- prev_per_hop: list[float] = []
- for hop_count in (1, 2, 3):
- combo = _find_pinned_route(all_nodes, rs, re, hop_count)
- if combo is None:
- continue # insufficient coverage for this hop count — skip
- request_body = json.dumps({
- "model": model,
- "messages": [{"role": "user", "content": prompt}],
- "max_tokens": max_new_tokens,
- "route": [n.node_id for n in combo],
- }).encode()
- req = urllib.request.Request(
- f"{self_url}/v1/chat/completions",
- data=request_body,
- headers={"Content-Type": "application/json", "Authorization": auth},
- method="POST",
- )
- started = time.monotonic()
- try:
- with urllib.request.urlopen(req, timeout=300.0) as resp:
- payload = json.loads(resp.read())
- except Exception as exc:
- route_results.append({
- "route": [n.node_id for n in combo],
- "error": str(exc),
- })
- continue
- total_ms = (time.monotonic() - started) * 1000.0
- if prev_total_ms is not None and len(prev_per_hop) == hop_count - 1:
- per_hop_ms = prev_per_hop + [max(0.0, total_ms - prev_total_ms)]
- else:
- per_hop_ms = [total_ms / hop_count] * hop_count
- prev_total_ms = total_ms
- prev_per_hop = per_hop_ms
- route_results.append({
- "route": [n.node_id for n in combo],
- "total_ms": total_ms,
- "per_hop_ms": per_hop_ms,
- "tokens_generated": _usage_total_tokens(payload) or 0,
- })
-
- record = {
- "timestamp": time.time(),
- "model": model,
- "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
- "routes": route_results,
- }
- with server.benchmark_lock:
- existing: list = []
- if os.path.exists(server.benchmark_results_path):
- try:
- with open(server.benchmark_results_path, encoding="utf-8") as fh:
- existing = json.load(fh)
- except (json.JSONDecodeError, OSError):
- existing = []
- if not isinstance(existing, list):
- existing = []
- existing.append(record)
- with open(server.benchmark_results_path, "w", encoding="utf-8") as fh:
- json.dump(existing, fh, indent=2)
- self._send_json(200, record)
-
- def _handle_benchmark_results(self):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- if not self._require_role("admin", "validator"):
- return
- results: list = []
- with server.benchmark_lock:
- if os.path.exists(server.benchmark_results_path):
- try:
- with open(server.benchmark_results_path, encoding="utf-8") as fh:
- results = json.load(fh)
- except (json.JSONDecodeError, OSError):
- results = []
- self._send_json(200, {"results": results if isinstance(results, list) else []})
-
- def _handle_assign(self, parsed: urllib.parse.ParseResult):
- """Return an optimal shard assignment for a node given its hardware profile.
-
- Query params:
- model — model preset name (default: first preset)
- device — "cuda" | "cpu"
- vram_mb — integer VRAM in MB (0 for CPU)
- ram_mb — integer system RAM in MB, used when vram_mb=0
-
- The greedy strategy: find the first gap in current layer coverage
- and assign it. If no gap exists, assign the full model range so the
- node provides redundant coverage.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- params = urllib.parse.parse_qs(parsed.query)
-
- model_list = params.get("model")
- if not model_list:
- model = next(iter(server.model_presets), None)
- if model is None:
- self._send_json(503, {"error": "no model presets configured"})
- return
- else:
- model = model_list[0]
-
- resolved_name, preset = _resolve_model_preset(server.model_presets, model)
- if preset is None:
- self._send_json(404, {"error": f"unknown model preset: {model!r}"})
- return
-
- required_start, required_end = _preset_layer_bounds(preset)
-
- with server.lock:
- self._purge_expired_nodes()
- alive = [
- node for node in server.registry.values()
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- ]
- if server.contracts is not None:
- alive = [
- node for node in alive
- if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
- ]
-
- device = params.get("device", ["cpu"])[0]
- try:
- vram_mb = int(params.get("vram_mb", ["0"])[0])
- except ValueError:
- vram_mb = 0
- try:
- ram_mb = int(params.get("ram_mb", ["0"])[0])
- except ValueError:
- ram_mb = 0
- max_layers = required_end - required_start + 1
- memory_mb = vram_mb if vram_mb > 0 else ram_mb
- if memory_mb > 0:
- layer_bytes = _preset_bytes_per_layer(preset).get("bfloat16", 30 * 1024 * 1024)
- max_layers = min(max_layers, max(1, int(((memory_mb * 1024 * 1024) * 0.8) // layer_bytes)))
- elif device != "cuda" or vram_mb < 8192:
- max_layers = min(max_layers, 16)
-
- # Collect covered intervals sorted by start layer.
- covered = sorted(
- [
- (n.shard_start, n.shard_end)
- for n in alive
- if n.shard_start is not None and n.shard_end is not None
- ],
- key=lambda t: t[0],
- )
-
- # Walk from required_start to find the first uncovered layer.
- gap_start = required_start
- for s, e in covered:
- if s <= gap_start:
- gap_start = max(gap_start, e + 1)
- else:
- break # gap found before this node
-
- if gap_start > required_end:
- # Full coverage exists — assign the full range for redundancy.
- shard_start = required_start
- shard_end = min(required_end, shard_start + max_layers - 1)
- else:
- shard_start = gap_start
- shard_end = min(required_end, shard_start + max_layers - 1)
-
- peers = [
- {"endpoint": node.endpoint, "checksum": node.shard_checksum}
- for node in alive
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- and node.shard_start == shard_start
- and node.shard_end == shard_end
- and node.shard_checksum
- ]
-
- self._send_json(200, {
- "shard_start": shard_start,
- "shard_end": shard_end,
- "model": resolved_name,
- "model_layers_end": required_end,
- "peers": peers,
- **({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
- })
-
- def _handle_network_assign(self, parsed: urllib.parse.ParseResult):
- """Assign a new node to fill the biggest uncovered shard gap across HF-model nodes.
-
- Query params:
- vram_mb — integer VRAM in MB (0 = CPU-only node)
- ram_mb — integer system RAM in MB, used when vram_mb=0
- device — "cuda" | "cpu"
- hf_repo — optional; if set, restrict search to this repo only
-
- Returns:
- {hf_repo, shard_start, shard_end, num_layers, gap_found}
- gap_found=true means a real uncovered gap was assigned; false means redundancy.
- """
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- params = urllib.parse.parse_qs(parsed.query)
- try:
- vram_mb = int(params.get("vram_mb", ["0"])[0])
- except ValueError:
- vram_mb = 0
- try:
- ram_mb = int(params.get("ram_mb", ["0"])[0])
- except ValueError:
- ram_mb = 0
- device = params.get("device", ["cpu"])[0]
- filter_repo = params.get("hf_repo", [None])[0] # optional repo filter
-
- with server.lock:
- self._purge_expired_nodes()
- all_nodes = list(server.registry.values())
-
- # Collect only nodes that registered a real HF model (have hf_repo + shard bounds).
- hf_nodes = [
- n for n in all_nodes
- if n.hf_repo
- and n.shard_start is not None
- and n.shard_end is not None
- and n.num_layers is not None
- and (filter_repo is None or n.hf_repo == filter_repo)
- ]
-
- if not hf_nodes:
- resolved_name = None
- preset = None
- if filter_repo:
- resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
- else:
- deployable = [
- (name, preset)
- for name, preset in server.model_presets.items()
- if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"]
- ]
- if deployable:
- resolved_name, preset = deployable[0]
- if preset is not None and preset.get("hf_repo"):
- required_start, required_end = _preset_layer_bounds(preset)
- total_l = required_end - required_start + 1
- memory_mb = vram_mb if vram_mb > 0 else ram_mb
- max_layers = _max_layers_for_memory(memory_mb, total_l, preset)
- shard_start = required_start
- shard_end = min(required_end, shard_start + max_layers - 1)
- self._send_json(200, {
- "hf_repo": preset["hf_repo"],
- "model": resolved_name,
- "shard_start": shard_start,
- "shard_end": shard_end,
- "num_layers": total_l,
- "gap_found": True,
- "price_per_token": 0.0,
- "deployment": _deployment_summary(all_nodes, preset),
- })
- return
-
- msg = (
- f"no HF-model nodes registered for {filter_repo!r}"
- if filter_repo
- else "no HF-model nodes registered; cannot assign shards"
- )
- self._send_json(503, {"error": msg})
- return
-
- # Group by hf_repo; pick the one with the largest total_layers and biggest gap.
- from collections import defaultdict
- repo_groups: dict = defaultdict(list)
- repo_layers: dict = {}
- for n in hf_nodes:
- repo_groups[n.hf_repo].append(n)
- # Use the largest num_layers seen for this repo.
- if n.hf_repo not in repo_layers or n.num_layers > repo_layers[n.hf_repo]:
- repo_layers[n.hf_repo] = n.num_layers
-
- # Smart scoring: demand_rpm × coverage_deficit
- # coverage_deficit = uncovered_layers / total_layers (0 = fully covered)
- # demand_rpm comes from the stats collector; defaults to 0.0 when no traffic yet.
- demand_rpms: dict[str, float] = {}
- if server.stats is not None:
- local_rpms = server.stats.get_local_rpms()
- for repo in repo_groups:
- demand_rpms[repo] = local_rpms.get(repo, {}).get("rpm_last_hour", 0.0)
-
- best_repo = None
- best_score = -1.0
- best_gap_size = -1
- best_gap_start = 0
- best_num_layers = 0
-
- for repo, nodes in repo_groups.items():
- total = repo_layers[repo]
- covered = sorted(
- [(n.shard_start, n.shard_end) for n in nodes],
- key=lambda t: t[0],
- )
- # Walk from 0 to find first uncovered layer.
- gap_start = 0
- for s, e in covered:
- if s <= gap_start:
- gap_start = max(gap_start, e + 1)
- else:
- break
- gap_size = max(0, (total - 1) - gap_start + 1)
- coverage_deficit = gap_size / max(total, 1)
- demand = demand_rpms.get(repo, 0.0)
- # +1.0 floor on demand so models with no recorded traffic still compete by coverage.
- score = (demand + 1.0) * (coverage_deficit + 0.01)
- if score > best_score or (score == best_score and gap_size > best_gap_size):
- best_score = score
- best_gap_size = gap_size
- best_gap_start = gap_start
- best_repo = repo
- best_num_layers = total
-
- gap_found = best_gap_size > 0
- if not gap_found:
- # All shards covered — assign to highest-demand model for redundancy.
- best_repo = max(repo_groups, key=lambda r: demand_rpms.get(r, 0.0))
- best_gap_start = 0
- best_num_layers = repo_layers[best_repo]
-
- # Capacity: use the same 80%-of-memory rule as registered node planning.
- total_l = best_num_layers
- memory_mb = vram_mb if vram_mb > 0 else ram_mb
- _resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
- if memory_mb > 0:
- max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
- elif device == "cuda" and vram_mb >= 8192:
- max_layers = total_l
- else:
- max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
-
- shard_start = best_gap_start
- shard_end = min(total_l - 1, shard_start + max_layers - 1)
-
- self._send_json(200, {
- "hf_repo": best_repo,
- "shard_start": shard_start,
- "shard_end": shard_end,
- "num_layers": total_l,
- "gap_found": gap_found,
- "price_per_token": 0.0,
- })
-
- def _handle_route(self, parsed: urllib.parse.ParseResult):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- params = urllib.parse.parse_qs(parsed.query)
- model_list = params.get("model")
- if not model_list:
- self._send_json(400, {"error": "missing 'model' query parameter"})
- return
-
- model = model_list[0]
- resolved_name, preset = _resolve_model_preset(server.model_presets, model)
-
- with server.lock:
- self._purge_expired_nodes()
- if preset is not None:
- # Preset-based routing (stub-model system).
- alive = [
- node for node in server.registry.values()
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- ]
- required_start, required_end = _preset_layer_bounds(preset)
- else:
- # HF model routing: match by hf_repo (full) or model short name.
- alive = [
- node for node in server.registry.values()
- if _node_matches_model(node, model)
- and node.shard_start is not None
- and node.shard_end is not None
- and node.num_layers is not None
- ]
- if not alive:
- self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
- return
- required_start = 0
- required_end = max(n.num_layers for n in alive) - 1 # type: ignore[type-var]
-
- if server.contracts is not None:
- alive = [
- node for node in alive
- if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
- ]
-
- route, error = _select_route(alive, required_start, required_end)
- if error:
- self._send_json(503, {"error": error})
- return
-
- covered_up_to = required_start - 1
- route_with_start: list[tuple] = []
- for rn in route:
- route_with_start.append((rn, covered_up_to + 1))
- covered_up_to = rn.shard_end if rn.shard_end is not None else covered_up_to
-
- self._send_json(200, {
- "route": [e.endpoint for e, _ in route_with_start],
- "nodes": [
- {
- "node_id": e.node_id,
- "endpoint": e.endpoint,
- "start_layer": start,
- "relay_addr": e.relay_addr,
- "peer_id": e.peer_id,
- "wallet_address": e.wallet_address,
- "shard_start": e.shard_start,
- "shard_end": e.shard_end,
- "model": e.model,
- "hf_repo": e.hf_repo,
- "num_layers": e.num_layers,
- "model_metadata": dict(e.model_metadata),
- "shard_checksum": e.shard_checksum,
- "score": e.score,
- }
- for e, start in route_with_start
- ],
- })
-
- def _handle_routes(self, parsed: urllib.parse.ParseResult):
- server: _TrackerHTTPServer = self.server # type: ignore[assignment]
- params = urllib.parse.parse_qs(parsed.query)
- model_list = params.get("model")
- if not model_list:
- self._send_json(400, {"error": "missing 'model' query parameter"})
- return
-
- try:
- redundancy = int(params.get("redundancy", ["1"])[0])
- except ValueError:
- self._send_json(400, {"error": "redundancy must be an integer"})
- return
- if redundancy < 1:
- self._send_json(400, {"error": "redundancy must be at least 1"})
- return
-
- model = model_list[0]
- resolved_name, preset = _resolve_model_preset(server.model_presets, model)
- if preset is None:
- self._send_json(404, {"error": f"unknown model preset: {model!r}"})
- return
-
- required_start, required_end = _preset_layer_bounds(preset)
-
- with server.lock:
- self._purge_expired_nodes()
- candidates = [
- node for node in server.registry.values()
- if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
- ]
- if server.contracts is not None:
- candidates = [
- node for node in candidates
- if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
- ]
-
- routes = []
- remaining = list(candidates)
- for _ in range(redundancy):
- route, error = _select_route(remaining, required_start, required_end)
- if error:
- self._send_json(503, {"error": error})
- return
- route_endpoints = {node.endpoint for node in route}
- remaining = [node for node in remaining if node.endpoint not in route_endpoints]
- routes.append({
- "route": [e.endpoint for e in route],
- "nodes": [
- {
- "node_id": e.node_id,
- "endpoint": e.endpoint,
- "relay_addr": e.relay_addr,
- "peer_id": e.peer_id,
- "wallet_address": e.wallet_address,
- "shard_start": e.shard_start,
- "shard_end": e.shard_end,
- "model": e.model,
- "shard_checksum": e.shard_checksum,
- "score": e.score,
- }
- for e in route
- ],
- })
-
- self._send_json(200, {"routes": routes})
-
-
-class TrackerServer:
- """HTTP tracker that manages node registration and resolves inference routes.
-
- Nodes register via POST /v1/nodes/register and keep themselves alive with
- POST /v1/nodes//heartbeat. The gateway queries GET /v1/route?model=
- to obtain an ordered list of node endpoints whose shards cover all layers.
- """
-
- def __init__(
- self,
- host: str = "127.0.0.1",
- port: int = 0,
- heartbeat_timeout: float = 90.0,
- rebalance_interval: float = 30.0,
- model_presets: dict | None = None,
- contracts: Any | None = None,
- minimum_stake: int = 0,
- cluster_peers: list[str] | None = None,
- cluster_self_url: str | None = None,
- stats_db: str | None = None,
- relay_url: str | None = None,
- billing: BillingLedger | None = None,
- enable_billing: bool = False,
- billing_db: str | None = None,
- accounts: AccountStore | None = None,
- accounts_db: str | None = None,
- benchmark_results_path: str | None = None,
- treasury: Any | None = None,
- deposit_poll_interval: float = 15.0,
- settle_period: float = 86400.0,
- payout_threshold: float = 5.0,
- payout_dust_floor: float = 0.01,
- settlement_check_interval: float | None = None,
- validator_service_token: str | None = None,
- hive_secret: str | None = None,
- ) -> None:
- self._host = host
- self._requested_port = port
- self._heartbeat_timeout = heartbeat_timeout
- self._rebalance_interval = rebalance_interval
- self._model_presets: dict = (
- model_presets if model_presets is not None else dict(DEFAULT_MODEL_PRESETS)
- )
- self._contracts = contracts
- self._minimum_stake = minimum_stake
- self._cluster_peers: list[str] = list(cluster_peers) if cluster_peers else []
- self._cluster_self_url: str | None = cluster_self_url
- self._relay_url = relay_url
- self._registry: dict[str, _NodeEntry] = {}
- self._lock = threading.Lock()
- self._server: _TrackerHTTPServer | None = None
- self._thread: threading.Thread | None = None
- self._rebalance_stop = threading.Event()
- self._rebalance_thread: threading.Thread | None = None
- self._raft: RaftNode | None = None
- self._gossip: NodeGossip | None = None
- self._stats: _StatsCollector | None = _StatsCollector(db_path=stats_db) if stats_db else _StatsCollector()
- self._stats_stop = threading.Event()
- self._stats_thread: threading.Thread | None = None
- if billing is None:
- db_path = billing_db
- if db_path is None and enable_billing:
- db_path = DEFAULT_BILLING_DB_PATH
- if db_path:
- preset_prices = {
- name: float(preset["price_per_1k_tokens"])
- for name, preset in self._model_presets.items()
- if isinstance(preset, dict) and "price_per_1k_tokens" in preset
- }
- billing = BillingLedger(db_path=db_path, prices=preset_prices)
- self._billing: BillingLedger | None = billing
- self._billing_gossip_cursor = 0
- if accounts is None:
- accounts_path = accounts_db
- if accounts_path is None and enable_billing:
- accounts_path = DEFAULT_ACCOUNTS_DB_PATH
- if accounts_path:
- accounts = AccountStore(db_path=accounts_path)
- self._accounts: AccountStore | None = accounts
- self._accounts_gossip_cursor = 0
- self._benchmark_results_path = benchmark_results_path
- self._treasury = treasury
- self._deposit_poll_interval = deposit_poll_interval
- self._deposit_stop = threading.Event()
- self._deposit_thread: threading.Thread | None = None
- self._settle_period = settle_period
- self._payout_threshold = payout_threshold
- self._payout_dust_floor = payout_dust_floor
- self._settlement_check_interval = settlement_check_interval or max(
- 1.0, min(settle_period / 4.0, 15.0)
- )
- self._settlement_stop = threading.Event()
- self._settlement_thread: threading.Thread | None = None
- self._validator_service_token = (
- validator_service_token
- if validator_service_token is not None
- else os.environ.get("MESHNET_VALIDATOR_SERVICE_TOKEN") or None
- )
- self._hive_secret = (
- hive_secret
- if hive_secret is not None
- else os.environ.get("MESHNET_HIVE_SECRET") or None
- )
- self.port: int | None = None
-
- def start(self) -> int:
- if self._server is not None:
- raise RuntimeError("TrackerServer is already running")
-
- effective_relay_url = (
- self._relay_url
- or derive_relay_url_from_public_tracker_url(self._cluster_self_url)
- )
-
- # Start HTTP server first so we know our port
- self._server = _TrackerHTTPServer(
- (self._host, self._requested_port),
- _TrackerHandler,
- self._registry,
- self._lock,
- self._heartbeat_timeout,
- self._model_presets,
- self._contracts,
- self._minimum_stake,
- relay_url=effective_relay_url,
- stats=self._stats,
- billing=self._billing,
- accounts=self._accounts,
- benchmark_results_path=self._benchmark_results_path,
- validator_service_token=self._validator_service_token,
- hive_secret=self._hive_secret,
- )
- self.port = self._server.server_address[1]
-
- # Start Raft + gossip if cluster peers are configured
- if self._cluster_peers:
- self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
- self._raft = RaftNode(
- self_url=self_url,
- peers=self._cluster_peers,
- apply_fn=self._raft_apply,
- )
- self._gossip = NodeGossip(peers=self._cluster_peers)
- self._server.raft = self._raft
- self._server.gossip = self._gossip
- self._raft.start()
- self._gossip.start()
-
- self._rebalance_stop.clear()
- self._stats_stop.clear()
- self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
- self._thread.start()
- self._rebalance_thread = threading.Thread(target=self._rebalance_loop, daemon=True)
- self._rebalance_thread.start()
- self._stats_thread = threading.Thread(target=self._stats_loop, daemon=True)
- self._stats_thread.start()
- if self._treasury is not None and self._billing is not None:
- self._deposit_stop.clear()
- self._deposit_thread = threading.Thread(target=self._deposit_loop, daemon=True)
- self._deposit_thread.start()
- self._settlement_stop.clear()
- self._settlement_thread = threading.Thread(target=self._settlement_loop, daemon=True)
- self._settlement_thread.start()
- return self.port
-
- def _settlement_loop(self) -> None:
- """Leader-only on-chain settlement (US-033, ADR-0015).
-
- Pay a node when pending ≥ threshold OR its pending age ≥ max period,
- with a dust floor. Pending is debited (payout events, replicated)
- before the transaction is sent; unconfirmed batches are resent with
- the same settlement id, so retries never double-pay.
- """
- billing = self._billing
- treasury = self._treasury
- assert billing is not None and treasury is not None
- while not self._settlement_stop.wait(self._settlement_check_interval):
- if self._raft is not None and not self._raft.is_leader:
- continue # followers replicate the ledger but never sign
- # resend batches whose transaction never confirmed
- for settlement_id, payouts in billing.unconfirmed_settlements().items():
- self._send_settlement(settlement_id, payouts)
- banned: set[str] = set()
- if self._server is not None and self._server.contracts is not None:
- registry = self._server.contracts.registry
- banned = {
- wallet for wallet, _ in billing.payables(
- threshold=0.0, max_period=0.0, dust_floor=0.0,
- )
- if registry.get_wallet(wallet).banned
- }
- due = billing.payables(
- threshold=self._payout_threshold,
- max_period=self._settle_period,
- dust_floor=self._payout_dust_floor,
- exclude=banned,
- )
- if not due:
- continue
- settlement_id = uuid.uuid4().hex
- for wallet, amount in due:
- billing.settle_node_payout(wallet, amount, reference=settlement_id)
- self._send_settlement(settlement_id, due)
-
- def _send_settlement(self, settlement_id: str, payouts: list) -> None:
- billing = self._billing
- treasury = self._treasury
- assert billing is not None and treasury is not None
- try:
- signature = treasury.send_payouts([(w, a) for w, a in payouts])
- except Exception as exc:
- print(
- f"[tracker] settlement {settlement_id} failed (will retry): {exc}",
- flush=True,
- )
- return
- billing.confirm_settlement(settlement_id, signature)
- total = sum(a for _, a in payouts)
- print(
- f"[tracker] settled {settlement_id}: {len(payouts)} payout(s), "
- f"{total:.6f} USDT total (tx {signature})",
- flush=True,
- )
-
- def _deposit_loop(self) -> None:
- """Poll the treasury token account and credit confirmed deposits (US-032)."""
- billing = self._billing
- treasury = self._treasury
- assert billing is not None and treasury is not None
- while not self._deposit_stop.wait(self._deposit_poll_interval):
- try:
- deposits = treasury.list_new_deposits(
- lambda sig: billing.has_event(f"deposit-{sig}")
- )
- except Exception as exc:
- print(f"[tracker] deposit poll failed: {exc}", flush=True)
- continue
- for deposit in deposits:
- api_key = billing.api_key_for_wallet(deposit.sender_wallet)
- if api_key is None:
- print(
- f"[tracker] unattributed deposit {deposit.signature}: "
- f"{deposit.amount_usdt} USDT from unbound wallet "
- f"{deposit.sender_wallet} — register via /v1/wallet/register",
- flush=True,
- )
- continue
- credited = billing.credit_deposit(
- api_key, deposit.amount_usdt, deposit.signature
- )
- if credited is not None:
- print(
- f"[tracker] deposit credited: {deposit.amount_usdt} USDT "
- f"-> api_key …{api_key[-6:]} (tx {deposit.signature})",
- flush=True,
- )
-
- def _raft_apply(self, command: str, payload: dict) -> None:
- """Called by RaftNode when a log entry is committed — replicate to local registry."""
- if command != "register":
- return
- # Re-apply the registration to our local registry (follower path).
- # On the leader this is a no-op since it already registered locally.
- node_id = payload.get("node_id")
- if not node_id or node_id in self._registry:
- return # already present (leader case) or malformed
- endpoint = payload.get("endpoint", "")
- try:
- shard_start = int(payload["shard_start"]) if payload.get("shard_start") is not None else None
- shard_end = int(payload["shard_end"]) if payload.get("shard_end") is not None else None
- except (TypeError, ValueError):
- return
- entry = _NodeEntry(
- node_id=node_id,
- endpoint=endpoint.rstrip("/"),
- shard_start=shard_start,
- shard_end=shard_end,
- model=payload.get("model", "stub-model"),
- shard_checksum=payload.get("shard_checksum"),
- hardware_profile=payload.get("hardware_profile", {}),
- wallet_address=payload.get("wallet_address"),
- score=float(payload.get("score", 1.0)),
- tracker_mode=bool(payload.get("tracker_mode", False)),
- hf_repo=payload.get("hf_repo"),
- num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
- model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
- )
- with self._lock:
- self._registry[node_id] = entry
-
- def _rebalance_loop(self) -> None:
- while not self._rebalance_stop.wait(self._rebalance_interval):
- server = self._server
- if server is None:
- return
- with self._lock:
- _purge_expired_nodes_locked(server)
- _rebalance_all_locked(server)
-
- def _push_to_peers(self, path: str, body: bytes) -> bool:
- """POST a hive-signed payload to every cluster peer; True if all succeeded."""
- headers = {"Content-Type": "application/json"}
- if self._hive_secret:
- headers.update(sign_hive_request(self._hive_secret, body))
- delivered_all = True
- for peer in self._cluster_peers:
- try:
- req = urllib.request.Request(
- f"{peer}{path}", data=body, headers=headers, method="POST",
- )
- with urllib.request.urlopen(req, timeout=2.0) as r:
- r.read()
- except Exception:
- delivered_all = False
- return delivered_all
-
- def _stats_loop(self) -> None:
- """Periodically save stats/billing to DB and push local slices to cluster peers."""
- while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
- if self._stats is not None:
- self._stats.save_to_db()
- if self._billing is not None:
- self._billing.save_to_db()
- if self._accounts is not None:
- self._accounts.save_to_db()
- if self._cluster_peers and not self._hive_secret:
- print(
- "[tracker] WARNING: cluster peers configured without --hive-secret — "
- "gossip pushes will be rejected (ADR-0017)",
- flush=True,
- )
- if self._stats is not None and self._cluster_peers:
- self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
- local_rpms = self._stats.get_local_rpms()
- body = json.dumps({"tracker_url": self_url, "stats": local_rpms}).encode()
- self._push_to_peers("/v1/stats/gossip", body)
- if self._billing is not None and self._cluster_peers:
- events, cursor = self._billing.events_since(self._billing_gossip_cursor)
- if events:
- body = json.dumps({"events": events}).encode()
- # Only advance past events every peer has seen; unreachable
- # peers get the full backlog on the next tick (idempotent
- # via event-id dedupe on the receiving side).
- if self._push_to_peers("/v1/billing/gossip", body):
- self._billing_gossip_cursor = cursor
- if self._accounts is not None and self._cluster_peers:
- events, cursor = self._accounts.events_since(self._accounts_gossip_cursor)
- if events:
- body = json.dumps({"events": events}).encode()
- if self._push_to_peers("/v1/accounts/gossip", body):
- self._accounts_gossip_cursor = cursor
-
- def stop(self) -> None:
- if self._raft is not None:
- self._raft.stop()
- if self._gossip is not None:
- self._gossip.stop()
- if self._server is None:
- return
- self._rebalance_stop.set()
- self._stats_stop.set()
- self._deposit_stop.set()
- self._settlement_stop.set()
- if self._stats is not None:
- self._stats.save_to_db()
- if self._billing is not None:
- self._billing.save_to_db()
- if self._accounts is not None:
- self._accounts.save_to_db()
- self._server.shutdown()
- self._server.server_close()
- if self._thread is not None:
- self._thread.join(timeout=1)
- if self._rebalance_thread is not None:
- self._rebalance_thread.join(timeout=1)
- if self._stats_thread is not None:
- self._stats_thread.join(timeout=1)
- if self._deposit_thread is not None:
- self._deposit_thread.join(timeout=1)
- self._deposit_thread = None
- if self._settlement_thread is not None:
- self._settlement_thread.join(timeout=1)
- self._settlement_thread = None
- self._server = None
- self._thread = None
- self._rebalance_thread = None
- self._stats_thread = None
- self._raft = None
- self._gossip = None
- self.port = None
+"""Tracker HTTP server — node registry and route selection for the inference mesh.
+
+HTTP API contract:
+- POST /v1/nodes/register
+ Request JSON: {
+ "endpoint": "http://host:port", "shard_start": int, "shard_end": int,
+ "model": str optional, "shard_checksum": str optional,
+ "hardware_profile": object, "wallet_address": str optional,
+ "score": number optional
+ }
+ Response 200: {"node_id": str}
+ Response 400: {"error": str}
+- POST /v1/nodes/{node_id}/heartbeat
+ Response 200: {}
+ Response 404: {"error": "node not found"}
+- GET /v1/route?model=
+ Response 200: {"route": ["http://node-a", "http://node-b"]}
+ Response 400/404/503: {"error": str}
+- GET /v1/routes?model=&redundancy=
+ Response 200: {"routes": [{"route": [...], "nodes": [...]}]}
+ Response 400/404/503: {"error": str}
+"""
+
+import http.server
+import hashlib
+import itertools
+import json
+import os
+import socketserver
+import sqlite3
+import threading
+import time
+import urllib.parse
+import urllib.request
+import uuid
+from importlib.resources import files
+from typing import Any
+
+from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
+from .auth import is_validator_token, sign_hive_request, verify_hive_request
+from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
+from .gossip import NodeGossip
+from .raft import RaftNode
+
+
+def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
+ """Return wss://host/ws when url is a public HTTPS tracker origin."""
+ if not url:
+ return None
+ parsed = urllib.parse.urlparse(url)
+ if parsed.scheme != "https":
+ return None
+ host = parsed.hostname
+ if not host or host in ("127.0.0.1", "localhost"):
+ return None
+ return f"wss://{parsed.netloc}/ws"
+
+
+def _load_model_presets_from_data() -> dict[str, dict]:
+ """Load recommended model presets from package JSON data."""
+ try:
+ raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
+ data = json.loads(raw)
+ except Exception:
+ return {}
+ models = data.get("models", {})
+ if not isinstance(models, dict):
+ return {}
+ return {
+ str(name): preset
+ for name, preset in models.items()
+ if isinstance(preset, dict)
+ }
+
+
+DEFAULT_MODEL_PRESETS: dict[str, dict] = {
+ "stub-model": {
+ "layers_start": 0,
+ "layers_end": 31,
+ "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
+ },
+ "openai-community/gpt2": {
+ "layers_start": 0,
+ "layers_end": 11,
+ "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
+ },
+ **_load_model_presets_from_data(),
+}
+
+DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
+DEFAULT_RAM_BYTES = 16 * 1024 * 1024 * 1024
+DEFAULT_QUANTIZATIONS = ["bfloat16"]
+DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
+
+
+def _model_aliases(model: str | None) -> set[str]:
+ """Return stable lookup aliases for a model repo or display name."""
+ if not model:
+ return set()
+ aliases = {model}
+ short = model.rsplit("/", 1)[-1]
+ aliases.add(short)
+ lowered = short.lower()
+ aliases.add(lowered)
+ if lowered.endswith("-instruct"):
+ aliases.add(lowered.removesuffix("-instruct"))
+ return aliases
+
+
+def _preset_aliases(name: str, preset: dict | None) -> set[str]:
+ aliases = _model_aliases(name)
+ if not preset:
+ return aliases
+ hf_repo = preset.get("hf_repo")
+ if isinstance(hf_repo, str):
+ aliases |= _model_aliases(hf_repo)
+ for alias in preset.get("aliases", []) or []:
+ if isinstance(alias, str):
+ aliases |= _model_aliases(alias)
+ return aliases
+
+
+def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
+ requested = _model_aliases(model)
+ for name, preset in model_presets.items():
+ if requested & _preset_aliases(name, preset):
+ return name, preset
+ return None, None
+
+
+def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
+ requested = _model_aliases(model)
+ if not requested:
+ return False
+ return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
+
+
+def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
+ requested = _preset_aliases(name, preset)
+ return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))
+
+
+class _RollingCounter:
+ """Circular-bucket request counter.
+
+ Tracks events in `num_buckets` slots, each covering `bucket_seconds` of wall time.
+ Old buckets are silently discarded when their epoch expires.
+ """
+
+ def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
+ self._num = num_buckets
+ self._bsec = bucket_seconds
+ self._counts: list[int] = [0] * num_buckets
+ self._epochs: list[int] = [-1] * num_buckets # which epoch each slot holds
+
+ def _epoch(self, now: float) -> int:
+ return int(now) // self._bsec
+
+ def record(self, now: float | None = None) -> None:
+ t = now if now is not None else time.time()
+ ep = self._epoch(t)
+ idx = ep % self._num
+ if self._epochs[idx] != ep:
+ self._counts[idx] = 0
+ self._epochs[idx] = ep
+ self._counts[idx] += 1
+
+ def rpm(self, now: float | None = None) -> float:
+ t = now if now is not None else time.time()
+ cutoff = self._epoch(t) - self._num # epochs <= this are stale
+ total = sum(self._counts[i] for i in range(self._num) if self._epochs[i] > cutoff)
+ window_minutes = (self._num * self._bsec) / 60.0
+ return total / window_minutes if window_minutes > 0 else 0.0
+
+ def buckets(self) -> list[tuple[int, int]]:
+ return [(self._epochs[i], self._counts[i]) for i in range(self._num)]
+
+ def restore_buckets(self, data: list[tuple[int, int]]) -> None:
+ for i, (ep, cnt) in enumerate(data):
+ if i < self._num:
+ self._epochs[i] = ep
+ self._counts[i] = cnt
+
+
+class _RollingThroughput:
+ """Circular buckets for observed tokens/sec.
+
+ Each bucket stores total output tokens and total elapsed seconds. TPS for a
+ window is the ratio across non-stale buckets, which avoids over-weighting
+ small fast requests.
+ """
+
+ def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
+ self._num = num_buckets
+ self._bsec = bucket_seconds
+ self._tokens: list[int] = [0] * num_buckets
+ self._seconds: list[float] = [0.0] * num_buckets
+ self._samples: list[int] = [0] * num_buckets
+ self._epochs: list[int] = [-1] * num_buckets
+
+ def _epoch(self, now: float) -> int:
+ return int(now) // self._bsec
+
+ def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
+ if total_tokens <= 0 or elapsed_seconds <= 0:
+ return
+ t = now if now is not None else time.time()
+ ep = self._epoch(t)
+ idx = ep % self._num
+ if self._epochs[idx] != ep:
+ self._tokens[idx] = 0
+ self._seconds[idx] = 0.0
+ self._samples[idx] = 0
+ self._epochs[idx] = ep
+ self._tokens[idx] += int(total_tokens)
+ self._seconds[idx] += float(elapsed_seconds)
+ self._samples[idx] += 1
+
+ def stats(self, now: float | None = None) -> dict:
+ t = now if now is not None else time.time()
+ cutoff = self._epoch(t) - self._num
+ tokens = 0
+ seconds = 0.0
+ samples = 0
+ for i in range(self._num):
+ if self._epochs[i] > cutoff:
+ tokens += self._tokens[i]
+ seconds += self._seconds[i]
+ samples += self._samples[i]
+ return {
+ "tokens_per_sec": round(tokens / seconds, 4) if seconds > 0 else None,
+ "tokens": tokens,
+ "seconds": round(seconds, 6),
+ "sample_count": samples,
+ }
+
+ def buckets(self) -> list[tuple[int, int, float, int]]:
+ return [
+ (self._epochs[i], self._tokens[i], self._seconds[i], self._samples[i])
+ for i in range(self._num)
+ ]
+
+ def restore_buckets(self, data: list[tuple[int, int, float, int]]) -> None:
+ for i, (ep, tokens, seconds, samples) in enumerate(data):
+ if i < self._num:
+ self._epochs[i] = ep
+ self._tokens[i] = tokens
+ self._seconds[i] = seconds
+ self._samples[i] = samples
+
+
+class _ModelStats:
+ """Three rolling windows for one model: last hour, last day, last month."""
+
+ def __init__(self) -> None:
+ self.per_minute = _RollingCounter(60, 60) # 60 × 1-min buckets = 1 hour
+ self.per_hour = _RollingCounter(24, 3600) # 24 × 1-hr buckets = 1 day
+ self.per_day = _RollingCounter(30, 86400) # 30 × 1-day buckets = ~1 month
+
+ def record(self, now: float | None = None) -> None:
+ t = now if now is not None else time.time()
+ self.per_minute.record(t)
+ self.per_hour.record(t)
+ self.per_day.record(t)
+
+ def rpms(self, now: float | None = None) -> dict:
+ t = now if now is not None else time.time()
+ return {
+ "rpm_last_hour": round(self.per_minute.rpm(t), 4),
+ "rpm_last_day": round(self.per_hour.rpm(t), 4),
+ "rpm_last_month": round(self.per_day.rpm(t), 4),
+ }
+
+
+class _NodeModelThroughput:
+ """Observed throughput windows for one node/model pair."""
+
+ def __init__(self) -> None:
+ self.per_minute = _RollingThroughput(60, 60)
+ self.per_hour = _RollingThroughput(24, 3600)
+ self.per_day = _RollingThroughput(30, 86400)
+
+ def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
+ t = now if now is not None else time.time()
+ self.per_minute.record(total_tokens, elapsed_seconds, t)
+ self.per_hour.record(total_tokens, elapsed_seconds, t)
+ self.per_day.record(total_tokens, elapsed_seconds, t)
+
+ def stats(self, now: float | None = None) -> dict:
+ hour = self.per_minute.stats(now)
+ day = self.per_hour.stats(now)
+ month = self.per_day.stats(now)
+ return {
+ "tokens_per_sec_last_hour": hour["tokens_per_sec"],
+ "tokens_per_sec_last_day": day["tokens_per_sec"],
+ "tokens_per_sec_last_month": month["tokens_per_sec"],
+ "sample_count_last_hour": hour["sample_count"],
+ "tokens_last_hour": hour["tokens"],
+ "seconds_last_hour": hour["seconds"],
+ }
+
+
+class _StatsCollector:
+ """Thread-safe model request stats with SQLite persistence and peer slice merging."""
+
+ SAVE_INTERVAL = 60.0 # seconds between DB saves
+
+ def __init__(self, db_path: str | None = None) -> None:
+ self._lock = threading.Lock()
+ self._local: dict[str, _ModelStats] = {}
+ self._node_model: dict[tuple[str, str], _NodeModelThroughput] = {}
+ self._peer_rpms: dict[str, dict[str, dict]] = {} # tracker_url -> model -> rpms
+ self._db_path = db_path
+ if db_path:
+ self._init_db()
+ self._load_from_db()
+
+ # ---------- public API ----------
+
+ def record_request(self, model: str, now: float | None = None) -> None:
+ t = now if now is not None else time.time()
+ with self._lock:
+ if model not in self._local:
+ self._local[model] = _ModelStats()
+ self._local[model].record(t)
+
+ def record_node_throughput(
+ self,
+ node_id: str,
+ model: str,
+ total_tokens: int,
+ elapsed_seconds: float,
+ now: float | None = None,
+ ) -> None:
+ if not node_id or not model or total_tokens <= 0 or elapsed_seconds <= 0:
+ return
+ t = now if now is not None else time.time()
+ with self._lock:
+ key = (node_id, model)
+ if key not in self._node_model:
+ self._node_model[key] = _NodeModelThroughput()
+ self._node_model[key].record(total_tokens, elapsed_seconds, t)
+
+ def get_local_rpms(self, now: float | None = None) -> dict[str, dict]:
+ t = now if now is not None else time.time()
+ with self._lock:
+ return {m: s.rpms(t) for m, s in self._local.items()}
+
+ def get_node_model_stats(self, node_id: str, model: str, now: float | None = None) -> dict:
+ t = now if now is not None else time.time()
+ empty = {
+ "tokens_per_sec_last_hour": None,
+ "tokens_per_sec_last_day": None,
+ "tokens_per_sec_last_month": None,
+ "sample_count_last_hour": 0,
+ "tokens_last_hour": 0,
+ "seconds_last_hour": 0.0,
+ }
+ with self._lock:
+ stat = self._node_model.get((node_id, model))
+ return stat.stats(t) if stat is not None else dict(empty)
+
+ def get_node_throughput_stats(self, now: float | None = None) -> dict[str, dict]:
+ t = now if now is not None else time.time()
+ with self._lock:
+ result: dict[str, dict] = {}
+ for (node_id, model), stat in self._node_model.items():
+ result.setdefault(node_id, {"models": {}})["models"][model] = stat.stats(t)
+ return result
+
+ def merge_peer_rpms(self, tracker_url: str, rpms: dict[str, dict]) -> None:
+ with self._lock:
+ self._peer_rpms[tracker_url] = dict(rpms)
+
+ def get_combined_stats(self, now: float | None = None) -> dict:
+ t = now if now is not None else time.time()
+ with self._lock:
+ combined: dict[str, dict] = {}
+ for model, ms in self._local.items():
+ combined[model] = ms.rpms(t)
+ for _url, peer in self._peer_rpms.items():
+ for model, rpms in peer.items():
+ if model not in combined:
+ combined[model] = {"rpm_last_hour": 0.0, "rpm_last_day": 0.0, "rpm_last_month": 0.0}
+ for key in ("rpm_last_hour", "rpm_last_day", "rpm_last_month"):
+ combined[model][key] = round(combined[model].get(key, 0.0) + rpms.get(key, 0.0), 4)
+ return combined
+
+ # ---------- persistence ----------
+
+ def _init_db(self) -> None:
+ con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
+ con.execute(
+ "CREATE TABLE IF NOT EXISTS model_rpm_buckets "
+ "(model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, count INTEGER, "
+ "PRIMARY KEY (model, window, bucket_idx))"
+ )
+ con.execute(
+ "CREATE TABLE IF NOT EXISTS node_model_tps_buckets "
+ "(node_id TEXT, model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, "
+ "tokens INTEGER, seconds REAL, samples INTEGER, "
+ "PRIMARY KEY (node_id, model, window, bucket_idx))"
+ )
+ con.commit()
+ con.close()
+
+ def save_to_db(self) -> None:
+ if not self._db_path:
+ return
+ with self._lock:
+ rows = []
+ for model, ms in self._local.items():
+ for window_name, counter in (
+ ("hour", ms.per_minute),
+ ("day", ms.per_hour),
+ ("month", ms.per_day),
+ ):
+ for idx, (ep, cnt) in enumerate(counter.buckets()):
+ if ep >= 0:
+ rows.append((model, window_name, idx, ep, cnt))
+ tps_rows = []
+ for (node_id, model), stat in self._node_model.items():
+ for window_name, counter in (
+ ("hour", stat.per_minute),
+ ("day", stat.per_hour),
+ ("month", stat.per_day),
+ ):
+ for idx, (ep, tokens, seconds, samples) in enumerate(counter.buckets()):
+ if ep >= 0:
+ tps_rows.append((node_id, model, window_name, idx, ep, tokens, seconds, samples))
+ con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
+ con.executemany(
+ "INSERT OR REPLACE INTO model_rpm_buckets VALUES (?,?,?,?,?)", rows
+ )
+ con.executemany(
+ "INSERT OR REPLACE INTO node_model_tps_buckets VALUES (?,?,?,?,?,?,?,?)",
+ tps_rows,
+ )
+ con.commit()
+ con.close()
+
+ def _load_from_db(self) -> None:
+ con = sqlite3.connect(self._db_path) # type: ignore[arg-type]
+ rows = con.execute("SELECT model, window, bucket_idx, bucket_epoch, count FROM model_rpm_buckets").fetchall()
+ tps_rows = con.execute(
+ "SELECT node_id, model, window, bucket_idx, bucket_epoch, tokens, seconds, samples "
+ "FROM node_model_tps_buckets"
+ ).fetchall()
+ con.close()
+ grouped: dict[str, dict[str, list[tuple[int, int]]]] = {}
+ for model, window, idx, ep, cnt in rows:
+ grouped.setdefault(model, {}).setdefault(window, []).append((idx, ep, cnt))
+ for model, windows in grouped.items():
+ ms = _ModelStats()
+ for window_name, entries in windows.items():
+ counter = {"hour": ms.per_minute, "day": ms.per_hour, "month": ms.per_day}.get(window_name)
+ if counter is None:
+ continue
+ data = [(0, -1)] * counter._num
+ for idx, ep, cnt in entries:
+ if 0 <= idx < counter._num:
+ data[idx] = (ep, cnt)
+ counter.restore_buckets(data)
+ self._local[model] = ms
+ tps_grouped: dict[tuple[str, str], dict[str, list[tuple[int, int, int, float, int]]]] = {}
+ for node_id, model, window, idx, ep, tokens, seconds, samples in tps_rows:
+ tps_grouped.setdefault((node_id, model), {}).setdefault(window, []).append(
+ (idx, ep, tokens, seconds, samples)
+ )
+ for key, windows in tps_grouped.items():
+ stat = _NodeModelThroughput()
+ for window_name, entries in windows.items():
+ counter = {"hour": stat.per_minute, "day": stat.per_hour, "month": stat.per_day}.get(window_name)
+ if counter is None:
+ continue
+ data = [(0, 0, 0.0, 0)] * counter._num
+ for idx, ep, tokens, seconds, samples in entries:
+ if 0 <= idx < counter._num:
+ data[idx] = (ep, int(tokens), float(seconds), int(samples))
+ counter.restore_buckets(data)
+ self._node_model[key] = stat
+
+
+class _NodeEntry:
+ __slots__ = (
+ "node_id", "endpoint", "shard_start", "shard_end",
+ "model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "hardware_profile", "wallet_address",
+ "score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
+ "benchmark_tokens_per_sec", "quantization", "managed_assignment",
+ "model_tokens_per_sec",
+ "pending_directives", "last_heartbeat", "tracker_mode",
+ "relay_addr", "cert_fingerprint", "peer_id",
+ # heartbeat stats (reported by node, cumulative)
+ "total_requests", "failed_requests", "queue_depth", "uptime_seconds",
+ "status", # "ready" | "loading"
+ "heartbeats_expected", "heartbeats_received",
+ # dynamic reassignment queued by the tracker
+ "pending_new_assignment",
+ )
+
+ def __init__(
+ self,
+ node_id: str,
+ endpoint: str,
+ shard_start: int | None,
+ shard_end: int | None,
+ model: str | None,
+ shard_checksum: str | None,
+ hardware_profile: dict,
+ wallet_address: str | None,
+ score: float,
+ vram_bytes: int = DEFAULT_VRAM_BYTES,
+ ram_bytes: int = DEFAULT_RAM_BYTES,
+ quantizations: list[str] | None = None,
+ max_loaded_shards: int = 1,
+ benchmark_tokens_per_sec: float = DEFAULT_BENCHMARK_TOKENS_PER_SEC,
+ quantization: str | None = None,
+ managed_assignment: bool = False,
+ tracker_mode: bool = False,
+ hf_repo: str | None = None,
+ num_layers: int | None = None,
+ model_metadata: dict | None = None,
+ relay_addr: str | None = None,
+ cert_fingerprint: str | None = None,
+ peer_id: str | None = None,
+ ) -> None:
+ self.node_id = node_id
+ self.endpoint = endpoint
+ self.shard_start = shard_start
+ self.shard_end = shard_end
+ self.model = model
+ self.shard_checksum = shard_checksum
+ self.hardware_profile = hardware_profile
+ self.wallet_address = wallet_address
+ self.score = score
+ self.vram_bytes = vram_bytes
+ self.ram_bytes = ram_bytes
+ self.quantizations = quantizations or list(DEFAULT_QUANTIZATIONS)
+ self.max_loaded_shards = max_loaded_shards
+ self.benchmark_tokens_per_sec = benchmark_tokens_per_sec
+ self.quantization = quantization
+ self.managed_assignment = managed_assignment
+ self.model_tokens_per_sec: dict[str, float] = {}
+ self.tracker_mode = tracker_mode
+ self.hf_repo = hf_repo
+ self.num_layers = num_layers
+ self.model_metadata = dict(model_metadata or {})
+ self.relay_addr = relay_addr
+ self.cert_fingerprint = cert_fingerprint
+ self.peer_id = peer_id
+ self.pending_directives: list[dict] = []
+ self.last_heartbeat: float = time.monotonic()
+ self.total_requests: int = 0
+ self.failed_requests: int = 0
+ self.queue_depth: int = 0
+ self.uptime_seconds: float = 0.0
+ self.status: str = "ready"
+ self.heartbeats_expected: int = 0
+ self.heartbeats_received: int = 0
+ self.pending_new_assignment: dict | None = None
+
+
+def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
+ """Effective tokens/s accounting for current queue depth."""
+ observed = None
+ if model:
+ observed = node.model_tokens_per_sec.get(model)
+ if observed is None:
+ for alias in _model_aliases(model):
+ observed = node.model_tokens_per_sec.get(alias)
+ if observed is not None:
+ break
+ base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
+ return base / (node.queue_depth + 1)
+
+
+def _select_route(
+ nodes: list[_NodeEntry],
+ required_start: int,
+ required_end: int,
+ model: str | None = None,
+) -> tuple[list[_NodeEntry], str]:
+ """Greedy interval-cover biased toward fast, lightly-loaded nodes.
+
+ Among nodes that equally advance coverage, prefer the one with higher
+ effective throughput: observed per-model tokens/sec / (queue_depth + 1),
+ falling back to startup benchmark_tokens_per_sec until observations exist.
+ Tiebreak: higher shard_end (fewer hops).
+ """
+ candidates = sorted(
+ [node for node in nodes if node.shard_start is not None and node.shard_end is not None],
+ key=lambda n: (n.shard_start, -n.shard_end), # type: ignore[operator]
+ )
+ route: list[_NodeEntry] = []
+ covered_up_to = required_start - 1
+
+ while covered_up_to < required_end:
+ best: _NodeEntry | None = None
+ for node in candidates:
+ if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
+ if best is None:
+ best = node
+ elif node.shard_end > best.shard_end:
+ best = node
+ elif node.shard_end == best.shard_end and _effective_throughput(node, model) > _effective_throughput(best, model):
+ best = node
+ if best is None:
+ missing = covered_up_to + 1
+ return [], f"no route available: no registered node covers layer {missing}"
+ route.append(best)
+ covered_up_to = best.shard_end
+ candidates = [n for n in candidates if n is not best]
+
+ return route, ""
+
+
+def _coverage_percentage(
+ nodes: list[_NodeEntry],
+ required_start: int,
+ required_end: int,
+) -> float:
+ required_layers = required_end - required_start + 1
+ if required_layers <= 0:
+ return 0.0
+
+ intervals = sorted(
+ (
+ (max(required_start, node.shard_start), min(required_end, node.shard_end))
+ for node in nodes
+ if node.shard_start is not None
+ and node.shard_end is not None
+ if node.shard_end >= required_start and node.shard_start <= required_end
+ ),
+ key=lambda interval: interval[0],
+ )
+ covered = 0
+ covered_until = required_start - 1
+ for start, end in intervals:
+ if end <= covered_until:
+ continue
+ next_start = max(start, covered_until + 1)
+ if next_start > end:
+ continue
+ covered += end - next_start + 1
+ covered_until = end
+ return round((covered / required_layers) * 100, 2)
+
+
+def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
+ start = int(preset.get("layers_start", 0))
+ if "layers_end" in preset:
+ return start, int(preset["layers_end"])
+ return start, start + int(preset["total_layers"]) - 1
+
+
+def _preset_bytes_per_layer(preset: dict) -> dict[str, int]:
+ raw = preset.get("bytes_per_layer", preset.get("bytes_per_layer_at_quant", {}))
+ if isinstance(raw, dict) and raw:
+ return {str(quant): int(value) for quant, value in raw.items()}
+ return {"bfloat16": 30 * 1024 * 1024}
+
+
+def _node_quantization(node: _NodeEntry, preset: dict) -> str:
+ bytes_per_layer = _preset_bytes_per_layer(preset)
+ if node.quantization in bytes_per_layer:
+ return node.quantization
+ for quantization in node.quantizations:
+ if quantization in bytes_per_layer:
+ return quantization
+ return next(iter(bytes_per_layer))
+
+
+def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
+ """Return the memory pool used for shard-capacity planning."""
+ if node.vram_bytes > 0:
+ return node.vram_bytes, "vram"
+ if node.ram_bytes > 0:
+ return node.ram_bytes, "ram"
+ return DEFAULT_RAM_BYTES, "ram-default"
+
+
+def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
+ bytes_per_layer = _preset_bytes_per_layer(preset)
+ quantization = _node_quantization(node, preset)
+ layer_bytes = bytes_per_layer[quantization]
+ if layer_bytes <= 0:
+ return 0
+ memory_budget_bytes, _ = _node_memory_budget_bytes(node)
+ return int((memory_budget_bytes * 0.8) // layer_bytes)
+
+
+def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
+ """Operator-facing capacity fields for inspection endpoints."""
+ memory_budget_bytes, memory_budget_source = _node_memory_budget_bytes(node)
+ summary = {
+ "vram_bytes": node.vram_bytes,
+ "ram_bytes": node.ram_bytes,
+ "memory_budget_bytes": memory_budget_bytes,
+ "memory_budget_source": memory_budget_source,
+ "max_loaded_shards": node.max_loaded_shards,
+ "quantizations": list(node.quantizations),
+ "quantization": node.quantization,
+ "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
+ "effective_throughput": round(_effective_throughput(node), 4),
+ }
+ if preset is not None:
+ summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
+ return summary
+
+
+def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
+ budget, _source = _node_memory_budget_bytes(node)
+ if preset is None:
+ return int(budget * 0.8)
+ return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))
+
+
+def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
+ total_vram = sum(max(0, node.vram_bytes) for node in nodes)
+ total_ram = sum(max(0, node.ram_bytes) for node in nodes)
+ total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
+ effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
+ return {
+ "node_count": len(nodes),
+ "total_vram_bytes": total_vram,
+ "total_ram_bytes": total_ram,
+ "total_memory_budget_bytes": total_budget,
+ "effective_assignable_memory_bytes": effective_budget,
+ "total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
+ "total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
+ }
+
+
+def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
+ if preset is None:
+ return {"recommended": False}
+ pool = _pool_summary(nodes, preset)
+ required = int(preset.get("required_model_bytes", 0) or 0)
+ deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
+ missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
+ return {
+ "recommended": bool(preset.get("recommended", False)),
+ "status": preset.get("deployment_status", "available"),
+ "required_model_bytes": required or None,
+ "download_size_bytes": preset.get("download_size_bytes"),
+ "native_quantization": preset.get("native_quantization"),
+ "pool": pool,
+ "deployable": deployable,
+ "missing_effective_memory_bytes": missing,
+ }
+
+
+def _max_layers_for_memory(memory_mb: int, total_layers: int, preset: dict | None = None) -> int:
+ if total_layers <= 0:
+ return 0
+ if memory_mb <= 0:
+ return max(1, total_layers // 2)
+ bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
+ return min(
+ total_layers,
+ max(1, int(((memory_mb * 1024 * 1024) * 0.8) // bytes_per_layer)),
+ )
+
+
+def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
+ metadata: dict = {}
+ for node in nodes:
+ if node.model_metadata:
+ metadata.update(node.model_metadata)
+ if "num_layers" not in metadata:
+ layers = [node.num_layers for node in nodes if node.num_layers is not None]
+ if layers:
+ metadata["num_layers"] = max(layers)
+ return metadata
+
+
+def _coverage_map(
+ nodes: list[_NodeEntry],
+ required_start: int,
+ required_end: int,
+) -> list[dict]:
+ layer_counts = []
+ for layer in range(required_start, required_end + 1):
+ count = 0
+ for node in nodes:
+ if node.shard_start is None or node.shard_end is None:
+ continue
+ if node.shard_start <= layer <= node.shard_end:
+ count += 1
+ layer_counts.append((layer, count))
+
+ coverage: list[dict] = []
+ for layer, count in layer_counts:
+ if coverage and coverage[-1]["node_count"] == count and coverage[-1]["end_layer"] == layer - 1:
+ coverage[-1]["end_layer"] = layer
+ else:
+ coverage.append({"start_layer": layer, "end_layer": layer, "node_count": count})
+ return coverage
+
+
+def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
+ """Per-node health detail for the availability map."""
+ age = time.monotonic() - node.last_heartbeat
+ alive = age <= heartbeat_timeout
+ hb_expected = max(1, round(node.uptime_seconds / 20.0)) # assume ~20s interval
+ hb_rate = round(min(1.0, node.heartbeats_received / hb_expected), 4) if node.heartbeats_received else 0.0
+ total = node.total_requests
+ inf_rate = round((total - node.failed_requests) / total, 4) if total > 0 else 1.0
+ return {
+ "node_id": node.node_id,
+ "endpoint": node.endpoint,
+ "alive": alive,
+ "last_seen_seconds_ago": round(age, 1),
+ "status": node.status,
+ "queue_depth": node.queue_depth,
+ "total_requests": node.total_requests,
+ "heartbeat_success_rate": hb_rate,
+ "inference_success_rate": inf_rate,
+ "capacity": _node_capacity_summary(node),
+ }
+
+
+def _coverage_map_detailed(
+ nodes: list["_NodeEntry"],
+ required_start: int,
+ required_end: int,
+ heartbeat_timeout: float,
+) -> list[dict]:
+ """Like _coverage_map but with per-node identity and health in each band.
+
+ Includes all nodes (alive and stale) so operators can see both coverage
+ holes and which dead nodes used to fill them.
+ """
+ now = time.monotonic()
+
+ def covers(node: "_NodeEntry", layer: int) -> bool:
+ return (
+ node.shard_start is not None
+ and node.shard_end is not None
+ and node.shard_start <= layer <= node.shard_end
+ )
+
+ # Build per-layer list of nodes (with alive flag)
+ layer_nodes: list[list[tuple["_NodeEntry", bool]]] = []
+ for layer in range(required_start, required_end + 1):
+ ns = [
+ (n, (now - n.last_heartbeat) <= heartbeat_timeout)
+ for n in nodes
+ if covers(n, layer)
+ ]
+ layer_nodes.append(ns)
+
+ # Merge consecutive layers with identical node-set (same node_ids, same alive)
+ coverage: list[dict] = []
+ for i, layer_idx in enumerate(range(required_start, required_end + 1)):
+ ns = layer_nodes[i]
+ node_ids = [n.node_id for n, _ in ns]
+ alive_flags = [a for _, a in ns]
+ # Compare with last band
+ if (
+ coverage
+ and coverage[-1]["end_layer"] == layer_idx - 1
+ and [nd["node_id"] for nd in coverage[-1]["nodes"]] == node_ids
+ and [nd["alive"] for nd in coverage[-1]["nodes"]] == alive_flags
+ ):
+ coverage[-1]["end_layer"] = layer_idx
+ else:
+ coverage.append({
+ "start_layer": layer_idx,
+ "end_layer": layer_idx,
+ "node_count": len(ns),
+ "nodes": [_node_health(n, heartbeat_timeout) for n, _ in ns],
+ })
+ return coverage
+
+
+def _coverage_gaps(coverage: list[dict]) -> list[tuple[int, int]]:
+ return [
+ (segment["start_layer"], segment["end_layer"])
+ for segment in coverage
+ if segment["node_count"] == 0
+ ]
+
+
+def _relay_http_request(
+ relay_addr: str,
+ path: str,
+ body: bytes,
+ headers: dict[str, str],
+ timeout: float = 310.0,
+) -> dict | None:
+ """Send an HTTP-shaped request through a relay RPC WebSocket."""
+ try:
+ import websockets.sync.client as wsc # type: ignore[import]
+
+ request_id = str(uuid.uuid4())
+ with wsc.connect(relay_addr, open_timeout=10, close_timeout=5) as ws:
+ ws.send(json.dumps({
+ "request_id": request_id,
+ "method": "POST",
+ "path": path,
+ "headers": headers,
+ "body": body.decode(errors="replace"),
+ }))
+ raw = ws.recv(timeout=timeout)
+ response = json.loads(raw)
+ if response.get("request_id") not in {None, request_id}:
+ return None
+ return response
+ except Exception:
+ return None
+
+
+def _find_pinned_route(
+ nodes: list[_NodeEntry],
+ required_start: int,
+ required_end: int,
+ hop_count: int,
+) -> list[_NodeEntry] | None:
+ """First combination of exactly ``hop_count`` distinct nodes covering the
+ layer range, where every node extends coverage (US-030 benchmark routes)."""
+ for combo in itertools.permutations(nodes, hop_count):
+ covered = required_start - 1
+ valid = True
+ for candidate in combo:
+ if candidate.shard_start is None or candidate.shard_end is None:
+ valid = False
+ break
+ if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
+ valid = False
+ break
+ covered = candidate.shard_end
+ if valid and covered >= required_end:
+ return list(combo)
+ return None
+
+
+def _nodes_and_bounds_for_model(
+ server: "_TrackerHTTPServer",
+ model: str,
+) -> tuple[list[_NodeEntry], int, int] | None:
+ resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+ if preset is not None:
+ required_start, required_end = _preset_layer_bounds(preset)
+ return [
+ node for node in server.registry.values()
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ ], required_start, required_end
+
+ nodes = [
+ node for node in server.registry.values()
+ if _node_matches_model(node, model)
+ and node.shard_start is not None
+ and node.shard_end is not None
+ and node.num_layers is not None
+ ]
+ if not nodes:
+ return None
+ return nodes, 0, max(node.num_layers for node in nodes) - 1
+
+
+def _load_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
+ return {
+ "action": "LOAD_SHARD",
+ "model": model,
+ "start_layer": start,
+ "end_layer": end,
+ "shard_start": start,
+ "shard_end": end,
+ "quantization": quantization,
+ }
+
+
+def _drop_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
+ return {
+ "action": "DROP_SHARD",
+ "model": model,
+ "start_layer": start,
+ "end_layer": end,
+ "shard_start": start,
+ "shard_end": end,
+ "quantization": quantization,
+ }
+
+
+def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
+ now = time.monotonic()
+ expired_ids = [
+ node_id for node_id, entry in server.registry.items()
+ if (now - entry.last_heartbeat) > server.heartbeat_timeout
+ ]
+ for node_id in expired_ids:
+ entry = server.registry.pop(node_id)
+ print(
+ f"[tracker] node expired: {node_id} {entry.endpoint} "
+ f"(no heartbeat for >{server.heartbeat_timeout:.0f}s)",
+ flush=True,
+ )
+ if expired_ids:
+ _rebalance_all_locked(server)
+ return expired_ids
+
+
+def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
+ resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+ if preset is None:
+ return
+ required_start, required_end = _preset_layer_bounds(preset)
+ total_layers = required_end - required_start + 1
+ model_nodes = [
+ node for node in server.registry.values()
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ ]
+ managed_nodes = [node for node in model_nodes if node.managed_assignment]
+ if not managed_nodes:
+ return
+
+ previous_ranges = {
+ node.node_id: (node.shard_start, node.shard_end, node.quantization)
+ for node in managed_nodes
+ }
+ for node in managed_nodes:
+ node.shard_start = None
+ node.shard_end = None
+
+ managed_nodes.sort(
+ key=lambda node: (
+ -node.benchmark_tokens_per_sec,
+ -_node_layer_capacity(node, preset),
+ node.node_id,
+ )
+ )
+ base_nodes = [node for node in model_nodes if not node.managed_assignment]
+ coverage = _coverage_map(base_nodes, required_start, required_end)
+ gaps = _coverage_gaps(coverage)
+ if not gaps:
+ gaps = [(required_start, required_end)]
+
+ eligible_nodes = [
+ node for node in managed_nodes
+ if _node_layer_capacity(node, preset) > 0
+ ]
+ node_index = 0
+ for gap_start, gap_end in gaps:
+ cursor = gap_start
+ while cursor <= gap_end and node_index < len(eligible_nodes):
+ node = eligible_nodes[node_index]
+ remaining_layers = gap_end - cursor + 1
+ remaining_nodes_after = len(eligible_nodes) - node_index - 1
+ capacity = min(
+ _node_layer_capacity(node, preset),
+ total_layers,
+ max(1, remaining_layers - remaining_nodes_after),
+ )
+ if capacity <= 0:
+ node_index += 1
+ continue
+ quantization = _node_quantization(node, preset)
+ node.quantization = quantization
+ node.shard_start = cursor
+ node.shard_end = min(gap_end, cursor + capacity - 1)
+ cursor = node.shard_end + 1
+ node_index += 1
+
+ for node in managed_nodes:
+ previous_start, previous_end, previous_quantization = previous_ranges[node.node_id]
+ current_range = (node.shard_start, node.shard_end, node.quantization)
+ if node.shard_start is None or node.shard_end is None or current_range == previous_ranges[node.node_id]:
+ continue
+ if previous_start is not None and previous_end is not None:
+ node.pending_directives.append(
+ _drop_directive(
+ node,
+ model,
+ previous_start,
+ previous_end,
+ previous_quantization or _node_quantization(node, preset),
+ )
+ )
+ node.pending_directives.append(
+ _load_directive(
+ node,
+ model,
+ node.shard_start,
+ node.shard_end,
+ node.quantization or _node_quantization(node, preset),
+ )
+ )
+
+
+def _hf_rebalance_preset(nodes: list[_NodeEntry]) -> dict:
+ total_layers = max(node.num_layers or 0 for node in nodes)
+ return {
+ "layers_start": 0,
+ "layers_end": total_layers - 1,
+ "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
+ }
+
+
+def _rebalance_hf_model_locked(server: "_TrackerHTTPServer", hf_repo: str) -> None:
+ model_nodes = [
+ node for node in server.registry.values()
+ if node.hf_repo == hf_repo
+ and node.num_layers is not None
+ ]
+ managed_nodes = [node for node in model_nodes if node.managed_assignment]
+ if not model_nodes or not managed_nodes:
+ return
+
+ preset = _hf_rebalance_preset(model_nodes)
+ required_start, required_end = _preset_layer_bounds(preset)
+ total_layers = required_end - required_start + 1
+ if total_layers <= 0:
+ return
+
+ previous_ranges = {
+ node.node_id: (node.shard_start, node.shard_end, node.quantization)
+ for node in managed_nodes
+ }
+ for node in managed_nodes:
+ node.shard_start = None
+ node.shard_end = None
+
+ managed_nodes.sort(
+ key=lambda node: (
+ -node.benchmark_tokens_per_sec,
+ -_node_layer_capacity(node, preset),
+ node.node_id,
+ )
+ )
+ base_nodes = [node for node in model_nodes if not node.managed_assignment]
+ coverage = _coverage_map(base_nodes, required_start, required_end)
+ gaps = _coverage_gaps(coverage)
+ if not gaps:
+ gaps = [(required_start, required_end)]
+
+ eligible_nodes = [
+ node for node in managed_nodes
+ if _node_layer_capacity(node, preset) > 0
+ ]
+ node_index = 0
+ for gap_start, gap_end in gaps:
+ cursor = gap_start
+ while cursor <= gap_end and node_index < len(eligible_nodes):
+ node = eligible_nodes[node_index]
+ remaining_layers = gap_end - cursor + 1
+ remaining_nodes_after = len(eligible_nodes) - node_index - 1
+ capacity = min(
+ _node_layer_capacity(node, preset),
+ total_layers,
+ max(1, remaining_layers - remaining_nodes_after),
+ )
+ if capacity <= 0:
+ node_index += 1
+ continue
+ quantization = _node_quantization(node, preset)
+ node.quantization = quantization
+ node.shard_start = cursor
+ node.shard_end = min(gap_end, cursor + capacity - 1)
+ cursor = node.shard_end + 1
+ node_index += 1
+
+ for node in managed_nodes:
+ previous_start, previous_end, previous_quantization = previous_ranges[node.node_id]
+ current_range = (node.shard_start, node.shard_end, node.quantization)
+ if node.shard_start is None or node.shard_end is None or current_range == previous_ranges[node.node_id]:
+ continue
+ if previous_start is not None and previous_end is not None:
+ node.pending_directives.append(
+ _drop_directive(
+ node,
+ hf_repo,
+ previous_start,
+ previous_end,
+ previous_quantization or _node_quantization(node, preset),
+ )
+ )
+ node.pending_directives.append(
+ _load_directive(
+ node,
+ hf_repo,
+ node.shard_start,
+ node.shard_end,
+ node.quantization or _node_quantization(node, preset),
+ )
+ )
+
+
+def _rebalance_all_locked(server: "_TrackerHTTPServer") -> None:
+ for model in list(server.model_presets):
+ _rebalance_model_locked(server, model)
+ for hf_repo in sorted({node.hf_repo for node in server.registry.values() if node.hf_repo}):
+ _rebalance_hf_model_locked(server, hf_repo)
+
+
+def _api_key_from_headers(headers) -> str | None:
+ auth = headers.get("Authorization")
+ if not auth:
+ return None
+ if auth.lower().startswith("bearer "):
+ return auth.split(" ", 1)[1].strip() or None
+ return auth.strip() or None
+
+
+def _usage_total_tokens(payload: dict) -> int | None:
+ usage = payload.get("usage")
+ if not isinstance(usage, dict):
+ return None
+ total = usage.get("total_tokens")
+ if isinstance(total, (int, float)):
+ return int(total)
+ prompt = usage.get("prompt_tokens")
+ completion = usage.get("completion_tokens")
+ if isinstance(prompt, (int, float)) or isinstance(completion, (int, float)):
+ return int(prompt or 0) + int(completion or 0)
+ return None
+
+
+def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
+ if contracts is None or not wallet_address:
+ return None
+ if contracts.registry.get_wallet(wallet_address).banned:
+ return "wallet is banned"
+ return None
+
+
+def _node_id_for_registration(
+ endpoint: str,
+ model: str,
+ wallet_address: str | None,
+ shard_start: int | None,
+ shard_end: int | None,
+ hf_repo: str | None,
+) -> str:
+ wallet_prefix = wallet_address[:8] if wallet_address else "anon"
+ stable_key = "|".join([
+ wallet_address or "",
+ endpoint.rstrip("/"),
+ model,
+ hf_repo or "",
+ "" if shard_start is None else str(shard_start),
+ "" if shard_end is None else str(shard_end),
+ ])
+ digest = hashlib.sha256(stable_key.encode()).hexdigest()[:12]
+ return f"{wallet_prefix}-{digest}"
+
+
+class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
+ daemon_threads = True
+
+ def __init__(
+ self,
+ addr: tuple,
+ handler,
+ registry: dict,
+ lock: threading.Lock,
+ heartbeat_timeout: float,
+ model_presets: dict,
+ contracts: Any | None,
+ minimum_stake: int,
+ relay_url: str | None = None,
+ raft: "RaftNode | None" = None,
+ gossip: "NodeGossip | None" = None,
+ stats: "_StatsCollector | None" = None,
+ billing: "BillingLedger | None" = None,
+ accounts: "AccountStore | None" = None,
+ benchmark_results_path: str | None = None,
+ validator_service_token: str | None = None,
+ hive_secret: str | None = None,
+ ) -> None:
+ super().__init__(addr, handler)
+ self.registry = registry
+ self.lock = lock
+ self.heartbeat_timeout = heartbeat_timeout
+ self.model_presets = model_presets
+ self.contracts = contracts
+ self.minimum_stake = minimum_stake
+ self.relay_url = relay_url.rstrip("/") if relay_url else None
+ self.raft = raft
+ self.gossip = gossip
+ self.stats: _StatsCollector | None = stats
+ self.billing: BillingLedger | None = billing
+ self.accounts: AccountStore | None = accounts
+ self.benchmark_results_path = benchmark_results_path or os.path.join(
+ os.getcwd(), "benchmark_results.json"
+ )
+ self.benchmark_lock = threading.Lock()
+ self.validator_service_token = validator_service_token
+ self.hive_secret = hive_secret
+
+
+class _TrackerHandler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, fmt, *args): # noqa: suppress request logs in tests
+ pass
+
+ def _send_json(self, status: int, data: dict) -> None:
+ body = json.dumps(data).encode()
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ try:
+ self.wfile.write(body)
+ except BrokenPipeError:
+ pass
+
+ # ---- unified auth boundary (ADR-0017) ----
+
+ def _resolve_identity(self) -> tuple[str | None, dict | None]:
+ """Resolve the caller to (role, account).
+
+ Roles: "validator" (service token), "admin"/"user" (session token).
+ Client API keys resolve to no privileged role — they authorize
+ inference and wallet binding only, never operator endpoints.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ token = _api_key_from_headers(self.headers)
+ if not token:
+ return None, None
+ if is_validator_token(token, server.validator_service_token):
+ return "validator", None
+ if server.accounts is not None:
+ account = server.accounts.session_account(token)
+ if account is not None:
+ return account.get("role", "user"), account
+ return None, None
+
+ def _require_role(self, *allowed: str) -> bool:
+ """Gate a privileged handler; sends 401/403 and returns False on failure.
+
+ 401 when no credential was presented; 403 when a credential was
+ presented but does not resolve to an allowed role — this covers
+ client API keys and garbage bearer strings on operator endpoints.
+ """
+ role, _account = self._resolve_identity()
+ if role in allowed:
+ return True
+ if _api_key_from_headers(self.headers) is None:
+ self._send_json(401, {"error": "authentication required (admin session or service token)"})
+ else:
+ self._send_json(403, {"error": "this endpoint requires an admin session or service token"})
+ return False
+
+ def _read_hive_authenticated_body(self) -> dict | None:
+ """Read + verify a hive gossip body (HMAC per ADR-0017 §3).
+
+ Fails closed: without a configured --hive-secret no gossip is
+ accepted. Sends the error response itself and returns None on failure.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ length = int(self.headers.get("Content-Length", 0))
+ raw = self.rfile.read(length) if length else b"{}"
+ if not verify_hive_request(server.hive_secret, self.headers, raw):
+ self._send_json(401, {"error": "valid hive signature required"})
+ return None
+ try:
+ body = json.loads(raw or b"{}")
+ except json.JSONDecodeError:
+ self._send_json(400, {"error": "invalid JSON body"})
+ return None
+ if not isinstance(body, dict):
+ self._send_json(400, {"error": "JSON body must be an object"})
+ return None
+ return body
+
+ def _read_json_body(self) -> dict | None:
+ length = int(self.headers.get("Content-Length", 0))
+ try:
+ body = json.loads(self.rfile.read(length) or b"{}")
+ except json.JSONDecodeError:
+ self._send_json(400, {"error": "invalid JSON body"})
+ return None
+ if not isinstance(body, dict):
+ self._send_json(400, {"error": "JSON body must be an object"})
+ return None
+ return body
+
+ def _purge_expired_nodes(self) -> None:
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ _purge_expired_nodes_locked(server)
+
+ def do_POST(self):
+ if self.path == "/v1/chat/completions":
+ self._handle_proxy_chat()
+ return
+ if self.path == "/v1/nodes/register":
+ self._handle_register()
+ return
+ if self.path == "/v1/raft/vote":
+ self._handle_raft_vote()
+ return
+ if self.path == "/v1/raft/append":
+ self._handle_raft_append()
+ return
+ if self.path == "/v1/gossip":
+ self._handle_gossip()
+ return
+ if self.path == "/v1/stats/gossip":
+ self._handle_stats_gossip()
+ return
+ if self.path == "/v1/billing/gossip":
+ self._handle_billing_gossip()
+ return
+ if self.path == "/v1/billing/forfeit":
+ self._handle_billing_forfeit()
+ return
+ if self.path == "/v1/auth/register":
+ self._handle_auth_register()
+ return
+ if self.path == "/v1/auth/login":
+ self._handle_auth_login()
+ return
+ if self.path == "/v1/auth/logout":
+ self._handle_auth_logout()
+ return
+ if self.path == "/v1/account/keys":
+ self._handle_account_key_create()
+ return
+ if self.path == "/v1/account/keys/revoke":
+ self._handle_account_key_revoke()
+ return
+ if self.path == "/v1/accounts/gossip":
+ self._handle_accounts_gossip()
+ return
+ if self.path == "/v1/benchmark/hop-penalty":
+ self._handle_benchmark_hop_penalty()
+ return
+ if self.path == "/v1/wallet/register":
+ self._handle_wallet_register()
+ return
+ parts = self.path.split("/")
+ # /v1/nodes//heartbeat -> ['', 'v1', 'nodes', '', 'heartbeat']
+ if len(parts) == 5 and parts[1] == "v1" and parts[2] == "nodes" and parts[4] == "heartbeat":
+ self._handle_heartbeat(parts[3])
+ return
+ self.send_response(404)
+ self.end_headers()
+
+ def do_GET(self):
+ parsed = urllib.parse.urlparse(self.path)
+ if parsed.path == "/v1/route":
+ self._handle_route(parsed)
+ elif parsed.path == "/v1/routes":
+ self._handle_routes(parsed)
+ elif parsed.path == "/v1/nodes/assign":
+ self._handle_assign(parsed)
+ elif parsed.path == "/v1/network/assign":
+ self._handle_network_assign(parsed)
+ elif parsed.path == "/v1/network/map":
+ self._handle_network_map()
+ elif parsed.path == "/v1/models":
+ self._handle_models()
+ elif parsed.path.startswith("/v1/coverage/"):
+ model = urllib.parse.unquote(parsed.path.removeprefix("/v1/coverage/"))
+ self._handle_coverage(model)
+ elif parsed.path.startswith("/v1/tracker-nodes/"):
+ model = urllib.parse.unquote(parsed.path.removeprefix("/v1/tracker-nodes/"))
+ self._handle_tracker_nodes(model)
+ elif parsed.path.startswith("/v1/head-workers/"):
+ model = urllib.parse.unquote(parsed.path.removeprefix("/v1/head-workers/"))
+ self._handle_tracker_nodes(model)
+ elif parsed.path == "/v1/raft/status":
+ self._handle_raft_status()
+ elif parsed.path == "/v1/stats":
+ self._handle_stats()
+ elif parsed.path == "/v1/billing/summary":
+ self._handle_billing_summary()
+ elif parsed.path == "/v1/billing/settlements":
+ self._handle_billing_settlements()
+ elif parsed.path == "/v1/account":
+ self._handle_account_me()
+ elif parsed.path == "/v1/admin/accounts":
+ self._handle_admin_accounts()
+ elif parsed.path == "/v1/benchmark/results":
+ self._handle_benchmark_results()
+ elif parsed.path == "/v1/registry/wallets":
+ self._handle_registry_wallets()
+ elif parsed.path in ("/dashboard", "/dashboard/"):
+ self._handle_dashboard()
+ elif parsed.path == "/v1/health":
+ self._send_json(200, {"status": "ok"})
+ else:
+ self.send_response(404)
+ self.end_headers()
+
+ def _handle_models(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ created = int(time.time())
+ with server.lock:
+ self._purge_expired_nodes()
+ alive = list(server.registry.values())
+ if server.contracts is not None:
+ alive = [
+ node for node in alive
+ if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
+ ]
+ data = []
+ seen_ids: set[str] = set()
+ for name, preset in server.model_presets.items():
+ model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
+ if not model_nodes and not preset.get("recommended"):
+ continue
+ required_start, required_end = _preset_layer_bounds(preset)
+ coverage = _coverage_percentage(
+ model_nodes,
+ required_start,
+ required_end,
+ )
+ aliases = [name]
+ hf_repo = preset.get("hf_repo")
+ if hf_repo and hf_repo not in aliases:
+ aliases.append(hf_repo)
+ for alias in preset.get("aliases", []) or []:
+ if isinstance(alias, str) and alias not in aliases:
+ aliases.append(alias)
+ data.append({
+ "id": name,
+ "object": "model",
+ "created": created,
+ "owned_by": "meshnet",
+ "name": name,
+ "hf_repo": hf_repo,
+ "aliases": aliases,
+ "metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
+ "recommended": bool(preset.get("recommended", False)),
+ "deployment": _deployment_summary(alive, preset),
+ "shard_coverage_percentage": coverage,
+ })
+ seen_ids.add(name)
+ if hf_repo:
+ seen_ids.add(hf_repo)
+
+ hf_model_ids = sorted({
+ node.hf_repo or node.model
+ for node in alive
+ if node.model is not None
+ and node.model not in server.model_presets
+ and node.shard_start is not None
+ and node.shard_end is not None
+ and node.num_layers is not None
+ })
+ for model_id in hf_model_ids:
+ if model_id is None or model_id in seen_ids:
+ continue
+ model_nodes = [
+ node for node in alive
+ if node.shard_start is not None
+ and node.shard_end is not None
+ and node.num_layers is not None
+ and (node.hf_repo == model_id or (node.hf_repo is None and node.model == model_id))
+ ]
+ if not model_nodes:
+ continue
+ short_names = sorted({node.model for node in model_nodes if node.model})
+ aliases = [model_id, *[name for name in short_names if name != model_id]]
+ required_start = 0
+ required_end = max(node.num_layers for node in model_nodes) - 1
+ data.append({
+ "id": model_id,
+ "object": "model",
+ "created": created,
+ "owned_by": "meshnet",
+ "name": short_names[0] if short_names else model_id,
+ "hf_repo": model_id if any(node.hf_repo == model_id for node in model_nodes) else None,
+ "aliases": aliases,
+ "metadata": _model_metadata_from_nodes(model_nodes),
+ "shard_coverage_percentage": _coverage_percentage(
+ model_nodes,
+ required_start,
+ required_end,
+ ),
+ })
+ seen_ids.add(model_id)
+ self._send_json(200, {"object": "list", "data": data})
+
+ def _handle_coverage(self, model: str):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ # Do NOT purge before coverage — dead nodes are included with alive=false
+ # so operators can see what was covering each layer band before failure.
+ with server.lock:
+ resolved = _nodes_and_bounds_for_model(server, model)
+ if resolved is None:
+ self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
+ return
+ all_nodes, required_start, required_end = resolved
+ if server.contracts is not None:
+ all_nodes = [
+ node for node in all_nodes
+ if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
+ ]
+ coverage = _coverage_map_detailed(all_nodes, required_start, required_end, server.heartbeat_timeout)
+ self._send_json(200, {"model": model, "coverage": coverage})
+
+ def _handle_tracker_nodes(self, model: str):
+ """Return head workers: worker nodes that can start inference for a model.
+
+ The historical endpoint name is /v1/tracker-nodes, but these are not
+ tracker processes and they are the only machines that load model shards.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+ if preset is None:
+ self._send_json(404, {"error": f"unknown model preset: {model!r}"})
+ return
+ required_start, _ = _preset_layer_bounds(preset)
+ with server.lock:
+ self._purge_expired_nodes()
+ alive = [
+ node for node in server.registry.values()
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ ]
+ if server.contracts is not None:
+ alive = [
+ node for node in alive
+ if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
+ ]
+ tracker_nodes = [
+ node for node in alive
+ if node.shard_start is not None
+ and node.shard_start == required_start
+ and node.tracker_mode
+ ]
+ self._send_json(200, {
+ "model": resolved_name,
+ "head_workers": [
+ {
+ "node_id": node.node_id,
+ "endpoint": node.endpoint,
+ "relay_addr": node.relay_addr,
+ "peer_id": node.peer_id,
+ }
+ for node in tracker_nodes
+ ],
+ "tracker_nodes": [
+ {
+ "node_id": node.node_id,
+ "endpoint": node.endpoint,
+ "relay_addr": node.relay_addr,
+ "peer_id": node.peer_id,
+ "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
+ }
+ for node in tracker_nodes
+ ],
+ })
+
+ def _handle_network_map(self) -> None:
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ with server.lock:
+ self._purge_expired_nodes()
+ nodes = list(server.registry.values())
+
+ def capacity_for(node: _NodeEntry) -> dict:
+ preset = None
+ if node.model:
+ preset = server.model_presets.get(node.model)
+ if preset is None and node.hf_repo and node.num_layers:
+ preset = _hf_rebalance_preset([node])
+ return _node_capacity_summary(node, preset)
+
+ def throughput_for(node: _NodeEntry) -> dict:
+ if server.stats is None:
+ return {}
+ models = [m for m in (node.hf_repo, node.model) if m]
+ result = {}
+ for model in models:
+ result[model] = server.stats.get_node_model_stats(node.node_id, model)
+ return result
+
+ self._send_json(200, {
+ "relay_url": server.relay_url,
+ "pool": _pool_summary(nodes),
+ "recommended_models": [
+ {
+ "id": name,
+ "hf_repo": preset.get("hf_repo"),
+ "aliases": list(preset.get("aliases", []) or []),
+ "metadata": dict(preset.get("metadata") or {}),
+ "deployment": _deployment_summary(nodes, preset),
+ }
+ for name, preset in server.model_presets.items()
+ if preset.get("recommended")
+ ],
+ "nodes": [
+ {
+ "node_id": node.node_id,
+ "endpoint": node.endpoint,
+ "relay_addr": node.relay_addr,
+ "peer_id": node.peer_id,
+ "model": node.model,
+ "hf_repo": node.hf_repo,
+ "num_layers": node.num_layers,
+ "model_metadata": dict(node.model_metadata),
+ "shard_start": node.shard_start,
+ "shard_end": node.shard_end,
+ "tracker_mode": node.tracker_mode,
+ "last_heartbeat": node.last_heartbeat,
+ "capacity": capacity_for(node),
+ "throughput": throughput_for(node),
+ "stats": _node_health(node, server.heartbeat_timeout),
+ }
+ for node in nodes
+ ],
+ })
+
+ # ---------------------------------------------------------------- OpenAI proxy
+
+ def _handle_proxy_chat(self) -> None:
+ """Proxy POST /v1/chat/completions to a tracker-mode (first-shard) node.
+
+ Picks a live tracker-mode node for the requested model using round-robin,
+ then forwards the request verbatim and relays the response (including
+ streaming SSE chunks) back to the caller.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+
+ length = int(self.headers.get("Content-Length", 0))
+ raw_body = self.rfile.read(length) if length else b"{}"
+
+ try:
+ body = json.loads(raw_body)
+ except json.JSONDecodeError:
+ self._send_json(400, {"error": {"message": "invalid JSON", "type": "invalid_request_error", "code": "invalid_request"}})
+ return
+
+ model: str = body.get("model", "")
+ is_stream: bool = bool(body.get("stream", False))
+
+ if model and server.stats is not None:
+ server.stats.record_request(model)
+
+ # Billing gate (ADR-0015): reject before any routing — no free work.
+ api_key = _api_key_from_headers(self.headers)
+ if server.billing is not None:
+ if api_key is None:
+ self._send_json(401, {"error": {
+ "message": "missing API key: send Authorization: Bearer ",
+ "type": "invalid_request_error",
+ "code": "missing_api_key",
+ }})
+ return
+ if server.accounts is not None and server.accounts.is_key_revoked(api_key):
+ self._send_json(401, {"error": {
+ "message": "API key has been revoked",
+ "type": "invalid_request_error",
+ "code": "invalid_api_key",
+ }})
+ return
+ if not server.billing.has_funds(api_key):
+ self._send_json(402, {"error": {
+ "message": "insufficient balance: deposit USDT to continue",
+ "type": "insufficient_quota",
+ "code": "insufficient_balance",
+ }})
+ return
+
+ # US-030: optional pinned route — "route": [node_id, ...] uses those
+ # nodes in order instead of auto-selection. Absent field: unchanged.
+ pinned_ids = body.get("route")
+ pinned_nodes: list[_NodeEntry] | None = None
+ if pinned_ids is not None:
+ if (
+ not isinstance(pinned_ids, list)
+ or not pinned_ids
+ or not all(isinstance(nid, str) and nid for nid in pinned_ids)
+ ):
+ self._send_json(400, {"error": {
+ "message": "route must be a non-empty list of node id strings",
+ "type": "invalid_request_error",
+ "code": "invalid_route",
+ }})
+ return
+ with server.lock:
+ self._purge_expired_nodes()
+ missing = [nid for nid in pinned_ids if nid not in server.registry]
+ if missing:
+ self._send_json(400, {"error": {
+ "message": f"unknown node ids in route: {missing}",
+ "type": "invalid_request_error",
+ "code": "unknown_route_nodes",
+ }})
+ return
+ pinned_nodes = [server.registry[nid] for nid in pinned_ids]
+
+ if pinned_nodes is not None:
+ node = pinned_nodes[0]
+ else:
+ # Find a live tracker-mode node for this model
+ with server.lock:
+ self._purge_expired_nodes()
+ candidates = [
+ n for n in server.registry.values()
+ if n.tracker_mode and _node_matches_model(n, model)
+ ]
+
+ if not candidates:
+ # Fall back: any node serving shard_start=0 for this model
+ with server.lock:
+ candidates = [
+ n for n in server.registry.values()
+ if n.shard_start == 0 and _node_matches_model(n, model)
+ ]
+
+ if not candidates:
+ self._send_json(503, {"error": {
+ "message": f"no nodes available for model {model!r}",
+ "type": "service_unavailable",
+ "code": "model_not_available",
+ }})
+ return
+
+ node = max(candidates, key=lambda n: _effective_throughput(n, model))
+ target_url = f"{node.endpoint}/v1/chat/completions"
+ request_id = str(body.get("id") or f"req-{time.time_ns():x}")
+
+ # Pre-resolve the downstream route so the first-shard node skips its own
+ # tracker query. We already hold the full registry picture — no need for
+ # a second round-trip.
+ route_model = node.hf_repo or node.model or model
+ with server.lock:
+ if route_model in server.model_presets:
+ preset = server.model_presets[route_model]
+ rs, re = _preset_layer_bounds(preset)
+ all_nodes: list = [n for n in server.registry.values() if n.model == route_model]
+ else:
+ all_nodes = [
+ n for n in server.registry.values()
+ if _node_matches_model(n, route_model)
+ and n.shard_start is not None and n.num_layers is not None
+ ]
+ rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
+ if pinned_nodes is not None:
+ route_nodes = pinned_nodes
+ else:
+ route_nodes, _ = _select_route(all_nodes, rs, re, model=route_model)
+ # Compute start_layer for each hop: each node begins where the previous ended + 1.
+ # This allows overlapping shard registrations without double-computation.
+ covered_up_to = rs - 1
+ route_hops: list[dict] = []
+ node_work: list[tuple[str | None, int]] = []
+ for rn in route_nodes:
+ hop: dict = {"endpoint": rn.endpoint, "start_layer": covered_up_to + 1}
+ if rn.relay_addr:
+ hop["relay_addr"] = rn.relay_addr
+ route_hops.append(hop)
+ effective_end = rn.shard_end if rn.shard_end is not None else covered_up_to
+ node_work.append((rn.wallet_address, max(0, effective_end - covered_up_to)))
+ covered_up_to = effective_end
+ # Strip the first-shard node we're about to proxy to — it's already handling the request.
+ downstream_hops = [h for h in route_hops if h["endpoint"].rstrip("/") != node.endpoint.rstrip("/")]
+ downstream_urls = json.dumps(downstream_hops)
+ route_debug = " -> ".join(
+ f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
+ for n in route_nodes
+ )
+ print(
+ f"[tracker] proxy route {request_id}: model={model!r} "
+ f"head={node.node_id}@{node.endpoint} downstream={downstream_urls} "
+ f"route={route_debug or ''}",
+ flush=True,
+ )
+
+ req = urllib.request.Request(
+ target_url,
+ data=raw_body,
+ headers={
+ "Content-Type": "application/json",
+ "X-Meshnet-Route": downstream_urls,
+ },
+ method="POST",
+ )
+ # Copy Authorization header from client if present
+ auth = self.headers.get("Authorization")
+ if auth:
+ req.add_header("Authorization", auth)
+
+ relay_headers = {
+ "Content-Type": "application/json",
+ "X-Meshnet-Route": downstream_urls,
+ **({"Authorization": auth} if auth else {}),
+ }
+
+ if node.relay_addr:
+ print(
+ f"[tracker] proxy via relay {request_id}: {node.relay_addr} "
+ f"(direct endpoint {target_url})",
+ flush=True,
+ )
+ started = time.monotonic()
+ relayed = _relay_http_request(
+ node.relay_addr,
+ path="/v1/chat/completions",
+ body=raw_body,
+ headers=relay_headers,
+ )
+ elapsed = time.monotonic() - started
+ if relayed is not None:
+ self._send_relayed_response(relayed)
+ if int(relayed.get("status", 503)) < 400:
+ body_text = relayed.get("body") or ""
+ try:
+ tokens = _usage_total_tokens(json.loads(body_text)) or 0
+ except (json.JSONDecodeError, TypeError):
+ tokens = 0
+ self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
+ self._bill_completed(api_key, model, tokens, node_work)
+ return
+ print(
+ f"[tracker] relay proxy failed {request_id}: {node.relay_addr}; "
+ f"trying direct {target_url}",
+ flush=True,
+ )
+
+ try:
+ started = time.monotonic()
+ upstream = urllib.request.urlopen(req, timeout=300.0)
+ print(f"[tracker] proxy connected {request_id}: {target_url}", flush=True)
+ except urllib.error.HTTPError as exc:
+ # Relay error status + body from node
+ err_body = exc.read()
+ self.send_response(exc.code)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(err_body)))
+ self.end_headers()
+ try:
+ self.wfile.write(err_body)
+ except BrokenPipeError:
+ pass
+ return
+ except Exception as exc:
+ if node.relay_addr:
+ print(
+ f"[tracker] direct proxy failed {request_id}: {target_url}: {exc}; "
+ f"relay already attempted for {node.relay_addr}",
+ flush=True,
+ )
+ else:
+ print(f"[tracker] proxy failed {request_id}: {target_url}: {exc}", flush=True)
+ self._send_json(503, {"error": {
+ "message": f"upstream node unreachable: {exc}",
+ "type": "service_unavailable",
+ "code": "upstream_error",
+ }})
+ return
+
+ with upstream:
+ content_type = upstream.headers.get("Content-Type", "application/json")
+ if is_stream or "text/event-stream" in content_type:
+ # Relay SSE stream chunk-by-chunk
+ self.send_response(200)
+ self.send_header("Content-Type", "text/event-stream; charset=utf-8")
+ self.send_header("Cache-Control", "no-cache")
+ self.end_headers()
+ stream_tokens: int | None = None
+ chunk_count = 0
+ try:
+ while True:
+ line = upstream.readline()
+ if not line:
+ break
+ self.wfile.write(line)
+ self.wfile.flush()
+ if line.startswith(b"data:"):
+ payload = line[5:].strip()
+ if payload and payload != b"[DONE]":
+ chunk_count += 1
+ if b'"usage"' in payload:
+ try:
+ found = _usage_total_tokens(json.loads(payload))
+ if found:
+ stream_tokens = found
+ except json.JSONDecodeError:
+ pass
+ except BrokenPipeError:
+ pass
+ elapsed = time.monotonic() - started
+ # Bill even on client disconnect — the nodes did the work.
+ # Chunk count approximates generated tokens when the stream
+ # carries no usage record.
+ observed_tokens = stream_tokens if stream_tokens is not None else chunk_count
+ self._record_observed_throughput(model, route_model, observed_tokens, elapsed, route_nodes)
+ self._bill_completed(
+ api_key, model,
+ observed_tokens,
+ node_work,
+ )
+ else:
+ # Non-streaming: buffer and relay
+ resp_body = upstream.read()
+ elapsed = time.monotonic() - started
+ print(
+ f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
+ flush=True,
+ )
+ self.send_response(200)
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(resp_body)))
+ self.end_headers()
+ try:
+ self.wfile.write(resp_body)
+ except BrokenPipeError:
+ pass
+ try:
+ tokens = _usage_total_tokens(json.loads(resp_body)) or 0
+ except json.JSONDecodeError:
+ tokens = 0
+ self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
+ self._bill_completed(api_key, model, tokens, node_work)
+
+ def _record_observed_throughput(
+ self,
+ requested_model: str,
+ route_model: str,
+ total_tokens: int,
+ elapsed_seconds: float,
+ route_nodes: list[_NodeEntry],
+ ) -> None:
+ """Record observed route TPS for participating nodes.
+
+ The tracker sees end-to-end request duration, not per-hop timings, so
+ each hop gets the same route-level observation for now. Per-hop telemetry
+ can refine this later without changing the external stats shape.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.stats is None or total_tokens <= 0 or elapsed_seconds <= 0:
+ return
+ models = [m for m in (requested_model, route_model) if m]
+ if len(models) == 2 and models[0] == models[1]:
+ models = [models[0]]
+ for node in route_nodes:
+ for model in models:
+ server.stats.record_node_throughput(
+ node.node_id,
+ model,
+ total_tokens=total_tokens,
+ elapsed_seconds=elapsed_seconds,
+ )
+ stats = server.stats.get_node_model_stats(node.node_id, model)
+ observed = stats.get("tokens_per_sec_last_hour")
+ if observed is not None:
+ node.model_tokens_per_sec[model] = float(observed)
+
+ def _bill_completed(
+ self,
+ api_key: str | None,
+ model: str,
+ total_tokens: int,
+ node_work: list[tuple[str | None, int]],
+ ) -> None:
+ """Charge a completed request against the billing ledger (ADR-0015)."""
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.billing is None or api_key is None:
+ return
+ # Probationary period (issue 08): a wallet's first N jobs earn nothing —
+ # its share stays in the protocol cut. Job counts live in the registry
+ # contract, so this only applies when the tracker runs with contracts.
+ if server.contracts is not None:
+ adjusted: list[tuple[str | None, int]] = []
+ for wallet, work in node_work:
+ if wallet:
+ in_probation = server.contracts.registry.probationary_jobs_remaining(wallet) > 0
+ server.contracts.registry.record_completed_job(wallet)
+ if in_probation:
+ wallet = None
+ adjusted.append((wallet, work))
+ node_work = adjusted
+ try:
+ event = server.billing.charge_request(api_key, model, total_tokens, node_work)
+ print(
+ f"[tracker] billed api_key=…{api_key[-6:]}: model={model!r} "
+ f"tokens={total_tokens} cost={event['cost']:.6f} USDT "
+ f"shares={event['shares']}",
+ flush=True,
+ )
+ except Exception as exc:
+ print(f"[tracker] billing failed for model={model!r}: {exc}", flush=True)
+
+ def _send_relayed_response(self, response: dict) -> None:
+ status = int(response.get("status", 503))
+ headers = response.get("headers") if isinstance(response.get("headers"), dict) else {}
+ body_text = response.get("body") or ""
+ body = body_text.encode() if isinstance(body_text, str) else bytes(body_text)
+ self.send_response(status)
+ self.send_header("Content-Type", headers.get("Content-Type", "application/json"))
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ try:
+ self.wfile.write(body)
+ except BrokenPipeError:
+ pass
+
+ def _handle_register(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_json_body()
+ if body is None:
+ return
+
+ # --- Raft cluster mode: forward to leader or propose via Raft ---
+ if server.raft is not None:
+ if not server.raft.is_leader:
+ leader = server.raft.leader()
+ if leader is None:
+ self._send_json(503, {"error": "no leader elected — retry in a moment"})
+ return
+ # Proxy to leader
+ try:
+ data = json.dumps(body).encode()
+ req = urllib.request.Request(
+ f"{leader}/v1/nodes/register",
+ data=data,
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+ with urllib.request.urlopen(req, timeout=5.0) as r:
+ resp_body = r.read()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(resp_body)))
+ self.end_headers()
+ self.wfile.write(resp_body)
+ except Exception as exc:
+ self._send_json(503, {"error": f"leader proxy failed: {exc}"})
+ return
+ # Leader path: fall through to normal registration, then replicate via Raft.
+ # We let the registration run first (to generate node_id), then asynchronously
+ # propose to the Raft log so followers can replicate the entry.
+ # Raft proposal happens after the response is sent (fire-and-forget replication).
+
+ endpoint = body.get("endpoint")
+ if not isinstance(endpoint, str) or not endpoint:
+ self._send_json(400, {"error": "endpoint is required"})
+ return
+ parsed_endpoint = urllib.parse.urlparse(endpoint)
+ if parsed_endpoint.scheme not in {"http", "https"} or not parsed_endpoint.netloc:
+ self._send_json(400, {"error": "endpoint must be an http(s) URL"})
+ return
+
+ shard_start: int | None
+ shard_end: int | None
+ explicit_shard = "shard_start" in body or "shard_end" in body
+ if explicit_shard:
+ try:
+ shard_start = int(body["shard_start"])
+ shard_end = int(body["shard_end"])
+ except (KeyError, TypeError, ValueError):
+ self._send_json(400, {"error": "shard_start and shard_end must be numeric"})
+ return
+ if shard_start < 0 or shard_end < 0 or shard_start > shard_end:
+ self._send_json(400, {"error": "shard range must be non-negative and ordered"})
+ return
+ else:
+ shard_start = None
+ shard_end = None
+ try:
+ score = float(body.get("score", 1.0))
+ except (TypeError, ValueError):
+ self._send_json(400, {"error": "score must be numeric"})
+ return
+
+ hardware_profile = body.get("hardware_profile", {})
+ if not isinstance(hardware_profile, dict):
+ self._send_json(400, {"error": "hardware_profile must be an object"})
+ return
+ model = body.get("model")
+ if model is None:
+ model = "stub-model"
+ if not isinstance(model, str):
+ self._send_json(400, {"error": "model must be a string"})
+ return
+ shard_checksum = body.get("shard_checksum")
+ if shard_checksum is not None and not isinstance(shard_checksum, str):
+ self._send_json(400, {"error": "shard_checksum must be a string"})
+ return
+ try:
+ vram_bytes = int(body.get("vram_bytes", DEFAULT_VRAM_BYTES))
+ ram_bytes = int(body.get("ram_bytes", DEFAULT_RAM_BYTES))
+ max_loaded_shards = int(body.get("max_loaded_shards", 1))
+ benchmark_tokens_per_sec = float(
+ body.get("benchmark_tokens_per_sec", DEFAULT_BENCHMARK_TOKENS_PER_SEC)
+ )
+ except (TypeError, ValueError):
+ self._send_json(400, {"error": "vram_bytes, ram_bytes, max_loaded_shards, and benchmark_tokens_per_sec must be numeric"})
+ return
+ if vram_bytes < 0 or ram_bytes < 0 or max_loaded_shards < 1 or benchmark_tokens_per_sec <= 0:
+ self._send_json(400, {"error": "capability values must be positive"})
+ return
+ quantizations_body = body.get("quantizations", DEFAULT_QUANTIZATIONS)
+ if not (
+ isinstance(quantizations_body, list)
+ and quantizations_body
+ and all(isinstance(item, str) and item for item in quantizations_body)
+ ):
+ self._send_json(400, {"error": "quantizations must be a non-empty string array"})
+ return
+ quantizations = list(quantizations_body)
+ quantization = body.get("quantization")
+ if quantization is not None and not isinstance(quantization, str):
+ self._send_json(400, {"error": "quantization must be a string"})
+ return
+ wallet_address = body.get("wallet_address")
+ if wallet_address is not None and not isinstance(wallet_address, str):
+ self._send_json(400, {"error": "wallet_address must be a string"})
+ return
+ ban_error = _registration_ban_error(server.contracts, wallet_address)
+ if ban_error:
+ self._send_json(403, {"error": ban_error})
+ return
+
+ tracker_mode = bool(body.get("tracker_mode", False))
+ managed_assignment = bool(body.get("managed_assignment", False))
+ hf_repo = body.get("hf_repo")
+ if hf_repo is not None and not isinstance(hf_repo, str):
+ self._send_json(400, {"error": "hf_repo must be a string"})
+ return
+ num_layers_body = body.get("num_layers")
+ num_layers: int | None = None
+ if num_layers_body is not None:
+ try:
+ num_layers = int(num_layers_body)
+ except (TypeError, ValueError):
+ self._send_json(400, {"error": "num_layers must be an integer"})
+ return
+ model_metadata = body.get("model_metadata", {})
+ if model_metadata is None:
+ model_metadata = {}
+ if not isinstance(model_metadata, dict):
+ self._send_json(400, {"error": "model_metadata must be an object"})
+ return
+ relay_addr = body.get("relay_addr") or None
+ cert_fingerprint = body.get("cert_fingerprint") or None
+ peer_id = body.get("peer_id") or None
+
+ node_id = _node_id_for_registration(
+ endpoint,
+ model,
+ wallet_address,
+ shard_start,
+ shard_end,
+ hf_repo,
+ )
+ entry = _NodeEntry(
+ node_id=node_id,
+ endpoint=endpoint.rstrip("/"),
+ shard_start=shard_start,
+ shard_end=shard_end,
+ model=model,
+ shard_checksum=shard_checksum,
+ hardware_profile=hardware_profile,
+ wallet_address=wallet_address,
+ score=score,
+ vram_bytes=vram_bytes,
+ ram_bytes=ram_bytes,
+ quantizations=quantizations,
+ max_loaded_shards=max_loaded_shards,
+ benchmark_tokens_per_sec=benchmark_tokens_per_sec,
+ quantization=quantization,
+ managed_assignment=managed_assignment or not explicit_shard,
+ tracker_mode=tracker_mode,
+ hf_repo=hf_repo,
+ num_layers=num_layers,
+ model_metadata=model_metadata,
+ relay_addr=relay_addr,
+ cert_fingerprint=cert_fingerprint,
+ peer_id=peer_id,
+ )
+ with server.lock:
+ self._purge_expired_nodes()
+ # Dedup: if this endpoint is already registered, remove the old entry first.
+ stale_ids = [
+ eid for eid, e in server.registry.items()
+ if e.endpoint == entry.endpoint.rstrip("/")
+ ]
+ for eid in stale_ids:
+ old = server.registry.pop(eid)
+ print(
+ f"[tracker] node re-registered: replaced {eid} with {node_id}"
+ f" {old.endpoint}",
+ flush=True,
+ )
+ server.registry[node_id] = entry
+ if entry.managed_assignment:
+ if entry.hf_repo:
+ _rebalance_hf_model_locked(server, entry.hf_repo)
+ else:
+ _rebalance_model_locked(server, model)
+ assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
+ if assignment_directive is not None:
+ entry.pending_directives.clear()
+
+ shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
+ repo_info = f" [{hf_repo}]" if hf_repo else ""
+ budget_bytes, budget_source = _node_memory_budget_bytes(entry)
+ budget_gb = budget_bytes / (1024 ** 3)
+ print(
+ f"[tracker] node registered: {node_id} {endpoint} {model}{repo_info} {shard_info} "
+ f"capacity={budget_gb:.1f}GB {budget_source} slots={max_loaded_shards}",
+ flush=True,
+ )
+
+ payload = {"node_id": node_id}
+ if assignment_directive is not None:
+ payload["directive"] = assignment_directive
+ self._send_json(200, payload)
+
+ # Raft replication: leader proposes this registration to followers so their
+ # registries stay in sync. Fire-and-forget (async) — the client already
+ # got its node_id; replication happens in the background.
+ if server.raft is not None and server.raft.is_leader:
+ raft_payload = dict(body)
+ raft_payload["node_id"] = node_id # include the generated ID
+ threading.Thread(
+ target=server.raft.propose,
+ args=("register", raft_payload),
+ daemon=True,
+ ).start()
+
+ def _handle_heartbeat(self, node_id: str):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body: dict = {}
+ content_length = int(self.headers.get("Content-Length", 0))
+ if content_length > 0:
+ try:
+ body = json.loads(self.rfile.read(content_length))
+ except Exception:
+ pass
+ with server.lock:
+ self._purge_expired_nodes()
+ entry = server.registry.get(node_id)
+ if entry is None:
+ self._send_json(404, {"error": "node not found"})
+ return
+ entry.last_heartbeat = time.monotonic()
+ entry.heartbeats_received += 1
+ # P2P metadata
+ if body.get("relay_addr"):
+ entry.relay_addr = body["relay_addr"]
+ if body.get("cert_fingerprint"):
+ entry.cert_fingerprint = body["cert_fingerprint"]
+ if body.get("peer_id"):
+ entry.peer_id = body["peer_id"]
+ # Node stats (cumulative — node always sends totals, tracker replaces)
+ if "total_requests" in body:
+ entry.total_requests = int(body["total_requests"])
+ if "failed_requests" in body:
+ entry.failed_requests = int(body["failed_requests"])
+ if "queue_depth" in body:
+ entry.queue_depth = int(body["queue_depth"])
+ if "uptime_seconds" in body:
+ entry.uptime_seconds = float(body["uptime_seconds"])
+ if "status" in body and body["status"] in ("ready", "loading"):
+ entry.status = body["status"]
+ if entry.hf_repo:
+ _rebalance_hf_model_locked(server, entry.hf_repo)
+ else:
+ _rebalance_model_locked(server, entry.model or "stub-model")
+ directives = list(entry.pending_directives)
+ entry.pending_directives.clear()
+ new_assignment = entry.pending_new_assignment
+ if new_assignment is not None:
+ entry.pending_new_assignment = None
+ if server.gossip is not None:
+ server.gossip.record(node_id)
+ resp: dict = {}
+ if directives:
+ resp["directives"] = directives
+ if new_assignment is not None:
+ resp["new_assignment"] = new_assignment
+ self._send_json(200, resp)
+
+ # ---------------------------------------------------------------- Raft handlers
+
+ def _handle_raft_vote(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_json_body()
+ if body is None:
+ return
+ if server.raft is None:
+ self._send_json(503, {"error": "raft not enabled"})
+ return
+ result = server.raft.handle_request_vote(body)
+ self._send_json(200, result)
+
+ def _handle_raft_append(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_json_body()
+ if body is None:
+ return
+ if server.raft is None:
+ self._send_json(503, {"error": "raft not enabled"})
+ return
+ result = server.raft.handle_append_entries(body)
+ self._send_json(200, result)
+
+ def _handle_raft_status(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.raft is None:
+ self._send_json(200, {"role": "standalone", "term": 0, "leader": None})
+ return
+ self._send_json(200, server.raft.status())
+
+ def _handle_gossip(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_json_body()
+ if body is None:
+ return
+ if server.gossip is not None and isinstance(body, dict):
+ server.gossip.merge({k: float(v) for k, v in body.items()})
+ self._send_json(200, {})
+
+ def _handle_stats(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.stats is None:
+ self._send_json(200, {"models": {}, "nodes": {}})
+ return
+ self._send_json(200, {
+ "models": server.stats.get_combined_stats(),
+ "nodes": server.stats.get_node_throughput_stats(),
+ })
+
+ def _handle_stats_gossip(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_hive_authenticated_body()
+ if body is None:
+ return
+ tracker_url = body.get("tracker_url", "")
+ rpms = body.get("stats", {})
+ if server.stats is not None and tracker_url and isinstance(rpms, dict):
+ server.stats.merge_peer_rpms(tracker_url, rpms)
+ self._send_json(200, {})
+
+ def _handle_billing_summary(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if not self._require_role("admin"):
+ return
+ if server.billing is None:
+ self._send_json(404, {"error": "billing is not enabled on this tracker"})
+ return
+ self._send_json(200, server.billing.snapshot())
+
+ def _handle_dashboard(self):
+ """Serve the read-only web dashboard (US-035). Any tracker in the
+ mesh — leader or follower — serves it from its replicated state."""
+ try:
+ html = files("meshnet_tracker").joinpath("dashboard.html").read_text()
+ except (FileNotFoundError, OSError):
+ self._send_json(404, {"error": "dashboard asset missing"})
+ return
+ body = html.encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "text/html; charset=utf-8")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ try:
+ self.wfile.write(body)
+ except BrokenPipeError:
+ pass
+
+ def _handle_registry_wallets(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if not self._require_role("admin"):
+ return
+ if server.contracts is None:
+ self._send_json(200, {"wallets": {}})
+ return
+ wallets = server.contracts.registry.list_wallets()
+ self._send_json(200, {"wallets": {
+ wallet: {
+ "stake_balance": state.stake_balance,
+ "strike_count": state.strike_count,
+ "banned": state.banned,
+ "completed_jobs": state.completed_job_count,
+ }
+ for wallet, state in wallets.items()
+ }})
+
+ def _handle_billing_settlements(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if not self._require_role("admin"):
+ return
+ if server.billing is None:
+ self._send_json(404, {"error": "billing is not enabled on this tracker"})
+ return
+ self._send_json(200, {"settlements": server.billing.settlement_history()})
+
+ def _handle_billing_gossip(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_hive_authenticated_body()
+ if body is None:
+ return
+ if server.billing is None:
+ self._send_json(200, {"applied": 0})
+ return
+ events = body.get("events")
+ if not isinstance(events, list):
+ self._send_json(400, {"error": "events must be a list"})
+ return
+ applied = server.billing.apply_events([e for e in events if isinstance(e, dict)])
+ self._send_json(200, {"applied": applied})
+
+ def _handle_billing_forfeit(self):
+ """Privileged: forfeit a node's pending balance + record a strike (US-034).
+
+ ADR-0017 §4: validator service token or admin session only. Client
+ API keys — valid or not — are rejected.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if not self._require_role("validator", "admin"):
+ return
+ if server.billing is None:
+ self._send_json(404, {"error": "billing is not enabled on this tracker"})
+ return
+ body = self._read_json_body()
+ if body is None:
+ return
+ wallet = body.get("wallet")
+ if not wallet or not isinstance(wallet, str):
+ self._send_json(400, {"error": "wallet is required"})
+ return
+ reason = body.get("reason") or "fraud"
+ event = server.billing.forfeit_pending(wallet, reason=str(reason))
+ strike_state: dict = {}
+ if server.contracts is not None:
+ server.contracts.registry.record_strike(wallet)
+ wallet_state = server.contracts.registry.get_wallet(wallet)
+ strike_state = {
+ "strike_count": wallet_state.strike_count,
+ "banned": wallet_state.banned,
+ }
+ print(
+ f"[tracker] forfeited pending balance of {wallet}: "
+ f"{event['amount']:.6f} USDT ({reason}) strikes={strike_state.get('strike_count', 'n/a')}",
+ flush=True,
+ )
+ self._send_json(200, {"forfeited": event["amount"], **strike_state})
+
+ # ---- user accounts (registration, login, API keys) ----
+
+ def _session_account(self) -> dict | None:
+ """Resolve the session token in the Authorization header, or None."""
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.accounts is None:
+ return None
+ return server.accounts.session_account(_api_key_from_headers(self.headers))
+
+ def _require_accounts(self) -> "AccountStore | None":
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.accounts is None:
+ self._send_json(404, {"error": "accounts are not enabled on this tracker"})
+ return None
+ return server.accounts
+
+ def _handle_auth_register(self):
+ accounts = self._require_accounts()
+ if accounts is None:
+ return
+ body = self._read_json_body()
+ if body is None:
+ return
+ try:
+ account = accounts.register(
+ email=body.get("email"),
+ wallet=body.get("wallet"),
+ password=str(body.get("password") or ""),
+ )
+ except ValueError as exc:
+ self._send_json(400, {"error": str(exc)})
+ return
+ token = accounts.create_session(account["account_id"])
+ api_key = accounts.create_api_key(account["account_id"])
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.billing is not None:
+ server.billing.ensure_client(api_key) # grant Caller Credit up front
+ print(
+ f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
+ f"role={account['role']}", flush=True,
+ )
+ self._send_json(200, {"account": account, "session_token": token, "api_key": api_key})
+
+ def _handle_auth_login(self):
+ accounts = self._require_accounts()
+ if accounts is None:
+ return
+ body = self._read_json_body()
+ if body is None:
+ return
+ identifier = body.get("identifier") or body.get("email") or body.get("wallet") or ""
+ account = accounts.verify_login(str(identifier), str(body.get("password") or ""))
+ if account is None:
+ self._send_json(401, {"error": "invalid credentials"})
+ return
+ token = accounts.create_session(account["account_id"])
+ self._send_json(200, {"account": account, "session_token": token})
+
+ def _handle_auth_logout(self):
+ accounts = self._require_accounts()
+ if accounts is None:
+ return
+ accounts.destroy_session(_api_key_from_headers(self.headers))
+ self._send_json(200, {"ok": True})
+
+ def _handle_account_me(self):
+ """Balance, usage, and API keys for the logged-in account."""
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if self._require_accounts() is None:
+ return
+ account = self._session_account()
+ if account is None:
+ self._send_json(401, {"error": "login required"})
+ return
+ keys = server.accounts.keys_for(account["account_id"]) # type: ignore[union-attr]
+ balances = {}
+ usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0, "recent": []}
+ if server.billing is not None:
+ balances = {key: server.billing.get_client_balance(key) for key in keys}
+ usage = server.billing.usage_for(keys)
+ self._send_json(200, {
+ "account": account,
+ "api_keys": keys,
+ "balances": balances,
+ "total_balance": sum(balances.values()),
+ "usage": usage,
+ })
+
+ def _handle_account_key_create(self):
+ accounts = self._require_accounts()
+ if accounts is None:
+ return
+ account = self._session_account()
+ if account is None:
+ self._send_json(401, {"error": "login required"})
+ return
+ api_key = accounts.create_api_key(account["account_id"])
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if server.billing is not None:
+ server.billing.ensure_client(api_key)
+ self._send_json(200, {"api_key": api_key})
+
+ def _handle_account_key_revoke(self):
+ accounts = self._require_accounts()
+ if accounts is None:
+ return
+ account = self._session_account()
+ if account is None:
+ self._send_json(401, {"error": "login required"})
+ return
+ body = self._read_json_body()
+ if body is None:
+ return
+ api_key = body.get("api_key")
+ if not api_key or not isinstance(api_key, str):
+ self._send_json(400, {"error": "api_key is required"})
+ return
+ if not accounts.revoke_api_key(account["account_id"], api_key):
+ self._send_json(404, {"error": "key not found on this account"})
+ return
+ self._send_json(200, {"revoked": api_key})
+
+ def _handle_admin_accounts(self):
+ """Admin-only: all accounts with their keys and balances."""
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ accounts = self._require_accounts()
+ if accounts is None:
+ return
+ account = self._session_account()
+ if account is None:
+ self._send_json(401, {"error": "login required"})
+ return
+ if account["role"] != "admin":
+ self._send_json(403, {"error": "admin role required"})
+ return
+ listing = accounts.list_accounts()
+ if server.billing is not None:
+ for entry in listing:
+ entry["balances"] = {
+ key: server.billing.get_client_balance(key)
+ for key in entry.get("api_keys", [])
+ }
+ self._send_json(200, {"accounts": listing})
+
+ def _handle_accounts_gossip(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ body = self._read_hive_authenticated_body()
+ if body is None:
+ return
+ if server.accounts is None:
+ self._send_json(200, {"applied": 0})
+ return
+ events = body.get("events")
+ if not isinstance(events, list):
+ self._send_json(400, {"error": "events must be a list"})
+ return
+ applied = server.accounts.apply_events([e for e in events if isinstance(e, dict)])
+ self._send_json(200, {"applied": applied})
+
+ def _handle_wallet_register(self):
+ """Bind the caller's client wallet pubkey to their API key (US-032).
+
+ Deposits from that wallet into the treasury are then credited to the
+ API key's ledger balance by the deposit watcher.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ api_key = _api_key_from_headers(self.headers)
+ if not api_key:
+ self._send_json(401, {"error": "Authorization header required"})
+ return
+ if server.billing is None:
+ self._send_json(404, {"error": "billing is not enabled on this tracker"})
+ return
+ body = self._read_json_body()
+ if body is None:
+ return
+ wallet = body.get("wallet")
+ if not wallet or not isinstance(wallet, str):
+ self._send_json(400, {"error": "wallet is required"})
+ return
+ server.billing.bind_wallet(api_key, wallet)
+ print(f"[tracker] wallet bound: {wallet} -> api_key …{api_key[-6:]}", flush=True)
+ self._send_json(200, {"wallet": wallet, "bound": True})
+
+ def _handle_benchmark_hop_penalty(self):
+ """Privileged: run the same prompt through 1/2/3-node pinned routes (US-030).
+
+ Data collection only — the routing algorithm is unchanged. Per-hop
+ latency is derived incrementally: the k-node route's final hop penalty
+ is its total minus the (k-1)-node route's total.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if not self._require_role("admin", "validator"):
+ return
+ auth = self.headers.get("Authorization")
+ body = self._read_json_body()
+ if body is None:
+ return
+ model = body.get("model", "")
+ if not model:
+ self._send_json(400, {"error": "model is required"})
+ return
+ prompt = body.get("prompt") or "Benchmark: say OK."
+ max_new_tokens = int(body.get("max_new_tokens", 64))
+
+ with server.lock:
+ self._purge_expired_nodes()
+ resolved = _nodes_and_bounds_for_model(server, model)
+ if resolved is None or not resolved[0]:
+ self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
+ return
+ all_nodes, rs, re = resolved
+
+ self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
+ route_results: list[dict] = []
+ prev_total_ms: float | None = None
+ prev_per_hop: list[float] = []
+ for hop_count in (1, 2, 3):
+ combo = _find_pinned_route(all_nodes, rs, re, hop_count)
+ if combo is None:
+ continue # insufficient coverage for this hop count — skip
+ request_body = json.dumps({
+ "model": model,
+ "messages": [{"role": "user", "content": prompt}],
+ "max_tokens": max_new_tokens,
+ "route": [n.node_id for n in combo],
+ }).encode()
+ req = urllib.request.Request(
+ f"{self_url}/v1/chat/completions",
+ data=request_body,
+ headers={"Content-Type": "application/json", "Authorization": auth},
+ method="POST",
+ )
+ started = time.monotonic()
+ try:
+ with urllib.request.urlopen(req, timeout=300.0) as resp:
+ payload = json.loads(resp.read())
+ except Exception as exc:
+ route_results.append({
+ "route": [n.node_id for n in combo],
+ "error": str(exc),
+ })
+ continue
+ total_ms = (time.monotonic() - started) * 1000.0
+ if prev_total_ms is not None and len(prev_per_hop) == hop_count - 1:
+ per_hop_ms = prev_per_hop + [max(0.0, total_ms - prev_total_ms)]
+ else:
+ per_hop_ms = [total_ms / hop_count] * hop_count
+ prev_total_ms = total_ms
+ prev_per_hop = per_hop_ms
+ route_results.append({
+ "route": [n.node_id for n in combo],
+ "total_ms": total_ms,
+ "per_hop_ms": per_hop_ms,
+ "tokens_generated": _usage_total_tokens(payload) or 0,
+ })
+
+ record = {
+ "timestamp": time.time(),
+ "model": model,
+ "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
+ "routes": route_results,
+ }
+ with server.benchmark_lock:
+ existing: list = []
+ if os.path.exists(server.benchmark_results_path):
+ try:
+ with open(server.benchmark_results_path, encoding="utf-8") as fh:
+ existing = json.load(fh)
+ except (json.JSONDecodeError, OSError):
+ existing = []
+ if not isinstance(existing, list):
+ existing = []
+ existing.append(record)
+ with open(server.benchmark_results_path, "w", encoding="utf-8") as fh:
+ json.dump(existing, fh, indent=2)
+ self._send_json(200, record)
+
+ def _handle_benchmark_results(self):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ if not self._require_role("admin", "validator"):
+ return
+ results: list = []
+ with server.benchmark_lock:
+ if os.path.exists(server.benchmark_results_path):
+ try:
+ with open(server.benchmark_results_path, encoding="utf-8") as fh:
+ results = json.load(fh)
+ except (json.JSONDecodeError, OSError):
+ results = []
+ self._send_json(200, {"results": results if isinstance(results, list) else []})
+
+ def _handle_assign(self, parsed: urllib.parse.ParseResult):
+ """Return an optimal shard assignment for a node given its hardware profile.
+
+ Query params:
+ model — model preset name (default: first preset)
+ device — "cuda" | "cpu"
+ vram_mb — integer VRAM in MB (0 for CPU)
+ ram_mb — integer system RAM in MB, used when vram_mb=0
+
+ The greedy strategy: find the first gap in current layer coverage
+ and assign it. If no gap exists, assign the full model range so the
+ node provides redundant coverage.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ params = urllib.parse.parse_qs(parsed.query)
+
+ model_list = params.get("model")
+ if not model_list:
+ model = next(iter(server.model_presets), None)
+ if model is None:
+ self._send_json(503, {"error": "no model presets configured"})
+ return
+ else:
+ model = model_list[0]
+
+ resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+ if preset is None:
+ self._send_json(404, {"error": f"unknown model preset: {model!r}"})
+ return
+
+ required_start, required_end = _preset_layer_bounds(preset)
+
+ with server.lock:
+ self._purge_expired_nodes()
+ alive = [
+ node for node in server.registry.values()
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ ]
+ if server.contracts is not None:
+ alive = [
+ node for node in alive
+ if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
+ ]
+
+ device = params.get("device", ["cpu"])[0]
+ try:
+ vram_mb = int(params.get("vram_mb", ["0"])[0])
+ except ValueError:
+ vram_mb = 0
+ try:
+ ram_mb = int(params.get("ram_mb", ["0"])[0])
+ except ValueError:
+ ram_mb = 0
+ max_layers = required_end - required_start + 1
+ memory_mb = vram_mb if vram_mb > 0 else ram_mb
+ if memory_mb > 0:
+ layer_bytes = _preset_bytes_per_layer(preset).get("bfloat16", 30 * 1024 * 1024)
+ max_layers = min(max_layers, max(1, int(((memory_mb * 1024 * 1024) * 0.8) // layer_bytes)))
+ elif device != "cuda" or vram_mb < 8192:
+ max_layers = min(max_layers, 16)
+
+ # Collect covered intervals sorted by start layer.
+ covered = sorted(
+ [
+ (n.shard_start, n.shard_end)
+ for n in alive
+ if n.shard_start is not None and n.shard_end is not None
+ ],
+ key=lambda t: t[0],
+ )
+
+ # Walk from required_start to find the first uncovered layer.
+ gap_start = required_start
+ for s, e in covered:
+ if s <= gap_start:
+ gap_start = max(gap_start, e + 1)
+ else:
+ break # gap found before this node
+
+ if gap_start > required_end:
+ # Full coverage exists — assign the full range for redundancy.
+ shard_start = required_start
+ shard_end = min(required_end, shard_start + max_layers - 1)
+ else:
+ shard_start = gap_start
+ shard_end = min(required_end, shard_start + max_layers - 1)
+
+ peers = [
+ {"endpoint": node.endpoint, "checksum": node.shard_checksum}
+ for node in alive
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ and node.shard_start == shard_start
+ and node.shard_end == shard_end
+ and node.shard_checksum
+ ]
+
+ self._send_json(200, {
+ "shard_start": shard_start,
+ "shard_end": shard_end,
+ "model": resolved_name,
+ "model_layers_end": required_end,
+ "peers": peers,
+ **({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
+ })
+
+ def _handle_network_assign(self, parsed: urllib.parse.ParseResult):
+ """Assign a new node to fill the biggest uncovered shard gap across HF-model nodes.
+
+ Query params:
+ vram_mb — integer VRAM in MB (0 = CPU-only node)
+ ram_mb — integer system RAM in MB, used when vram_mb=0
+ device — "cuda" | "cpu"
+ hf_repo — optional; if set, restrict search to this repo only
+
+ Returns:
+ {hf_repo, shard_start, shard_end, num_layers, gap_found}
+ gap_found=true means a real uncovered gap was assigned; false means redundancy.
+ """
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ params = urllib.parse.parse_qs(parsed.query)
+ try:
+ vram_mb = int(params.get("vram_mb", ["0"])[0])
+ except ValueError:
+ vram_mb = 0
+ try:
+ ram_mb = int(params.get("ram_mb", ["0"])[0])
+ except ValueError:
+ ram_mb = 0
+ device = params.get("device", ["cpu"])[0]
+ filter_repo = params.get("hf_repo", [None])[0] # optional repo filter
+
+ with server.lock:
+ self._purge_expired_nodes()
+ all_nodes = list(server.registry.values())
+
+ # Collect only nodes that registered a real HF model (have hf_repo + shard bounds).
+ hf_nodes = [
+ n for n in all_nodes
+ if n.hf_repo
+ and n.shard_start is not None
+ and n.shard_end is not None
+ and n.num_layers is not None
+ and (filter_repo is None or n.hf_repo == filter_repo)
+ ]
+
+ if not hf_nodes:
+ resolved_name = None
+ preset = None
+ if filter_repo:
+ resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
+ else:
+ deployable = [
+ (name, preset)
+ for name, preset in server.model_presets.items()
+ if preset.get("recommended") and _deployment_summary(all_nodes, preset)["deployable"]
+ ]
+ if deployable:
+ resolved_name, preset = deployable[0]
+ if preset is not None and preset.get("hf_repo"):
+ required_start, required_end = _preset_layer_bounds(preset)
+ total_l = required_end - required_start + 1
+ memory_mb = vram_mb if vram_mb > 0 else ram_mb
+ max_layers = _max_layers_for_memory(memory_mb, total_l, preset)
+ shard_start = required_start
+ shard_end = min(required_end, shard_start + max_layers - 1)
+ self._send_json(200, {
+ "hf_repo": preset["hf_repo"],
+ "model": resolved_name,
+ "shard_start": shard_start,
+ "shard_end": shard_end,
+ "num_layers": total_l,
+ "gap_found": True,
+ "price_per_token": 0.0,
+ "deployment": _deployment_summary(all_nodes, preset),
+ })
+ return
+
+ msg = (
+ f"no HF-model nodes registered for {filter_repo!r}"
+ if filter_repo
+ else "no HF-model nodes registered; cannot assign shards"
+ )
+ self._send_json(503, {"error": msg})
+ return
+
+ # Group by hf_repo; pick the one with the largest total_layers and biggest gap.
+ from collections import defaultdict
+ repo_groups: dict = defaultdict(list)
+ repo_layers: dict = {}
+ for n in hf_nodes:
+ repo_groups[n.hf_repo].append(n)
+ # Use the largest num_layers seen for this repo.
+ if n.hf_repo not in repo_layers or n.num_layers > repo_layers[n.hf_repo]:
+ repo_layers[n.hf_repo] = n.num_layers
+
+ # Smart scoring: demand_rpm × coverage_deficit
+ # coverage_deficit = uncovered_layers / total_layers (0 = fully covered)
+ # demand_rpm comes from the stats collector; defaults to 0.0 when no traffic yet.
+ demand_rpms: dict[str, float] = {}
+ if server.stats is not None:
+ local_rpms = server.stats.get_local_rpms()
+ for repo in repo_groups:
+ demand_rpms[repo] = local_rpms.get(repo, {}).get("rpm_last_hour", 0.0)
+
+ best_repo = None
+ best_score = -1.0
+ best_gap_size = -1
+ best_gap_start = 0
+ best_num_layers = 0
+
+ for repo, nodes in repo_groups.items():
+ total = repo_layers[repo]
+ covered = sorted(
+ [(n.shard_start, n.shard_end) for n in nodes],
+ key=lambda t: t[0],
+ )
+ # Walk from 0 to find first uncovered layer.
+ gap_start = 0
+ for s, e in covered:
+ if s <= gap_start:
+ gap_start = max(gap_start, e + 1)
+ else:
+ break
+ gap_size = max(0, (total - 1) - gap_start + 1)
+ coverage_deficit = gap_size / max(total, 1)
+ demand = demand_rpms.get(repo, 0.0)
+ # +1.0 floor on demand so models with no recorded traffic still compete by coverage.
+ score = (demand + 1.0) * (coverage_deficit + 0.01)
+ if score > best_score or (score == best_score and gap_size > best_gap_size):
+ best_score = score
+ best_gap_size = gap_size
+ best_gap_start = gap_start
+ best_repo = repo
+ best_num_layers = total
+
+ gap_found = best_gap_size > 0
+ if not gap_found:
+ # All shards covered — assign to highest-demand model for redundancy.
+ best_repo = max(repo_groups, key=lambda r: demand_rpms.get(r, 0.0))
+ best_gap_start = 0
+ best_num_layers = repo_layers[best_repo]
+
+ # Capacity: use the same 80%-of-memory rule as registered node planning.
+ total_l = best_num_layers
+ memory_mb = vram_mb if vram_mb > 0 else ram_mb
+ _resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
+ if memory_mb > 0:
+ max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
+ elif device == "cuda" and vram_mb >= 8192:
+ max_layers = total_l
+ else:
+ max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset)
+
+ shard_start = best_gap_start
+ shard_end = min(total_l - 1, shard_start + max_layers - 1)
+
+ self._send_json(200, {
+ "hf_repo": best_repo,
+ "shard_start": shard_start,
+ "shard_end": shard_end,
+ "num_layers": total_l,
+ "gap_found": gap_found,
+ "price_per_token": 0.0,
+ })
+
+ def _handle_route(self, parsed: urllib.parse.ParseResult):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ params = urllib.parse.parse_qs(parsed.query)
+ model_list = params.get("model")
+ if not model_list:
+ self._send_json(400, {"error": "missing 'model' query parameter"})
+ return
+
+ model = model_list[0]
+ resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+
+ with server.lock:
+ self._purge_expired_nodes()
+ if preset is not None:
+ # Preset-based routing (stub-model system).
+ alive = [
+ node for node in server.registry.values()
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ ]
+ required_start, required_end = _preset_layer_bounds(preset)
+ else:
+ # HF model routing: match by hf_repo (full) or model short name.
+ alive = [
+ node for node in server.registry.values()
+ if _node_matches_model(node, model)
+ and node.shard_start is not None
+ and node.shard_end is not None
+ and node.num_layers is not None
+ ]
+ if not alive:
+ self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
+ return
+ required_start = 0
+ required_end = max(n.num_layers for n in alive) - 1 # type: ignore[type-var]
+
+ if server.contracts is not None:
+ alive = [
+ node for node in alive
+ if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
+ ]
+
+ route, error = _select_route(alive, required_start, required_end)
+ if error:
+ self._send_json(503, {"error": error})
+ return
+
+ covered_up_to = required_start - 1
+ route_with_start: list[tuple] = []
+ for rn in route:
+ route_with_start.append((rn, covered_up_to + 1))
+ covered_up_to = rn.shard_end if rn.shard_end is not None else covered_up_to
+
+ self._send_json(200, {
+ "route": [e.endpoint for e, _ in route_with_start],
+ "nodes": [
+ {
+ "node_id": e.node_id,
+ "endpoint": e.endpoint,
+ "start_layer": start,
+ "relay_addr": e.relay_addr,
+ "peer_id": e.peer_id,
+ "wallet_address": e.wallet_address,
+ "shard_start": e.shard_start,
+ "shard_end": e.shard_end,
+ "model": e.model,
+ "hf_repo": e.hf_repo,
+ "num_layers": e.num_layers,
+ "model_metadata": dict(e.model_metadata),
+ "shard_checksum": e.shard_checksum,
+ "score": e.score,
+ }
+ for e, start in route_with_start
+ ],
+ })
+
+ def _handle_routes(self, parsed: urllib.parse.ParseResult):
+ server: _TrackerHTTPServer = self.server # type: ignore[assignment]
+ params = urllib.parse.parse_qs(parsed.query)
+ model_list = params.get("model")
+ if not model_list:
+ self._send_json(400, {"error": "missing 'model' query parameter"})
+ return
+
+ try:
+ redundancy = int(params.get("redundancy", ["1"])[0])
+ except ValueError:
+ self._send_json(400, {"error": "redundancy must be an integer"})
+ return
+ if redundancy < 1:
+ self._send_json(400, {"error": "redundancy must be at least 1"})
+ return
+
+ model = model_list[0]
+ resolved_name, preset = _resolve_model_preset(server.model_presets, model)
+ if preset is None:
+ self._send_json(404, {"error": f"unknown model preset: {model!r}"})
+ return
+
+ required_start, required_end = _preset_layer_bounds(preset)
+
+ with server.lock:
+ self._purge_expired_nodes()
+ candidates = [
+ node for node in server.registry.values()
+ if _node_matches_preset(node, resolved_name, preset) # type: ignore[arg-type]
+ ]
+ if server.contracts is not None:
+ candidates = [
+ node for node in candidates
+ if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
+ ]
+
+ routes = []
+ remaining = list(candidates)
+ for _ in range(redundancy):
+ route, error = _select_route(remaining, required_start, required_end)
+ if error:
+ self._send_json(503, {"error": error})
+ return
+ route_endpoints = {node.endpoint for node in route}
+ remaining = [node for node in remaining if node.endpoint not in route_endpoints]
+ routes.append({
+ "route": [e.endpoint for e in route],
+ "nodes": [
+ {
+ "node_id": e.node_id,
+ "endpoint": e.endpoint,
+ "relay_addr": e.relay_addr,
+ "peer_id": e.peer_id,
+ "wallet_address": e.wallet_address,
+ "shard_start": e.shard_start,
+ "shard_end": e.shard_end,
+ "model": e.model,
+ "shard_checksum": e.shard_checksum,
+ "score": e.score,
+ }
+ for e in route
+ ],
+ })
+
+ self._send_json(200, {"routes": routes})
+
+
+class TrackerServer:
+ """HTTP tracker that manages node registration and resolves inference routes.
+
+ Nodes register via POST /v1/nodes/register and keep themselves alive with
+ POST /v1/nodes//heartbeat. The gateway queries GET /v1/route?model=
+ to obtain an ordered list of node endpoints whose shards cover all layers.
+ """
+
+ def __init__(
+ self,
+ host: str = "127.0.0.1",
+ port: int = 0,
+ heartbeat_timeout: float = 90.0,
+ rebalance_interval: float = 30.0,
+ model_presets: dict | None = None,
+ contracts: Any | None = None,
+ minimum_stake: int = 0,
+ cluster_peers: list[str] | None = None,
+ cluster_self_url: str | None = None,
+ stats_db: str | None = None,
+ relay_url: str | None = None,
+ billing: BillingLedger | None = None,
+ enable_billing: bool = False,
+ billing_db: str | None = None,
+ accounts: AccountStore | None = None,
+ accounts_db: str | None = None,
+ benchmark_results_path: str | None = None,
+ treasury: Any | None = None,
+ deposit_poll_interval: float = 15.0,
+ settle_period: float = 86400.0,
+ payout_threshold: float = 5.0,
+ payout_dust_floor: float = 0.01,
+ settlement_check_interval: float | None = None,
+ validator_service_token: str | None = None,
+ hive_secret: str | None = None,
+ ) -> None:
+ self._host = host
+ self._requested_port = port
+ self._heartbeat_timeout = heartbeat_timeout
+ self._rebalance_interval = rebalance_interval
+ self._model_presets: dict = (
+ model_presets if model_presets is not None else dict(DEFAULT_MODEL_PRESETS)
+ )
+ self._contracts = contracts
+ self._minimum_stake = minimum_stake
+ self._cluster_peers: list[str] = list(cluster_peers) if cluster_peers else []
+ self._cluster_self_url: str | None = cluster_self_url
+ self._relay_url = relay_url
+ self._registry: dict[str, _NodeEntry] = {}
+ self._lock = threading.Lock()
+ self._server: _TrackerHTTPServer | None = None
+ self._thread: threading.Thread | None = None
+ self._rebalance_stop = threading.Event()
+ self._rebalance_thread: threading.Thread | None = None
+ self._raft: RaftNode | None = None
+ self._gossip: NodeGossip | None = None
+ self._stats: _StatsCollector | None = _StatsCollector(db_path=stats_db) if stats_db else _StatsCollector()
+ self._stats_stop = threading.Event()
+ self._stats_thread: threading.Thread | None = None
+ if billing is None:
+ db_path = billing_db
+ if db_path is None and enable_billing:
+ db_path = DEFAULT_BILLING_DB_PATH
+ if db_path:
+ preset_prices = {
+ name: float(preset["price_per_1k_tokens"])
+ for name, preset in self._model_presets.items()
+ if isinstance(preset, dict) and "price_per_1k_tokens" in preset
+ }
+ billing = BillingLedger(db_path=db_path, prices=preset_prices)
+ self._billing: BillingLedger | None = billing
+ self._billing_gossip_cursor = 0
+ if accounts is None:
+ accounts_path = accounts_db
+ if accounts_path is None and enable_billing:
+ accounts_path = DEFAULT_ACCOUNTS_DB_PATH
+ if accounts_path:
+ accounts = AccountStore(db_path=accounts_path)
+ self._accounts: AccountStore | None = accounts
+ self._accounts_gossip_cursor = 0
+ self._benchmark_results_path = benchmark_results_path
+ self._treasury = treasury
+ self._deposit_poll_interval = deposit_poll_interval
+ self._deposit_stop = threading.Event()
+ self._deposit_thread: threading.Thread | None = None
+ self._settle_period = settle_period
+ self._payout_threshold = payout_threshold
+ self._payout_dust_floor = payout_dust_floor
+ self._settlement_check_interval = settlement_check_interval or max(
+ 1.0, min(settle_period / 4.0, 15.0)
+ )
+ self._settlement_stop = threading.Event()
+ self._settlement_thread: threading.Thread | None = None
+ self._validator_service_token = (
+ validator_service_token
+ if validator_service_token is not None
+ else os.environ.get("MESHNET_VALIDATOR_SERVICE_TOKEN") or None
+ )
+ self._hive_secret = (
+ hive_secret
+ if hive_secret is not None
+ else os.environ.get("MESHNET_HIVE_SECRET") or None
+ )
+ self.port: int | None = None
+
+ def start(self) -> int:
+ if self._server is not None:
+ raise RuntimeError("TrackerServer is already running")
+
+ effective_relay_url = (
+ self._relay_url
+ or derive_relay_url_from_public_tracker_url(self._cluster_self_url)
+ )
+
+ # Start HTTP server first so we know our port
+ self._server = _TrackerHTTPServer(
+ (self._host, self._requested_port),
+ _TrackerHandler,
+ self._registry,
+ self._lock,
+ self._heartbeat_timeout,
+ self._model_presets,
+ self._contracts,
+ self._minimum_stake,
+ relay_url=effective_relay_url,
+ stats=self._stats,
+ billing=self._billing,
+ accounts=self._accounts,
+ benchmark_results_path=self._benchmark_results_path,
+ validator_service_token=self._validator_service_token,
+ hive_secret=self._hive_secret,
+ )
+ self.port = self._server.server_address[1]
+
+ # Start Raft + gossip if cluster peers are configured
+ if self._cluster_peers:
+ self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
+ self._raft = RaftNode(
+ self_url=self_url,
+ peers=self._cluster_peers,
+ apply_fn=self._raft_apply,
+ )
+ self._gossip = NodeGossip(peers=self._cluster_peers)
+ self._server.raft = self._raft
+ self._server.gossip = self._gossip
+ self._raft.start()
+ self._gossip.start()
+
+ self._rebalance_stop.clear()
+ self._stats_stop.clear()
+ self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+ self._thread.start()
+ self._rebalance_thread = threading.Thread(target=self._rebalance_loop, daemon=True)
+ self._rebalance_thread.start()
+ self._stats_thread = threading.Thread(target=self._stats_loop, daemon=True)
+ self._stats_thread.start()
+ if self._treasury is not None and self._billing is not None:
+ self._deposit_stop.clear()
+ self._deposit_thread = threading.Thread(target=self._deposit_loop, daemon=True)
+ self._deposit_thread.start()
+ self._settlement_stop.clear()
+ self._settlement_thread = threading.Thread(target=self._settlement_loop, daemon=True)
+ self._settlement_thread.start()
+ return self.port
+
+ def _settlement_loop(self) -> None:
+ """Leader-only on-chain settlement (US-033, ADR-0015).
+
+ Pay a node when pending ≥ threshold OR its pending age ≥ max period,
+ with a dust floor. Pending is debited (payout events, replicated)
+ before the transaction is sent; unconfirmed batches are resent with
+ the same settlement id, so retries never double-pay.
+ """
+ billing = self._billing
+ treasury = self._treasury
+ assert billing is not None and treasury is not None
+ while not self._settlement_stop.wait(self._settlement_check_interval):
+ if self._raft is not None and not self._raft.is_leader:
+ continue # followers replicate the ledger but never sign
+ # resend batches whose transaction never confirmed
+ for settlement_id, payouts in billing.unconfirmed_settlements().items():
+ self._send_settlement(settlement_id, payouts)
+ banned: set[str] = set()
+ if self._server is not None and self._server.contracts is not None:
+ registry = self._server.contracts.registry
+ banned = {
+ wallet for wallet, _ in billing.payables(
+ threshold=0.0, max_period=0.0, dust_floor=0.0,
+ )
+ if registry.get_wallet(wallet).banned
+ }
+ due = billing.payables(
+ threshold=self._payout_threshold,
+ max_period=self._settle_period,
+ dust_floor=self._payout_dust_floor,
+ exclude=banned,
+ )
+ if not due:
+ continue
+ settlement_id = uuid.uuid4().hex
+ for wallet, amount in due:
+ billing.settle_node_payout(wallet, amount, reference=settlement_id)
+ self._send_settlement(settlement_id, due)
+
+ def _send_settlement(self, settlement_id: str, payouts: list) -> None:
+ billing = self._billing
+ treasury = self._treasury
+ assert billing is not None and treasury is not None
+ try:
+ signature = treasury.send_payouts([(w, a) for w, a in payouts])
+ except Exception as exc:
+ print(
+ f"[tracker] settlement {settlement_id} failed (will retry): {exc}",
+ flush=True,
+ )
+ return
+ billing.confirm_settlement(settlement_id, signature)
+ total = sum(a for _, a in payouts)
+ print(
+ f"[tracker] settled {settlement_id}: {len(payouts)} payout(s), "
+ f"{total:.6f} USDT total (tx {signature})",
+ flush=True,
+ )
+
+ def _deposit_loop(self) -> None:
+ """Poll the treasury token account and credit confirmed deposits (US-032)."""
+ billing = self._billing
+ treasury = self._treasury
+ assert billing is not None and treasury is not None
+ while not self._deposit_stop.wait(self._deposit_poll_interval):
+ try:
+ deposits = treasury.list_new_deposits(
+ lambda sig: billing.has_event(f"deposit-{sig}")
+ )
+ except Exception as exc:
+ print(f"[tracker] deposit poll failed: {exc}", flush=True)
+ continue
+ for deposit in deposits:
+ api_key = billing.api_key_for_wallet(deposit.sender_wallet)
+ if api_key is None:
+ print(
+ f"[tracker] unattributed deposit {deposit.signature}: "
+ f"{deposit.amount_usdt} USDT from unbound wallet "
+ f"{deposit.sender_wallet} — register via /v1/wallet/register",
+ flush=True,
+ )
+ continue
+ credited = billing.credit_deposit(
+ api_key, deposit.amount_usdt, deposit.signature
+ )
+ if credited is not None:
+ print(
+ f"[tracker] deposit credited: {deposit.amount_usdt} USDT "
+ f"-> api_key …{api_key[-6:]} (tx {deposit.signature})",
+ flush=True,
+ )
+
+ def _raft_apply(self, command: str, payload: dict) -> None:
+ """Called by RaftNode when a log entry is committed — replicate to local registry."""
+ if command != "register":
+ return
+ # Re-apply the registration to our local registry (follower path).
+ # On the leader this is a no-op since it already registered locally.
+ node_id = payload.get("node_id")
+ if not node_id or node_id in self._registry:
+ return # already present (leader case) or malformed
+ endpoint = payload.get("endpoint", "")
+ try:
+ shard_start = int(payload["shard_start"]) if payload.get("shard_start") is not None else None
+ shard_end = int(payload["shard_end"]) if payload.get("shard_end") is not None else None
+ except (TypeError, ValueError):
+ return
+ entry = _NodeEntry(
+ node_id=node_id,
+ endpoint=endpoint.rstrip("/"),
+ shard_start=shard_start,
+ shard_end=shard_end,
+ model=payload.get("model", "stub-model"),
+ shard_checksum=payload.get("shard_checksum"),
+ hardware_profile=payload.get("hardware_profile", {}),
+ wallet_address=payload.get("wallet_address"),
+ score=float(payload.get("score", 1.0)),
+ tracker_mode=bool(payload.get("tracker_mode", False)),
+ hf_repo=payload.get("hf_repo"),
+ num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
+ model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
+ )
+ with self._lock:
+ self._registry[node_id] = entry
+
+ def _rebalance_loop(self) -> None:
+ while not self._rebalance_stop.wait(self._rebalance_interval):
+ server = self._server
+ if server is None:
+ return
+ with self._lock:
+ _purge_expired_nodes_locked(server)
+ _rebalance_all_locked(server)
+
+ def _push_to_peers(self, path: str, body: bytes) -> bool:
+ """POST a hive-signed payload to every cluster peer; True if all succeeded."""
+ headers = {"Content-Type": "application/json"}
+ if self._hive_secret:
+ headers.update(sign_hive_request(self._hive_secret, body))
+ delivered_all = True
+ for peer in self._cluster_peers:
+ try:
+ req = urllib.request.Request(
+ f"{peer}{path}", data=body, headers=headers, method="POST",
+ )
+ with urllib.request.urlopen(req, timeout=2.0) as r:
+ r.read()
+ except Exception:
+ delivered_all = False
+ return delivered_all
+
+ def _stats_loop(self) -> None:
+ """Periodically save stats/billing to DB and push local slices to cluster peers."""
+ while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
+ if self._stats is not None:
+ self._stats.save_to_db()
+ if self._billing is not None:
+ self._billing.save_to_db()
+ if self._accounts is not None:
+ self._accounts.save_to_db()
+ if self._cluster_peers and not self._hive_secret:
+ print(
+ "[tracker] WARNING: cluster peers configured without --hive-secret — "
+ "gossip pushes will be rejected (ADR-0017)",
+ flush=True,
+ )
+ if self._stats is not None and self._cluster_peers:
+ self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
+ local_rpms = self._stats.get_local_rpms()
+ body = json.dumps({"tracker_url": self_url, "stats": local_rpms}).encode()
+ self._push_to_peers("/v1/stats/gossip", body)
+ if self._billing is not None and self._cluster_peers:
+ events, cursor = self._billing.events_since(self._billing_gossip_cursor)
+ if events:
+ body = json.dumps({"events": events}).encode()
+ # Only advance past events every peer has seen; unreachable
+ # peers get the full backlog on the next tick (idempotent
+ # via event-id dedupe on the receiving side).
+ if self._push_to_peers("/v1/billing/gossip", body):
+ self._billing_gossip_cursor = cursor
+ if self._accounts is not None and self._cluster_peers:
+ events, cursor = self._accounts.events_since(self._accounts_gossip_cursor)
+ if events:
+ body = json.dumps({"events": events}).encode()
+ if self._push_to_peers("/v1/accounts/gossip", body):
+ self._accounts_gossip_cursor = cursor
+
+ def stop(self) -> None:
+ if self._raft is not None:
+ self._raft.stop()
+ if self._gossip is not None:
+ self._gossip.stop()
+ if self._server is None:
+ return
+ self._rebalance_stop.set()
+ self._stats_stop.set()
+ self._deposit_stop.set()
+ self._settlement_stop.set()
+ if self._stats is not None:
+ self._stats.save_to_db()
+ if self._billing is not None:
+ self._billing.save_to_db()
+ if self._accounts is not None:
+ self._accounts.save_to_db()
+ self._server.shutdown()
+ self._server.server_close()
+ if self._thread is not None:
+ self._thread.join(timeout=1)
+ if self._rebalance_thread is not None:
+ self._rebalance_thread.join(timeout=1)
+ if self._stats_thread is not None:
+ self._stats_thread.join(timeout=1)
+ if self._deposit_thread is not None:
+ self._deposit_thread.join(timeout=1)
+ self._deposit_thread = None
+ if self._settlement_thread is not None:
+ self._settlement_thread.join(timeout=1)
+ self._settlement_thread = None
+ self._server = None
+ self._thread = None
+ self._rebalance_thread = None
+ self._stats_thread = None
+ self._raft = None
+ self._gossip = None
+ self.port = None
diff --git a/tests/test_accounts.py b/tests/test_accounts.py
index d3c9d8b..809b70f 100644
--- a/tests/test_accounts.py
+++ b/tests/test_accounts.py
@@ -1,221 +1,231 @@
-"""Dashboard user accounts: registration, login, roles, API keys, usage.
-
-Unit tests for AccountStore plus HTTP integration on the tracker:
-register/login/logout, per-account balance and usage, API-key lifecycle
-(revoked keys rejected by the OpenAI proxy), and the admin listing.
-"""
-
-import json
-import urllib.error
-import urllib.request
-
-import pytest
-
-from meshnet_tracker.accounts import AccountStore
-from meshnet_tracker.billing import BillingLedger
-from meshnet_tracker.server import TrackerServer
-
-
-# ---------------------------------------------------------------- unit tests
-
-
-def test_first_account_is_admin_then_users():
- store = AccountStore()
- first = store.register(email="admin@example.com", password="secret-123")
- second = store.register(email="user@example.com", password="secret-123")
- assert first["role"] == "admin"
- assert second["role"] == "user"
-
-
-def test_register_requires_email_or_wallet_and_password_length():
- store = AccountStore()
- with pytest.raises(ValueError, match="email or a wallet"):
- store.register(password="secret-123")
- with pytest.raises(ValueError, match="invalid email"):
- store.register(email="not-an-email", password="secret-123")
- with pytest.raises(ValueError, match="at least 8"):
- store.register(email="a@b.co", password="short")
-
-
-def test_register_rejects_duplicate_identifiers():
- store = AccountStore()
- store.register(email="dup@example.com", password="secret-123")
- with pytest.raises(ValueError, match="already exists"):
- store.register(email="DUP@example.com", password="other-secret")
-
-
-def test_login_by_email_or_wallet():
- store = AccountStore()
- account = store.register(
- email="both@example.com", wallet="WalletXYZ", password="secret-123"
- )
- assert store.verify_login("both@example.com", "secret-123")["account_id"] == account["account_id"]
- assert store.verify_login("WalletXYZ", "secret-123")["account_id"] == account["account_id"]
- assert store.verify_login("both@example.com", "wrong-password") is None
- assert store.verify_login("nobody@example.com", "secret-123") is None
-
-
-def test_sessions_resolve_and_destroy():
- store = AccountStore()
- account = store.register(email="s@example.com", password="secret-123")
- token = store.create_session(account["account_id"])
- assert store.session_account(token)["account_id"] == account["account_id"]
- store.destroy_session(token)
- assert store.session_account(token) is None
- assert store.session_account("bogus") is None
-
-
-def test_api_key_lifecycle():
- store = AccountStore()
- account = store.register(email="k@example.com", password="secret-123")
- other = store.register(email="other@example.com", password="secret-123")
- key = store.create_api_key(account["account_id"])
- assert key.startswith("sk-mesh-")
- assert store.keys_for(account["account_id"]) == [key]
- # someone else's account cannot revoke it
- assert store.revoke_api_key(other["account_id"], key) is False
- assert store.revoke_api_key(account["account_id"], key) is True
- assert store.keys_for(account["account_id"]) == []
- assert store.is_key_revoked(key)
-
-
-def test_accounts_persist_across_restart(tmp_path):
- db = str(tmp_path / "accounts.db")
- store = AccountStore(db_path=db)
- account = store.register(email="p@example.com", password="secret-123")
- key = store.create_api_key(account["account_id"])
- store.save_to_db()
-
- reloaded = AccountStore(db_path=db)
- assert reloaded.verify_login("p@example.com", "secret-123") is not None
- assert reloaded.keys_for(account["account_id"]) == [key]
-
-
-def test_account_events_replicate_and_dedupe():
- leader = AccountStore()
- follower = AccountStore()
- account = leader.register(email="r@example.com", password="secret-123")
- key = leader.create_api_key(account["account_id"])
- leader.revoke_api_key(account["account_id"], key)
-
- events, cursor = leader.events_since(0)
- assert follower.apply_events(events) == len(events)
- assert follower.apply_events(events) == 0 # replay is a no-op
- assert follower.verify_login("r@example.com", "secret-123") is not None
- assert follower.is_key_revoked(key)
- more, _ = leader.events_since(cursor)
- assert more == []
-
-
-# ---------------------------------------------------------- HTTP integration
-
-
-def _call(url, method="GET", body=None, token=None):
- headers = {"Content-Type": "application/json"}
- if token:
- headers["Authorization"] = f"Bearer {token}"
- data = json.dumps(body).encode() if body is not None else None
- req = urllib.request.Request(url, data=data, headers=headers, method=method)
- with urllib.request.urlopen(req) as r:
- return json.loads(r.read())
-
-
-@pytest.fixture
-def account_tracker():
- ledger = BillingLedger(starting_credit=0.5, default_price_per_1k=0.02)
- tracker = TrackerServer(billing=ledger, accounts=AccountStore())
- port = tracker.start()
- yield f"http://127.0.0.1:{port}", ledger
- tracker.stop()
-
-
-def test_register_login_and_account_view(account_tracker):
- url, _ = account_tracker
- reg = _call(f"{url}/v1/auth/register", "POST",
- {"email": "admin@example.com", "password": "secret-123"})
- assert reg["account"]["role"] == "admin"
- assert reg["api_key"].startswith("sk-mesh-")
- assert reg["session_token"]
-
- login = _call(f"{url}/v1/auth/login", "POST",
- {"identifier": "admin@example.com", "password": "secret-123"})
- me = _call(f"{url}/v1/account", token=login["session_token"])
- assert me["account"]["email"] == "admin@example.com"
- assert me["api_keys"] == [reg["api_key"]]
- # registration granted Caller Credit on the ledger
- assert me["total_balance"] == pytest.approx(0.5)
- assert me["usage"]["requests"] == 0
-
-
-def test_bad_credentials_and_missing_session_are_401(account_tracker):
- url, _ = account_tracker
- _call(f"{url}/v1/auth/register", "POST",
- {"email": "a@example.com", "password": "secret-123"})
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _call(f"{url}/v1/auth/login", "POST",
- {"identifier": "a@example.com", "password": "wrong-pass"})
- assert exc_info.value.code == 401
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _call(f"{url}/v1/account")
- assert exc_info.value.code == 401
-
-
-def test_key_create_revoke_and_revoked_key_rejected_by_proxy(account_tracker):
- url, _ = account_tracker
- reg = _call(f"{url}/v1/auth/register", "POST",
- {"email": "k@example.com", "password": "secret-123"})
- token = reg["session_token"]
-
- new_key = _call(f"{url}/v1/account/keys", "POST", {}, token=token)["api_key"]
- me = _call(f"{url}/v1/account", token=token)
- assert sorted(me["api_keys"]) == sorted([reg["api_key"], new_key])
-
- _call(f"{url}/v1/account/keys/revoke", "POST", {"api_key": new_key}, token=token)
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _call(f"{url}/v1/chat/completions", "POST",
- {"model": "any", "messages": []}, token=new_key)
- assert exc_info.value.code == 401
- assert "revoked" in exc_info.value.read().decode()
-
-
-def test_admin_listing_requires_admin_role(account_tracker):
- url, _ = account_tracker
- admin = _call(f"{url}/v1/auth/register", "POST",
- {"email": "admin@example.com", "password": "secret-123"})
- user = _call(f"{url}/v1/auth/register", "POST",
- {"wallet": "WalletUser1", "password": "secret-123"})
-
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _call(f"{url}/v1/admin/accounts", token=user["session_token"])
- assert exc_info.value.code == 403
-
- listing = _call(f"{url}/v1/admin/accounts", token=admin["session_token"])
- accounts = listing["accounts"]
- assert len(accounts) == 2
- assert accounts[0]["role"] == "admin"
- assert accounts[1]["wallet"] == "WalletUser1"
- assert "balances" in accounts[0]
-
-
-def test_accounts_gossip_endpoint_applies_events(account_tracker):
- url, _ = account_tracker
- peer = AccountStore()
- peer.register(email="remote@example.com", password="secret-123")
- events, _ = peer.events_since(0)
- result = _call(f"{url}/v1/accounts/gossip", "POST", {"events": events})
- assert result["applied"] == len(events)
- login = _call(f"{url}/v1/auth/login", "POST",
- {"identifier": "remote@example.com", "password": "secret-123"})
- assert login["account"]["email"] == "remote@example.com"
-
-
-def test_accounts_endpoints_404_when_disabled():
- tracker = TrackerServer() # no accounts, no billing
- port = tracker.start()
- try:
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _call(f"http://127.0.0.1:{port}/v1/auth/register", "POST",
- {"email": "x@example.com", "password": "secret-123"})
- assert exc_info.value.code == 404
- finally:
- tracker.stop()
+"""Dashboard user accounts: registration, login, roles, API keys, usage.
+
+Unit tests for AccountStore plus HTTP integration on the tracker:
+register/login/logout, per-account balance and usage, API-key lifecycle
+(revoked keys rejected by the OpenAI proxy), and the admin listing.
+"""
+
+import json
+import urllib.error
+import urllib.request
+
+import pytest
+
+from meshnet_tracker.accounts import AccountStore
+from meshnet_tracker.auth import sign_hive_request
+from meshnet_tracker.billing import BillingLedger
+from meshnet_tracker.server import TrackerServer
+
+HIVE_SECRET = "test-hive-secret"
+
+
+# ---------------------------------------------------------------- unit tests
+
+
+def test_first_account_is_admin_then_users():
+ store = AccountStore()
+ first = store.register(email="admin@example.com", password="secret-123")
+ second = store.register(email="user@example.com", password="secret-123")
+ assert first["role"] == "admin"
+ assert second["role"] == "user"
+
+
+def test_register_requires_email_or_wallet_and_password_length():
+ store = AccountStore()
+ with pytest.raises(ValueError, match="email or a wallet"):
+ store.register(password="secret-123")
+ with pytest.raises(ValueError, match="invalid email"):
+ store.register(email="not-an-email", password="secret-123")
+ with pytest.raises(ValueError, match="at least 8"):
+ store.register(email="a@b.co", password="short")
+
+
+def test_register_rejects_duplicate_identifiers():
+ store = AccountStore()
+ store.register(email="dup@example.com", password="secret-123")
+ with pytest.raises(ValueError, match="already exists"):
+ store.register(email="DUP@example.com", password="other-secret")
+
+
+def test_login_by_email_or_wallet():
+ store = AccountStore()
+ account = store.register(
+ email="both@example.com", wallet="WalletXYZ", password="secret-123"
+ )
+ assert store.verify_login("both@example.com", "secret-123")["account_id"] == account["account_id"]
+ assert store.verify_login("WalletXYZ", "secret-123")["account_id"] == account["account_id"]
+ assert store.verify_login("both@example.com", "wrong-password") is None
+ assert store.verify_login("nobody@example.com", "secret-123") is None
+
+
+def test_sessions_resolve_and_destroy():
+ store = AccountStore()
+ account = store.register(email="s@example.com", password="secret-123")
+ token = store.create_session(account["account_id"])
+ assert store.session_account(token)["account_id"] == account["account_id"]
+ store.destroy_session(token)
+ assert store.session_account(token) is None
+ assert store.session_account("bogus") is None
+
+
+def test_api_key_lifecycle():
+ store = AccountStore()
+ account = store.register(email="k@example.com", password="secret-123")
+ other = store.register(email="other@example.com", password="secret-123")
+ key = store.create_api_key(account["account_id"])
+ assert key.startswith("sk-mesh-")
+ assert store.keys_for(account["account_id"]) == [key]
+ # someone else's account cannot revoke it
+ assert store.revoke_api_key(other["account_id"], key) is False
+ assert store.revoke_api_key(account["account_id"], key) is True
+ assert store.keys_for(account["account_id"]) == []
+ assert store.is_key_revoked(key)
+
+
+def test_accounts_persist_across_restart(tmp_path):
+ db = str(tmp_path / "accounts.db")
+ store = AccountStore(db_path=db)
+ account = store.register(email="p@example.com", password="secret-123")
+ key = store.create_api_key(account["account_id"])
+ store.save_to_db()
+
+ reloaded = AccountStore(db_path=db)
+ assert reloaded.verify_login("p@example.com", "secret-123") is not None
+ assert reloaded.keys_for(account["account_id"]) == [key]
+
+
+def test_account_events_replicate_and_dedupe():
+ leader = AccountStore()
+ follower = AccountStore()
+ account = leader.register(email="r@example.com", password="secret-123")
+ key = leader.create_api_key(account["account_id"])
+ leader.revoke_api_key(account["account_id"], key)
+
+ events, cursor = leader.events_since(0)
+ assert follower.apply_events(events) == len(events)
+ assert follower.apply_events(events) == 0 # replay is a no-op
+ assert follower.verify_login("r@example.com", "secret-123") is not None
+ assert follower.is_key_revoked(key)
+ more, _ = leader.events_since(cursor)
+ assert more == []
+
+
+# ---------------------------------------------------------- HTTP integration
+
+
+def _call(url, method="GET", body=None, token=None):
+ headers = {"Content-Type": "application/json"}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+ data = json.dumps(body).encode() if body is not None else None
+ req = urllib.request.Request(url, data=data, headers=headers, method=method)
+ with urllib.request.urlopen(req) as r:
+ return json.loads(r.read())
+
+
+@pytest.fixture
+def account_tracker():
+ ledger = BillingLedger(starting_credit=0.5, default_price_per_1k=0.02)
+ tracker = TrackerServer(billing=ledger, accounts=AccountStore(), hive_secret=HIVE_SECRET)
+ port = tracker.start()
+ yield f"http://127.0.0.1:{port}", ledger
+ tracker.stop()
+
+
+def test_register_login_and_account_view(account_tracker):
+ url, _ = account_tracker
+ reg = _call(f"{url}/v1/auth/register", "POST",
+ {"email": "admin@example.com", "password": "secret-123"})
+ assert reg["account"]["role"] == "admin"
+ assert reg["api_key"].startswith("sk-mesh-")
+ assert reg["session_token"]
+
+ login = _call(f"{url}/v1/auth/login", "POST",
+ {"identifier": "admin@example.com", "password": "secret-123"})
+ me = _call(f"{url}/v1/account", token=login["session_token"])
+ assert me["account"]["email"] == "admin@example.com"
+ assert me["api_keys"] == [reg["api_key"]]
+ # registration granted Caller Credit on the ledger
+ assert me["total_balance"] == pytest.approx(0.5)
+ assert me["usage"]["requests"] == 0
+
+
+def test_bad_credentials_and_missing_session_are_401(account_tracker):
+ url, _ = account_tracker
+ _call(f"{url}/v1/auth/register", "POST",
+ {"email": "a@example.com", "password": "secret-123"})
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _call(f"{url}/v1/auth/login", "POST",
+ {"identifier": "a@example.com", "password": "wrong-pass"})
+ assert exc_info.value.code == 401
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _call(f"{url}/v1/account")
+ assert exc_info.value.code == 401
+
+
+def test_key_create_revoke_and_revoked_key_rejected_by_proxy(account_tracker):
+ url, _ = account_tracker
+ reg = _call(f"{url}/v1/auth/register", "POST",
+ {"email": "k@example.com", "password": "secret-123"})
+ token = reg["session_token"]
+
+ new_key = _call(f"{url}/v1/account/keys", "POST", {}, token=token)["api_key"]
+ me = _call(f"{url}/v1/account", token=token)
+ assert sorted(me["api_keys"]) == sorted([reg["api_key"], new_key])
+
+ _call(f"{url}/v1/account/keys/revoke", "POST", {"api_key": new_key}, token=token)
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _call(f"{url}/v1/chat/completions", "POST",
+ {"model": "any", "messages": []}, token=new_key)
+ assert exc_info.value.code == 401
+ assert "revoked" in exc_info.value.read().decode()
+
+
+def test_admin_listing_requires_admin_role(account_tracker):
+ url, _ = account_tracker
+ admin = _call(f"{url}/v1/auth/register", "POST",
+ {"email": "admin@example.com", "password": "secret-123"})
+ user = _call(f"{url}/v1/auth/register", "POST",
+ {"wallet": "WalletUser1", "password": "secret-123"})
+
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _call(f"{url}/v1/admin/accounts", token=user["session_token"])
+ assert exc_info.value.code == 403
+
+ listing = _call(f"{url}/v1/admin/accounts", token=admin["session_token"])
+ accounts = listing["accounts"]
+ assert len(accounts) == 2
+ assert accounts[0]["role"] == "admin"
+ assert accounts[1]["wallet"] == "WalletUser1"
+ assert "balances" in accounts[0]
+
+
+def test_accounts_gossip_endpoint_applies_events(account_tracker):
+ url, _ = account_tracker
+ peer = AccountStore()
+ peer.register(email="remote@example.com", password="secret-123")
+ events, _ = peer.events_since(0)
+ body = json.dumps({"events": events}).encode()
+ req = urllib.request.Request(
+ f"{url}/v1/accounts/gossip", data=body,
+ headers={"Content-Type": "application/json", **sign_hive_request(HIVE_SECRET, body)},
+ method="POST",
+ )
+ with urllib.request.urlopen(req) as r:
+ result = json.loads(r.read())
+ assert result["applied"] == len(events)
+ login = _call(f"{url}/v1/auth/login", "POST",
+ {"identifier": "remote@example.com", "password": "secret-123"})
+ assert login["account"]["email"] == "remote@example.com"
+
+
+def test_accounts_endpoints_404_when_disabled():
+ tracker = TrackerServer() # no accounts, no billing
+ port = tracker.start()
+ try:
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _call(f"http://127.0.0.1:{port}/v1/auth/register", "POST",
+ {"email": "x@example.com", "password": "secret-123"})
+ assert exc_info.value.code == 404
+ finally:
+ tracker.stop()
diff --git a/tests/test_billing_ledger.py b/tests/test_billing_ledger.py
index c9d8daa..734b0e7 100644
--- a/tests/test_billing_ledger.py
+++ b/tests/test_billing_ledger.py
@@ -1,315 +1,320 @@
-"""US-031: billing ledger — per-token pricing, 90/10 split, pending balances.
-
-Unit tests for BillingLedger math/persistence/replication, plus HTTP
-integration: 401 without an API key, billed 200 with one, 402 once the
-Caller Credit is exhausted (rejected before routing — no free work).
-"""
-
-import http.server
-import json
-import socketserver
-import threading
-import urllib.error
-import urllib.request
-
-import pytest
-
-from meshnet_tracker.billing import BillingLedger
-from meshnet_tracker.server import TrackerServer
-
-MODEL = "openai-community/gpt2"
-
-
-# ---------------------------------------------------------------- unit tests
-
-
-def test_charge_single_node_gets_90_percent():
- ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
- ledger.ensure_client("key-a")
- event = ledger.charge_request("key-a", MODEL, total_tokens=1000, node_work=[("wallet-1", 12)])
- assert event["cost"] == pytest.approx(0.02)
- assert ledger.get_client_balance("key-a") == pytest.approx(1.0 - 0.02)
- assert ledger.get_node_pending("wallet-1") == pytest.approx(0.02 * 0.90)
- assert ledger.snapshot()["protocol_cut"] == pytest.approx(0.02 * 0.10)
-
-
-def test_charge_three_node_split_by_work_units():
- ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
- ledger.ensure_client("key-a")
- ledger.charge_request(
- "key-a", MODEL, total_tokens=1000,
- node_work=[("wallet-1", 6), ("wallet-2", 3), ("wallet-3", 3)],
- )
- pool = 0.02 * 0.90
- assert ledger.get_node_pending("wallet-1") == pytest.approx(pool * 6 / 12)
- assert ledger.get_node_pending("wallet-2") == pytest.approx(pool * 3 / 12)
- assert ledger.get_node_pending("wallet-3") == pytest.approx(pool * 3 / 12)
- assert ledger.snapshot()["protocol_cut"] == pytest.approx(0.02 * 0.10)
-
-
-def test_walletless_node_share_accrues_to_protocol_cut():
- ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
- ledger.ensure_client("key-a")
- ledger.charge_request(
- "key-a", MODEL, total_tokens=1000,
- node_work=[("wallet-1", 6), (None, 6)],
- )
- pool = 0.02 * 0.90
- assert ledger.get_node_pending("wallet-1") == pytest.approx(pool / 2)
- # walletless half of the pool + the 10% cut both land in protocol_cut
- assert ledger.snapshot()["protocol_cut"] == pytest.approx(pool / 2 + 0.02 * 0.10)
-
-
-def test_per_model_price_override():
- ledger = BillingLedger(
- starting_credit=1.0,
- default_price_per_1k=0.02,
- prices={MODEL: 0.10},
- )
- ledger.ensure_client("key-a")
- event = ledger.charge_request("key-a", MODEL, 500, [("wallet-1", 1)])
- assert event["cost"] == pytest.approx(0.10 * 500 / 1000)
- event = ledger.charge_request("key-a", "other-model", 500, [("wallet-1", 1)])
- assert event["cost"] == pytest.approx(0.02 * 500 / 1000)
-
-
-def test_payout_and_forfeit_hooks():
- ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
- ledger.charge_request("key-a", MODEL, 1000, [("wallet-1", 1)])
- pending = ledger.get_node_pending("wallet-1")
- ledger.settle_node_payout("wallet-1", pending, reference="tx-sig-1")
- assert ledger.get_node_pending("wallet-1") == pytest.approx(0.0)
-
- ledger.charge_request("key-a", MODEL, 1000, [("wallet-1", 1)])
- cut_before = ledger.snapshot()["protocol_cut"]
- forfeited = ledger.forfeit_pending("wallet-1")["amount"]
- assert forfeited == pytest.approx(0.02 * 0.90)
- assert ledger.get_node_pending("wallet-1") == pytest.approx(0.0)
- assert ledger.snapshot()["protocol_cut"] == pytest.approx(cut_before + forfeited)
-
-
-def test_restart_persistence(tmp_path):
- db = str(tmp_path / "billing.db")
- ledger = BillingLedger(db_path=db, starting_credit=1.0, default_price_per_1k=0.02)
- ledger.credit_client("key-a", 5.0)
- ledger.charge_request("key-a", MODEL, 1000, [("wallet-1", 12)])
- ledger.save_to_db()
-
- reloaded = BillingLedger(db_path=db, starting_credit=1.0, default_price_per_1k=0.02)
- assert reloaded.get_client_balance("key-a") == pytest.approx(
- ledger.get_client_balance("key-a")
- )
- assert reloaded.get_node_pending("wallet-1") == pytest.approx(0.02 * 0.90)
- assert reloaded.snapshot()["protocol_cut"] == pytest.approx(0.02 * 0.10)
-
-
-def test_tracker_enables_billing_with_default_db_when_requested(tmp_path, monkeypatch):
- from meshnet_tracker.billing import DEFAULT_BILLING_DB_PATH
-
- monkeypatch.chdir(tmp_path)
- tracker = TrackerServer(enable_billing=True)
- port = tracker.start()
- try:
- summary = json.loads(
- urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/billing/summary").read()
- )
- assert "protocol_cut" in summary
- finally:
- tracker.stop()
- assert (tmp_path / DEFAULT_BILLING_DB_PATH).exists()
-
-
-def test_event_replication_converges_and_dedupes():
- leader = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
- follower = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
-
- leader.credit_client("key-a", 10.0)
- leader.charge_request("key-a", MODEL, 2000, [("wallet-1", 8), ("wallet-2", 4)])
-
- events, cursor = leader.events_since(0)
- assert follower.apply_events(events) == len(events)
- # replaying the same batch must be a no-op
- assert follower.apply_events(events) == 0
-
- assert follower.get_client_balance("key-a") == pytest.approx(
- leader.get_client_balance("key-a")
- )
- assert follower.get_node_pending("wallet-1") == pytest.approx(
- leader.get_node_pending("wallet-1")
- )
- assert follower.snapshot()["protocol_cut"] == pytest.approx(
- leader.snapshot()["protocol_cut"]
- )
- # incremental cursor: nothing new after full sync
- more, _ = leader.events_since(cursor)
- assert more == []
-
-
-# ---------------------------------------------------------- HTTP integration
-
-
-class _UsageStubNode:
- """Minimal head node returning a chat completion with real usage numbers."""
-
- def __init__(self, total_tokens: int = 1000):
- self.total_tokens = total_tokens
- outer = self
-
- class Handler(http.server.BaseHTTPRequestHandler):
- def log_message(self, fmt, *args):
- pass
-
- def do_POST(self):
- length = int(self.headers.get("Content-Length", 0))
- self.rfile.read(length)
- body = json.dumps({
- "id": "chatcmpl-stub",
- "object": "chat.completion",
- "model": MODEL,
- "choices": [{
- "index": 0,
- "message": {"role": "assistant", "content": "ok"},
- "finish_reason": "stop",
- }],
- "usage": {
- "prompt_tokens": 0,
- "completion_tokens": outer.total_tokens,
- "total_tokens": outer.total_tokens,
- },
- }).encode()
- self.send_response(200)
- self.send_header("Content-Type", "application/json")
- self.send_header("Content-Length", str(len(body)))
- self.end_headers()
- self.wfile.write(body)
-
- self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
- self._server.daemon_threads = True
- self._thread: threading.Thread | None = None
-
- def start(self) -> int:
- self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
- self._thread.start()
- return self._server.server_address[1]
-
- def stop(self):
- self._server.shutdown()
- self._server.server_close()
-
-
-@pytest.fixture
-def billed_tracker():
- ledger = BillingLedger(starting_credit=0.03, default_price_per_1k=0.02)
- tracker = TrackerServer(
- model_presets={
- MODEL: {
- "layers_start": 0,
- "layers_end": 11,
- "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
- }
- },
- billing=ledger,
- )
- port = tracker.start()
- tracker_url = f"http://127.0.0.1:{port}"
-
- stub = _UsageStubNode(total_tokens=1000)
- stub_port = stub.start()
- reg = json.dumps({
- "endpoint": f"http://127.0.0.1:{stub_port}",
- "shard_start": 0,
- "shard_end": 11,
- "model": MODEL,
- "hardware_profile": {},
- "score": 1.0,
- "tracker_mode": True,
- "wallet_address": "wallet-head",
- }).encode()
- req = urllib.request.Request(
- f"{tracker_url}/v1/nodes/register",
- data=reg,
- headers={"Content-Type": "application/json"},
- method="POST",
- )
- with urllib.request.urlopen(req) as r:
- r.read()
-
- yield tracker_url, ledger
-
- stub.stop()
- tracker.stop()
-
-
-def _chat(tracker_url: str, api_key: str | None):
- data = json.dumps({
- "model": MODEL,
- "messages": [{"role": "user", "content": "hi"}],
- }).encode()
- headers = {"Content-Type": "application/json"}
- if api_key:
- headers["Authorization"] = f"Bearer {api_key}"
- req = urllib.request.Request(
- f"{tracker_url}/v1/chat/completions",
- data=data,
- headers=headers,
- method="POST",
- )
- with urllib.request.urlopen(req) as r:
- return json.loads(r.read())
-
-
-def test_proxy_chat_requires_api_key_when_billing_enabled(billed_tracker):
- tracker_url, _ = billed_tracker
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _chat(tracker_url, api_key=None)
- assert exc_info.value.code == 401
-
-
-def test_proxy_chat_bills_client_and_credits_node(billed_tracker):
- tracker_url, ledger = billed_tracker
- _chat(tracker_url, api_key="client-1")
- # 1000 tokens at 0.02/1K = 0.02 USDT; starting credit 0.03
- assert ledger.get_client_balance("client-1") == pytest.approx(0.03 - 0.02)
- assert ledger.get_node_pending("wallet-head") == pytest.approx(0.02 * 0.90)
-
- summary = json.loads(
- urllib.request.urlopen(f"{tracker_url}/v1/billing/summary").read()
- )
- assert summary["node_pending"]["wallet-head"] == pytest.approx(0.02 * 0.90)
- assert summary["protocol_cut"] == pytest.approx(0.02 * 0.10)
-
- stats = json.loads(urllib.request.urlopen(f"{tracker_url}/v1/stats").read())
- node_stats = next(iter(stats["nodes"].values()))["models"][MODEL]
- assert node_stats["tokens_per_sec_last_hour"] is not None
- assert node_stats["sample_count_last_hour"] == 1
-
-
-def test_proxy_chat_402_when_balance_exhausted(billed_tracker):
- tracker_url, ledger = billed_tracker
- _chat(tracker_url, api_key="client-2") # 0.03 -> 0.01
- _chat(tracker_url, api_key="client-2") # 0.01 -> -0.01 (post-pay drift)
- assert ledger.get_client_balance("client-2") == pytest.approx(-0.01)
- with pytest.raises(urllib.error.HTTPError) as exc_info:
- _chat(tracker_url, api_key="client-2")
- assert exc_info.value.code == 402
- # rejected before routing: nothing further was billed
- assert ledger.get_client_balance("client-2") == pytest.approx(-0.01)
-
-
-def test_billing_gossip_endpoint_applies_events(billed_tracker):
- tracker_url, ledger = billed_tracker
- peer = BillingLedger(starting_credit=0.03, default_price_per_1k=0.02)
- peer.credit_client("remote-client", 7.0)
- events, _ = peer.events_since(0)
- body = json.dumps({"events": events}).encode()
- req = urllib.request.Request(
- f"{tracker_url}/v1/billing/gossip",
- data=body,
- headers={"Content-Type": "application/json"},
- method="POST",
- )
- with urllib.request.urlopen(req) as r:
- assert json.loads(r.read())["applied"] == len(events)
- # only the replicated credit event lands here — Caller Credit was granted
- # on the peer that first saw the key, and its event replicates separately
- assert ledger.get_client_balance("remote-client") == pytest.approx(7.0)
+"""US-031: billing ledger — per-token pricing, 90/10 split, pending balances.
+
+Unit tests for BillingLedger math/persistence/replication, plus HTTP
+integration: 401 without an API key, billed 200 with one, 402 once the
+Caller Credit is exhausted (rejected before routing — no free work).
+"""
+
+import http.server
+import json
+import socketserver
+import threading
+import urllib.error
+import urllib.request
+
+import pytest
+
+from meshnet_tracker.auth import sign_hive_request
+from meshnet_tracker.billing import BillingLedger
+from meshnet_tracker.server import TrackerServer
+
+MODEL = "openai-community/gpt2"
+HIVE_SECRET = "test-hive-secret"
+
+
+# ---------------------------------------------------------------- unit tests
+
+
+def test_charge_single_node_gets_90_percent():
+ ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
+ ledger.ensure_client("key-a")
+ event = ledger.charge_request("key-a", MODEL, total_tokens=1000, node_work=[("wallet-1", 12)])
+ assert event["cost"] == pytest.approx(0.02)
+ assert ledger.get_client_balance("key-a") == pytest.approx(1.0 - 0.02)
+ assert ledger.get_node_pending("wallet-1") == pytest.approx(0.02 * 0.90)
+ assert ledger.snapshot()["protocol_cut"] == pytest.approx(0.02 * 0.10)
+
+
+def test_charge_three_node_split_by_work_units():
+ ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
+ ledger.ensure_client("key-a")
+ ledger.charge_request(
+ "key-a", MODEL, total_tokens=1000,
+ node_work=[("wallet-1", 6), ("wallet-2", 3), ("wallet-3", 3)],
+ )
+ pool = 0.02 * 0.90
+ assert ledger.get_node_pending("wallet-1") == pytest.approx(pool * 6 / 12)
+ assert ledger.get_node_pending("wallet-2") == pytest.approx(pool * 3 / 12)
+ assert ledger.get_node_pending("wallet-3") == pytest.approx(pool * 3 / 12)
+ assert ledger.snapshot()["protocol_cut"] == pytest.approx(0.02 * 0.10)
+
+
+def test_walletless_node_share_accrues_to_protocol_cut():
+ ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
+ ledger.ensure_client("key-a")
+ ledger.charge_request(
+ "key-a", MODEL, total_tokens=1000,
+ node_work=[("wallet-1", 6), (None, 6)],
+ )
+ pool = 0.02 * 0.90
+ assert ledger.get_node_pending("wallet-1") == pytest.approx(pool / 2)
+ # walletless half of the pool + the 10% cut both land in protocol_cut
+ assert ledger.snapshot()["protocol_cut"] == pytest.approx(pool / 2 + 0.02 * 0.10)
+
+
+def test_per_model_price_override():
+ ledger = BillingLedger(
+ starting_credit=1.0,
+ default_price_per_1k=0.02,
+ prices={MODEL: 0.10},
+ )
+ ledger.ensure_client("key-a")
+ event = ledger.charge_request("key-a", MODEL, 500, [("wallet-1", 1)])
+ assert event["cost"] == pytest.approx(0.10 * 500 / 1000)
+ event = ledger.charge_request("key-a", "other-model", 500, [("wallet-1", 1)])
+ assert event["cost"] == pytest.approx(0.02 * 500 / 1000)
+
+
+def test_payout_and_forfeit_hooks():
+ ledger = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
+ ledger.charge_request("key-a", MODEL, 1000, [("wallet-1", 1)])
+ pending = ledger.get_node_pending("wallet-1")
+ ledger.settle_node_payout("wallet-1", pending, reference="tx-sig-1")
+ assert ledger.get_node_pending("wallet-1") == pytest.approx(0.0)
+
+ ledger.charge_request("key-a", MODEL, 1000, [("wallet-1", 1)])
+ cut_before = ledger.snapshot()["protocol_cut"]
+ forfeited = ledger.forfeit_pending("wallet-1")["amount"]
+ assert forfeited == pytest.approx(0.02 * 0.90)
+ assert ledger.get_node_pending("wallet-1") == pytest.approx(0.0)
+ assert ledger.snapshot()["protocol_cut"] == pytest.approx(cut_before + forfeited)
+
+
+def test_restart_persistence(tmp_path):
+ db = str(tmp_path / "billing.db")
+ ledger = BillingLedger(db_path=db, starting_credit=1.0, default_price_per_1k=0.02)
+ ledger.credit_client("key-a", 5.0)
+ ledger.charge_request("key-a", MODEL, 1000, [("wallet-1", 12)])
+ ledger.save_to_db()
+
+ reloaded = BillingLedger(db_path=db, starting_credit=1.0, default_price_per_1k=0.02)
+ assert reloaded.get_client_balance("key-a") == pytest.approx(
+ ledger.get_client_balance("key-a")
+ )
+ assert reloaded.get_node_pending("wallet-1") == pytest.approx(0.02 * 0.90)
+ assert reloaded.snapshot()["protocol_cut"] == pytest.approx(0.02 * 0.10)
+
+
+def test_tracker_enables_billing_with_default_db_when_requested(tmp_path, monkeypatch):
+ from meshnet_tracker.billing import DEFAULT_BILLING_DB_PATH
+
+ monkeypatch.chdir(tmp_path)
+ tracker = TrackerServer(enable_billing=True)
+ port = tracker.start()
+ try:
+ # /v1/billing/summary is admin-gated now; just confirm the server is up.
+ health = json.loads(
+ urllib.request.urlopen(f"http://127.0.0.1:{port}/v1/health").read()
+ )
+ assert health["status"] == "ok"
+ finally:
+ tracker.stop()
+ # enabling billing creates the ledger DB in the working directory
+ assert (tmp_path / DEFAULT_BILLING_DB_PATH).exists()
+
+
+def test_event_replication_converges_and_dedupes():
+ leader = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
+ follower = BillingLedger(starting_credit=1.0, default_price_per_1k=0.02)
+
+ leader.credit_client("key-a", 10.0)
+ leader.charge_request("key-a", MODEL, 2000, [("wallet-1", 8), ("wallet-2", 4)])
+
+ events, cursor = leader.events_since(0)
+ assert follower.apply_events(events) == len(events)
+ # replaying the same batch must be a no-op
+ assert follower.apply_events(events) == 0
+
+ assert follower.get_client_balance("key-a") == pytest.approx(
+ leader.get_client_balance("key-a")
+ )
+ assert follower.get_node_pending("wallet-1") == pytest.approx(
+ leader.get_node_pending("wallet-1")
+ )
+ assert follower.snapshot()["protocol_cut"] == pytest.approx(
+ leader.snapshot()["protocol_cut"]
+ )
+ # incremental cursor: nothing new after full sync
+ more, _ = leader.events_since(cursor)
+ assert more == []
+
+
+# ---------------------------------------------------------- HTTP integration
+
+
+class _UsageStubNode:
+ """Minimal head node returning a chat completion with real usage numbers."""
+
+ def __init__(self, total_tokens: int = 1000):
+ self.total_tokens = total_tokens
+ outer = self
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def log_message(self, fmt, *args):
+ pass
+
+ def do_POST(self):
+ length = int(self.headers.get("Content-Length", 0))
+ self.rfile.read(length)
+ body = json.dumps({
+ "id": "chatcmpl-stub",
+ "object": "chat.completion",
+ "model": MODEL,
+ "choices": [{
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }],
+ "usage": {
+ "prompt_tokens": 0,
+ "completion_tokens": outer.total_tokens,
+ "total_tokens": outer.total_tokens,
+ },
+ }).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ self._server = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
+ self._server.daemon_threads = True
+ self._thread: threading.Thread | None = None
+
+ def start(self) -> int:
+ self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
+ self._thread.start()
+ return self._server.server_address[1]
+
+ def stop(self):
+ self._server.shutdown()
+ self._server.server_close()
+
+
+@pytest.fixture
+def billed_tracker():
+ ledger = BillingLedger(starting_credit=0.03, default_price_per_1k=0.02)
+ tracker = TrackerServer(
+ model_presets={
+ MODEL: {
+ "layers_start": 0,
+ "layers_end": 11,
+ "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
+ }
+ },
+ billing=ledger,
+ hive_secret=HIVE_SECRET,
+ )
+ port = tracker.start()
+ tracker_url = f"http://127.0.0.1:{port}"
+
+ stub = _UsageStubNode(total_tokens=1000)
+ stub_port = stub.start()
+ reg = json.dumps({
+ "endpoint": f"http://127.0.0.1:{stub_port}",
+ "shard_start": 0,
+ "shard_end": 11,
+ "model": MODEL,
+ "hardware_profile": {},
+ "score": 1.0,
+ "tracker_mode": True,
+ "wallet_address": "wallet-head",
+ }).encode()
+ req = urllib.request.Request(
+ f"{tracker_url}/v1/nodes/register",
+ data=reg,
+ headers={"Content-Type": "application/json"},
+ method="POST",
+ )
+ with urllib.request.urlopen(req) as r:
+ r.read()
+
+ yield tracker_url, ledger
+
+ stub.stop()
+ tracker.stop()
+
+
+def _chat(tracker_url: str, api_key: str | None):
+ data = json.dumps({
+ "model": MODEL,
+ "messages": [{"role": "user", "content": "hi"}],
+ }).encode()
+ headers = {"Content-Type": "application/json"}
+ if api_key:
+ headers["Authorization"] = f"Bearer {api_key}"
+ req = urllib.request.Request(
+ f"{tracker_url}/v1/chat/completions",
+ data=data,
+ headers=headers,
+ method="POST",
+ )
+ with urllib.request.urlopen(req) as r:
+ return json.loads(r.read())
+
+
+def test_proxy_chat_requires_api_key_when_billing_enabled(billed_tracker):
+ tracker_url, _ = billed_tracker
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _chat(tracker_url, api_key=None)
+ assert exc_info.value.code == 401
+
+
+def test_proxy_chat_bills_client_and_credits_node(billed_tracker):
+ tracker_url, ledger = billed_tracker
+ _chat(tracker_url, api_key="client-1")
+ # 1000 tokens at 0.02/1K = 0.02 USDT; starting credit 0.03
+ assert ledger.get_client_balance("client-1") == pytest.approx(0.03 - 0.02)
+ assert ledger.get_node_pending("wallet-head") == pytest.approx(0.02 * 0.90)
+
+ # /v1/billing/summary is admin-gated now (ADR-0017); auth is covered in
+ # test_auth_boundary.py, so verify the numbers via the ledger snapshot.
+ summary = ledger.snapshot()
+ assert summary["node_pending"]["wallet-head"] == pytest.approx(0.02 * 0.90)
+ assert summary["protocol_cut"] == pytest.approx(0.02 * 0.10)
+
+ stats = json.loads(urllib.request.urlopen(f"{tracker_url}/v1/stats").read())
+ node_stats = next(iter(stats["nodes"].values()))["models"][MODEL]
+ assert node_stats["tokens_per_sec_last_hour"] is not None
+ assert node_stats["sample_count_last_hour"] == 1
+
+
+def test_proxy_chat_402_when_balance_exhausted(billed_tracker):
+ tracker_url, ledger = billed_tracker
+ _chat(tracker_url, api_key="client-2") # 0.03 -> 0.01
+ _chat(tracker_url, api_key="client-2") # 0.01 -> -0.01 (post-pay drift)
+ assert ledger.get_client_balance("client-2") == pytest.approx(-0.01)
+ with pytest.raises(urllib.error.HTTPError) as exc_info:
+ _chat(tracker_url, api_key="client-2")
+ assert exc_info.value.code == 402
+ # rejected before routing: nothing further was billed
+ assert ledger.get_client_balance("client-2") == pytest.approx(-0.01)
+
+
+def test_billing_gossip_endpoint_applies_events(billed_tracker):
+ tracker_url, ledger = billed_tracker
+ peer = BillingLedger(starting_credit=0.03, default_price_per_1k=0.02)
+ peer.credit_client("remote-client", 7.0)
+ events, _ = peer.events_since(0)
+ body = json.dumps({"events": events}).encode()
+ req = urllib.request.Request(
+ f"{tracker_url}/v1/billing/gossip",
+ data=body,
+ headers={"Content-Type": "application/json", **sign_hive_request(HIVE_SECRET, body)},
+ method="POST",
+ )
+ with urllib.request.urlopen(req) as r:
+ assert json.loads(r.read())["applied"] == len(events)
+ # only the replicated credit event lands here — Caller Credit was granted
+ # on the peer that first saw the key, and its event replicates separately
+ assert ledger.get_client_balance("remote-client") == pytest.approx(7.0)
diff --git a/tests/test_dashboard.py b/tests/test_dashboard.py
index b0e3c27..d0247a1 100644
--- a/tests/test_dashboard.py
+++ b/tests/test_dashboard.py
@@ -4,6 +4,7 @@ import json
import urllib.request
from meshnet_contracts import LocalSolanaContracts
+from meshnet_tracker.accounts import AccountStore
from meshnet_tracker.billing import BillingLedger
from meshnet_tracker.server import TrackerServer
@@ -48,12 +49,17 @@ def test_registry_wallets_endpoint():
contracts = LocalSolanaContracts()
contracts.registry.submit_stake("wallet-a", 100)
contracts.registry.record_strike("wallet-a")
- tracker = TrackerServer(contracts=contracts)
+ accounts = AccountStore()
+ admin = accounts.register(email="admin@example.com", password="admin-pass-123")
+ session = accounts.create_session(admin["account_id"])
+ tracker = TrackerServer(contracts=contracts, accounts=accounts)
port = tracker.start()
try:
- data = json.loads(urllib.request.urlopen(
- f"http://127.0.0.1:{port}/v1/registry/wallets"
- ).read())
+ req = urllib.request.Request(
+ f"http://127.0.0.1:{port}/v1/registry/wallets",
+ headers={"Authorization": f"Bearer {session}"},
+ )
+ data = json.loads(urllib.request.urlopen(req).read())
assert data["wallets"]["wallet-a"]["strike_count"] == 1
assert data["wallets"]["wallet-a"]["banned"] is False
finally:
diff --git a/tests/test_forfeiture_penalty.py b/tests/test_forfeiture_penalty.py
index 0c4ddc9..9893409 100644
--- a/tests/test_forfeiture_penalty.py
+++ b/tests/test_forfeiture_penalty.py
@@ -157,7 +157,9 @@ def test_forfeit_endpoint_requires_auth_and_forfeits():
ledger.charge_request("client", MODEL, 1000, [("wallet-x", 12)])
pending = ledger.get_node_pending("wallet-x")
- tracker = TrackerServer(contracts=contracts, billing=ledger)
+ tracker = TrackerServer(
+ contracts=contracts, billing=ledger, validator_service_token="test-svc-token",
+ )
port = tracker.start()
url = f"http://127.0.0.1:{port}/v1/billing/forfeit"
body = json.dumps({"wallet": "wallet-x", "reason": "fraud"}).encode()
@@ -171,7 +173,7 @@ def test_forfeit_endpoint_requires_auth_and_forfeits():
req = urllib.request.Request(
url, data=body,
- headers={"Content-Type": "application/json", "Authorization": "Bearer validator"},
+ headers={"Content-Type": "application/json", "Authorization": "Bearer test-svc-token"},
method="POST",
)
with urllib.request.urlopen(req) as r:
diff --git a/tests/test_manual_route_benchmark.py b/tests/test_manual_route_benchmark.py
index 10237a5..87a3393 100644
--- a/tests/test_manual_route_benchmark.py
+++ b/tests/test_manual_route_benchmark.py
@@ -41,6 +41,7 @@ def benchmark_setup(tmp_path):
}
},
benchmark_results_path=results_path,
+ validator_service_token="bench",
)
port = tracker.start()
tracker_url = f"http://127.0.0.1:{port}"
diff --git a/tests/test_settlement_loop.py b/tests/test_settlement_loop.py
index e105988..5082be8 100644
--- a/tests/test_settlement_loop.py
+++ b/tests/test_settlement_loop.py
@@ -73,9 +73,9 @@ def test_threshold_triggers_payout_and_zeroes_pending():
assert treasury.batches[0] == [("wallet-a", pytest.approx(0.018))]
assert ledger.get_node_pending("wallet-a") == pytest.approx(0.0)
- history = json.loads(urllib.request.urlopen(
- f"http://127.0.0.1:{port}/v1/billing/settlements"
- ).read())["settlements"]
+ # /v1/billing/settlements is admin-gated now (ADR-0017, covered in
+ # test_auth_boundary.py); verify content via the ledger directly.
+ history = ledger.settlement_history()
assert len(history) == 1
assert history[0]["signature"] == "fake-tx-1"
assert history[0]["payouts"] == [
diff --git a/tests/test_tracker_routing.py b/tests/test_tracker_routing.py
index a6a3dce..0237ed1 100644
--- a/tests/test_tracker_routing.py
+++ b/tests/test_tracker_routing.py
@@ -12,8 +12,11 @@ import pytest
from meshnet_gateway.server import GatewayServer, _banned_route_wallet
from meshnet_node.server import StubNodeServer
from meshnet_contracts import LocalSolanaContracts
+from meshnet_tracker.auth import sign_hive_request
from meshnet_tracker.server import TrackerServer, _NodeEntry, _registration_ban_error
+_TEST_HIVE_SECRET = "test-hive-secret"
+
def _post_json(url: str, payload: dict) -> dict:
data = json.dumps(payload).encode()
@@ -1558,7 +1561,7 @@ def test_stats_sqlite_persistence_survives_restart(tmp_path):
def test_stats_gossip_endpoint_merges_peer_slice():
- tracker = TrackerServer()
+ tracker = TrackerServer(hive_secret=_TEST_HIVE_SECRET)
port = tracker.start()
try:
peer_payload = {
@@ -1571,7 +1574,14 @@ def test_stats_gossip_endpoint_merges_peer_slice():
}
},
}
- _post_json(f"http://127.0.0.1:{port}/v1/stats/gossip", peer_payload)
+ body = json.dumps(peer_payload).encode()
+ req = urllib.request.Request(
+ f"http://127.0.0.1:{port}/v1/stats/gossip", data=body,
+ headers={"Content-Type": "application/json", **sign_hive_request(_TEST_HIVE_SECRET, body)},
+ method="POST",
+ )
+ with urllib.request.urlopen(req) as r:
+ r.read()
combined = _get_json(f"http://127.0.0.1:{port}/v1/stats")
assert "remote-model" in combined["models"]