neuron-tai/packages/tracker/meshnet_tracker/server.py

"""Tracker HTTP server — node registry and route selection for the inference mesh.

HTTP API contract:
- POST /v1/nodes/register
  Request JSON: {
    "endpoint": "http://host:port", "shard_start": int, "shard_end": int,
    "model": str optional, "shard_checksum": str optional,
    "hardware_profile": object, "wallet_address": str optional,
    "score": number optional,
    "capability_report": object optional,  # ADR-0023 proof of this exact shard
    "recipe_id": str optional, "recipe_version": str optional
  }
  Response 200: {"node_id": str, "capability": {"state": str, "routable": bool, ...}}
  Response 400: {"error": str}
  A node whose proof does not admit what it advertises still registers (so the
  operator can see why it is dark) but is not routable -- see `capability.py`.
- POST /v1/nodes/{node_id}/heartbeat
  Response 200: {}
  Response 404: {"error": "node not found"}
- GET /v1/route?model=<preset>
  Response 200: {"route": ["http://node-a", "http://node-b"]}
  Response 400/404/503: {"error": str}
- GET /v1/routes?model=<preset>&redundancy=<n>
  Response 200: {"routes": [{"route": [...], "nodes": [...]}]}
  Response 400/404/503: {"error": str}
- GET /v1/routing?model=<name>  (ADR-0021 learned route table)
  Response 200: {"config": {...}, "models": {model: {"epoch": int, "routes": [...]}}}
"""

import http.cookies
import http.server
import hashlib
import itertools
import json
import os
import random
import select
import socketserver
import sqlite3
import sys
import tarfile
import threading
import time
import urllib.parse
import urllib.request
import uuid
from collections import deque
from dataclasses import dataclass, field
from importlib.resources import files
from pathlib import Path
from typing import Any

from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
from .auth import is_validator_token, sign_hive_request, verify_hive_request
from .capability import (
    DEFAULT_POLICY as DEFAULT_CAPABILITY_POLICY,
    POLICY_COMPAT,
    POLICY_ENFORCE,
    STATE_ABSENT,
    STATE_ADMITTED,
    STATE_MODEL_MISMATCH,
    STATE_SHARD_MISMATCH,
    CapabilityState,
    absent_state,
    evaluate_report,
    normalize_policy,
    policy_from_env,
)
from .wallet_proof import binding_message, verify_wallet_signature
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
from .calibration import DEFAULT_CALIBRATION_DB_PATH, ToplocCalibrationStore
from .hf_pricing import DEFAULT_HF_PRICING_LOG_DB_PATH, HfPricingLog, refresh_preset_price
from .gossip import NodeGossip
from .logging_setup import tracker_logger
from .test_runner import ENABLE_ENV_VAR as TEST_RUNNER_ENABLE_ENV_VAR
from .test_runner import TestRunManager, TestRunnerError
from .routing_stats import (
    RouteCandidate,
    RouteStatsStore,
    RoutingConfig,
    choose_route,
    route_signature,
    route_table,
)
from .model_files import files_for_layer_range, snapshot_dir_for_repo
from .raft import RaftNode


_CONSOLE_LIMIT = 300
_PROXY_PROGRESS_LOG_INTERVAL = 5.0
_SESSION_COOKIE_NAME = "meshnet_session"


def _preset_price_keys(name: str, preset: dict) -> set[str]:
    """All model strings a client may bill under for one preset.

    ``BillingLedger.price_for`` is keyed by the raw ``model`` string in the
    request, so the preset price must be registered under the preset name,
    its ``hf_repo``, and every alias — otherwise ``unsloth/Qwen…`` style
    requests silently fall back to the default rate.
    """
    keys = {name}
    hf_repo = preset.get("hf_repo")
    if isinstance(hf_repo, str) and hf_repo:
        keys.add(hf_repo)
    for alias in preset.get("aliases") or []:
        if isinstance(alias, str) and alias:
            keys.add(alias)
    return keys


def derive_relay_url_from_public_tracker_url(url: str | None) -> str | None:
    """Return wss://host/ws when url is a public HTTPS tracker origin."""
    if not url:
        return None
    parsed = urllib.parse.urlparse(url)
    if parsed.scheme != "https":
        return None
    host = parsed.hostname
    if not host or host in ("127.0.0.1", "localhost"):
        return None
    return f"wss://{parsed.netloc}/ws"


def _local_relay_url(host: str, port: int) -> str:
    """Return a node-consumable ws:// URL for an embedded relay bind address."""
    advertise_host = "127.0.0.1" if host in {"", "0.0.0.0", "::"} else host
    return f"ws://{advertise_host}:{port}/ws"


def _load_model_presets_from_data() -> dict[str, dict]:
    """Load recommended model presets from package JSON data."""
    try:
        raw = files("meshnet_tracker").joinpath("model_presets.json").read_text()
        data = json.loads(raw)
    except Exception:
        return {}
    models = data.get("models", {})
    if not isinstance(models, dict):
        return {}
    return {
        str(name): preset
        for name, preset in models.items()
        if isinstance(preset, dict)
    }


DEFAULT_MODEL_PRESETS: dict[str, dict] = {
    "stub-model": {
        "layers_start": 0,
        "layers_end": 31,
        "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024},
    },
    "openai-community/gpt2": {
        "layers_start": 0,
        "layers_end": 11,
        "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
    },
    **_load_model_presets_from_data(),
}

def _clone_model_presets(presets: dict[str, dict]) -> dict[str, dict]:
    """Shallow-copy each preset dict so a runtime mutation (e.g. issue 23's
    dynamic pricing refresh writing hf_last_price_per_1k/hf_last_updated)
    never leaks into the shared module-level DEFAULT_MODEL_PRESETS and from
    there into other TrackerServer instances in the same process."""
    return {name: dict(preset) for name, preset in presets.items()}


DEFAULT_VRAM_BYTES = 8 * 1024 * 1024 * 1024
DEFAULT_RAM_BYTES = 16 * 1024 * 1024 * 1024


def _snapshot_regular_files(snapshot_dir: Path) -> list[str]:
    return sorted(
        path.relative_to(snapshot_dir).as_posix()
        for path in snapshot_dir.rglob("*")
        if path.is_file()
    )
DEFAULT_QUANTIZATIONS = ["bfloat16"]
DEFAULT_BENCHMARK_TOKENS_PER_SEC = 1.0
# US-039/US-040 — single source of truth for the credit defaults (referenced by
# TrackerServer, _TrackerHTTPServer, and the CLI). Alpha runs devnet-friendly;
# flip both to 0 before any deployment holding a mainnet treasury.
DEFAULT_CALLER_CREDIT_USDT = 1.0
DEFAULT_DEVNET_TOPUP_USDT = 1.0


def _endpoint_key(url: str) -> str:
    """Normalize http(s) endpoints for host:port comparison."""
    parsed = urllib.parse.urlparse(url.rstrip("/"))
    host = (parsed.hostname or "").lower()
    if not host:
        return url.rstrip("/").lower()
    port = parsed.port
    if port is None:
        port = 443 if parsed.scheme == "https" else 80
    return f"{host}:{port}"


def _model_aliases(model: str | None) -> set[str]:
    """Return stable lookup aliases for a model repo or display name."""
    if not model:
        return set()
    normalized = model.strip()
    if not normalized:
        return set()
    aliases = {normalized}
    short = normalized.rsplit("/", 1)[-1]
    aliases.add(short)
    lowered = short.lower()
    aliases.add(lowered)
    if lowered.endswith("-instruct"):
        aliases.add(lowered.removesuffix("-instruct"))
    return aliases


def _preset_aliases(name: str, preset: dict | None) -> set[str]:
    aliases = _model_aliases(name)
    if not preset:
        return aliases
    hf_repo = preset.get("hf_repo")
    if isinstance(hf_repo, str):
        aliases |= _model_aliases(hf_repo)
    for alias in preset.get("aliases", []) or []:
        if isinstance(alias, str):
            aliases |= _model_aliases(alias)
    return aliases


def _resolve_model_preset(model_presets: dict, model: str) -> tuple[str, dict] | tuple[None, None]:
    requested = _model_aliases(model)
    for name, preset in model_presets.items():
        if requested & _preset_aliases(name, preset):
            return name, preset
    return None, None


def _node_matches_model(node: "_NodeEntry", model: str) -> bool:
    requested = _model_aliases(model)
    if not requested:
        return False
    return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))


def _node_matches_preset(node: "_NodeEntry", name: str, preset: dict) -> bool:
    requested = _preset_aliases(name, preset)
    return bool(requested & (_model_aliases(node.model) | _model_aliases(node.hf_repo)))


class _RollingCounter:
    """Circular-bucket request counter.

    Tracks events in `num_buckets` slots, each covering `bucket_seconds` of wall time.
    Old buckets are silently discarded when their epoch expires.
    """

    def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
        self._num = num_buckets
        self._bsec = bucket_seconds
        self._counts: list[int] = [0] * num_buckets
        self._epochs: list[int] = [-1] * num_buckets  # which epoch each slot holds

    def _epoch(self, now: float) -> int:
        return int(now) // self._bsec

    def record(self, now: float | None = None) -> None:
        t = now if now is not None else time.time()
        ep = self._epoch(t)
        idx = ep % self._num
        if self._epochs[idx] != ep:
            self._counts[idx] = 0
            self._epochs[idx] = ep
        self._counts[idx] += 1

    def rpm(self, now: float | None = None) -> float:
        t = now if now is not None else time.time()
        cutoff = self._epoch(t) - self._num  # epochs <= this are stale
        total = sum(self._counts[i] for i in range(self._num) if self._epochs[i] > cutoff)
        window_minutes = (self._num * self._bsec) / 60.0
        return total / window_minutes if window_minutes > 0 else 0.0

    def buckets(self) -> list[tuple[int, int]]:
        return [(self._epochs[i], self._counts[i]) for i in range(self._num)]

    def restore_buckets(self, data: list[tuple[int, int]]) -> None:
        for i, (ep, cnt) in enumerate(data):
            if i < self._num:
                self._epochs[i] = ep
                self._counts[i] = cnt


class _RollingThroughput:
    """Circular buckets for observed tokens/sec.

    Each bucket stores total output tokens and total elapsed seconds. TPS for a
    window is the ratio across non-stale buckets, which avoids over-weighting
    small fast requests.
    """

    def __init__(self, num_buckets: int, bucket_seconds: int) -> None:
        self._num = num_buckets
        self._bsec = bucket_seconds
        self._tokens: list[int] = [0] * num_buckets
        self._seconds: list[float] = [0.0] * num_buckets
        self._samples: list[int] = [0] * num_buckets
        self._epochs: list[int] = [-1] * num_buckets

    def _epoch(self, now: float) -> int:
        return int(now) // self._bsec

    def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
        if total_tokens <= 0 or elapsed_seconds <= 0:
            return
        t = now if now is not None else time.time()
        ep = self._epoch(t)
        idx = ep % self._num
        if self._epochs[idx] != ep:
            self._tokens[idx] = 0
            self._seconds[idx] = 0.0
            self._samples[idx] = 0
            self._epochs[idx] = ep
        self._tokens[idx] += int(total_tokens)
        self._seconds[idx] += float(elapsed_seconds)
        self._samples[idx] += 1

    def stats(self, now: float | None = None) -> dict:
        t = now if now is not None else time.time()
        cutoff = self._epoch(t) - self._num
        tokens = 0
        seconds = 0.0
        samples = 0
        for i in range(self._num):
            if self._epochs[i] > cutoff:
                tokens += self._tokens[i]
                seconds += self._seconds[i]
                samples += self._samples[i]
        return {
            "tokens_per_sec": round(tokens / seconds, 4) if seconds > 0 else None,
            "tokens": tokens,
            "seconds": round(seconds, 6),
            "sample_count": samples,
        }

    def buckets(self) -> list[tuple[int, int, float, int]]:
        return [
            (self._epochs[i], self._tokens[i], self._seconds[i], self._samples[i])
            for i in range(self._num)
        ]

    def restore_buckets(self, data: list[tuple[int, int, float, int]]) -> None:
        for i, (ep, tokens, seconds, samples) in enumerate(data):
            if i < self._num:
                self._epochs[i] = ep
                self._tokens[i] = tokens
                self._seconds[i] = seconds
                self._samples[i] = samples


class _ModelStats:
    """Three rolling windows for one model: last hour, last day, last month."""

    def __init__(self) -> None:
        self.per_minute = _RollingCounter(60, 60)     # 60 × 1-min buckets = 1 hour
        self.per_hour   = _RollingCounter(24, 3600)   # 24 × 1-hr  buckets = 1 day
        self.per_day    = _RollingCounter(30, 86400)  # 30 × 1-day buckets = ~1 month

    def record(self, now: float | None = None) -> None:
        t = now if now is not None else time.time()
        self.per_minute.record(t)
        self.per_hour.record(t)
        self.per_day.record(t)

    def rpms(self, now: float | None = None) -> dict:
        t = now if now is not None else time.time()
        return {
            "rpm_last_hour":  round(self.per_minute.rpm(t), 4),
            "rpm_last_day":   round(self.per_hour.rpm(t), 4),
            "rpm_last_month": round(self.per_day.rpm(t), 4),
        }


class _NodeModelThroughput:
    """Observed throughput windows for one node/model pair."""

    def __init__(self) -> None:
        self.per_minute = _RollingThroughput(60, 60)
        self.per_hour = _RollingThroughput(24, 3600)
        self.per_day = _RollingThroughput(30, 86400)

    def record(self, total_tokens: int, elapsed_seconds: float, now: float | None = None) -> None:
        t = now if now is not None else time.time()
        self.per_minute.record(total_tokens, elapsed_seconds, t)
        self.per_hour.record(total_tokens, elapsed_seconds, t)
        self.per_day.record(total_tokens, elapsed_seconds, t)

    def stats(self, now: float | None = None) -> dict:
        hour = self.per_minute.stats(now)
        day = self.per_hour.stats(now)
        month = self.per_day.stats(now)
        return {
            "tokens_per_sec_last_hour": hour["tokens_per_sec"],
            "tokens_per_sec_last_day": day["tokens_per_sec"],
            "tokens_per_sec_last_month": month["tokens_per_sec"],
            "sample_count_last_hour": hour["sample_count"],
            "tokens_last_hour": hour["tokens"],
            "seconds_last_hour": hour["seconds"],
        }


class _StatsCollector:
    """Thread-safe model request stats with SQLite persistence and peer slice merging."""

    SAVE_INTERVAL = 60.0  # seconds between DB saves

    def __init__(self, db_path: str | None = None) -> None:
        self._lock = threading.Lock()
        self._local: dict[str, _ModelStats] = {}
        self._node_model: dict[tuple[str, str], _NodeModelThroughput] = {}
        self._peer_rpms: dict[str, dict[str, dict]] = {}  # tracker_url -> model -> rpms
        self._db_path = db_path
        if db_path:
            self._init_db()
            self._load_from_db()

    # ---------- public API ----------

    def record_request(self, model: str, now: float | None = None) -> None:
        t = now if now is not None else time.time()
        with self._lock:
            if model not in self._local:
                self._local[model] = _ModelStats()
            self._local[model].record(t)

    def record_node_throughput(
        self,
        node_id: str,
        model: str,
        total_tokens: int,
        elapsed_seconds: float,
        now: float | None = None,
    ) -> None:
        if not node_id or not model or total_tokens <= 0 or elapsed_seconds <= 0:
            return
        t = now if now is not None else time.time()
        with self._lock:
            key = (node_id, model)
            if key not in self._node_model:
                self._node_model[key] = _NodeModelThroughput()
            self._node_model[key].record(total_tokens, elapsed_seconds, t)

    def get_local_rpms(self, now: float | None = None) -> dict[str, dict]:
        t = now if now is not None else time.time()
        with self._lock:
            return {m: s.rpms(t) for m, s in self._local.items()}

    def get_node_model_stats(self, node_id: str, model: str, now: float | None = None) -> dict:
        t = now if now is not None else time.time()
        empty = {
            "tokens_per_sec_last_hour": None,
            "tokens_per_sec_last_day": None,
            "tokens_per_sec_last_month": None,
            "sample_count_last_hour": 0,
            "tokens_last_hour": 0,
            "seconds_last_hour": 0.0,
        }
        with self._lock:
            stat = self._node_model.get((node_id, model))
            return stat.stats(t) if stat is not None else dict(empty)

    def get_node_throughput_stats(self, now: float | None = None) -> dict[str, dict]:
        t = now if now is not None else time.time()
        with self._lock:
            result: dict[str, dict] = {}
            for (node_id, model), stat in self._node_model.items():
                result.setdefault(node_id, {"models": {}})["models"][model] = stat.stats(t)
            return result

    def merge_peer_rpms(self, tracker_url: str, rpms: dict[str, dict]) -> None:
        with self._lock:
            self._peer_rpms[tracker_url] = dict(rpms)

    def get_combined_stats(self, now: float | None = None) -> dict:
        t = now if now is not None else time.time()
        with self._lock:
            combined: dict[str, dict] = {}
            for model, ms in self._local.items():
                combined[model] = ms.rpms(t)
            for _url, peer in self._peer_rpms.items():
                for model, rpms in peer.items():
                    if model not in combined:
                        combined[model] = {"rpm_last_hour": 0.0, "rpm_last_day": 0.0, "rpm_last_month": 0.0}
                    for key in ("rpm_last_hour", "rpm_last_day", "rpm_last_month"):
                        combined[model][key] = round(combined[model].get(key, 0.0) + rpms.get(key, 0.0), 4)
        return combined

    # ---------- persistence ----------

    def _init_db(self) -> None:
        con = sqlite3.connect(self._db_path)  # type: ignore[arg-type]
        con.execute(
            "CREATE TABLE IF NOT EXISTS model_rpm_buckets "
            "(model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, count INTEGER, "
            "PRIMARY KEY (model, window, bucket_idx))"
        )
        con.execute(
            "CREATE TABLE IF NOT EXISTS node_model_tps_buckets "
            "(node_id TEXT, model TEXT, window TEXT, bucket_idx INTEGER, bucket_epoch INTEGER, "
            "tokens INTEGER, seconds REAL, samples INTEGER, "
            "PRIMARY KEY (node_id, model, window, bucket_idx))"
        )
        con.commit()
        con.close()

    def save_to_db(self) -> None:
        if not self._db_path:
            return
        with self._lock:
            rows = []
            for model, ms in self._local.items():
                for window_name, counter in (
                    ("hour", ms.per_minute),
                    ("day", ms.per_hour),
                    ("month", ms.per_day),
                ):
                    for idx, (ep, cnt) in enumerate(counter.buckets()):
                        if ep >= 0:
                            rows.append((model, window_name, idx, ep, cnt))
            tps_rows = []
            for (node_id, model), stat in self._node_model.items():
                for window_name, counter in (
                    ("hour", stat.per_minute),
                    ("day", stat.per_hour),
                    ("month", stat.per_day),
                ):
                    for idx, (ep, tokens, seconds, samples) in enumerate(counter.buckets()):
                        if ep >= 0:
                            tps_rows.append((node_id, model, window_name, idx, ep, tokens, seconds, samples))
        con = sqlite3.connect(self._db_path)  # type: ignore[arg-type]
        con.executemany(
            "INSERT OR REPLACE INTO model_rpm_buckets VALUES (?,?,?,?,?)", rows
        )
        con.executemany(
            "INSERT OR REPLACE INTO node_model_tps_buckets VALUES (?,?,?,?,?,?,?,?)",
            tps_rows,
        )
        con.commit()
        con.close()

    def _load_from_db(self) -> None:
        con = sqlite3.connect(self._db_path)  # type: ignore[arg-type]
        rows = con.execute("SELECT model, window, bucket_idx, bucket_epoch, count FROM model_rpm_buckets").fetchall()
        tps_rows = con.execute(
            "SELECT node_id, model, window, bucket_idx, bucket_epoch, tokens, seconds, samples "
            "FROM node_model_tps_buckets"
        ).fetchall()
        con.close()
        grouped: dict[str, dict[str, list[tuple[int, int]]]] = {}
        for model, window, idx, ep, cnt in rows:
            grouped.setdefault(model, {}).setdefault(window, []).append((idx, ep, cnt))
        for model, windows in grouped.items():
            ms = _ModelStats()
            for window_name, entries in windows.items():
                counter = {"hour": ms.per_minute, "day": ms.per_hour, "month": ms.per_day}.get(window_name)
                if counter is None:
                    continue
                data = [(0, -1)] * counter._num
                for idx, ep, cnt in entries:
                    if 0 <= idx < counter._num:
                        data[idx] = (ep, cnt)
                counter.restore_buckets(data)
            self._local[model] = ms
        tps_grouped: dict[tuple[str, str], dict[str, list[tuple[int, int, int, float, int]]]] = {}
        for node_id, model, window, idx, ep, tokens, seconds, samples in tps_rows:
            tps_grouped.setdefault((node_id, model), {}).setdefault(window, []).append(
                (idx, ep, tokens, seconds, samples)
            )
        for key, windows in tps_grouped.items():
            stat = _NodeModelThroughput()
            for window_name, entries in windows.items():
                counter = {"hour": stat.per_minute, "day": stat.per_hour, "month": stat.per_day}.get(window_name)
                if counter is None:
                    continue
                data = [(0, 0, 0.0, 0)] * counter._num
                for idx, ep, tokens, seconds, samples in entries:
                    if 0 <= idx < counter._num:
                        data[idx] = (ep, int(tokens), float(seconds), int(samples))
                counter.restore_buckets(data)
            self._node_model[key] = stat


class _NodeEntry:
    __slots__ = (
        "node_id", "endpoint", "shard_start", "shard_end",
        "model", "hf_repo", "num_layers", "model_metadata", "shard_checksum", "downloaded_models", "hardware_profile", "wallet_address",
        "score", "vram_bytes", "ram_bytes", "quantizations", "max_loaded_shards",
        "benchmark_tokens_per_sec", "quantization", "managed_assignment",
        "model_tokens_per_sec",
        "pending_directives", "last_heartbeat", "tracker_mode",
        "relay_addr", "cert_fingerprint", "peer_id", "friendly_name",
        # heartbeat stats (reported by node, cumulative)
        "total_requests", "failed_requests", "queue_depth", "proxy_inflight", "uptime_seconds",
        "current_requests",
        "status",  # "ready" | "loading"
        "heartbeats_expected", "heartbeats_received",
        # dynamic reassignment queued by the tracker
        "pending_new_assignment",
        # the tracker's verdict on the capability proof this node presented
        "capability",
    )

    def __init__(
        self,
        node_id: str,
        endpoint: str,
        shard_start: int | None,
        shard_end: int | None,
        model: str | None,
        shard_checksum: str | None,
        hardware_profile: dict,
        wallet_address: str | None,
        score: float,
        vram_bytes: int = DEFAULT_VRAM_BYTES,
        ram_bytes: int = DEFAULT_RAM_BYTES,
        quantizations: list[str] | None = None,
        max_loaded_shards: int = 1,
        benchmark_tokens_per_sec: float = DEFAULT_BENCHMARK_TOKENS_PER_SEC,
        quantization: str | None = None,
        managed_assignment: bool = False,
        tracker_mode: bool = False,
        hf_repo: str | None = None,
        num_layers: int | None = None,
        model_metadata: dict | None = None,
        downloaded_models: list[dict] | None = None,
        relay_addr: str | None = None,
        cert_fingerprint: str | None = None,
        peer_id: str | None = None,
        friendly_name: str | None = None,
        capability: "CapabilityState | None" = None,
    ) -> None:
        self.node_id = node_id
        self.endpoint = endpoint
        self.shard_start = shard_start
        self.shard_end = shard_end
        self.model = model
        self.shard_checksum = shard_checksum
        self.hardware_profile = hardware_profile
        self.wallet_address = wallet_address
        self.score = score
        self.vram_bytes = vram_bytes
        self.ram_bytes = ram_bytes
        self.quantizations = quantizations or list(DEFAULT_QUANTIZATIONS)
        self.max_loaded_shards = max_loaded_shards
        self.benchmark_tokens_per_sec = benchmark_tokens_per_sec
        self.quantization = quantization
        self.managed_assignment = managed_assignment
        self.model_tokens_per_sec: dict[str, float] = {}
        self.tracker_mode = tracker_mode
        self.hf_repo = hf_repo
        self.num_layers = num_layers
        self.model_metadata = dict(model_metadata or {})
        self.downloaded_models = [dict(item) for item in (downloaded_models or []) if isinstance(item, dict)]
        self.relay_addr = relay_addr
        self.cert_fingerprint = cert_fingerprint
        self.peer_id = peer_id
        self.friendly_name = friendly_name
        # No proof presented is `absent`, never `admitted` — a node can only earn
        # `admitted` by presenting a report that covers what it advertises.
        self.capability: CapabilityState = capability or absent_state()
        self.pending_directives: list[dict] = []
        self.last_heartbeat: float = time.monotonic()
        self.total_requests: int = 0
        self.failed_requests: int = 0
        self.queue_depth: int = 0
        self.proxy_inflight: int = 0
        self.current_requests: list[dict] = []
        self.uptime_seconds: float = 0.0
        self.status: str = "ready"
        self.heartbeats_expected: int = 0
        self.heartbeats_received: int = 0
        self.pending_new_assignment: dict | None = None


def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float:
    """Effective tokens/s accounting for current queue depth."""
    observed = None
    if model:
        observed = node.model_tokens_per_sec.get(model)
        if observed is None:
            for alias in _model_aliases(model):
                observed = node.model_tokens_per_sec.get(alias)
                if observed is not None:
                    break
    base = observed if observed is not None and observed > 0 else node.benchmark_tokens_per_sec
    return base / (_effective_queue_depth(node) + 1)


def _effective_queue_depth(node: "_NodeEntry") -> int:
    """Best current load estimate: heartbeat queue or tracker-routed in-flight."""
    return max(node.queue_depth, node.proxy_inflight)


_CURRENT_REQUEST_FIELDS = frozenset({
    "request_id", "model", "kind", "tokens", "tokens_per_sec",
    "elapsed_seconds", "routing_complete",
})


def _normalize_current_requests(items: object, *, limit: int = 32) -> list[dict]:
    """Sanitize node-reported in-flight request snapshots from heartbeats."""
    if not isinstance(items, list):
        return []
    out: list[dict] = []
    for item in items[:limit]:
        if not isinstance(item, dict):
            continue
        request_id = item.get("request_id")
        if not request_id:
            continue
        rec: dict = {"request_id": str(request_id)}
        for key in _CURRENT_REQUEST_FIELDS:
            if key == "request_id" or key not in item:
                continue
            value = item[key]
            if key in {"tokens"}:
                rec[key] = int(value)
            elif key in {"tokens_per_sec", "elapsed_seconds"}:
                rec[key] = float(value)
            elif key == "routing_complete":
                rec[key] = bool(value)
            else:
                rec[key] = str(value)
        out.append(rec)
    return out


def _record_proxy_inflight(
    server: "_TrackerHTTPServer",
    nodes: list["_NodeEntry"],
    delta: int,
) -> None:
    with server.lock:
        for node in nodes:
            node.proxy_inflight = max(0, node.proxy_inflight + delta)


def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
    """ADR-0018 §6: veteran, good-standing nodes route more -- earnings scale
    with tenure/reputation. Maps reputation [0, 1] to a [0.5, 1.0] routing
    weight so a damaged-but-not-banned wallet loses traffic gradually rather
    than being cut off outright (banning is handled separately)."""
    if contracts is None or not node.wallet_address:
        return 1.0
    reputation = contracts.registry.get_wallet(node.wallet_address).reputation
    return 0.5 + 0.5 * reputation


def _node_admission(node: "_NodeEntry") -> CapabilityState:
    """The node's recorded verdict, re-checked against what it advertises *now*.

    A proof covers one model/shard combination. The tracker can move a node to a
    different range after it registered (rebalance, `pending_new_assignment`), and
    the proof does not travel with it: until the node re-registers with a proof
    for the range it now advertises, it is shard-mismatched and not routable.
    """
    state = node.capability
    if not state.proven:
        return state
    if state.model_id and not _node_matches_model(node, state.model_id):
        return state.with_state(
            STATE_MODEL_MISMATCH,
            f"proof is for model {state.model_id!r}, but the node now serves "
            f"{node.hf_repo or node.model!r}",
        )
    if (
        node.shard_start is not None
        and node.shard_end is not None
        and (state.shard_start, state.shard_end) != (node.shard_start, node.shard_end)
    ):
        return state.with_state(
            STATE_SHARD_MISMATCH,
            f"proof is for layers {state.shard_start}–{state.shard_end}, but the "
            f"node now serves layers {node.shard_start}–{node.shard_end}",
        )
    return state


def _capability_from_registration(
    payload: dict,
    *,
    model: str | None,
    hf_repo: str | None,
    shard_start: int | None,
    shard_end: int | None,
) -> CapabilityState:
    """The tracker's verdict on the proof carried by one registration payload.

    Used by the register handler and by the Raft follower path, so a replicated
    registration lands with the same verdict as the one the leader recorded.
    """
    aliases = _model_aliases(model) | _model_aliases(hf_repo)
    recipe_id = payload.get("recipe_id")
    recipe_version = payload.get("recipe_version")
    return evaluate_report(
        payload.get("capability_report"),
        model_matches=lambda reported: bool(_model_aliases(reported) & aliases),
        advertised_model=hf_repo or model,
        shard_start=shard_start,
        shard_end=shard_end,
        declared_recipe_id=recipe_id if isinstance(recipe_id, str) else None,
        declared_recipe_version=(
            recipe_version if isinstance(recipe_version, str) else None
        ),
    )


def _capability_routable(node: "_NodeEntry", policy: str) -> bool:
    """May this node carry traffic under the tracker's capability policy?"""
    return _node_admission(node).routable_under(policy)


def _admitted_nodes(nodes: list["_NodeEntry"], policy: str | None) -> list["_NodeEntry"]:
    """Drop every candidate whose capability proof does not admit it (ADR-0023).

    This is the single gate every route path goes through. It removes candidates;
    it never reorders or reweights them, so coverage-first selection and
    throughput-weighted preference among the survivors are unchanged.
    """
    effective = normalize_policy(policy)
    return [node for node in nodes if _capability_routable(node, effective)]


def _select_route(
    nodes: list[_NodeEntry],
    required_start: int,
    required_end: int,
    model: str | None = None,
    contracts: Any | None = None,
    policy: str | None = None,
) -> tuple[list[_NodeEntry], str]:
    """Greedy interval-cover biased toward fast, lightly-loaded, reputable nodes.

    Among nodes that equally advance coverage, prefer the one with higher
    effective throughput: observed per-model tokens/sec / (queue_depth + 1),
    falling back to startup benchmark_tokens_per_sec until observations exist,
    weighted by the node's reputation multiplier (see `_reputation_multiplier`).
    Tiebreak: higher shard_end (fewer hops).
    """
    candidates = sorted(
        [
            node for node in _admitted_nodes(nodes, policy)
            if node.shard_start is not None and node.shard_end is not None
        ],
        key=lambda n: (n.shard_start, -n.shard_end),  # type: ignore[operator]
    )
    route: list[_NodeEntry] = []
    covered_up_to = required_start - 1

    def _routing_score(node: "_NodeEntry") -> float:
        return _effective_throughput(node, model) * _reputation_multiplier(node, contracts)

    while covered_up_to < required_end:
        best: _NodeEntry | None = None
        for node in candidates:
            if node.shard_start <= covered_up_to + 1 and node.shard_end > covered_up_to:
                if best is None:
                    best = node
                elif node.shard_end > best.shard_end:
                    best = node
                elif node.shard_end == best.shard_end and _routing_score(node) > _routing_score(best):
                    best = node
        if best is None:
            missing = covered_up_to + 1
            return [], f"no route available: no registered node covers layer {missing}"
        route.append(best)
        covered_up_to = best.shard_end
        candidates = [n for n in candidates if n is not best]

    return route, ""


def _enumerate_routes(
    nodes: list["_NodeEntry"],
    required_start: int,
    required_end: int,
    model: str | None = None,
    contracts: Any | None = None,
    max_candidates: int = 8,
    policy: str | None = None,
) -> list["RouteCandidate"]:
    """Enumerate viable route candidates for bandit selection (ADR-0021).

    One candidate per distinct head (a node that can embed the prompt, i.e.
    covers `required_start` from layer 0 of the range), each greedily completed
    with the longest-advancing hops. The route's prior throughput estimate is
    its bottleneck hop's queue-adjusted effective throughput — used only until
    observed route samples exist.

    Candidates that are not admitted by their capability proof are dropped before
    enumeration, so an unproven node cannot appear in any route candidate — not
    even as a scouted one.
    """
    sharded = [
        n for n in _admitted_nodes(nodes, policy)
        if n.shard_start is not None and n.shard_end is not None
    ]
    # Heads must start the pipeline at the first required layer (they tokenize
    # and embed the prompt — same condition as tracker_mode registration).
    heads = [n for n in sharded if n.shard_start == required_start]
    candidates: dict[str, RouteCandidate] = {}
    for head in heads:
        route = [head]
        covered_up_to = head.shard_end
        pool = [n for n in sharded if n is not head]
        while covered_up_to < required_end:
            best = None
            for n in pool:
                if n.shard_start <= covered_up_to + 1 and n.shard_end > covered_up_to:
                    if best is None or n.shard_end > best.shard_end or (
                        n.shard_end == best.shard_end
                        and _effective_throughput(n, model) * _reputation_multiplier(n, contracts)
                        > _effective_throughput(best, model) * _reputation_multiplier(best, contracts)
                    ):
                        best = n
            if best is None:
                route = []
                break
            route.append(best)
            covered_up_to = best.shard_end
            pool = [n for n in pool if n is not best]
        if not route:
            continue
        signature = route_signature(model or "?", route)
        if signature in candidates:
            continue
        prior = min(
            _effective_throughput(n, model) * _reputation_multiplier(n, contracts)
            for n in route
        )
        candidates[signature] = RouteCandidate(nodes=route, signature=signature, prior_tps=prior)
    ranked = sorted(candidates.values(), key=lambda c: -c.prior_tps)
    return ranked[:max_candidates]


def _node_route_summary(nodes: list["_NodeEntry"]) -> list[dict]:
    return [
        {
            "node_id": node.node_id,
            "model": node.model,
            "hf_repo": node.hf_repo,
            "endpoint": node.endpoint,
            "shard": f"{node.shard_start}-{node.shard_end}",
            "num_layers": node.num_layers,
            "queue_depth": _effective_queue_depth(node),
            "heartbeat_queue_depth": node.queue_depth,
            "proxy_inflight": node.proxy_inflight,
            "current_requests": list(node.current_requests),
        }
        for node in nodes
    ]


def _coverage_percentage(
    nodes: list[_NodeEntry],
    required_start: int,
    required_end: int,
) -> float:
    required_layers = required_end - required_start + 1
    if required_layers <= 0:
        return 0.0

    intervals = sorted(
        (
            (max(required_start, node.shard_start), min(required_end, node.shard_end))
            for node in _serviceable_nodes(nodes)
            if node.shard_start is not None
            and node.shard_end is not None
            if node.shard_end >= required_start and node.shard_start <= required_end
        ),
        key=lambda interval: interval[0],
    )
    covered = 0
    covered_until = required_start - 1
    for start, end in intervals:
        if end <= covered_until:
            continue
        next_start = max(start, covered_until + 1)
        if next_start > end:
            continue
        covered += end - next_start + 1
        covered_until = end
    return round((covered / required_layers) * 100, 2)


def _served_model_copies(
    nodes: list[_NodeEntry],
    required_start: int,
    required_end: int,
) -> float:
    required_layers = required_end - required_start + 1
    if required_layers <= 0:
        return 0.0

    layer_counts = []
    for layer in range(required_start, required_end + 1):
        count = 0
        for node in _serviceable_nodes(nodes):
            if node.shard_start is None or node.shard_end is None:
                continue
            if node.shard_start <= layer <= node.shard_end:
                count += 1
        layer_counts.append(count)

    if not layer_counts:
        return 0.0

    complete_copies = min(layer_counts)
    residual_layers = sum(1 for count in layer_counts if count > complete_copies)
    return round(complete_copies + (residual_layers / required_layers), 2)


def _model_health_summary(
    server: "_TrackerHTTPServer",
    model: str | None,
    hf_repo: str | None = None,
) -> dict:
    lookup = hf_repo or model
    if not lookup:
        return {}

    resolved_name, preset = _resolve_model_preset(server.model_presets, lookup)
    if preset is not None:
        required_start, required_end = _preset_layer_bounds(preset)
        model_nodes = [
            node for node in server.registry.values()
            if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
        ]
        model_id = resolved_name or lookup
    else:
        model_nodes = [
            node for node in server.registry.values()
            if _node_matches_model(node, lookup)
            and node.shard_start is not None
            and node.shard_end is not None
            and node.num_layers is not None
        ]
        if not model_nodes:
            return {
                "model": lookup,
                "served_model_copies": 0.0,
                "coverage_percentage": 0.0,
                "node_count": 0,
            }
        required_start = 0
        required_end = max(node.num_layers for node in model_nodes if node.num_layers is not None) - 1
        model_id = hf_repo or model or lookup

    return {
        "model": model_id,
        "required_start": required_start,
        "required_end": required_end,
        "served_model_copies": _served_model_copies(model_nodes, required_start, required_end),
        "coverage_percentage": _coverage_percentage(model_nodes, required_start, required_end),
        "node_count": len(model_nodes),
    }


def _preset_layer_bounds(preset: dict) -> tuple[int, int]:
    start = int(preset.get("layers_start", 0))
    if "layers_end" in preset:
        return start, int(preset["layers_end"])
    return start, start + int(preset["total_layers"]) - 1


def _preset_bytes_per_layer(preset: dict) -> dict[str, int]:
    raw = preset.get("bytes_per_layer", preset.get("bytes_per_layer_at_quant", {}))
    if isinstance(raw, dict) and raw:
        return {str(quant): int(value) for quant, value in raw.items()}
    return {"bfloat16": 30 * 1024 * 1024}


_QUANTIZATION_QUALITY = {"nf4": 0, "int8": 1, "bfloat16": 2}


def _normalize_quantization(value: object) -> str | None:
    """Normalize public precision names to the node/backend vocabulary."""
    if not isinstance(value, str):
        return None
    normalized = value.strip().lower()
    if normalized == "bf16":
        normalized = "bfloat16"
    return normalized if normalized in _QUANTIZATION_QUALITY else None


def _registration_quantization(body: dict, quantizations: list[str]) -> str | None:
    """Resolve the active precision a registration is routable at.

    Only the raw body can tell an *absent* `quantization` from an explicit null,
    and the two mean opposite things, so this must run at the registration seam
    rather than in `_NodeEntry` (both reach the constructor as None).

    An absent field predates the protocol adding it: it means "unknown", not
    "unsupported", so the node keeps the best precision it advertises and stays
    routable. Anything the node states explicitly is taken at its word -- a null,
    a non-string, or an unsupported name leaves it with no usable precision and
    routing excludes it.
    """
    if "quantization" in body:
        return _normalize_quantization(body["quantization"])
    supported = [
        normalized for value in quantizations
        if (normalized := _normalize_quantization(value)) is not None
    ]
    if not supported:
        return None
    return max(supported, key=lambda value: _QUANTIZATION_QUALITY[value])


def _has_usable_quantization(node: _NodeEntry) -> bool:
    """Whether a node declared a precision it can actually serve.

    `quantization` is None only for a registration that explicitly declared a
    null, malformed, or unsupported precision -- a legacy registration that omits
    the field is resolved to a concrete precision at the registration seam. Such a
    node is already dark to routing, and shard placement has to keep it that way:
    the assignment paths pick a precision via `_node_quantization`, which falls
    back to the advertised list and then to a preset default, so an unguarded
    assignment would overwrite the declared-unusable precision and silently make
    the node routable again.
    """
    return node.quantization is not None


def _serviceable_nodes(nodes: list[_NodeEntry]) -> list[_NodeEntry]:
    """The nodes a coverage calculation is allowed to count.

    A node with no usable precision is excluded from every routing path, so it
    serves none of the layers it claims. Counting its shard range as covered
    would report a complete model that no request can reach, and would hide the
    hole from the rebalancer whose whole job is to fill it -- the managed nodes
    that could cover those layers stay idle because the gap never appears.
    """
    return [node for node in nodes if _has_usable_quantization(node)]


def _quantization_satisfies(available: object, requested: object) -> bool:
    """Return whether a shard meets a request's minimum precision."""
    available_normalized = _normalize_quantization(available)
    requested_normalized = _normalize_quantization(requested)
    return (
        available_normalized is not None
        and requested_normalized is not None
        and _QUANTIZATION_QUALITY[available_normalized]
        >= _QUANTIZATION_QUALITY[requested_normalized]
    )


def _available_quantizations(
    nodes: list["_NodeEntry"],
    required_start: int,
    required_end: int,
) -> list[dict]:
    """Summarize selectable minimum-precision variants for a model.

    A request may use higher-precision shards, so each row counts coverage from
    every node whose active quantization satisfies that row's minimum.
    """
    rows = []
    for quantization in ("bfloat16", "int8", "nf4"):
        qualifying = [
            node for node in nodes
            if _quantization_satisfies(node.quantization, quantization)
        ]
        coverage = _coverage_percentage(qualifying, required_start, required_end)
        rows.append({
            "id": quantization,
            "shard_coverage_percentage": coverage,
            "served_model_copies": _served_model_copies(
                qualifying, required_start, required_end,
            ),
            "selectable": coverage >= 100.0,
        })
    return rows


def _node_quantization(node: _NodeEntry, preset: dict) -> str:
    bytes_per_layer = _preset_bytes_per_layer(preset)
    if node.quantization in bytes_per_layer:
        return node.quantization
    for quantization in node.quantizations:
        if quantization in bytes_per_layer:
            return quantization
    return next(iter(bytes_per_layer))


def _node_memory_budget_bytes(node: _NodeEntry) -> tuple[int, str]:
    """Return the memory pool used for shard-capacity planning."""
    if node.vram_bytes > 0:
        return node.vram_bytes, "vram"
    if node.ram_bytes > 0:
        return node.ram_bytes, "ram"
    return DEFAULT_RAM_BYTES, "ram-default"


def _node_layer_capacity(node: _NodeEntry, preset: dict) -> int:
    bytes_per_layer = _preset_bytes_per_layer(preset)
    quantization = _node_quantization(node, preset)
    layer_bytes = bytes_per_layer[quantization]
    if layer_bytes <= 0:
        return 0
    memory_budget_bytes, _ = _node_memory_budget_bytes(node)
    return int((memory_budget_bytes * 0.8) // layer_bytes)


def _node_capacity_summary(node: _NodeEntry, preset: dict | None = None) -> dict:
    """Operator-facing capacity fields for inspection endpoints."""
    memory_budget_bytes, memory_budget_source = _node_memory_budget_bytes(node)
    summary = {
        "vram_bytes": node.vram_bytes,
        "ram_bytes": node.ram_bytes,
        "memory_budget_bytes": memory_budget_bytes,
        "memory_budget_source": memory_budget_source,
        "max_loaded_shards": node.max_loaded_shards,
        "quantizations": list(node.quantizations),
        "quantization": node.quantization,
        "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
        "effective_throughput": round(_effective_throughput(node), 4),
    }
    if preset is not None:
        summary["max_assignable_layers"] = _node_layer_capacity(node, preset)
    return summary


def _node_memory_budget_for_preset(node: _NodeEntry, preset: dict | None = None) -> int:
    budget, _source = _node_memory_budget_bytes(node)
    if preset is None:
        return int(budget * 0.8)
    return _node_layer_capacity(node, preset) * max(1, next(iter(_preset_bytes_per_layer(preset).values())))


def _pool_summary(nodes: list[_NodeEntry], preset: dict | None = None) -> dict:
    total_vram = sum(max(0, node.vram_bytes) for node in nodes)
    total_ram = sum(max(0, node.ram_bytes) for node in nodes)
    total_budget = sum(_node_memory_budget_bytes(node)[0] for node in nodes)
    effective_budget = sum(_node_memory_budget_for_preset(node, preset) for node in nodes)
    return {
        "node_count": len(nodes),
        "total_vram_bytes": total_vram,
        "total_ram_bytes": total_ram,
        "total_memory_budget_bytes": total_budget,
        "effective_assignable_memory_bytes": effective_budget,
        "total_benchmark_tokens_per_sec": round(sum(node.benchmark_tokens_per_sec for node in nodes), 4),
        "total_effective_throughput": round(sum(_effective_throughput(node) for node in nodes), 4),
    }


def _repo_demand_rpm(
    server: "_TrackerHTTPServer",
    repo: str,
    local_rpms: dict[str, dict],
) -> float:
    """Look up recent request rate by hf_repo or preset alias."""
    if repo in local_rpms:
        return float(local_rpms[repo].get("rpm_last_hour", 0.0) or 0.0)
    resolved_name, _preset = _resolve_model_preset(server.model_presets, repo)
    if resolved_name and resolved_name in local_rpms:
        return float(local_rpms[resolved_name].get("rpm_last_hour", 0.0) or 0.0)
    return 0.0


def _preset_for_node(server: "_TrackerHTTPServer", node: _NodeEntry) -> dict | None:
    if node.hf_repo:
        _resolved, preset = _resolve_model_preset(server.model_presets, node.hf_repo)
        if preset is not None:
            return preset
    if node.model:
        preset = server.model_presets.get(node.model)
        if preset is not None:
            return preset
    if node.hf_repo and node.num_layers:
        return _hf_rebalance_preset([node])
    return None


def _assignment_memory_bytes(node: _NodeEntry, preset: dict | None) -> int:
    if preset is None or node.shard_start is None or node.shard_end is None:
        return 0
    layers = node.shard_end - node.shard_start + 1
    if layers <= 0:
        return 0
    quantization = _node_quantization(node, preset)
    layer_bytes = _preset_bytes_per_layer(preset).get(quantization)
    if not layer_bytes:
        layer_bytes = next(iter(_preset_bytes_per_layer(preset).values()))
    return layers * int(layer_bytes)


def _endpoint_memory_pool(
    server: "_TrackerHTTPServer",
    nodes: list[_NodeEntry],
) -> dict:
    """Per-host RAM/VRAM budget, usage, spare capacity, and loaded assignments."""
    if not nodes:
        return {}
    host = nodes[0]
    budget_bytes, budget_source = _node_memory_budget_bytes(host)
    reserve_bytes = int(budget_bytes * 0.8)
    loaded: list[dict] = []
    used_bytes = 0
    for node in nodes:
        preset = _preset_for_node(server, node)
        assignment_bytes = _assignment_memory_bytes(node, preset)
        used_bytes += assignment_bytes
        loaded.append({
            "node_id": node.node_id,
            "model": node.model,
            "hf_repo": node.hf_repo,
            "shard_start": node.shard_start,
            "shard_end": node.shard_end,
            "memory_bytes": assignment_bytes,
            "status": node.status,
        })
    max_slots = max(node.max_loaded_shards for node in nodes)
    loaded_slots = len([node for node in nodes if node.shard_start is not None])
    spare_bytes = max(0, reserve_bytes - used_bytes)
    return {
        "endpoint": host.endpoint,
        "memory_budget_bytes": budget_bytes,
        "memory_budget_source": budget_source,
        "memory_reserve_bytes": reserve_bytes,
        "memory_used_bytes": used_bytes,
        "memory_spare_bytes": spare_bytes,
        "loaded_slots": loaded_slots,
        "max_loaded_shards": max_slots,
        "spare_slots": max(0, max_slots - loaded_slots),
        "loaded": loaded,
    }


def _memory_pool_map(server: "_TrackerHTTPServer") -> dict:
    """Aggregate and per-endpoint view of assignable RAM across the hive."""
    from collections import defaultdict

    groups: dict[str, list[_NodeEntry]] = defaultdict(list)
    for node in server.registry.values():
        groups[node.endpoint.rstrip("/")].append(node)

    hosts = [_endpoint_memory_pool(server, group) for group in groups.values()]
    hosts.sort(key=lambda item: (-item.get("memory_spare_bytes", 0), item.get("endpoint", "")))
    total_budget = sum(item["memory_budget_bytes"] for item in hosts)
    total_used = sum(item["memory_used_bytes"] for item in hosts)
    total_spare = sum(item["memory_spare_bytes"] for item in hosts)
    total_slots = sum(item["max_loaded_shards"] for item in hosts)
    loaded_slots = sum(item["loaded_slots"] for item in hosts)
    return {
        "total_memory_budget_bytes": total_budget,
        "total_memory_used_bytes": total_used,
        "total_memory_spare_bytes": total_spare,
        "total_loaded_slots": loaded_slots,
        "total_max_loaded_shards": total_slots,
        "total_spare_slots": max(0, total_slots - loaded_slots),
        "hosts": hosts,
    }


def _add_shard_directive(
    node: _NodeEntry,
    model: str,
    start: int,
    end: int,
    quantization: str,
    *,
    model_sources: list[dict] | None = None,
) -> dict:
    directive = {
        "action": "ADD_SHARD",
        "model": model,
        "start_layer": start,
        "end_layer": end,
        "shard_start": start,
        "shard_end": end,
        "quantization": quantization,
    }
    if model_sources:
        directive["model_sources"] = model_sources
    return directive


def _model_demand_and_supply(
    server: "_TrackerHTTPServer",
    model_key: str,
    preset: dict,
) -> dict:
    resolved_name, _ = _resolve_model_preset(server.model_presets, model_key)
    lookup = preset.get("hf_repo") or resolved_name or model_key
    required_start, required_end = _preset_layer_bounds(preset)
    model_nodes = [
        node for node in server.registry.values()
        if _node_matches_preset(node, resolved_name or model_key, preset)  # type: ignore[arg-type]
    ]
    health = _model_health_summary(server, model_key, preset.get("hf_repo"))
    local_rpms = server.stats.get_local_rpms() if server.stats is not None else {}
    demand_rpm = _repo_demand_rpm(server, str(lookup), local_rpms)
    return {
        "model": resolved_name or model_key,
        "hf_repo": preset.get("hf_repo"),
        "preset": preset,
        "demand_rpm": demand_rpm,
        "served_model_copies": float(health.get("served_model_copies") or 0.0),
        "coverage_percentage": float(health.get("coverage_percentage") or 0.0),
        "required_start": required_start,
        "required_end": required_end,
        "model_nodes": model_nodes,
    }


def _scale_demanded_models_locked(server: "_TrackerHTTPServer") -> None:
    """Use spare host RAM/slots to add copies of high-demand models."""
    pool = _memory_pool_map(server)
    candidates: list[tuple[float, str, dict]] = []
    for model_key, preset in server.model_presets.items():
        if not isinstance(preset, dict) or not preset.get("hf_repo"):
            continue
        info = _model_demand_and_supply(server, model_key, preset)
        demand = info["demand_rpm"]
        copies = info["served_model_copies"]
        coverage = info["coverage_percentage"]
        if coverage < 100.0:
            continue
        target_copies = max(1.0, demand / 5.0)
        if copies >= target_copies:
            continue
        candidates.append((demand, model_key, info))

    if not candidates:
        return

    candidates.sort(key=lambda item: (-item[0], item[2]["served_model_copies"]))
    for _demand, model_key, info in candidates:
        preset = info["preset"]
        hf_repo = str(info["hf_repo"])
        required_start = info["required_start"]
        required_end = info["required_end"]
        total_layers = required_end - required_start + 1
        layer_bytes = next(iter(_preset_bytes_per_layer(preset).values()))
        full_copy_bytes = total_layers * layer_bytes

        for host in pool["hosts"]:
            if host["spare_slots"] <= 0 or host["memory_spare_bytes"] < full_copy_bytes:
                continue
            host_nodes = [
                server.registry[node["node_id"]]
                for node in host["loaded"]
                if node["node_id"] in server.registry
            ]
            if not host_nodes:
                continue
            if any((n.hf_repo or n.model) == hf_repo for n in host_nodes):
                continue
            placeable = [n for n in host_nodes if _has_usable_quantization(n)]
            if not placeable:
                continue
            anchor = max(placeable, key=lambda n: n.benchmark_tokens_per_sec)
            if anchor.status != "ready" or anchor.pending_new_assignment is not None:
                continue
            capacity = min(_node_layer_capacity(anchor, preset), total_layers)
            if capacity <= 0:
                continue
            quantization = _node_quantization(anchor, preset)
            shard_end = min(required_end, required_start + capacity - 1)
            assignment = _add_shard_directive(
                anchor,
                hf_repo,
                required_start,
                shard_end,
                quantization,
            )
            anchor.pending_new_assignment = assignment
            anchor.pending_directives.append(assignment)
            _tracker_log(
                server,
                "info",
                "scale demanded model copy",
                endpoint=anchor.endpoint,
                node_id=anchor.node_id,
                model=model_key,
                hf_repo=hf_repo,
                shard=f"{required_start}-{shard_end}",
                demand_rpm=round(info["demand_rpm"], 4),
                served_model_copies=info["served_model_copies"],
                target_copies=round(max(1.0, info["demand_rpm"] / 5.0), 2),
                host_spare_bytes=host["memory_spare_bytes"],
            )
            break


def _request_model_load_locked(server: "_TrackerHTTPServer", model_key: str) -> dict | None:
    """Queue an explicitly requested model on the best available joined node."""
    resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
    if preset is None or not preset.get("hf_repo"):
        return None
    required_start, required_end = _preset_layer_bounds(preset)
    total_layers = required_end - required_start + 1
    required_bytes = int(preset.get("required_model_bytes") or 0)
    for host in _memory_pool_map(server)["hosts"]:
        # A host qualifies with a declared spare slot, or — for an explicit
        # operator request — with enough spare assignable memory for the model.
        has_spare_memory = required_bytes > 0 and host["memory_spare_bytes"] >= required_bytes
        if host["spare_slots"] <= 0 and not has_spare_memory:
            continue
        host_nodes = [server.registry[item["node_id"]] for item in host["loaded"] if item["node_id"] in server.registry]
        placeable = [node for node in host_nodes if _has_usable_quantization(node)]
        if not placeable:
            continue
        anchor = max(placeable, key=lambda node: node.benchmark_tokens_per_sec)
        if anchor.status != "ready" or anchor.pending_new_assignment is not None:
            continue
        capacity = min(_node_layer_capacity(anchor, preset), total_layers)
        if capacity <= 0:
            continue
        quantization = _node_quantization(anchor, preset)
        shard_end = min(required_end, required_start + capacity - 1)
        directive = _add_shard_directive(anchor, str(preset["hf_repo"]), required_start, shard_end, quantization)
        anchor.pending_new_assignment = directive
        anchor.pending_directives.append(directive)
        _tracker_log(server, "info", "model load requested", node_id=anchor.node_id, model=resolved_name, hf_repo=preset["hf_repo"], shard=f"{required_start}-{shard_end}")
        return {"node_id": anchor.node_id, "model": resolved_name, "hf_repo": preset["hf_repo"], "shard_start": required_start, "shard_end": shard_end}
    return None


def _preferred_node_quantization(
    node: _NodeEntry,
    preset: dict,
    requested: str,
) -> str | None:
    """Choose the highest supported precision that satisfies a request."""
    if not _has_usable_quantization(node):
        return None
    supported = {
        normalized
        for value in ([node.quantization] if node.quantization else []) + list(node.quantizations)
        if (normalized := _normalize_quantization(value)) in _preset_bytes_per_layer(preset)
    }
    qualifying = [
        value for value in supported if _quantization_satisfies(value, requested)
    ]
    return max(qualifying, key=lambda value: _QUANTIZATION_QUALITY[value]) if qualifying else None


def _queue_initial_model_placement_locked(
    server: "_TrackerHTTPServer",
    model_key: str,
    requested_quantization: str,
) -> list[dict]:
    """Queue a complete first copy on spare tracker-managed capacity.

    This intentionally never replaces an existing assignment. Replacement is
    a later demand-balancing action, while the first chat request may use only
    genuinely spare slots and memory.
    """
    resolved_name, preset = _resolve_model_preset(server.model_presets, model_key)
    if preset is None or not preset.get("recommended") or not preset.get("hf_repo"):
        return []
    required_start, required_end = _preset_layer_bounds(preset)
    qualifying = [
        node for node in server.registry.values()
        if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
        and _quantization_satisfies(node.quantization, requested_quantization)
    ]
    if _coverage_percentage(qualifying, required_start, required_end) >= 100.0:
        return []

    cursor = required_start
    queued: list[dict] = []
    for host in _memory_pool_map(server)["hosts"]:
        if cursor > required_end or host["spare_slots"] <= 0:
            break
        host_nodes = [
            server.registry[item["node_id"]]
            for item in host["loaded"]
            if item["node_id"] in server.registry
        ]
        candidates = [
            node for node in host_nodes
            if node.status == "ready" and node.pending_new_assignment is None
            and _preferred_node_quantization(node, preset, requested_quantization) is not None
        ]
        if not candidates:
            continue
        anchor = max(candidates, key=lambda node: node.benchmark_tokens_per_sec)
        quantization = _preferred_node_quantization(anchor, preset, requested_quantization)
        if quantization is None:
            continue
        layer_bytes = _preset_bytes_per_layer(preset)[quantization]
        capacity = int(host["memory_spare_bytes"] // layer_bytes)
        if capacity <= 0:
            continue
        end = min(required_end, cursor + capacity - 1)
        directive = _add_shard_directive(
            anchor, str(preset["hf_repo"]), cursor, end, quantization,
        )
        anchor.pending_new_assignment = directive
        anchor.pending_directives.append(directive)
        queued.append({
            "node_id": anchor.node_id,
            "shard_start": cursor,
            "shard_end": end,
            "quantization": quantization,
        })
        cursor = end + 1
    if cursor <= required_end:
        # The directives are not safe to issue without a complete route. Undo
        # only the directives created above; unrelated pending work is intact.
        for placement in queued:
            node = server.registry.get(placement["node_id"])
            if node is None:
                continue
            node.pending_directives = [
                directive for directive in node.pending_directives
                if directive is not node.pending_new_assignment
            ]
            node.pending_new_assignment = None
        return []
    _tracker_log(
        server, "info", "initial demanded model placement queued",
        model=resolved_name, hf_repo=preset["hf_repo"],
        requested_quantization=requested_quantization, placements=queued,
    )
    return queued


def _deployment_summary(nodes: list[_NodeEntry], preset: dict | None) -> dict:
    if preset is None:
        return {"recommended": False}
    pool = _pool_summary(nodes, preset)
    required = int(preset.get("required_model_bytes", 0) or 0)
    deployable = required > 0 and pool["effective_assignable_memory_bytes"] >= required
    missing = max(0, required - pool["effective_assignable_memory_bytes"]) if required > 0 else 0
    return {
        "recommended": bool(preset.get("recommended", False)),
        "status": preset.get("deployment_status", "available"),
        "required_model_bytes": required or None,
        "download_size_bytes": preset.get("download_size_bytes"),
        "native_quantization": preset.get("native_quantization"),
        "pool": pool,
        "deployable": deployable,
        "missing_effective_memory_bytes": missing,
    }


def _max_layers_for_memory(
    memory_mb: int,
    total_layers: int,
    preset: dict | None = None,
    *,
    device: str | None = None,
) -> int:
    if total_layers <= 0:
        return 0
    if memory_mb <= 0:
        return max(1, total_layers // 2)
    memory_bytes = memory_mb * 1024 * 1024
    bytes_per_layer = next(iter(_preset_bytes_per_layer(preset).values())) if preset is not None else 30 * 1024 * 1024
    safety_fraction = 0.55 if device == "cpu" else 0.8
    return min(
        total_layers,
        max(1, int((memory_bytes * safety_fraction) // bytes_per_layer)),
    )


def _model_metadata_from_nodes(nodes: list[_NodeEntry]) -> dict:
    metadata: dict = {}
    for node in nodes:
        if node.model_metadata:
            metadata.update(node.model_metadata)
    if "num_layers" not in metadata:
        layers = [node.num_layers for node in nodes if node.num_layers is not None]
        if layers:
            metadata["num_layers"] = max(layers)
    return metadata


def _coverage_map(
    nodes: list[_NodeEntry],
    required_start: int,
    required_end: int,
) -> list[dict]:
    layer_counts = []
    for layer in range(required_start, required_end + 1):
        count = 0
        for node in _serviceable_nodes(nodes):
            if node.shard_start is None or node.shard_end is None:
                continue
            if node.shard_start <= layer <= node.shard_end:
                count += 1
        layer_counts.append((layer, count))

    coverage: list[dict] = []
    for layer, count in layer_counts:
        if coverage and coverage[-1]["node_count"] == count and coverage[-1]["end_layer"] == layer - 1:
            coverage[-1]["end_layer"] = layer
        else:
            coverage.append({"start_layer": layer, "end_layer": layer, "node_count": count})
    return coverage


def _node_health(node: "_NodeEntry", heartbeat_timeout: float) -> dict:
    """Per-node health detail for the availability map."""
    age = time.monotonic() - node.last_heartbeat
    alive = age <= heartbeat_timeout
    hb_expected = max(1, round(node.uptime_seconds / 20.0))  # assume ~20s interval
    hb_rate = round(min(1.0, node.heartbeats_received / hb_expected), 4) if node.heartbeats_received else 0.0
    total = node.total_requests
    inf_rate = round((total - node.failed_requests) / total, 4) if total > 0 else 1.0
    return {
        "node_id": node.node_id,
        "endpoint": node.endpoint,
        "alive": alive,
        "last_seen_seconds_ago": round(age, 1),
        "status": node.status,
        "queue_depth": _effective_queue_depth(node),
        "heartbeat_queue_depth": node.queue_depth,
        "proxy_inflight": node.proxy_inflight,
        "current_requests": list(node.current_requests),
        "total_requests": node.total_requests,
        "heartbeat_success_rate": hb_rate,
        "inference_success_rate": inf_rate,
        "capacity": _node_capacity_summary(node),
    }


def _coverage_map_detailed(
    nodes: list["_NodeEntry"],
    required_start: int,
    required_end: int,
    heartbeat_timeout: float,
) -> list[dict]:
    """Like _coverage_map but with per-node identity and health in each band.

    Includes all nodes (alive and stale, serviceable or not) so operators can see
    both coverage holes and which nodes used to fill them. This is why it does not
    share `_coverage_map`'s serviceability filter: a node dark for a declared bad
    precision is exactly what an operator staring at a hole needs to see.
    """
    now = time.monotonic()

    def covers(node: "_NodeEntry", layer: int) -> bool:
        return (
            node.shard_start is not None
            and node.shard_end is not None
            and node.shard_start <= layer <= node.shard_end
        )

    # Build per-layer list of nodes (with alive flag)
    layer_nodes: list[list[tuple["_NodeEntry", bool]]] = []
    for layer in range(required_start, required_end + 1):
        ns = [
            (n, (now - n.last_heartbeat) <= heartbeat_timeout)
            for n in nodes
            if covers(n, layer)
        ]
        layer_nodes.append(ns)

    # Merge consecutive layers with identical node-set (same node_ids, same alive)
    coverage: list[dict] = []
    for i, layer_idx in enumerate(range(required_start, required_end + 1)):
        ns = layer_nodes[i]
        node_ids = [n.node_id for n, _ in ns]
        alive_flags = [a for _, a in ns]
        # Compare with last band
        if (
            coverage
            and coverage[-1]["end_layer"] == layer_idx - 1
            and [nd["node_id"] for nd in coverage[-1]["nodes"]] == node_ids
            and [nd["alive"] for nd in coverage[-1]["nodes"]] == alive_flags
        ):
            coverage[-1]["end_layer"] = layer_idx
        else:
            coverage.append({
                "start_layer": layer_idx,
                "end_layer": layer_idx,
                "node_count": len(ns),
                "nodes": [_node_health(n, heartbeat_timeout) for n, _ in ns],
            })
    return coverage


def _coverage_gaps(coverage: list[dict]) -> list[tuple[int, int]]:
    return [
        (segment["start_layer"], segment["end_layer"])
        for segment in coverage
        if segment["node_count"] == 0
    ]


def _unassigned_managed_nodes(nodes: list["_NodeEntry"]) -> list["_NodeEntry"]:
    return [
        node for node in nodes
        if node.managed_assignment
        and (node.shard_start is None or node.shard_end is None)
    ]


def _emit_shard_change_directives(
    node: "_NodeEntry",
    model: str,
    previous_range: tuple[int | None, int | None, str | None],
    preset: dict,
) -> None:
    """Queue DROP/LOAD directives when a managed node's shard assignment changes."""
    previous_start, previous_end, previous_quantization = previous_range
    current_range = (node.shard_start, node.shard_end, node.quantization)
    if node.shard_start is None or node.shard_end is None or current_range == previous_range:
        return
    if previous_start is not None and previous_end is not None:
        node.pending_directives.append(
            _drop_directive(
                node,
                model,
                previous_start,
                previous_end,
                previous_quantization or _node_quantization(node, preset),
            )
        )
    node.pending_directives.append(
        _load_directive(
            node,
            model,
            node.shard_start,
            node.shard_end,
            node.quantization or _node_quantization(node, preset),
        )
    )


def _assign_redundant_managed_nodes(
    managed_nodes: list["_NodeEntry"],
    model: str,
    preset: dict,
    required_start: int,
    required_end: int,
) -> None:
    """Give newly joined managed nodes their own copy without reshuffling incumbents."""
    total_layers = required_end - required_start + 1
    for node in sorted(
        managed_nodes,
        key=lambda entry: (
            -entry.benchmark_tokens_per_sec,
            -_node_layer_capacity(entry, preset),
            entry.node_id,
        ),
    ):
        if node.shard_start is not None and node.shard_end is not None:
            continue
        if not _has_usable_quantization(node):
            continue
        capacity = min(_node_layer_capacity(node, preset), total_layers)
        if capacity <= 0:
            continue
        previous_range = (node.shard_start, node.shard_end, node.quantization)
        quantization = _node_quantization(node, preset)
        node.quantization = quantization
        node.shard_start = required_start
        node.shard_end = min(required_end, required_start + capacity - 1)
        _emit_shard_change_directives(node, model, previous_range, preset)


# Activation tensors ride the relay as base64 inside one JSON frame, so the
# websockets default of 1 MiB rejects any real prefill (close code 1009).
DEFAULT_WS_MAX_BYTES = 256 * 1024 * 1024


def _ws_max_size() -> int | None:
    """Max inbound WebSocket frame size; MESHNET_WS_MAX_BYTES<=0 means unlimited."""
    raw = os.environ.get("MESHNET_WS_MAX_BYTES", "").strip()
    if not raw:
        return DEFAULT_WS_MAX_BYTES
    try:
        value = int(raw)
    except ValueError:
        return DEFAULT_WS_MAX_BYTES
    return None if value <= 0 else value


def _relay_http_request_frames(
    relay_addr: str,
    path: str,
    body: bytes,
    headers: dict[str, str],
    timeout: float = 310.0,
    idle_timeout: float = 120.0,
    *,
    cancel_event: threading.Event | None = None,
    ws_holder: list[Any] | None = None,
    # Quoted: threading.Lock is a factory function (not a class) before
    # Python 3.13, so an unquoted `| None` union crashes at import time.
    ws_lock: "threading.Lock | None" = None,
):
    """Send an HTTP-shaped request through a relay RPC WebSocket, yielding
    response frames until a terminal one (US-036).

    A frame with ``stream: true`` is part of a chunked SSE response ending with
    ``done: true``; a frame without ``stream`` is a complete single response.
    Yields nothing when the relay is unreachable; stops silently on idle or
    overall timeout (the caller bills whatever was observed).
    """
    try:
        import websockets.sync.client as wsc  # type: ignore[import]
    except Exception:
        return
    request_id = str(uuid.uuid4())
    deadline = time.monotonic() + timeout
    try:
        with wsc.connect(relay_addr, open_timeout=10, close_timeout=5, max_size=_ws_max_size()) as ws:
            if ws_holder is not None:
                if ws_lock is not None:
                    with ws_lock:
                        ws_holder.clear()
                        ws_holder.append(ws)
                else:
                    ws_holder.clear()
                    ws_holder.append(ws)
            ws.send(json.dumps({
                "request_id": request_id,
                "method": "POST",
                "path": path,
                "headers": headers,
                "body": body.decode(errors="replace"),
            }))
            while True:
                if cancel_event is not None and cancel_event.is_set():
                    return
                remaining = deadline - time.monotonic()
                if remaining <= 0:
                    return
                raw = ws.recv(timeout=min(idle_timeout, remaining))
                frame = json.loads(raw)
                if frame.get("request_id") not in {None, request_id}:
                    continue
                yield frame
                if not frame.get("stream") or frame.get("done"):
                    return
    except Exception:
        return


def _relay_http_request(
    relay_addr: str,
    path: str,
    body: bytes,
    headers: dict[str, str],
    timeout: float = 310.0,
) -> dict | None:
    """Send an HTTP-shaped request through a relay and buffer the response.

    Streamed frame sequences are collapsed into one response dict whose body is
    the concatenated SSE text — used by non-chat callers and kept for
    backward compatibility; the chat proxy streams frames directly.
    """
    frames = _relay_http_request_frames(relay_addr, path, body, headers, timeout=timeout)
    first = next(frames, None)
    if first is None:
        return None
    if not first.get("stream"):
        return first
    chunks = [first.get("chunk") or ""]
    for frame in frames:
        chunks.append(frame.get("chunk") or "")
    return {
        "request_id": first.get("request_id"),
        "status": first.get("status", 200),
        "headers": first.get("headers") or {},
        "body": "".join(chunks),
    }


def _usage_split(payload: dict) -> dict | None:
    """Parse a usage block into {"prompt", "completion", "total"} (ints or None)."""
    usage = payload.get("usage")
    if not isinstance(usage, dict):
        return None

    def _num(key: str) -> int | None:
        value = usage.get(key)
        return int(value) if isinstance(value, (int, float)) else None

    return {
        "prompt": _num("prompt_tokens"),
        "completion": _num("completion_tokens"),
        "total": _usage_total_tokens(payload),
    }


def _stream_line_tokens(line: bytes) -> tuple[int, dict | None]:
    """Token accounting for one SSE line: (observed output delta, usage split or None)."""
    if not line.startswith(b"data:"):
        return 0, None
    payload = line[5:].strip()
    if not payload or payload == b"[DONE]":
        return 0, None
    try:
        chunk_payload = json.loads(payload)
    except json.JSONDecodeError:
        return 1, None
    return _observed_stream_tokens(chunk_payload), _usage_split(chunk_payload)


def _stream_billable_split(
    observed_output: int, usage: dict | None, request_body: dict
) -> tuple[int, int]:
    """(input_tokens, output_tokens) for a streamed response (US-045).

    Output: observed deltas, capped by reported completion count when the
    stream carried a usage chunk. Input: reported prompt count, else the
    prompt estimate from the request body.
    """
    prompt = (usage or {}).get("prompt")
    completion = (usage or {}).get("completion")
    total = (usage or {}).get("total")
    if prompt is None:
        prompt = _estimate_prompt_tokens(request_body) or 0
    if completion is None and total is not None:
        completion = max(0, total - prompt)
    return max(0, prompt), _billable_stream_tokens(observed_output, completion)


def _billable_non_stream_split(payload: dict, request_body: dict) -> tuple[int, int]:
    """(input_tokens, output_tokens) for a buffered response (US-045).

    Prefers the response usage block; falls back to content estimates.
    Completion stays capped by the request's max-tokens bound, as before.
    """
    usage = _usage_split(payload)
    prompt_estimate = _estimate_prompt_tokens(request_body) or 0
    prompt = (usage or {}).get("prompt")
    completion = (usage or {}).get("completion")
    if prompt is None:
        prompt = prompt_estimate
    if completion is None:
        total = (usage or {}).get("total")
        if total is not None:
            completion = max(0, total - prompt)
        else:
            completion = _observed_non_stream_completion_tokens(payload)
    limit = _requested_completion_token_limit(request_body)
    if limit is not None and completion > limit:
        completion = min(completion, limit)
        prompt = max(prompt, prompt_estimate)
    return max(0, prompt), max(0, completion)


def _find_pinned_route(
    nodes: list[_NodeEntry],
    required_start: int,
    required_end: int,
    hop_count: int,
) -> list[_NodeEntry] | None:
    """First combination of exactly ``hop_count`` distinct nodes covering the
    layer range, where every node extends coverage (US-030 benchmark routes)."""
    for combo in itertools.permutations(nodes, hop_count):
        covered = required_start - 1
        valid = True
        for candidate in combo:
            if candidate.shard_start is None or candidate.shard_end is None:
                valid = False
                break
            if candidate.shard_start > covered + 1 or candidate.shard_end <= covered:
                valid = False
                break
            covered = candidate.shard_end
        if valid and covered >= required_end:
            return list(combo)
    return None


def _nodes_and_bounds_for_model(
    server: "_TrackerHTTPServer",
    model: str,
) -> tuple[list[_NodeEntry], int, int] | None:
    resolved_name, preset = _resolve_model_preset(server.model_presets, model)
    if preset is not None:
        required_start, required_end = _preset_layer_bounds(preset)
        return [
            node for node in server.registry.values()
            if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
        ], required_start, required_end

    nodes = [
        node for node in server.registry.values()
        if _node_matches_model(node, model)
        and node.shard_start is not None
        and node.shard_end is not None
        and node.num_layers is not None
    ]
    if not nodes:
        return None
    return nodes, 0, max(node.num_layers for node in nodes) - 1


def _fetch_toploc_commitment(
    node: _NodeEntry,
    *,
    session_id: str,
    model: str,
    messages: list[dict],
) -> dict | None:
    """Fetch a node's own on-demand TOPLOC boundary commitment (ADR-0018 §3),
    same protocol as `ValidatorProcess._fetch_hop_commitment`."""
    endpoint = node.endpoint
    if not isinstance(endpoint, str) or not endpoint:
        return None
    try:
        req = urllib.request.Request(
            f"{endpoint.rstrip('/')}/v1/audit/toploc/commitment",
            data=json.dumps({
                "session_id": session_id,
                "model": model,
                "messages": messages,
                "shard_start": node.shard_start,
                "shard_end": node.shard_end,
            }).encode(),
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        with urllib.request.urlopen(req, timeout=5.0) as resp:
            response = json.loads(resp.read())
    except (OSError, ValueError, json.JSONDecodeError):
        return None
    proof = response.get("toploc_proof") or response.get("activation_proof")
    token_ids = response.get("claimed_token_ids") or response.get("output_token_ids")
    if not isinstance(proof, dict):
        return None
    if not isinstance(token_ids, list) or not all(isinstance(t, int) for t in token_ids):
        return None
    return {"toploc_proof": proof, "claimed_token_ids": token_ids}


def _fetch_toploc_reference_activations(
    reference_node_url: str,
    *,
    model: str,
    messages: list[dict],
    claimed_token_ids: list[int],
    claim: Any,
) -> list | None:
    """Teacher-force the claimed tokens on the reference node (same contract
    as `ValidatorProcess._run_teacher_forced_prefill` / validator README's
    "TOPLOC audit contract")."""
    try:
        req = urllib.request.Request(
            f"{reference_node_url.rstrip('/')}/v1/audit/toploc",
            data=json.dumps({
                "model": model,
                "messages": messages,
                "claimed_token_ids": claimed_token_ids,
                "dtype": claim.dtype,
                "quantization": claim.quantization,
                "decode_batching_size": claim.decode_batching_size,
                "topk": claim.topk,
                "skip_prefill": claim.skip_prefill,
            }).encode(),
            headers={"Content-Type": "application/json"},
            method="POST",
        )
        with urllib.request.urlopen(req, timeout=300.0) as resp:
            response = json.loads(resp.read())
    except (OSError, ValueError, json.JSONDecodeError):
        return None
    activations = response.get("activations")
    if not isinstance(activations, list):
        return None
    return activations


def _load_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
    return {
        "action": "LOAD_SHARD",
        "model": model,
        "start_layer": start,
        "end_layer": end,
        "shard_start": start,
        "shard_end": end,
        "quantization": quantization,
    }


def _drop_directive(node: _NodeEntry, model: str, start: int, end: int, quantization: str) -> dict:
    return {
        "action": "DROP_SHARD",
        "model": model,
        "start_layer": start,
        "end_layer": end,
        "shard_start": start,
        "shard_end": end,
        "quantization": quantization,
    }


def _route_stats_keys(server: "_TrackerHTTPServer", entry: "_NodeEntry") -> list[str]:
    """All stats keys a node's routes may be recorded under (model, repo, resolved preset)."""
    keys = {entry.model, entry.hf_repo}
    resolved, _ = _resolve_model_preset(server.model_presets, entry.hf_repo or entry.model)
    keys.add(resolved)
    return [key for key in keys if key]


def _purge_expired_nodes_locked(server: "_TrackerHTTPServer") -> list[str]:
    now = time.monotonic()
    expired_ids = [
        node_id for node_id, entry in server.registry.items()
        if (now - entry.last_heartbeat) > server.heartbeat_timeout
    ]
    expired_entries: list[tuple[str, _NodeEntry]] = []
    for node_id in expired_ids:
        entry = server.registry.pop(node_id)
        expired_entries.append((node_id, entry))
    if expired_ids:
        _rebalance_all_locked(server)
        server.route_stats.bump_epoch(
            key
            for _, entry in expired_entries
            for key in _route_stats_keys(server, entry)
        )
    for node_id, entry in expired_entries:
        _tracker_log(
            server,
            "warn",
            "node expired",
            node_id=node_id,
            endpoint=entry.endpoint,
            model=entry.model,
            hf_repo=entry.hf_repo,
            shard=f"{entry.shard_start}-{entry.shard_end}",
            heartbeat_timeout_seconds=server.heartbeat_timeout,
            model_health=_model_health_summary(server, entry.model, entry.hf_repo),
        )
    return expired_ids


def _rebalance_model_locked(server: "_TrackerHTTPServer", model: str) -> None:
    resolved_name, preset = _resolve_model_preset(server.model_presets, model)
    if preset is None:
        return
    required_start, required_end = _preset_layer_bounds(preset)
    total_layers = required_end - required_start + 1
    model_nodes = [
        node for node in server.registry.values()
        if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
    ]
    managed_nodes = [node for node in model_nodes if node.managed_assignment]
    if not managed_nodes:
        return

    # Directives must name a repo the node can actually load, not the preset key.
    directive_model = str(preset.get("hf_repo") or model)

    coverage = _coverage_map(model_nodes, required_start, required_end)
    gaps = _coverage_gaps(coverage)
    unassigned = _unassigned_managed_nodes(managed_nodes)
    if not gaps and not unassigned:
        return
    if not gaps and unassigned:
        _assign_redundant_managed_nodes(
            unassigned, directive_model, preset, required_start, required_end,
        )
        return

    previous_ranges = {
        node.node_id: (node.shard_start, node.shard_end, node.quantization)
        for node in managed_nodes
    }
    for node in managed_nodes:
        node.shard_start = None
        node.shard_end = None

    managed_nodes.sort(
        key=lambda node: (
            -node.benchmark_tokens_per_sec,
            -_node_layer_capacity(node, preset),
            node.node_id,
        )
    )
    base_nodes = [node for node in model_nodes if not node.managed_assignment]
    base_coverage = _coverage_map(base_nodes, required_start, required_end)
    gaps = _coverage_gaps(base_coverage)

    eligible_nodes = [
        node for node in managed_nodes
        if _has_usable_quantization(node)
        and _node_layer_capacity(node, preset) > 0
    ]
    node_index = 0
    for gap_start, gap_end in gaps:
        cursor = gap_start
        while cursor <= gap_end and node_index < len(eligible_nodes):
            node = eligible_nodes[node_index]
            remaining_layers = gap_end - cursor + 1
            remaining_nodes_after = len(eligible_nodes) - node_index - 1
            capacity = min(
                _node_layer_capacity(node, preset),
                total_layers,
                max(1, remaining_layers - remaining_nodes_after),
            )
            if capacity <= 0:
                node_index += 1
                continue
            quantization = _node_quantization(node, preset)
            node.quantization = quantization
            node.shard_start = cursor
            node.shard_end = min(gap_end, cursor + capacity - 1)
            cursor = node.shard_end + 1
            node_index += 1

    for node in managed_nodes:
        _emit_shard_change_directives(
            node,
            directive_model,
            previous_ranges[node.node_id],
            preset,
        )


def _hf_rebalance_preset(nodes: list[_NodeEntry]) -> dict:
    total_layers = max(node.num_layers or 0 for node in nodes)
    return {
        "layers_start": 0,
        "layers_end": total_layers - 1,
        "bytes_per_layer": {"bfloat16": 30 * 1024 * 1024, "int8": 15 * 1024 * 1024, "nf4": 8 * 1024 * 1024},
    }


def _rebalance_hf_model_locked(server: "_TrackerHTTPServer", hf_repo: str) -> None:
    model_nodes = [
        node for node in server.registry.values()
        if node.hf_repo == hf_repo
        and node.num_layers is not None
    ]
    managed_nodes = [node for node in model_nodes if node.managed_assignment]
    if not model_nodes or not managed_nodes:
        return

    preset = _hf_rebalance_preset(model_nodes)
    required_start, required_end = _preset_layer_bounds(preset)
    total_layers = required_end - required_start + 1
    if total_layers <= 0:
        return

    coverage = _coverage_map(model_nodes, required_start, required_end)
    gaps = _coverage_gaps(coverage)
    unassigned = _unassigned_managed_nodes(managed_nodes)
    if not gaps and not unassigned:
        return
    if not gaps and unassigned:
        _assign_redundant_managed_nodes(
            unassigned, hf_repo, preset, required_start, required_end,
        )
        return

    previous_ranges = {
        node.node_id: (node.shard_start, node.shard_end, node.quantization)
        for node in managed_nodes
    }
    for node in managed_nodes:
        node.shard_start = None
        node.shard_end = None

    managed_nodes.sort(
        key=lambda node: (
            -node.benchmark_tokens_per_sec,
            -_node_layer_capacity(node, preset),
            node.node_id,
        )
    )
    base_nodes = [node for node in model_nodes if not node.managed_assignment]
    base_coverage = _coverage_map(base_nodes, required_start, required_end)
    gaps = _coverage_gaps(base_coverage)

    eligible_nodes = [
        node for node in managed_nodes
        if _has_usable_quantization(node)
        and _node_layer_capacity(node, preset) > 0
    ]
    node_index = 0
    for gap_start, gap_end in gaps:
        cursor = gap_start
        while cursor <= gap_end and node_index < len(eligible_nodes):
            node = eligible_nodes[node_index]
            remaining_layers = gap_end - cursor + 1
            remaining_nodes_after = len(eligible_nodes) - node_index - 1
            capacity = min(
                _node_layer_capacity(node, preset),
                total_layers,
                max(1, remaining_layers - remaining_nodes_after),
            )
            if capacity <= 0:
                node_index += 1
                continue
            quantization = _node_quantization(node, preset)
            node.quantization = quantization
            node.shard_start = cursor
            node.shard_end = min(gap_end, cursor + capacity - 1)
            cursor = node.shard_end + 1
            node_index += 1

    for node in managed_nodes:
        _emit_shard_change_directives(
            node,
            hf_repo,
            previous_ranges[node.node_id],
            preset,
        )


def _rebalance_all_locked(server: "_TrackerHTTPServer") -> None:
    for model in list(server.model_presets):
        _rebalance_model_locked(server, model)
    for hf_repo in sorted({node.hf_repo for node in server.registry.values() if node.hf_repo}):
        _rebalance_hf_model_locked(server, hf_repo)


def _api_key_from_headers(headers) -> str | None:
    auth = headers.get("Authorization")
    if not auth:
        return None
    if auth.lower().startswith("bearer "):
        return auth.split(" ", 1)[1].strip() or None
    return auth.strip() or None


def _session_token_from_headers(headers) -> str | None:
    token = _api_key_from_headers(headers)
    if token:
        return token
    cookie_header = headers.get("Cookie")
    if not cookie_header:
        return None
    cookie = http.cookies.SimpleCookie()
    try:
        cookie.load(cookie_header)
    except http.cookies.CookieError:
        return None
    morsel = cookie.get(_SESSION_COOKIE_NAME)
    if morsel is None:
        return None
    return morsel.value.strip() or None


def _session_cookie_header(token: str | None) -> str:
    cookie = http.cookies.SimpleCookie()
    cookie[_SESSION_COOKIE_NAME] = token or ""
    morsel = cookie[_SESSION_COOKIE_NAME]
    morsel["path"] = "/"
    morsel["httponly"] = True
    morsel["samesite"] = "Lax"
    if token:
        morsel["max-age"] = str(int(7 * 86400))
    else:
        morsel["max-age"] = "0"
    return morsel.OutputString()


def _usage_total_tokens(payload: dict) -> int | None:
    usage = payload.get("usage")
    if not isinstance(usage, dict):
        return None
    total = usage.get("total_tokens")
    if isinstance(total, (int, float)):
        return int(total)
    prompt = usage.get("prompt_tokens")
    completion = usage.get("completion_tokens")
    if isinstance(prompt, (int, float)) or isinstance(completion, (int, float)):
        return int(prompt or 0) + int(completion or 0)
    return None


def _estimate_text_tokens(value: Any) -> int | None:
    if isinstance(value, str):
        text = value.strip()
        if not text:
            return 0
        return len(text.split())
    if isinstance(value, list):
        total = 0
        found = False
        for item in value:
            if isinstance(item, str):
                estimated = _estimate_text_tokens(item)
            elif isinstance(item, dict):
                estimated = _estimate_text_tokens(item.get("text"))
            else:
                estimated = None
            if estimated is not None:
                total += estimated
                found = True
        return total if found else None
    return None


def _estimate_prompt_tokens(body: dict) -> int | None:
    messages = body.get("messages")
    if isinstance(messages, list):
        total = 0
        found = False
        for message in messages:
            if not isinstance(message, dict):
                continue
            estimated = _estimate_text_tokens(message.get("content"))
            if estimated is not None:
                total += estimated
                found = True
        return total if found else None
    prompt = body.get("prompt")
    return _estimate_text_tokens(prompt)


def _requested_completion_token_limit(body: dict) -> int | None:
    for field in ("max_completion_tokens", "max_tokens"):
        value = body.get(field)
        if isinstance(value, bool):
            return None
        if isinstance(value, (int, float)):
            return max(0, int(value))
    return None


def _request_total_token_upper_bound(body: dict) -> int | None:
    completion_limit = _requested_completion_token_limit(body)
    if completion_limit is None:
        return None
    prompt_estimate = _estimate_prompt_tokens(body)
    return completion_limit + (prompt_estimate or 0)


def _billable_non_stream_tokens(payload: dict, request_body: dict) -> int:
    reported = _usage_total_tokens(payload)
    upper_bound = _request_total_token_upper_bound(request_body)
    if reported is None:
        completion_estimate = _observed_non_stream_completion_tokens(payload)
        if completion_estimate > 0:
            billable = completion_estimate + (_estimate_prompt_tokens(request_body) or 0)
        elif upper_bound is not None:
            billable = upper_bound
        else:
            return 0
        return min(billable, upper_bound) if upper_bound is not None else billable
    billable = max(0, reported)
    if upper_bound is not None:
        billable = min(billable, upper_bound)
    return billable


def _observed_non_stream_completion_tokens(payload: dict) -> int:
    choices = payload.get("choices")
    if not isinstance(choices, list):
        return 0
    observed = 0
    for choice in choices:
        if not isinstance(choice, dict):
            continue
        message = choice.get("message")
        estimated: int | None = None
        if isinstance(message, dict):
            estimated = _estimate_text_tokens(message.get("content"))
        if estimated is None:
            estimated = _estimate_text_tokens(choice.get("text"))
        if estimated is not None:
            observed += estimated
        elif choice:
            observed += 1
    return observed


def _observed_output_from_non_stream_payload(payload: dict) -> str:
    choices = payload.get("choices")
    if not isinstance(choices, list) or not choices:
        return ""
    first = choices[0]
    if not isinstance(first, dict):
        return ""
    message = first.get("message")
    if isinstance(message, dict) and isinstance(message.get("content"), str):
        return message["content"]
    text = first.get("text")
    return text if isinstance(text, str) else ""


def _observed_stream_tokens(payload: dict) -> int:
    choices = payload.get("choices")
    if not isinstance(choices, list):
        return 0
    observed = 0
    for choice in choices:
        if not isinstance(choice, dict):
            continue
        delta = choice.get("delta")
        if isinstance(delta, dict):
            estimated = _estimate_text_tokens(delta.get("content"))
            if estimated:
                observed += estimated
                continue
        if choice:
            observed += 1
    return observed


def _billable_stream_tokens(observed_tokens: int, reported_tokens: int | None) -> int:
    observed = max(0, observed_tokens)
    if reported_tokens is None:
        return observed
    return min(observed, max(0, reported_tokens))


def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
    if contracts is None or not wallet_address:
        return None
    if contracts.registry.get_wallet(wallet_address).banned:
        return "wallet is banned"
    return None


_MAX_FRIENDLY_NAME_LENGTH = 64


def _normalize_friendly_name(value: object) -> str | None:
    if value is None:
        return None
    if not isinstance(value, str):
        raise ValueError("friendly_name must be a string")
    name = value.strip()
    if not name:
        return None
    if len(name) > _MAX_FRIENDLY_NAME_LENGTH:
        raise ValueError(f"friendly_name must be at most {_MAX_FRIENDLY_NAME_LENGTH} characters")
    return name


def _tracker_log(
    server: "_TrackerHTTPServer",
    level: str,
    message: str,
    *,
    stdout: bool = True,
    update_console_key: str | None = None,
    **fields: Any,
) -> None:
    log_level = {
        "debug": 10,
        "info": 20,
        "warn": 30,
        "warning": 30,
        "error": 40,
    }.get(level.lower(), 20)
    event = {
        "ts": time.time(),
        "level": level,
        "message": message,
        "fields": {
            key: value
            for key, value in fields.items()
            if value is not None
        },
    }
    with server.console_lock:
        if update_console_key is not None:
            updated = False
            for existing in reversed(server.console_events):
                if (
                    existing.get("message") == message
                    and existing.get("fields", {}).get("request_id") == update_console_key
                ):
                    existing["ts"] = event["ts"]
                    existing["fields"] = event["fields"]
                    updated = True
                    break
            if not updated:
                server.console_events.append(event)
        else:
            server.console_events.append(event)
    extras = " ".join(f"{key}={value}" for key, value in event["fields"].items())
    suffix = f"  {extras}" if extras else ""
    tracker_logger().log(log_level, f"{message}{suffix}")
    if stdout:
        print(f"[tracker] {level}: {message}{suffix}", flush=True)


@dataclass
class _ActiveProxyContext:
    request_id: str
    cancel_event: threading.Event = field(default_factory=threading.Event)
    upstream: Any | None = None
    upstream_lock: threading.Lock = field(default_factory=threading.Lock)
    relay_ws: Any | None = None
    relay_ws_lock: threading.Lock = field(default_factory=threading.Lock)


def _register_active_proxy(server: "_TrackerHTTPServer", request_id: str) -> _ActiveProxyContext:
    ctx = _ActiveProxyContext(request_id=request_id)
    with server.active_proxies_lock:
        server.active_proxies[request_id] = ctx
    return ctx


def _unregister_active_proxy(server: "_TrackerHTTPServer", request_id: str) -> None:
    with server.active_proxies_lock:
        server.active_proxies.pop(request_id, None)


def _request_proxy_cancel(server: "_TrackerHTTPServer", request_id: str) -> bool:
    with server.active_proxies_lock:
        ctx = server.active_proxies.get(request_id)
    if ctx is None:
        return False
    ctx.cancel_event.set()

    def _close_resources() -> None:
        with ctx.upstream_lock:
            upstream = ctx.upstream
        if upstream is not None:
            try:
                upstream.close()
            except Exception:
                pass
        with ctx.relay_ws_lock:
            relay_ws = ctx.relay_ws
        if relay_ws is not None:
            try:
                relay_ws.close()
            except Exception:
                pass

    threading.Thread(target=_close_resources, daemon=True).start()
    return True


def _upstream_socket(upstream: Any) -> Any | None:
    fp = getattr(upstream, "fp", None)
    raw = getattr(fp, "raw", None) if fp is not None else None
    return getattr(raw, "_sock", None) if raw is not None else None


def _set_upstream_read_timeout(upstream: Any, timeout: float | None) -> None:
    sock = _upstream_socket(upstream)
    if sock is not None:
        sock.settimeout(timeout)


def _clear_proxy_progress_log_state(server: "_TrackerHTTPServer", request_id: str) -> None:
    state = getattr(server, "_proxy_progress_log_state", None)
    if state is not None:
        state.pop(request_id, None)


def _tracker_log_proxy_progress(
    server: "_TrackerHTTPServer",
    *,
    request_id: str,
    model: str,
    route_model: str,
    tokens: int,
    started: float,
    route_nodes: list["_NodeEntry"],
    stream: bool = True,
    relay: bool = False,
) -> None:
    elapsed = time.monotonic() - started
    effective_elapsed = max(elapsed, 1e-6)
    now = time.monotonic()
    state = getattr(server, "_proxy_progress_log_state", None)
    if state is None:
        state = {}
        server._proxy_progress_log_state = state
    last_stdout = state.get(request_id)
    stdout = last_stdout is None or (now - last_stdout) >= _PROXY_PROGRESS_LOG_INTERVAL
    if stdout:
        state[request_id] = now
    _tracker_log(
        server,
        "info",
        "proxy progress",
        stdout=stdout,
        update_console_key=request_id,
        request_id=request_id,
        model=model,
        route_model=route_model,
        stream=stream,
        relay=relay or None,
        tokens=tokens,
        elapsed_seconds=round(elapsed, 4),
        tokens_per_sec=round(tokens / effective_elapsed, 4) if tokens > 0 else 0.0,
        route=_node_route_summary(route_nodes),
    )


def _node_id_for_registration(
    endpoint: str,
    model: str,
    wallet_address: str | None,
    shard_start: int | None,
    shard_end: int | None,
    hf_repo: str | None,
) -> str:
    wallet_prefix = wallet_address[:8] if wallet_address else "anon"
    stable_key = "|".join([
        wallet_address or "",
        endpoint.rstrip("/"),
        model,
        hf_repo or "",
        "" if shard_start is None else str(shard_start),
        "" if shard_end is None else str(shard_end),
    ])
    digest = hashlib.sha256(stable_key.encode()).hexdigest()[:12]
    return f"{wallet_prefix}-{digest}"


class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
    daemon_threads = True

    def __init__(
        self,
        addr: tuple,
        handler,
        registry: dict,
        lock: threading.Lock,
        heartbeat_timeout: float,
        model_presets: dict,
        contracts: Any | None,
        minimum_stake: int,
        relay_url: str | None = None,
        raft: "RaftNode | None" = None,
        gossip: "NodeGossip | None" = None,
        stats: "_StatsCollector | None" = None,
        billing: "BillingLedger | None" = None,
        accounts: "AccountStore | None" = None,
        benchmark_results_path: str | None = None,
        validator_service_token: str | None = None,
        hive_secret: str | None = None,
        max_charge_per_request: float | None = None,
        starting_credit: float = DEFAULT_CALLER_CREDIT_USDT,
        devnet_topup_amount: float = DEFAULT_DEVNET_TOPUP_USDT,
        toploc_calibration: "ToplocCalibrationStore | None" = None,
        toploc_reference_node_url: str | None = None,
        toploc_calibration_gate_min_hardware_profiles: int = 1,
        toploc_backend: Any | None = None,
        hf_pricing_log: "HfPricingLog | None" = None,
        models_dir: Path | None = None,
        route_stats: "RouteStatsStore | None" = None,
        relay_status: dict | None = None,
        test_runner: "TestRunManager | None" = None,
        capability_policy: str | None = None,
    ) -> None:
        super().__init__(addr, handler)
        self.registry = registry
        self.capability_policy = normalize_policy(
            capability_policy if capability_policy is not None else policy_from_env()
        )
        self.lock = lock
        self.heartbeat_timeout = heartbeat_timeout
        self.model_presets = model_presets
        self.contracts = contracts
        self.minimum_stake = minimum_stake
        self.relay_url = relay_url.rstrip("/") if relay_url else None
        self.raft = raft
        self.gossip = gossip
        self.stats: _StatsCollector | None = stats
        self.billing: BillingLedger | None = billing
        self.accounts: AccountStore | None = accounts
        self.benchmark_results_path = benchmark_results_path or os.path.join(
            os.getcwd(), "benchmark_results.json"
        )
        self.benchmark_lock = threading.Lock()
        self.validator_service_token = validator_service_token
        self.hive_secret = hive_secret
        self.max_charge_per_request = max_charge_per_request
        self.starting_credit = starting_credit
        self.devnet_topup_amount = devnet_topup_amount
        self.toploc_calibration: ToplocCalibrationStore | None = toploc_calibration
        self.toploc_reference_node_url = (
            toploc_reference_node_url.rstrip("/") if toploc_reference_node_url else None
        )
        self.toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
        self.toploc_backend = toploc_backend
        self.hf_pricing_log: HfPricingLog | None = hf_pricing_log
        self.models_dir = models_dir
        self.console_events = deque(maxlen=_CONSOLE_LIMIT)
        self.console_lock = threading.Lock()
        self.active_proxies: dict[str, _ActiveProxyContext] = {}
        self.active_proxies_lock = threading.Lock()
        self.route_stats: RouteStatsStore = route_stats or RouteStatsStore()
        self.route_rng = random.Random()
        self.relay_status = dict(
            relay_status or {"mode": "external" if self.relay_url else "off"}
        )
        self.test_runner: TestRunManager | None = test_runner


class _TrackerHandler(http.server.BaseHTTPRequestHandler):
    def log_message(self, fmt, *args):  # noqa: suppress request logs in tests
        pass

    def _send_json(self, status: int, data: dict, headers: dict[str, str] | None = None) -> None:
        body = json.dumps(data).encode()
        self.send_response(status)
        self.send_header("Content-Type", "application/json")
        self.send_header("Content-Length", str(len(body)))
        for name, value in (headers or {}).items():
            self.send_header(name, value)
        self.end_headers()
        try:
            self.wfile.write(body)
        except BrokenPipeError:
            pass

    # ---- unified auth boundary (ADR-0017) ----

    def _resolve_identity(self) -> tuple[str | None, dict | None]:
        """Resolve the caller to (role, account).

        Roles: "validator" (service token), "admin"/"user" (session token).
        Client API keys resolve to no privileged role — they authorize
        inference and wallet binding only, never operator endpoints.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        token = _session_token_from_headers(self.headers)
        if not token:
            return None, None
        if is_validator_token(token, server.validator_service_token):
            return "validator", None
        if server.accounts is not None:
            account = server.accounts.session_account(token)
            if account is not None:
                return account.get("role", "user"), account
        return None, None

    def _require_role(self, *allowed: str) -> bool:
        """Gate a privileged handler; sends 401/403 and returns False on failure.

        401 when no credential was presented; 403 when a credential was
        presented but does not resolve to an allowed role — this covers
        client API keys and garbage bearer strings on operator endpoints.
        """
        role, _account = self._resolve_identity()
        if role in allowed:
            return True
        if _api_key_from_headers(self.headers) is None:
            self._send_json(401, {"error": "authentication required (admin session or service token)"})
        else:
            self._send_json(403, {"error": "this endpoint requires an admin session or service token"})
        return False

    def _read_hive_authenticated_body(self) -> dict | None:
        """Read + verify a hive gossip body (HMAC per ADR-0017 §3).

        Fails closed: without a configured --hive-secret no gossip is
        accepted. Sends the error response itself and returns None on failure.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        length = int(self.headers.get("Content-Length", 0))
        raw = self.rfile.read(length) if length else b"{}"
        if not verify_hive_request(server.hive_secret, self.headers, raw):
            self._send_json(401, {"error": "valid hive signature required"})
            return None
        try:
            body = json.loads(raw or b"{}")
        except json.JSONDecodeError:
            self._send_json(400, {"error": "invalid JSON body"})
            return None
        if not isinstance(body, dict):
            self._send_json(400, {"error": "JSON body must be an object"})
            return None
        return body

    def _read_json_body(self) -> dict | None:
        length = int(self.headers.get("Content-Length", 0))
        try:
            body = json.loads(self.rfile.read(length) or b"{}")
        except json.JSONDecodeError:
            self._send_json(400, {"error": "invalid JSON body"})
            return None
        if not isinstance(body, dict):
            self._send_json(400, {"error": "JSON body must be an object"})
            return None
        return body

    def _purge_expired_nodes(self) -> None:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        _purge_expired_nodes_locked(server)

    def do_POST(self):
        if self.path == "/v1/chat/completions":
            self._handle_proxy_chat()
            return
        if self.path == "/v1/nodes/register":
            self._handle_register()
            return
        if self.path == "/v1/raft/vote":
            self._handle_raft_vote()
            return
        if self.path == "/v1/raft/append":
            self._handle_raft_append()
            return
        if self.path == "/v1/gossip":
            self._handle_gossip()
            return
        if self.path == "/v1/stats/gossip":
            self._handle_stats_gossip()
            return
        if self.path == "/v1/billing/gossip":
            self._handle_billing_gossip()
            return
        if self.path == "/v1/billing/forfeit":
            self._handle_billing_forfeit()
            return
        if self.path == "/v1/auth/register":
            self._handle_auth_register()
            return
        if self.path == "/v1/auth/login":
            self._handle_auth_login()
            return
        if self.path == "/v1/auth/logout":
            self._handle_auth_logout()
            return
        if self.path == "/v1/account/profile":
            self._handle_account_profile()
            return
        if self.path == "/v1/account/keys":
            self._handle_account_key_create()
            return
        if self.path == "/v1/account/keys/revoke":
            self._handle_account_key_revoke()
            return
        if self.path == "/v1/account/topup":
            self._handle_account_topup()
            return
        if self.path == "/v1/accounts/gossip":
            self._handle_accounts_gossip()
            return
        if self.path == "/v1/registry/gossip":
            self._handle_registry_gossip()
            return
        if self.path == "/v1/benchmark/hop-penalty":
            self._handle_benchmark_hop_penalty()
            return
        if self.path == "/v1/models/load":
            self._handle_model_load_request()
            return
        if self.path == "/v1/models/vote":
            self._handle_model_coverage_vote()
            return
        if self.path == "/v1/calibration/toploc/run":
            self._handle_toploc_calibration_run()
            return
        if self.path == "/v1/wallet/register":
            self._handle_wallet_register()
            return
        if self.path == "/v1/tests/run":
            self._handle_tests_run()
            return
        parts = self.path.split("/")
        # /v1/nodes/<node_id>/heartbeat -> ['', 'v1', 'nodes', '<id>', 'heartbeat']
        if len(parts) == 5 and parts[1] == "v1" and parts[2] == "nodes" and parts[4] == "heartbeat":
            self._handle_heartbeat(parts[3])
            return
        # /v1/proxy/requests/<request_id>/cancel
        if (
            len(parts) == 6
            and parts[1] == "v1"
            and parts[2] == "proxy"
            and parts[3] == "requests"
            and parts[5] == "cancel"
            and parts[4]
        ):
            self._handle_proxy_request_cancel(urllib.parse.unquote(parts[4]))
            return
        self.send_response(404)
        self.end_headers()

    def do_GET(self):
        parsed = urllib.parse.urlparse(self.path)
        if parsed.path == "/":
            self.send_response(302)
            self.send_header("Location", "/dashboard")
            self.end_headers()
        elif parsed.path == "/v1/route":
            self._handle_route(parsed)
        elif parsed.path == "/v1/routes":
            self._handle_routes(parsed)
        elif parsed.path == "/v1/routing":
            self._handle_routing(parsed)
        elif parsed.path == "/v1/model-speed":
            self._handle_model_speed(parsed)
        elif parsed.path == "/v1/nodes/assign":
            self._handle_assign(parsed)
        elif parsed.path == "/v1/network/assign":
            self._handle_network_assign(parsed)
        elif parsed.path == "/v1/network/map":
            self._handle_network_map()
        elif parsed.path == "/v1/models":
            self._handle_models()
        elif parsed.path == "/v1/model-files/download":
            self._handle_model_files_download(parsed)
        elif parsed.path.startswith("/v1/coverage/"):
            model = urllib.parse.unquote(parsed.path.removeprefix("/v1/coverage/"))
            self._handle_coverage(model)
        elif parsed.path.startswith("/v1/tracker-nodes/"):
            model = urllib.parse.unquote(parsed.path.removeprefix("/v1/tracker-nodes/"))
            self._handle_tracker_nodes(model)
        elif parsed.path.startswith("/v1/head-workers/"):
            model = urllib.parse.unquote(parsed.path.removeprefix("/v1/head-workers/"))
            self._handle_tracker_nodes(model)
        elif parsed.path == "/v1/raft/status":
            self._handle_raft_status()
        elif parsed.path == "/v1/stats":
            self._handle_stats()
        elif parsed.path == "/v1/console":
            self._handle_console()
        elif parsed.path == "/v1/billing/summary":
            self._handle_billing_summary()
        elif parsed.path == "/v1/billing/settlements":
            self._handle_billing_settlements()
        elif parsed.path == "/v1/account/usage":
            self._handle_account_usage(parsed)
        elif parsed.path == "/v1/account":
            self._handle_account_me()
        elif parsed.path == "/v1/admin/accounts":
            self._handle_admin_accounts()
        elif parsed.path == "/v1/benchmark/results":
            self._handle_benchmark_results()
        elif parsed.path == "/v1/calibration/toploc/results":
            self._handle_toploc_calibration_results()
        elif parsed.path == "/v1/pricing/hf/history":
            self._handle_hf_pricing_history(parsed)
        elif parsed.path == "/v1/tests":
            self._handle_tests_list(parsed)
        elif parsed.path == "/v1/tests/status":
            self._handle_tests_status()
        elif parsed.path == "/v1/registry/wallets":
            self._handle_registry_wallets()
        elif parsed.path in ("/dashboard", "/dashboard/"):
            self._handle_dashboard()
        elif parsed.path in ("/favicon.svg", "/favicon.ico"):
            self._handle_favicon()
        elif parsed.path == "/v1/health":
            self._send_json(200, {"status": "ok"})
        else:
            self.send_response(404)
            self.end_headers()

    def _model_pricing_payload(self, model: str) -> dict | None:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.billing is None:
            return None
        in_rate, out_rate = server.billing.prices_for(model)
        return {
            "input_per_1k_usdt": in_rate,
            "output_per_1k_usdt": out_rate,
        }

    def _handle_models(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        created = int(time.time())
        with server.lock:
            self._purge_expired_nodes()
            alive = list(server.registry.values())
        if server.contracts is not None:
            alive = [
                node for node in alive
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]
        data = []
        seen_ids: set[str] = set()
        for name, preset in server.model_presets.items():
            model_nodes = [node for node in alive if _node_matches_preset(node, name, preset)]
            if not model_nodes and not preset.get("recommended"):
                continue
            required_start, required_end = _preset_layer_bounds(preset)
            coverage = _coverage_percentage(
                model_nodes,
                required_start,
                required_end,
            )
            served_copies = _served_model_copies(
                model_nodes,
                required_start,
                required_end,
            )
            quantizations = _available_quantizations(
                model_nodes,
                required_start,
                required_end,
            )
            aliases = [name]
            hf_repo = preset.get("hf_repo")
            if hf_repo and hf_repo not in aliases:
                aliases.append(hf_repo)
            for alias in preset.get("aliases", []) or []:
                if isinstance(alias, str) and alias not in aliases:
                    aliases.append(alias)
            data.append({
                "id": name,
                "object": "model",
                "created": created,
                "owned_by": "meshnet",
                "name": name,
                "hf_repo": hf_repo,
                "revision": preset.get("revision"),
                "aliases": aliases,
                "metadata": dict(preset.get("metadata") or _model_metadata_from_nodes(model_nodes)),
                "recommended": bool(preset.get("recommended", False)),
                "deployment": _deployment_summary(alive, preset),
                "shard_coverage_percentage": coverage,
                "served_model_copies": served_copies,
                "quantizations": quantizations,
                "pricing": self._model_pricing_payload(name),
            })
            seen_ids.add(name)
            # Note: the preset's hf_repo is deliberately NOT added to seen_ids —
            # nodes registered with an explicit hf_repo also get their own
            # repo-keyed entry below, with the node's short name as an alias.

        hf_model_ids = sorted({
            node.hf_repo or node.model
            for node in alive
            if node.model is not None
            and node.model not in server.model_presets
            and node.shard_start is not None
            and node.shard_end is not None
            and node.num_layers is not None
        })
        for model_id in hf_model_ids:
            if model_id is None or model_id in seen_ids:
                continue
            model_nodes = [
                node for node in alive
                if node.shard_start is not None
                and node.shard_end is not None
                and node.num_layers is not None
                and (node.hf_repo == model_id or (node.hf_repo is None and node.model == model_id))
            ]
            if not model_nodes:
                continue
            short_names = sorted({node.model for node in model_nodes if node.model})
            aliases = [model_id, *[name for name in short_names if name != model_id]]
            required_start = 0
            required_end = max(node.num_layers for node in model_nodes) - 1
            coverage = _coverage_percentage(
                model_nodes,
                required_start,
                required_end,
            )
            data.append({
                "id": model_id,
                "object": "model",
                "created": created,
                "owned_by": "meshnet",
                "name": short_names[0] if short_names else model_id,
                "hf_repo": model_id if any(node.hf_repo == model_id for node in model_nodes) else None,
                "aliases": aliases,
                "metadata": _model_metadata_from_nodes(model_nodes),
                "shard_coverage_percentage": coverage,
                "served_model_copies": _served_model_copies(
                    model_nodes,
                    required_start,
                    required_end,
                ),
                "pricing": self._model_pricing_payload(model_id),
            })
            seen_ids.add(model_id)
        self._send_json(200, {"object": "list", "data": data})

    def _handle_coverage(self, model: str):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        # Do NOT purge before coverage — dead nodes are included with alive=false
        # so operators can see what was covering each layer band before failure.
        with server.lock:
            resolved = _nodes_and_bounds_for_model(server, model)
        if resolved is None:
            self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
            return
        all_nodes, required_start, required_end = resolved
        if server.contracts is not None:
            all_nodes = [
                node for node in all_nodes
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]
        coverage = _coverage_map_detailed(all_nodes, required_start, required_end, server.heartbeat_timeout)
        self._send_json(200, {"model": model, "coverage": coverage})

    def _handle_tracker_nodes(self, model: str):
        """Return head workers: worker nodes that can start inference for a model.

        The historical endpoint name is /v1/tracker-nodes, but these are not
        tracker processes and they are the only machines that load model shards.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        resolved_name, preset = _resolve_model_preset(server.model_presets, model)
        if preset is None:
            self._send_json(404, {"error": f"unknown model preset: {model!r}"})
            return
        required_start, _ = _preset_layer_bounds(preset)
        with server.lock:
            self._purge_expired_nodes()
            alive = [
                node for node in server.registry.values()
                if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
            ]
        if server.contracts is not None:
            alive = [
                node for node in alive
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]
        tracker_nodes = [
            node for node in alive
            if node.shard_start is not None
            and node.shard_start == required_start
            and node.tracker_mode
        ]
        self._send_json(200, {
            "model": resolved_name,
            "head_workers": [
                {
                    "node_id": node.node_id,
                    "endpoint": node.endpoint,
                    "relay_addr": node.relay_addr,
                    "peer_id": node.peer_id,
                }
                for node in tracker_nodes
            ],
            "tracker_nodes": [
                {
                    "node_id": node.node_id,
                    "endpoint": node.endpoint,
                    "relay_addr": node.relay_addr,
                    "peer_id": node.peer_id,
                    "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
                }
                for node in tracker_nodes
            ],
        })

    def _handle_network_map(self) -> None:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        with server.lock:
            self._purge_expired_nodes()
            nodes = list(server.registry.values())
            memory_pool = _memory_pool_map(server)

        def capacity_for(node: _NodeEntry) -> dict:
            preset = None
            if node.model:
                preset = server.model_presets.get(node.model)
            if preset is None and node.hf_repo and node.num_layers:
                preset = _hf_rebalance_preset([node])
            return _node_capacity_summary(node, preset)

        def throughput_for(node: _NodeEntry) -> dict:
            if server.stats is None:
                return {}
            models = [m for m in (node.hf_repo, node.model) if m]
            result = {}
            for model in models:
                result[model] = server.stats.get_node_model_stats(node.node_id, model)
            return result

        def model_supply_for(node: _NodeEntry) -> dict:
            return _model_health_summary(server, node.model, node.hf_repo)

        def capability_for(node: _NodeEntry) -> dict:
            # Re-verdicted against what the node advertises now, so the operator
            # view and the routing gate can never disagree.
            state = _node_admission(node)
            return {
                **state.to_dict(),
                "routable": state.routable_under(server.capability_policy),
            }

        self._send_json(200, {
            "relay_url": server.relay_url,
            "relay": dict(server.relay_status),
            "capability_policy": server.capability_policy,
            "pool": _pool_summary(nodes),
            "memory_pool": memory_pool,
            "recommended_models": [
                {
                    "id": name,
                    "hf_repo": preset.get("hf_repo"),
                    "revision": preset.get("revision"),
                    "aliases": list(preset.get("aliases", []) or []),
                    "metadata": dict(preset.get("metadata") or {}),
                    "deployment": _deployment_summary(nodes, preset),
                }
                for name, preset in server.model_presets.items()
                if preset.get("recommended")
            ],
            "nodes": [
                {
                    "node_id": node.node_id,
                    "friendly_name": node.friendly_name,
                    "endpoint": node.endpoint,
                    "relay_addr": node.relay_addr,
                    "peer_id": node.peer_id,
                    "model": node.model,
                    "hf_repo": node.hf_repo,
                    "num_layers": node.num_layers,
                    "model_metadata": dict(node.model_metadata),
                    "downloaded_models": [dict(item) for item in node.downloaded_models],
                    "shard_start": node.shard_start,
                    "shard_end": node.shard_end,
                    "tracker_mode": node.tracker_mode,
                    "last_heartbeat": node.last_heartbeat,
                    "capacity": capacity_for(node),
                    "model_supply": model_supply_for(node),
                    "throughput": throughput_for(node),
                    "capability": capability_for(node),
                    "stats": _node_health(node, server.heartbeat_timeout),
                }
                for node in nodes
            ],
        })

    # ---------------------------------------------------------------- OpenAI proxy

    def _handle_proxy_chat(self) -> None:
        """Proxy POST /v1/chat/completions to a tracker-mode (first-shard) node.

        Picks a live tracker-mode node for the requested model using round-robin,
        then forwards the request verbatim and relays the response (including
        streaming SSE chunks) back to the caller.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]

        length = int(self.headers.get("Content-Length", 0))
        raw_body = self.rfile.read(length) if length else b"{}"

        try:
            body = json.loads(raw_body)
        except json.JSONDecodeError:
            self._send_json(400, {"error": {"message": "invalid JSON", "type": "invalid_request_error", "code": "invalid_request"}})
            return

        model: str = body.get("model", "")
        is_stream: bool = bool(body.get("stream", False))
        requested_quantization = _normalize_quantization(
            body.get("quantization", "bfloat16")
        )
        if requested_quantization is None:
            self._send_json(400, {"error": {
                "message": "quantization must be one of bfloat16, int8, or nf4",
                "type": "invalid_request_error",
                "code": "invalid_quantization",
            }})
            return
        body["quantization"] = requested_quantization

        if model and server.stats is not None:
            server.stats.record_request(model)

        # Billing gate (ADR-0015): reject before any routing — no free work.
        api_key = _api_key_from_headers(self.headers)
        if server.billing is not None:
            if api_key is None:
                self._send_json(401, {"error": {
                    "message": "missing API key: send Authorization: Bearer <key>",
                    "type": "invalid_request_error",
                    "code": "missing_api_key",
                }})
                return
            if server.accounts is not None and server.accounts.is_key_revoked(api_key):
                self._send_json(401, {"error": {
                    "message": "API key has been revoked",
                    "type": "invalid_request_error",
                    "code": "invalid_api_key",
                }})
                return
            # US-039: with accounts enabled, only real account keys may spend —
            # arbitrary bearer strings must never become billable clients.
            if server.accounts is not None and not server.accounts.is_active_key(api_key):
                self._send_json(401, {"error": {
                    "message": "unknown API key: create one at /dashboard (register, then + new key)",
                    "type": "invalid_request_error",
                    "code": "invalid_api_key",
                }})
                return
            if not server.billing.has_funds(api_key):
                self._send_json(402, {"error": {
                    "message": "insufficient balance: deposit USDT to continue",
                    "type": "insufficient_quota",
                    "code": "insufficient_balance",
                }})
                return
            if server.max_charge_per_request is not None:
                token_limit = _requested_completion_token_limit(body)
                if token_limit is None:
                    self._send_json(400, {"error": {
                        "message": (
                            "max_charge_per_request is enabled; include max_tokens "
                            "or max_completion_tokens so this request can be capped"
                        ),
                        "type": "invalid_request_error",
                        "code": "missing_token_limit",
                    }})
                    return
                in_rate, out_rate = server.billing.prices_for(model)
                prompt_estimate = _estimate_prompt_tokens(body) or 0
                estimated_charge = (in_rate * prompt_estimate + out_rate * token_limit) / 1000.0
                if estimated_charge > server.max_charge_per_request:
                    self._send_json(402, {"error": {
                        "message": (
                            f"request exceeds max_charge_per_request "
                            f"({estimated_charge:.6f} USDT estimated > "
                            f"{server.max_charge_per_request:.6f} USDT cap)"
                        ),
                        "type": "insufficient_quota",
                        "code": "spend_cap_exceeded",
                    }})
                    return

        # US-030: optional pinned route — "route": [node_id, ...] uses those
        # nodes in order instead of auto-selection. Absent field: unchanged.
        pinned_ids = body.get("route")
        pinned_nodes: list[_NodeEntry] | None = None
        if pinned_ids is not None:
            if (
                not isinstance(pinned_ids, list)
                or not pinned_ids
                or not all(isinstance(nid, str) and nid for nid in pinned_ids)
            ):
                self._send_json(400, {"error": {
                    "message": "route must be a non-empty list of node id strings",
                    "type": "invalid_request_error",
                    "code": "invalid_route",
                }})
                return
            with server.lock:
                self._purge_expired_nodes()
                missing = [nid for nid in pinned_ids if nid not in server.registry]
                if missing:
                    self._send_json(400, {"error": {
                        "message": f"unknown node ids in route: {missing}",
                        "type": "invalid_request_error",
                        "code": "unknown_route_nodes",
                    }})
                    return
                pinned_nodes = [server.registry[nid] for nid in pinned_ids]

        if pinned_nodes is not None:
            if any(
                not _quantization_satisfies(node.quantization, requested_quantization)
                for node in pinned_nodes
            ):
                self._send_json(409, {"error": {
                    "message": "pinned route does not satisfy requested quantization",
                    "type": "invalid_request_error",
                    "code": "route_quantization_unavailable",
                }})
                return
            node = pinned_nodes[0]
        else:
            # Find a live tracker-mode node for this model
            with server.lock:
                self._purge_expired_nodes()
                candidates = [
                    n for n in server.registry.values()
                    if n.tracker_mode
                    and _node_matches_model(n, model)
                    and _quantization_satisfies(n.quantization, requested_quantization)
                    and _capability_routable(n, server.capability_policy)
                ]

            if not candidates:
                # Fall back: any node serving shard_start=0 for this model
                with server.lock:
                    candidates = [
                        n for n in server.registry.values()
                        if n.shard_start == 0
                        and _node_matches_model(n, model)
                        and _quantization_satisfies(n.quantization, requested_quantization)
                        and _capability_routable(n, server.capability_policy)
                    ]

            if not candidates:
                with server.lock:
                    resolved_name, preset = _resolve_model_preset(server.model_presets, model)
                    placements = (
                        _queue_initial_model_placement_locked(
                            server,
                            resolved_name or model,
                            requested_quantization,
                        )
                        if preset is not None and preset.get("recommended")
                        else []
                    )
                    if preset is not None:
                        required_start, required_end = _preset_layer_bounds(preset)
                        qualifying = [
                            n for n in server.registry.values()
                            if _node_matches_preset(n, resolved_name, preset)
                            and _quantization_satisfies(
                                n.quantization, requested_quantization,
                            )
                        ]
                        coverage = _coverage_percentage(
                            qualifying, required_start, required_end,
                        )
                    else:
                        coverage = 0.0
                    if placements:
                        self._send_json(503, {"error": {
                            "message": f"model {model!r} is being deployed",
                            "type": "service_unavailable",
                            "code": "model_loading",
                            "retry_after_seconds": 30,
                            "requested_quantization": requested_quantization,
                            "shard_coverage_percentage": coverage,
                            "placements": placements,
                        }})
                        return
                    registered = [
                        {
                            "node_id": n.node_id,
                            "model": n.model,
                            "hf_repo": n.hf_repo,
                            "shard": f"{n.shard_start}-{n.shard_end}",
                            "tracker_mode": n.tracker_mode,
                        }
                        for n in server.registry.values()
                    ]
                _tracker_log(
                    server,
                    "warn",
                    "no nodes available for model",
                    model=model,
                    registered_nodes=registered,
                )
                self._send_json(503, {"error": {
                    "message": f"no nodes available for model {model!r}",
                    "type": "service_unavailable",
                    "code": "model_not_available",
                }})
                return

            node = max(candidates, key=lambda n: _effective_throughput(n, model))
        target_url = f"{node.endpoint}/v1/chat/completions"
        request_id = str(body.get("id") or f"req-{time.time_ns():x}")
        body["id"] = request_id
        raw_body = json.dumps(body).encode()

        # Pre-resolve the downstream route so the first-shard node skips its own
        # tracker query.  We already hold the full registry picture — no need for
        # a second round-trip.
        route_model = node.hf_repo or node.model or model
        with server.lock:
            resolved_route_model, route_preset = _resolve_model_preset(server.model_presets, route_model)
            if route_preset is not None:
                route_model = resolved_route_model or route_model
                preset = route_preset
                rs, re = _preset_layer_bounds(preset)
                all_nodes: list = [
                    n for n in server.registry.values()
                    if _node_matches_preset(n, route_model, preset)
                    and n.shard_start is not None
                    and n.shard_end is not None
                    and _quantization_satisfies(n.quantization, requested_quantization)
                ]
            else:
                all_nodes = [
                    n for n in server.registry.values()
                    if _node_matches_model(n, route_model)
                    and n.shard_start is not None and n.num_layers is not None
                    and _quantization_satisfies(n.quantization, requested_quantization)
                ]
                rs, re = 0, (max((n.num_layers for n in all_nodes), default=1) - 1)
        if pinned_nodes is not None:
            route_nodes = pinned_nodes
            routing_decision = {"mode": "pinned"}
        else:
            # ADR-0021: enumerate viable routes and pick one bandit-style —
            # ε-scout among unproven routes, otherwise weighted ∝ observed tps^α.
            route_candidates = _enumerate_routes(
                all_nodes, rs, re,
                model=route_model,
                contracts=server.contracts,
                max_candidates=server.route_stats.config.max_candidate_routes,
                policy=server.capability_policy,
            )
            picked, routing_decision = choose_route(
                route_candidates, server.route_stats, route_model, rng=server.route_rng,
            )
            if picked is not None:
                route_nodes = picked.nodes
            else:
                # No head-anchored candidate — legacy greedy cover as fallback
                # (also produces the layer-gap error message).
                route_nodes, route_error = _select_route(
                    all_nodes, rs, re,
                    model=route_model,
                    contracts=server.contracts,
                    policy=server.capability_policy,
                )
                routing_decision = {"mode": "greedy-fallback"}
                if route_error:
                    _tracker_log(
                        server,
                        "warn",
                        "route unavailable",
                        model=model,
                        route_model=route_model,
                        error=route_error,
                        candidate_count=len(all_nodes),
                        candidates=_node_route_summary(all_nodes),
                    )
                    self._send_json(503, {"error": {
                        "message": route_error,
                        "type": "service_unavailable",
                        "code": "route_not_available",
                    }})
                    return
            # The proxy target must be the route's own head: an independently
            # chosen fastest node may not be part of the planned route, which
            # previously injected downstream hops with wrong start layers
            # (ADR-0020 mixed-topology flaw).
            if route_nodes:
                node = route_nodes[0]
                target_url = f"{node.endpoint}/v1/chat/completions"
        # Compute start_layer for each hop: each node begins where the previous ended + 1.
        # This allows overlapping shard registrations without double-computation.
        covered_up_to = rs - 1
        route_hops: list[dict] = []
        node_work: list[tuple[str | None, int]] = []
        for rn in route_nodes:
            hop: dict = {"endpoint": rn.endpoint, "start_layer": covered_up_to + 1}
            if rn.relay_addr:
                hop["relay_addr"] = rn.relay_addr
            route_hops.append(hop)
            effective_end = rn.shard_end if rn.shard_end is not None else covered_up_to
            node_work.append((rn.wallet_address, max(0, effective_end - covered_up_to)))
            covered_up_to = effective_end
        # Strip the first-shard node we're about to proxy to — it's already handling the request.
        downstream_hops = [
            h for h in route_hops
            if _endpoint_key(h["endpoint"]) != _endpoint_key(node.endpoint)
        ]
        downstream_urls = json.dumps(downstream_hops)
        route_debug = " -> ".join(
            f"{n.node_id}@{n.endpoint}[{n.shard_start}-{n.shard_end}]"
            for n in route_nodes
        )
        inflight_nodes = route_nodes or [node]
        inflight_recorded = True
        _record_proxy_inflight(server, inflight_nodes, 1)

        def finish_proxy_inflight() -> None:
            nonlocal inflight_recorded
            if inflight_recorded:
                _record_proxy_inflight(server, inflight_nodes, -1)
                inflight_recorded = False
            _unregister_active_proxy(server, request_id)

        proxy_ctx = _register_active_proxy(server, request_id)

        _tracker_log(
            server,
            "info",
            "proxy route selected",
            request_id=request_id,
            model=model,
            head_node_id=node.node_id,
            head_endpoint=node.endpoint,
            downstream=downstream_urls,
            route=route_debug or "<empty>",
            routing=routing_decision,
            nodes=_node_route_summary(route_nodes),
        )

        req = urllib.request.Request(
            target_url,
            data=raw_body,
            headers={
                "Content-Type": "application/json",
                "X-Meshnet-Route": downstream_urls,
                "X-Meshnet-Request-Id": request_id,
            },
            method="POST",
        )
        # Copy Authorization header from client if present
        auth = self.headers.get("Authorization")
        if auth:
            req.add_header("Authorization", auth)

        relay_headers = {
            "Content-Type": "application/json",
            "X-Meshnet-Route": downstream_urls,
            "X-Meshnet-Request-Id": request_id,
            **({"Authorization": auth} if auth else {}),
        }

        if node.relay_addr:
            _tracker_log(
                server,
                "info",
                "proxy via relay",
                request_id=request_id,
                relay_addr=node.relay_addr,
                direct_endpoint=target_url,
            )
            started = time.monotonic()
            relay_ws_holder: list[Any] = []
            frames = _relay_http_request_frames(
                node.relay_addr,
                path="/v1/chat/completions",
                body=raw_body,
                headers=relay_headers,
                cancel_event=proxy_ctx.cancel_event,
                ws_holder=relay_ws_holder,
                ws_lock=proxy_ctx.relay_ws_lock,
            )
            first = next(frames, None)
            with proxy_ctx.relay_ws_lock:
                proxy_ctx.relay_ws = relay_ws_holder[0] if relay_ws_holder else None
            if proxy_ctx.cancel_event.is_set():
                if self._finalize_proxy_cancel(
                    proxy_ctx=proxy_ctx,
                    server=server,
                    request_id=request_id,
                    started=started,
                    model=model,
                    route_model=route_model,
                    route_nodes=route_nodes,
                    api_key=api_key,
                    node_work=node_work,
                    body=body,
                    finish_proxy_inflight=finish_proxy_inflight,
                ):
                    return
            if first is not None and first.get("stream"):
                # Streamed response (US-036): forward SSE chunks as they arrive
                # and run the same token accounting as the direct stream path.
                self._stream_relayed_frames(
                    first, frames, started,
                    model, route_model, route_nodes, api_key, node_work,
                    request_body=body,
                    request_id=request_id,
                    proxy_ctx=proxy_ctx,
                    finish_proxy_inflight=finish_proxy_inflight,
                )
                finish_proxy_inflight()
                return
            if first is not None:
                elapsed = time.monotonic() - started
                self._send_relayed_response(first)
                if int(first.get("status", 503)) < 400:
                    body_text = first.get("body") or ""
                    try:
                        in_tokens, out_tokens = _billable_non_stream_split(json.loads(body_text), body)
                    except (json.JSONDecodeError, TypeError):
                        in_tokens, out_tokens = 0, 0
                    tokens = in_tokens + out_tokens
                    self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
                    _clear_proxy_progress_log_state(server, request_id)
                    _tracker_log(
                        server,
                        "info",
                        "proxy complete",
                        request_id=request_id,
                        model=model,
                        route_model=route_model,
                        status=int(first.get("status", 503)),
                        tokens=tokens,
                        elapsed_seconds=round(elapsed, 4),
                        tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
                        route=_node_route_summary(route_nodes),
                    )
                    self._bill_completed(
                        api_key, model, tokens, node_work,
                        input_tokens=in_tokens, output_tokens=out_tokens,
                    )
                finish_proxy_inflight()
                return
            _tracker_log(
                server,
                "warn",
                "relay proxy failed, trying direct",
                request_id=request_id,
                relay_addr=node.relay_addr,
                direct_endpoint=target_url,
            )

        try:
            started = time.monotonic()
            _tracker_log(
                server,
                "info",
                "proxy connecting",
                request_id=request_id,
                target_url=target_url,
                stream=is_stream or None,
            )
            upstream_result: list[Any] = []
            connect_errors: list[BaseException] = []

            def _connect_upstream() -> None:
                try:
                    upstream_result.append(urllib.request.urlopen(req, timeout=300.0))
                except BaseException as exc:
                    connect_errors.append(exc)

            connect_thread = threading.Thread(target=_connect_upstream, daemon=True)
            connect_thread.start()
            while connect_thread.is_alive():
                if proxy_ctx.cancel_event.is_set():
                    connect_thread.join(timeout=310.0)
                    if upstream_result:
                        try:
                            upstream_result[0].close()
                        except Exception:
                            pass
                    if self._finalize_proxy_cancel(
                        proxy_ctx=proxy_ctx,
                        server=server,
                        request_id=request_id,
                        started=started,
                        model=model,
                        route_model=route_model,
                        route_nodes=route_nodes,
                        api_key=api_key,
                        node_work=node_work,
                        body=body,
                        finish_proxy_inflight=finish_proxy_inflight,
                    ):
                        return
                connect_thread.join(0.2)

            if proxy_ctx.cancel_event.is_set():
                if upstream_result:
                    try:
                        upstream_result[0].close()
                    except Exception:
                        pass
                if self._finalize_proxy_cancel(
                    proxy_ctx=proxy_ctx,
                    server=server,
                    request_id=request_id,
                    started=started,
                    model=model,
                    route_model=route_model,
                    route_nodes=route_nodes,
                    api_key=api_key,
                    node_work=node_work,
                    body=body,
                    finish_proxy_inflight=finish_proxy_inflight,
                ):
                    return

            if connect_errors:
                raise connect_errors[0]

            upstream = upstream_result[0]
            with proxy_ctx.upstream_lock:
                proxy_ctx.upstream = upstream
            upstream_sock = _upstream_socket(upstream)
            if upstream_sock is not None:
                _set_upstream_read_timeout(upstream, None)
            else:
                _set_upstream_read_timeout(upstream, 0.5)
            _tracker_log(server, "info", "proxy connected", request_id=request_id, target_url=target_url)
        except urllib.error.HTTPError as exc:
            # Relay error status + body from node
            err_body = exc.read()
            self.send_response(exc.code)
            self.send_header("Content-Type", "application/json")
            self.send_header("Content-Length", str(len(err_body)))
            self.end_headers()
            try:
                self.wfile.write(err_body)
            except BrokenPipeError:
                pass
            _clear_proxy_progress_log_state(server, request_id)
            finish_proxy_inflight()
            return
        except Exception as exc:
            _clear_proxy_progress_log_state(server, request_id)
            if node.relay_addr:
                _tracker_log(
                    server,
                    "error",
                    "direct proxy failed after relay",
                    request_id=request_id,
                    target_url=target_url,
                    relay_addr=node.relay_addr,
                    error=repr(exc),
                )
            else:
                _tracker_log(
                    server,
                    "error",
                    "proxy failed",
                    request_id=request_id,
                    target_url=target_url,
                    error=repr(exc),
                )
            self._send_json(503, {"error": {
                "message": f"upstream node unreachable: {exc}",
                "type": "service_unavailable",
                "code": "upstream_error",
            }})
            finish_proxy_inflight()
            return

        with upstream:
            content_type = upstream.headers.get("Content-Type", "application/json")
            if is_stream or "text/event-stream" in content_type:
                # Relay SSE stream chunk-by-chunk
                self.send_response(200)
                self.send_header("Content-Type", "text/event-stream; charset=utf-8")
                self.send_header("Cache-Control", "no-cache")
                self.send_header("X-Accel-Buffering", "no")
                self.end_headers()
                stream_usage: dict | None = None
                observed_stream_tokens = 0
                client_gone = False
                try:
                    while True:
                        if proxy_ctx.cancel_event.is_set():
                            break
                        if upstream_sock is not None:
                            readable, _, _ = select.select([upstream_sock], [], [], 0.5)
                            if not readable:
                                continue
                        try:
                            line = upstream.readline()
                        except TimeoutError:
                            continue
                        if not line:
                            if proxy_ctx.cancel_event.is_set():
                                break
                            break
                        if not client_gone:
                            try:
                                self.wfile.write(line)
                                self.wfile.flush()
                            except (BrokenPipeError, ConnectionResetError):
                                # Keep draining upstream so completed node work is still billed.
                                client_gone = True
                        observed, usage = _stream_line_tokens(line)
                        observed_stream_tokens += observed
                        if observed:
                            _tracker_log_proxy_progress(
                                server,
                                request_id=request_id,
                                model=model,
                                route_model=route_model,
                                tokens=observed_stream_tokens,
                                started=started,
                                route_nodes=route_nodes,
                            )
                        if usage is not None:
                            stream_usage = usage
                except (BrokenPipeError, ConnectionResetError):
                    client_gone = True
                if self._finalize_proxy_cancel(
                    proxy_ctx=proxy_ctx,
                    server=server,
                    request_id=request_id,
                    started=started,
                    model=model,
                    route_model=route_model,
                    route_nodes=route_nodes,
                    api_key=api_key,
                    node_work=node_work,
                    body=body,
                    finish_proxy_inflight=finish_proxy_inflight,
                    observed_stream_tokens=observed_stream_tokens,
                    stream_usage=stream_usage,
                ):
                    return
                elapsed = time.monotonic() - started
                # Bill even on client disconnect — the nodes did the work.
                # Observed stream chunks are authoritative for the upper bound;
                # upstream usage may only lower that count.
                in_tokens, out_tokens = _stream_billable_split(
                    observed_stream_tokens, stream_usage, body
                )
                self._record_observed_throughput(
                    model, route_model, in_tokens + out_tokens, elapsed, route_nodes
                )
                tokens = in_tokens + out_tokens
                _clear_proxy_progress_log_state(server, request_id)
                _tracker_log(
                    server,
                    "info",
                    "proxy complete",
                    request_id=request_id,
                    model=model,
                    route_model=route_model,
                    status=200,
                    stream=True,
                    client_disconnected=client_gone,
                    tokens=tokens,
                    elapsed_seconds=round(elapsed, 4),
                    tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
                    route=_node_route_summary(route_nodes),
                )
                self._bill_completed(
                    api_key, model, tokens, node_work,
                    input_tokens=in_tokens, output_tokens=out_tokens,
                )
            else:
                # Non-streaming: buffer and relay
                resp_body = upstream.read()
                elapsed = time.monotonic() - started
                observed_output = ""
                try:
                    response_payload = json.loads(resp_body)
                    in_tokens, out_tokens = _billable_non_stream_split(response_payload, body)
                    observed_output = _observed_output_from_non_stream_payload(response_payload)
                except json.JSONDecodeError:
                    in_tokens, out_tokens = 0, 0
                tokens = in_tokens + out_tokens
                self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
                _clear_proxy_progress_log_state(server, request_id)
                _tracker_log(
                    server,
                    "info",
                    "proxy complete",
                    request_id=request_id,
                    model=model,
                    route_model=route_model,
                    target_url=target_url,
                    status=200,
                    bytes=len(resp_body),
                    tokens=tokens,
                    elapsed_seconds=round(elapsed, 4),
                    tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
                    route=_node_route_summary(route_nodes),
                )
                self._record_validation_event(request_id, model, body, observed_output, route_nodes)
                self._bill_completed(
                    api_key, model, tokens, node_work,
                    input_tokens=in_tokens, output_tokens=out_tokens,
                )
                self.send_response(200)
                self.send_header("Content-Type", content_type)
                self.send_header("Content-Length", str(len(resp_body)))
                self.end_headers()
                try:
                    self.wfile.write(resp_body)
                except (BrokenPipeError, ConnectionResetError):
                    pass
        finish_proxy_inflight()

    def _record_observed_throughput(
        self,
        requested_model: str,
        route_model: str,
        total_tokens: int,
        elapsed_seconds: float,
        route_nodes: list[_NodeEntry],
    ) -> None:
        """Record observed route TPS for participating nodes.

        The tracker sees end-to-end request duration, not per-hop timings, so
        each hop gets the same route-level observation for now. Per-hop telemetry
        can refine this later without changing the external stats shape.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if route_model and route_nodes:
            # Route-level bandit sample (ADR-0021); the store itself rejects
            # near-empty completions that would poison the arm.
            server.route_stats.record_sample(
                route_model,
                route_signature(route_model, route_nodes),
                total_tokens,
                elapsed_seconds,
            )
        if server.stats is None or total_tokens <= 0:
            return
        elapsed_seconds = max(elapsed_seconds, 1e-6)
        models = [m for m in (requested_model, route_model) if m]
        if len(models) == 2 and models[0] == models[1]:
            models = [models[0]]
        for node in route_nodes:
            for model in models:
                server.stats.record_node_throughput(
                    node.node_id,
                    model,
                    total_tokens=total_tokens,
                    elapsed_seconds=elapsed_seconds,
                )
                stats = server.stats.get_node_model_stats(node.node_id, model)
                observed = stats.get("tokens_per_sec_last_hour")
                if observed is not None:
                    node.model_tokens_per_sec[model] = float(observed)

    def _record_validation_event(
        self,
        session_id: str,
        model: str,
        request_body: dict,
        observed_output: str,
        route_nodes: list[_NodeEntry],
    ) -> None:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        validation = getattr(server.contracts, "validation", None) if server.contracts is not None else None
        if validation is None:
            return
        messages = request_body.get("messages")
        if not isinstance(messages, list):
            messages = []
        event_nodes = [
            {
                "node_id": node.node_id,
                "endpoint": node.endpoint,
                "wallet_address": node.wallet_address,
                "shard_start": node.shard_start,
                "shard_end": node.shard_end,
            }
            for node in route_nodes
        ]
        try:
            validation.record_completed_inference(
                session_id=session_id,
                model=model,
                messages=[dict(message) for message in messages if isinstance(message, dict)],
                observed_output=observed_output,
                route_nodes=event_nodes,
            )
        except Exception as exc:
            print(f"[tracker] validation event recording failed for {session_id}: {exc}", flush=True)

    def _bill_completed(
        self,
        api_key: str | None,
        model: str,
        total_tokens: int,
        node_work: list[tuple[str | None, int]],
        *,
        input_tokens: int | None = None,
        output_tokens: int | None = None,
    ) -> None:
        """Charge a completed request against the billing ledger (ADR-0015).

        With ``input_tokens``/``output_tokens`` the ledger bills each side at
        its own rate (US-045); ``total_tokens`` alone falls back to the
        blended rate.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.billing is None or api_key is None:
            return
        # Probationary period (issue 08): a wallet's first N jobs earn nothing —
        # its share stays in the protocol cut. Job counts live in the registry
        # contract, so this only applies when the tracker runs with contracts.
        if server.contracts is not None:
            adjusted: list[tuple[str | None, int]] = []
            for wallet, work in node_work:
                if wallet:
                    in_probation = server.contracts.registry.probationary_jobs_remaining(wallet) > 0
                    server.contracts.registry.record_completed_job(wallet)
                    if in_probation:
                        wallet = None
                adjusted.append((wallet, work))
            node_work = adjusted
        try:
            event = server.billing.charge_request(
                api_key, model, total_tokens, node_work,
                input_tokens=input_tokens, output_tokens=output_tokens,
            )
            print(
                f"[tracker] billed api_key=…{api_key[-6:]}: model={model!r} "
                f"tokens={event['total_tokens']} "
                f"(in={event.get('input_tokens', '?')} out={event.get('output_tokens', '?')}) "
                f"cost={event['cost']:.6f} USDT shares={event['shares']}",
                flush=True,
            )
        except Exception as exc:
            print(f"[tracker] billing failed for model={model!r}: {exc}", flush=True)

    def _stream_relayed_frames(
        self,
        first: dict,
        frames,
        started: float,
        model: str,
        route_model: str,
        route_nodes: list,
        api_key: str | None,
        node_work: list,
        request_body: dict,
        request_id: str,
        *,
        proxy_ctx: _ActiveProxyContext | None = None,
        finish_proxy_inflight: Any = None,
    ) -> None:
        """Forward a streamed relay response (US-036) to the client as SSE,
        billing with the same accounting as the direct stream path."""
        headers = first.get("headers") if isinstance(first.get("headers"), dict) else {}
        self.send_response(int(first.get("status", 200)))
        self.send_header("Content-Type", headers.get("Content-Type", "text/event-stream; charset=utf-8"))
        self.send_header("Cache-Control", "no-cache")
        self.send_header("X-Accel-Buffering", "no")
        self.end_headers()
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        stream_usage: dict | None = None
        observed_stream_tokens = 0
        client_gone = False
        for frame in itertools.chain([first], frames):
            if proxy_ctx is not None and proxy_ctx.cancel_event.is_set():
                break
            chunk = frame.get("chunk") or ""
            if not chunk:
                continue
            data = chunk.encode()
            if not client_gone:
                try:
                    self.wfile.write(data)
                    self.wfile.flush()
                except BrokenPipeError:
                    # Keep draining frames — the nodes did the work; bill it.
                    client_gone = True
            for line in data.splitlines():
                observed, usage = _stream_line_tokens(line)
                observed_stream_tokens += observed
                if observed:
                    _tracker_log_proxy_progress(
                        server,
                        request_id=request_id,
                        model=model,
                        route_model=route_model,
                        tokens=observed_stream_tokens,
                        started=started,
                        route_nodes=route_nodes,
                        relay=True,
                    )
                if usage is not None:
                    stream_usage = usage
        if (
            proxy_ctx is not None
            and finish_proxy_inflight is not None
            and self._finalize_proxy_cancel(
                proxy_ctx=proxy_ctx,
                server=server,
                request_id=request_id,
                started=started,
                model=model,
                route_model=route_model,
                route_nodes=route_nodes,
                api_key=api_key,
                node_work=node_work,
                body=request_body,
                finish_proxy_inflight=finish_proxy_inflight,
                observed_stream_tokens=observed_stream_tokens,
                stream_usage=stream_usage,
            )
        ):
            return
        elapsed = time.monotonic() - started
        in_tokens, out_tokens = _stream_billable_split(
            observed_stream_tokens, stream_usage, request_body
        )
        self._record_observed_throughput(
            model, route_model, in_tokens + out_tokens, elapsed, route_nodes
        )
        tokens = in_tokens + out_tokens
        _clear_proxy_progress_log_state(server, request_id)
        _tracker_log(
            server,
            "info",
            "proxy complete",
            request_id=request_id,
            model=model,
            route_model=route_model,
            status=200,
            stream=True,
            client_disconnected=client_gone,
            relay=True,
            tokens=tokens,
            elapsed_seconds=round(elapsed, 4),
            tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 else 0.0,
            route=_node_route_summary(route_nodes),
        )
        self._bill_completed(
            api_key, model, tokens, node_work,
            input_tokens=in_tokens, output_tokens=out_tokens,
        )

    def _send_relayed_response(self, response: dict) -> None:
        status = int(response.get("status", 503))
        headers = response.get("headers") if isinstance(response.get("headers"), dict) else {}
        body_text = response.get("body") or ""
        body = body_text.encode() if isinstance(body_text, str) else bytes(body_text)
        self.send_response(status)
        self.send_header("Content-Type", headers.get("Content-Type", "application/json"))
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        try:
            self.wfile.write(body)
        except BrokenPipeError:
            pass

    def _handle_register(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_json_body()
        if body is None:
            return

        # --- Raft cluster mode: forward to leader or propose via Raft ---
        if server.raft is not None:
            if not server.raft.is_leader:
                leader = server.raft.leader()
                if leader is None:
                    self._send_json(503, {"error": "no leader elected — retry in a moment"})
                    return
                # Proxy to leader
                try:
                    data = json.dumps(body).encode()
                    req = urllib.request.Request(
                        f"{leader}/v1/nodes/register",
                        data=data,
                        headers={"Content-Type": "application/json"},
                        method="POST",
                    )
                    with urllib.request.urlopen(req, timeout=5.0) as r:
                        resp_body = r.read()
                    self.send_response(200)
                    self.send_header("Content-Type", "application/json")
                    self.send_header("Content-Length", str(len(resp_body)))
                    self.end_headers()
                    self.wfile.write(resp_body)
                except Exception as exc:
                    self._send_json(503, {"error": f"leader proxy failed: {exc}"})
                return
            # Leader path: fall through to normal registration, then replicate via Raft.
            # We let the registration run first (to generate node_id), then asynchronously
            # propose to the Raft log so followers can replicate the entry.
            # Raft proposal happens after the response is sent (fire-and-forget replication).

        endpoint = body.get("endpoint")
        if not isinstance(endpoint, str) or not endpoint:
            self._send_json(400, {"error": "endpoint is required"})
            return
        parsed_endpoint = urllib.parse.urlparse(endpoint)
        if parsed_endpoint.scheme not in {"http", "https"} or not parsed_endpoint.netloc:
            self._send_json(400, {"error": "endpoint must be an http(s) URL"})
            return

        shard_start: int | None
        shard_end: int | None
        explicit_shard = "shard_start" in body or "shard_end" in body
        if explicit_shard:
            try:
                shard_start = int(body["shard_start"])
                shard_end = int(body["shard_end"])
            except (KeyError, TypeError, ValueError):
                self._send_json(400, {"error": "shard_start and shard_end must be numeric"})
                return
            if shard_start < 0 or shard_end < 0 or shard_start > shard_end:
                self._send_json(400, {"error": "shard range must be non-negative and ordered"})
                return
        else:
            shard_start = None
            shard_end = None
        try:
            score = float(body.get("score", 1.0))
        except (TypeError, ValueError):
            self._send_json(400, {"error": "score must be numeric"})
            return

        hardware_profile = body.get("hardware_profile", {})
        if not isinstance(hardware_profile, dict):
            self._send_json(400, {"error": "hardware_profile must be an object"})
            return
        model = body.get("model")
        if model is None:
            model = "stub-model"
        if not isinstance(model, str):
            self._send_json(400, {"error": "model must be a string"})
            return
        shard_checksum = body.get("shard_checksum")
        if shard_checksum is not None and not isinstance(shard_checksum, str):
            self._send_json(400, {"error": "shard_checksum must be a string"})
            return
        try:
            vram_bytes = int(body.get("vram_bytes", DEFAULT_VRAM_BYTES))
            ram_bytes = int(body.get("ram_bytes", DEFAULT_RAM_BYTES))
            max_loaded_shards = int(body.get("max_loaded_shards", 1))
            benchmark_tokens_per_sec = float(
                body.get("benchmark_tokens_per_sec", DEFAULT_BENCHMARK_TOKENS_PER_SEC)
            )
        except (TypeError, ValueError):
            self._send_json(400, {"error": "vram_bytes, ram_bytes, max_loaded_shards, and benchmark_tokens_per_sec must be numeric"})
            return
        if vram_bytes < 0 or ram_bytes < 0 or max_loaded_shards < 1 or benchmark_tokens_per_sec <= 0:
            self._send_json(400, {"error": "capability values must be positive"})
            return
        quantizations_body = body.get("quantizations", DEFAULT_QUANTIZATIONS)
        if not (
            isinstance(quantizations_body, list)
            and quantizations_body
            and all(isinstance(item, str) and item for item in quantizations_body)
        ):
            self._send_json(400, {"error": "quantizations must be a non-empty string array"})
            return
        quantizations = list(quantizations_body)
        if body.get("quantization") is not None and not isinstance(body["quantization"], str):
            self._send_json(400, {"error": "quantization must be a string"})
            return
        quantization = _registration_quantization(body, quantizations)
        wallet_address = body.get("wallet_address")
        if wallet_address is not None and not isinstance(wallet_address, str):
            self._send_json(400, {"error": "wallet_address must be a string"})
            return
        ban_error = _registration_ban_error(server.contracts, wallet_address)
        if ban_error:
            self._send_json(403, {"error": ban_error})
            return

        tracker_mode = bool(body.get("tracker_mode", False))
        managed_assignment = bool(body.get("managed_assignment", False))
        hf_repo = body.get("hf_repo")
        if hf_repo is not None and not isinstance(hf_repo, str):
            self._send_json(400, {"error": "hf_repo must be a string"})
            return
        num_layers_body = body.get("num_layers")
        num_layers: int | None = None
        if num_layers_body is not None:
            try:
                num_layers = int(num_layers_body)
            except (TypeError, ValueError):
                self._send_json(400, {"error": "num_layers must be an integer"})
                return
        model_metadata = body.get("model_metadata", {})
        if model_metadata is None:
            model_metadata = {}
        if not isinstance(model_metadata, dict):
            self._send_json(400, {"error": "model_metadata must be an object"})
            return
        downloaded_models = body.get("downloaded_models", [])
        if downloaded_models is None:
            downloaded_models = []
        if not (
            isinstance(downloaded_models, list)
            and all(isinstance(item, dict) for item in downloaded_models)
        ):
            self._send_json(400, {"error": "downloaded_models must be an array of objects"})
            return
        relay_addr = body.get("relay_addr") or None
        cert_fingerprint = body.get("cert_fingerprint") or None
        peer_id = body.get("peer_id") or None
        try:
            friendly_name = _normalize_friendly_name(body.get("friendly_name"))
        except ValueError as exc:
            self._send_json(400, {"error": str(exc)})
            return

        recipe_id = body.get("recipe_id")
        recipe_version = body.get("recipe_version")
        if (recipe_id is not None and not isinstance(recipe_id, str)) or (
            recipe_version is not None and not isinstance(recipe_version, str)
        ):
            self._send_json(400, {"error": "recipe_id and recipe_version must be strings"})
            return

        # The capability proof (ADR-0023). A bad proof does not fail registration --
        # the node is recorded with its verdict so the operator can see *why* it is
        # not routing -- but only an `admitted` verdict makes it routable.
        capability = _capability_from_registration(
            body,
            model=model,
            hf_repo=hf_repo,
            shard_start=shard_start,
            shard_end=shard_end,
        )

        node_id = _node_id_for_registration(
            endpoint,
            model,
            wallet_address,
            shard_start,
            shard_end,
            hf_repo,
        )
        entry = _NodeEntry(
            node_id=node_id,
            endpoint=endpoint.rstrip("/"),
            shard_start=shard_start,
            shard_end=shard_end,
            model=model,
            shard_checksum=shard_checksum,
            hardware_profile=hardware_profile,
            wallet_address=wallet_address,
            score=score,
            vram_bytes=vram_bytes,
            ram_bytes=ram_bytes,
            quantizations=quantizations,
            max_loaded_shards=max_loaded_shards,
            benchmark_tokens_per_sec=benchmark_tokens_per_sec,
            quantization=quantization,
            managed_assignment=managed_assignment or not explicit_shard,
            tracker_mode=tracker_mode,
            hf_repo=hf_repo,
            num_layers=num_layers,
            model_metadata=model_metadata,
            downloaded_models=downloaded_models,
            relay_addr=relay_addr,
            cert_fingerprint=cert_fingerprint,
            peer_id=peer_id,
            friendly_name=friendly_name,
            capability=capability,
        )
        with server.lock:
            self._purge_expired_nodes()
            # Dedup: replace the same node id or the same endpoint+model assignment.
            endpoint_key = entry.endpoint.rstrip("/")
            model_key = entry.hf_repo or entry.model
            stale_ids = [
                eid for eid, e in server.registry.items()
                if eid == node_id
                or (
                    e.endpoint.rstrip("/") == endpoint_key
                    and (e.hf_repo or e.model) == model_key
                )
            ]
            stale_entries: list[tuple[str, _NodeEntry]] = []
            for eid in stale_ids:
                old = server.registry.pop(eid)
                stale_entries.append((eid, old))
            if friendly_name is None:
                for _eid, old in stale_entries:
                    if old.friendly_name:
                        friendly_name = old.friendly_name
                        break
            entry.friendly_name = friendly_name
            is_topology_change = node_id not in stale_ids or any(
                (old.shard_start, old.shard_end) != (entry.shard_start, entry.shard_end)
                for eid, old in stale_entries
                if eid == node_id
            )
            server.registry[node_id] = entry
            if is_topology_change:
                server.route_stats.bump_epoch(_route_stats_keys(server, entry))
            if entry.managed_assignment and not explicit_shard:
                if entry.hf_repo:
                    _rebalance_hf_model_locked(server, entry.hf_repo)
                else:
                    _rebalance_model_locked(server, model)
            assignment_directive = entry.pending_directives[-1] if entry.pending_directives else None
            if assignment_directive is not None:
                entry.pending_directives.clear()

            model_health = _model_health_summary(server, entry.model, entry.hf_repo)
            for eid, old in stale_entries:
                _tracker_log(
                    server,
                    "info",
                    "node re-registered",
                    old_node_id=eid,
                    node_id=node_id,
                    endpoint=old.endpoint,
                    old_model=old.model,
                    old_hf_repo=old.hf_repo,
                    old_model_health=_model_health_summary(server, old.model, old.hf_repo),
                    model_health=model_health,
                )
            routable = _capability_routable(entry, server.capability_policy)
            _tracker_log(
                server,
                "info",
                "node registered",
                node_id=node_id,
                endpoint=entry.endpoint,
                model=entry.model,
                hf_repo=entry.hf_repo,
                shard=f"{entry.shard_start}-{entry.shard_end}",
                tracker_mode=entry.tracker_mode,
                model_health=model_health,
                capability=entry.capability.state,
                routable=routable,
            )
            if not routable:
                _tracker_log(
                    server,
                    "warn",
                    "node registered but is not routable",
                    node_id=node_id,
                    endpoint=entry.endpoint,
                    capability=entry.capability.state,
                    detail=entry.capability.detail,
                )

        shard_info = f"layers {shard_start}-{shard_end}" if shard_start is not None else "unsharded"
        repo_info = f" [{hf_repo}]" if hf_repo else ""
        budget_bytes, budget_source = _node_memory_budget_bytes(entry)
        budget_gb = budget_bytes / (1024 ** 3)
        print(
            f"[tracker] node registered: {node_id}  {endpoint}  {model}{repo_info}  {shard_info}  "
            f"capacity={budget_gb:.1f}GB {budget_source} slots={max_loaded_shards}",
            flush=True,
        )

        payload = {
            "node_id": node_id,
            # Tell the node what the tracker made of its proof: a node that is
            # registered but not routable must be able to see that it is dark.
            "capability": {
                "state": entry.capability.state,
                "detail": entry.capability.detail,
                "routable": routable,
                "policy": server.capability_policy,
            },
        }
        if assignment_directive is not None:
            payload["directive"] = assignment_directive
        self._send_json(200, payload)

        # Raft replication: leader proposes this registration to followers so their
        # registries stay in sync.  Fire-and-forget (async) — the client already
        # got its node_id; replication happens in the background.
        if server.raft is not None and server.raft.is_leader:
            raft_payload = dict(body)
            raft_payload["node_id"] = node_id  # include the generated ID
            threading.Thread(
                target=server.raft.propose,
                args=("register", raft_payload),
                daemon=True,
            ).start()

    def _handle_heartbeat(self, node_id: str):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body: dict = {}
        content_length = int(self.headers.get("Content-Length", 0))
        if content_length > 0:
            try:
                body = json.loads(self.rfile.read(content_length))
            except Exception:
                pass
        with server.lock:
            self._purge_expired_nodes()
            entry = server.registry.get(node_id)
            if entry is None:
                _tracker_log(server, "warn", "heartbeat for unknown node", node_id=node_id)
                self._send_json(404, {"error": "node not found"})
                return
            entry.last_heartbeat = time.monotonic()
            entry.heartbeats_received += 1
            # P2P metadata
            if body.get("relay_addr"):
                entry.relay_addr = body["relay_addr"]
            if body.get("cert_fingerprint"):
                entry.cert_fingerprint = body["cert_fingerprint"]
            if body.get("peer_id"):
                entry.peer_id = body["peer_id"]
            # Node stats (cumulative — node always sends totals, tracker replaces)
            if "total_requests" in body:
                entry.total_requests = int(body["total_requests"])
            if "failed_requests" in body:
                entry.failed_requests = int(body["failed_requests"])
            if "queue_depth" in body:
                entry.queue_depth = int(body["queue_depth"])
            if "current_requests" in body:
                entry.current_requests = _normalize_current_requests(body["current_requests"])
            if "uptime_seconds" in body:
                entry.uptime_seconds = float(body["uptime_seconds"])
            if "status" in body and body["status"] in ("ready", "loading"):
                entry.status = body["status"]
            if "friendly_name" in body:
                try:
                    entry.friendly_name = _normalize_friendly_name(body.get("friendly_name"))
                except ValueError as exc:
                    self._send_json(400, {"error": str(exc)})
                    return
            directives = list(entry.pending_directives)
            entry.pending_directives.clear()
            new_assignment = entry.pending_new_assignment
            if new_assignment is not None:
                entry.pending_new_assignment = None
        if server.gossip is not None:
            server.gossip.record(node_id)
        resp: dict = {}
        if directives:
            resp["directives"] = directives
        if new_assignment is not None:
            resp["new_assignment"] = new_assignment
        self._send_json(200, resp)

    # ---------------------------------------------------------------- Raft handlers

    def _handle_raft_vote(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_json_body()
        if body is None:
            return
        if server.raft is None:
            self._send_json(503, {"error": "raft not enabled"})
            return
        result = server.raft.handle_request_vote(body)
        self._send_json(200, result)

    def _handle_raft_append(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_json_body()
        if body is None:
            return
        if server.raft is None:
            self._send_json(503, {"error": "raft not enabled"})
            return
        result = server.raft.handle_append_entries(body)
        self._send_json(200, result)

    def _handle_raft_status(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.raft is None:
            self._send_json(200, {"role": "standalone", "term": 0, "leader": None})
            return
        self._send_json(200, server.raft.status())

    def _handle_gossip(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_json_body()
        if body is None:
            return
        if server.gossip is not None and isinstance(body, dict):
            server.gossip.merge({k: float(v) for k, v in body.items()})
        self._send_json(200, {})

    def _handle_model_load_request(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin", "validator"):
            return
        body = self._read_json_body()
        if body is None:
            return
        model = body.get("model")
        if not isinstance(model, str) or not model.strip():
            self._send_json(400, {"error": "model is required"})
            return
        with server.lock:
            self._purge_expired_nodes()
            assignment = _request_model_load_locked(server, model)
        if assignment is None:
            self._send_json(409, {"error": "no ready joined node has an available model slot and sufficient capacity"})
            return
        self._send_json(202, {"status": "queued", "assignment": assignment})

    def _handle_model_coverage_vote(self):
        """Record a rolling wish-list signal for an unavailable precision."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_json_body()
        if body is None:
            return
        model = body.get("model")
        quantization = _normalize_quantization(body.get("quantization"))
        if not isinstance(model, str) or not model.strip() or quantization is None:
            self._send_json(400, {"error": "model and valid quantization are required"})
            return
        resolved_name, preset = _resolve_model_preset(server.model_presets, model)
        if preset is None:
            self._send_json(404, {"error": "unknown model preset"})
            return
        vote_key = f"wishlist:{resolved_name}:{quantization}"
        if server.stats is not None:
            server.stats.record_request(vote_key)
        _tracker_log(server, "info", "model coverage vote", model=resolved_name, quantization=quantization)
        self._send_json(202, {"status": "recorded", "model": resolved_name, "quantization": quantization})

    def _handle_stats(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.stats is None:
            self._send_json(200, {"models": {}, "nodes": {}})
            return
        self._send_json(200, {
            "models": server.stats.get_combined_stats(),
            "nodes": server.stats.get_node_throughput_stats(),
        })

    def _handle_stats_gossip(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_hive_authenticated_body()
        if body is None:
            return
        tracker_url = body.get("tracker_url", "")
        rpms = body.get("stats", {})
        if server.stats is not None and tracker_url and isinstance(rpms, dict):
            server.stats.merge_peer_rpms(tracker_url, rpms)
        self._send_json(200, {})

    def _handle_billing_summary(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin"):
            return
        if server.billing is None:
            self._send_json(404, {"error": "billing is not enabled on this tracker"})
            return
        self._send_json(200, server.billing.snapshot())

    def _handle_dashboard(self):
        """Serve the read-only web dashboard (US-035). Any tracker in the
        mesh — leader or follower — serves it from its replicated state."""
        self._serve_package_asset("dashboard.html", "text/html; charset=utf-8")

    def _handle_favicon(self) -> None:
        """Serve the dashboard favicon (SVG; ICO path aliases the same asset)."""
        self._serve_package_asset("favicon.svg", "image/svg+xml")

    def _serve_package_asset(self, filename: str, content_type: str) -> None:
        try:
            body = files("meshnet_tracker").joinpath(filename).read_bytes()
        except (FileNotFoundError, OSError):
            self._send_json(404, {"error": f"{filename} missing"})
            return
        self.send_response(200)
        self.send_header("Content-Type", content_type)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        try:
            self.wfile.write(body)
        except BrokenPipeError:
            pass

    def _handle_console(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        with server.console_lock:
            events = [dict(event) for event in server.console_events]
        self._send_json(200, {"events": events})

    def _handle_proxy_request_cancel(self, request_id: str) -> None:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.accounts is not None and not self._require_role("admin"):
            return
        if not _request_proxy_cancel(server, request_id):
            self._send_json(404, {"error": f"no active proxy for request {request_id!r}"})
            return
        self._send_json(200, {"status": "canceled", "request_id": request_id})

    def _finalize_proxy_cancel(
        self,
        *,
        proxy_ctx: _ActiveProxyContext,
        server: "_TrackerHTTPServer",
        request_id: str,
        started: float,
        model: str,
        route_model: str,
        route_nodes: list,
        api_key: str | None,
        node_work: list,
        body: dict,
        finish_proxy_inflight: Any,
        observed_stream_tokens: int = 0,
        stream_usage: dict | None = None,
    ) -> bool:
        if not proxy_ctx.cancel_event.is_set():
            return False
        elapsed = time.monotonic() - started
        _clear_proxy_progress_log_state(server, request_id)
        tokens = observed_stream_tokens
        if observed_stream_tokens > 0:
            in_tokens, out_tokens = _stream_billable_split(
                observed_stream_tokens, stream_usage, body,
            )
            tokens = in_tokens + out_tokens
            self._record_observed_throughput(
                model, route_model, tokens, elapsed, route_nodes,
            )
        _tracker_log(
            server,
            "info",
            "proxy canceled",
            request_id=request_id,
            model=model,
            route_model=route_model,
            tokens=tokens,
            elapsed_seconds=round(elapsed, 4),
            tokens_per_sec=round(tokens / elapsed, 4) if elapsed > 0 and tokens > 0 else 0.0,
            route=_node_route_summary(route_nodes),
        )
        if observed_stream_tokens > 0:
            in_tokens, out_tokens = _stream_billable_split(
                observed_stream_tokens, stream_usage, body,
            )
            self._bill_completed(
                api_key, model, in_tokens + out_tokens, node_work,
                input_tokens=in_tokens, output_tokens=out_tokens,
            )
        finish_proxy_inflight()
        return True

    # ---- opt-in test runner (dashboard-test-runner US-001) ----

    def _test_runner_or_reject(self) -> "TestRunManager | None":
        """Admin session first, then the explicit enable gate — both fail closed."""
        if not self._require_role("admin"):
            return None
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.test_runner is None:
            self._send_json(403, {
                "error": (
                    "test runner disabled; start the tracker with "
                    f"--enable-test-runner or {TEST_RUNNER_ENABLE_ENV_VAR}=1"
                )
            })
            return None
        return server.test_runner

    def _handle_tests_list(self, parsed: urllib.parse.ParseResult) -> None:
        runner = self._test_runner_or_reject()
        if runner is None:
            return
        query = urllib.parse.parse_qs(parsed.query)
        refresh = query.get("refresh", ["0"])[0] in {"1", "true"}
        try:
            data = runner.collect(refresh=refresh)
        except TestRunnerError as exc:
            self._send_json(exc.status, {"error": str(exc)})
            return
        self._send_json(200, {"enabled": True, **data})

    def _handle_tests_run(self) -> None:
        runner = self._test_runner_or_reject()
        if runner is None:
            return
        body = self._read_json_body()
        if body is None:
            return
        target = body.get("target")
        try:
            state = runner.start(target if isinstance(target, str) else "")
        except TestRunnerError as exc:
            self._send_json(exc.status, {"error": str(exc)})
            return
        self._send_json(202, state)

    def _handle_tests_status(self) -> None:
        runner = self._test_runner_or_reject()
        if runner is None:
            return
        self._send_json(200, runner.status())

    def _handle_registry_wallets(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin"):
            return
        if server.contracts is None:
            self._send_json(200, {"wallets": {}})
            return
        wallets = server.contracts.registry.list_wallets()
        self._send_json(200, {"wallets": {
            wallet: {
                "stake_balance": state.stake_balance,
                "strike_count": state.strike_count,
                "banned": state.banned,
                "completed_jobs": state.completed_job_count,
                "reputation": state.reputation,
                "last_audit_ts": state.last_audit_ts,
                "probationary_jobs_remaining": (
                    server.contracts.registry.probationary_jobs_remaining(wallet)
                ),
            }
            for wallet, state in wallets.items()
        }})

    def _handle_billing_settlements(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin"):
            return
        if server.billing is None:
            self._send_json(404, {"error": "billing is not enabled on this tracker"})
            return
        self._send_json(200, {"settlements": server.billing.settlement_history()})

    def _handle_billing_gossip(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_hive_authenticated_body()
        if body is None:
            return
        if server.billing is None:
            self._send_json(200, {"applied": 0})
            return
        events = body.get("events")
        if not isinstance(events, list):
            self._send_json(400, {"error": "events must be a list"})
            return
        applied = server.billing.apply_events([e for e in events if isinstance(e, dict)])
        self._send_json(200, {"applied": applied})

    def _handle_billing_forfeit(self):
        """Privileged: forfeit a node's pending balance + record a strike (US-034).

        ADR-0017 §4: validator service token or admin session only. Client
        API keys — valid or not — are rejected.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("validator", "admin"):
            return
        if server.billing is None:
            self._send_json(404, {"error": "billing is not enabled on this tracker"})
            return
        body = self._read_json_body()
        if body is None:
            return
        wallet = body.get("wallet")
        if not wallet or not isinstance(wallet, str):
            self._send_json(400, {"error": "wallet is required"})
            return
        reason = body.get("reason") or "fraud"
        event = server.billing.forfeit_pending(wallet, reason=str(reason))
        strike_state: dict = {}
        if server.contracts is not None:
            server.contracts.registry.record_strike(wallet)
            wallet_state = server.contracts.registry.get_wallet(wallet)
            strike_state = {
                "strike_count": wallet_state.strike_count,
                "banned": wallet_state.banned,
            }
        print(
            f"[tracker] forfeited pending balance of {wallet}: "
            f"{event['amount']:.6f} USDT ({reason}) strikes={strike_state.get('strike_count', 'n/a')}",
            flush=True,
        )
        self._send_json(200, {"forfeited": event["amount"], **strike_state})

    # ---- user accounts (registration, login, API keys) ----

    def _session_account(self) -> dict | None:
        """Resolve the session token in the Authorization header, or None."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.accounts is None:
            return None
        return server.accounts.session_account(_session_token_from_headers(self.headers))

    def _require_accounts(self) -> "AccountStore | None":
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.accounts is None:
            self._send_json(404, {"error": "accounts are not enabled on this tracker"})
            return None
        return server.accounts

    def _handle_auth_register(self):
        accounts = self._require_accounts()
        if accounts is None:
            return
        body = self._read_json_body()
        if body is None:
            return
        try:
            account = accounts.register(
                email=body.get("email"),
                wallet=body.get("wallet"),
                password=str(body.get("password") or ""),
                nickname=body.get("nickname"),
            )
        except ValueError as exc:
            self._send_json(400, {"error": str(exc)})
            return
        token = accounts.create_session(account["account_id"])
        api_key = accounts.create_api_key(account["account_id"])
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.billing is not None:
            # US-039: registration creates the account's first key, so Caller
            # Credit lands here; the account-derived event id keeps later
            # grants (e.g. via /v1/account/keys) no-ops.
            server.billing.grant_caller_credit(
                api_key, account["account_id"], server.starting_credit
            )
            server.billing.ensure_client(api_key)
        print(
            f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
            f"role={account['role']}", flush=True,
        )
        self._send_json(
            200,
            {"account": account, "session_token": token, "api_key": api_key},
            headers={"Set-Cookie": _session_cookie_header(token)},
        )

    def _handle_auth_login(self):
        accounts = self._require_accounts()
        if accounts is None:
            return
        body = self._read_json_body()
        if body is None:
            return
        identifier = body.get("identifier") or body.get("email") or body.get("wallet") or ""
        account = accounts.verify_login(str(identifier), str(body.get("password") or ""))
        if account is None:
            self._send_json(401, {"error": "invalid credentials"})
            return
        token = accounts.create_session(account["account_id"])
        self._send_json(
            200,
            {"account": account, "session_token": token},
            headers={"Set-Cookie": _session_cookie_header(token)},
        )

    def _handle_auth_logout(self):
        accounts = self._require_accounts()
        if accounts is None:
            return
        accounts.destroy_session(_session_token_from_headers(self.headers))
        self._send_json(200, {"ok": True}, headers={"Set-Cookie": _session_cookie_header(None)})

    def _handle_account_me(self):
        """Balance, usage totals, and API keys for the logged-in account."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if self._require_accounts() is None:
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        keys = server.accounts.keys_for(account["account_id"])  # type: ignore[union-attr]
        balances = {}
        usage: dict = {"requests": 0, "total_tokens": 0, "total_cost": 0.0}
        if server.billing is not None:
            balances = {key: server.billing.get_client_balance(key) for key in keys}
            usage = server.billing.usage_totals_for(keys)
        self._send_json(200, {
            "account": account,
            "api_keys": keys,
            "balances": balances,
            "total_balance": sum(balances.values()),
            "usage": usage,
            "topup_amount": server.devnet_topup_amount if server.billing is not None else 0.0,
        })

    def _handle_account_usage(self, parsed: urllib.parse.ParseResult):
        """Per-request charge history for the logged-in account (billing tab)."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if self._require_accounts() is None:
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        keys = server.accounts.keys_for(account["account_id"])  # type: ignore[union-attr]
        if server.billing is None:
            self._send_json(200, {
                "requests": 0,
                "total_tokens": 0,
                "total_cost": 0.0,
                "records": [],
                "recent": [],
            })
            return
        limit = None
        raw_limit = urllib.parse.parse_qs(parsed.query).get("limit", [None])[0]
        if raw_limit is not None:
            try:
                limit = max(1, int(raw_limit))
            except ValueError:
                self._send_json(400, {"error": "limit must be an integer"})
                return
        self._send_json(200, server.billing.usage_for(keys, recent_limit=limit))

    def _handle_account_profile(self):
        accounts = self._require_accounts()
        if accounts is None:
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        body = self._read_json_body()
        if body is None:
            return
        try:
            updated = accounts.update_profile(
                account["account_id"],
                nickname=body.get("nickname"),
            )
        except ValueError as exc:
            self._send_json(400, {"error": str(exc)})
            return
        self._send_json(200, {"account": updated})

    def _handle_account_key_create(self):
        accounts = self._require_accounts()
        if accounts is None:
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        api_key = accounts.create_api_key(account["account_id"])
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        credited = False
        if server.billing is not None:
            # US-039: Caller Credit lands with the account's first key; the
            # account-derived event id keeps it once-per-account forever.
            credited = server.billing.grant_caller_credit(
                api_key, account["account_id"], server.starting_credit
            )
            server.billing.ensure_client(api_key)
        self._send_json(200, {"api_key": api_key, "caller_credit_granted": credited})

    def _handle_account_topup(self):
        """Devnet faucet (US-040): credit the configured amount to one of the
        logged-in account's keys. 404 unless the operator enabled it."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        accounts = self._require_accounts()
        if accounts is None:
            return
        if server.billing is None or server.devnet_topup_amount <= 0:
            self._send_json(404, {"error": "top-up is not enabled on this tracker"})
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        body = self._read_json_body()
        if body is None:
            return
        api_key = body.get("api_key")
        if not api_key or not isinstance(api_key, str):
            self._send_json(400, {"error": "api_key is required"})
            return
        if accounts.owner_of_key(api_key) != account["account_id"]:
            self._send_json(403, {"error": "key not found on this account"})
            return
        balance = server.billing.credit_client(
            api_key, server.devnet_topup_amount, note="devnet-topup"
        )
        self._send_json(200, {
            "api_key": api_key,
            "credited": server.devnet_topup_amount,
            "balance": balance,
        })

    def _handle_account_key_revoke(self):
        accounts = self._require_accounts()
        if accounts is None:
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        body = self._read_json_body()
        if body is None:
            return
        api_key = body.get("api_key")
        if not api_key or not isinstance(api_key, str):
            self._send_json(400, {"error": "api_key is required"})
            return
        if not accounts.revoke_api_key(account["account_id"], api_key):
            self._send_json(404, {"error": "key not found on this account"})
            return
        self._send_json(200, {"revoked": api_key})

    def _handle_admin_accounts(self):
        """Admin-only: all accounts with their keys and balances."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        accounts = self._require_accounts()
        if accounts is None:
            return
        account = self._session_account()
        if account is None:
            self._send_json(401, {"error": "login required"})
            return
        if account["role"] != "admin":
            self._send_json(403, {"error": "admin role required"})
            return
        listing = accounts.list_accounts()
        if server.billing is not None:
            for entry in listing:
                entry["balances"] = {
                    key: server.billing.get_client_balance(key)
                    for key in entry.get("api_keys", [])
                }
        self._send_json(200, {"accounts": listing})

    def _handle_accounts_gossip(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_hive_authenticated_body()
        if body is None:
            return
        if server.accounts is None:
            self._send_json(200, {"applied": 0})
            return
        events = body.get("events")
        if not isinstance(events, list):
            self._send_json(400, {"error": "events must be a list"})
            return
        applied = server.accounts.apply_events([e for e in events if isinstance(e, dict)])
        self._send_json(200, {"applied": applied})

    def _handle_registry_gossip(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        body = self._read_hive_authenticated_body()
        if body is None:
            return
        registry_log = getattr(server.contracts, "registry_log", None)
        if registry_log is None:
            self._send_json(200, {"applied": 0})
            return
        events = body.get("events")
        if not isinstance(events, list):
            self._send_json(400, {"error": "events must be a list"})
            return
        applied = registry_log.apply_events([e for e in events if isinstance(e, dict)])
        self._send_json(200, {"applied": applied})

    def _handle_wallet_register(self):
        """Bind a client wallet pubkey to an API key (US-032, C6).

        Deposits from that wallet into the treasury are then credited to the
        API key's ledger balance by the deposit watcher.

        Requires a signature proving ownership of the wallet's private key —
        the caller signs ``binding_message(api_key, wallet)`` with the
        wallet's ed25519 key. An admin session may force-rebind a wallet
        already bound to a different API key (documented override; no
        signed-release flow is implemented).
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.billing is None:
            self._send_json(404, {"error": "billing is not enabled on this tracker"})
            return
        role, _account = self._resolve_identity()
        body = self._read_json_body()
        if body is None:
            return
        wallet = body.get("wallet")
        if not wallet or not isinstance(wallet, str):
            self._send_json(400, {"error": "wallet is required"})
            return

        if role == "admin":
            api_key = body.get("api_key")
            if not api_key or not isinstance(api_key, str):
                self._send_json(400, {"error": "api_key is required for admin override"})
                return
            event = server.billing.bind_wallet(api_key, wallet, admin_override=True)
        else:
            api_key = _api_key_from_headers(self.headers)
            if not api_key:
                self._send_json(401, {"error": "Authorization header required"})
                return
            signature = body.get("signature")
            if not signature or not isinstance(signature, str):
                self._send_json(400, {"error": "signature is required to prove wallet ownership"})
                return
            if not verify_wallet_signature(wallet, binding_message(api_key, wallet), signature):
                self._send_json(401, {"error": "invalid wallet signature"})
                return
            event = server.billing.bind_wallet(api_key, wallet)

        if event.get("rejected"):
            self._send_json(409, {"error": "wallet already bound to a different API key"})
            return
        print(f"[tracker] wallet bound: {wallet} -> api_key …{api_key[-6:]}", flush=True)
        self._send_json(200, {"wallet": wallet, "bound": True})

    def _handle_benchmark_hop_penalty(self):
        """Privileged: run the same prompt through 1/2/3-node pinned routes (US-030).

        Data collection only — the routing algorithm is unchanged. Per-hop
        latency is derived incrementally: the k-node route's final hop penalty
        is its total minus the (k-1)-node route's total.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin", "validator"):
            return
        auth = self.headers.get("Authorization")
        body = self._read_json_body()
        if body is None:
            return
        model = body.get("model", "")
        if not model:
            self._send_json(400, {"error": "model is required"})
            return
        prompt = body.get("prompt") or "Benchmark: say OK."
        max_new_tokens = int(body.get("max_new_tokens", 64))

        with server.lock:
            self._purge_expired_nodes()
            resolved = _nodes_and_bounds_for_model(server, model)
        if resolved is None or not resolved[0]:
            self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
            return
        all_nodes, rs, re = resolved

        self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
        route_results: list[dict] = []
        prev_total_ms: float | None = None
        prev_per_hop: list[float] = []
        for hop_count in (1, 2, 3):
            combo = _find_pinned_route(all_nodes, rs, re, hop_count)
            if combo is None:
                continue  # insufficient coverage for this hop count — skip
            request_body = json.dumps({
                "model": model,
                "messages": [{"role": "user", "content": prompt}],
                "max_tokens": max_new_tokens,
                "route": [n.node_id for n in combo],
            }).encode()
            req = urllib.request.Request(
                f"{self_url}/v1/chat/completions",
                data=request_body,
                headers={"Content-Type": "application/json", "Authorization": auth},
                method="POST",
            )
            started = time.monotonic()
            try:
                with urllib.request.urlopen(req, timeout=300.0) as resp:
                    payload = json.loads(resp.read())
            except Exception as exc:
                route_results.append({
                    "route": [n.node_id for n in combo],
                    "error": str(exc),
                })
                continue
            total_ms = (time.monotonic() - started) * 1000.0
            if prev_total_ms is not None and len(prev_per_hop) == hop_count - 1:
                per_hop_ms = prev_per_hop + [max(0.0, total_ms - prev_total_ms)]
            else:
                per_hop_ms = [total_ms / hop_count] * hop_count
            prev_total_ms = total_ms
            prev_per_hop = per_hop_ms
            route_results.append({
                "route": [n.node_id for n in combo],
                "total_ms": total_ms,
                "per_hop_ms": per_hop_ms,
                "tokens_generated": _usage_total_tokens(payload) or 0,
            })

        record = {
            "timestamp": time.time(),
            "model": model,
            "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
            "routes": route_results,
        }
        with server.benchmark_lock:
            existing: list = []
            if os.path.exists(server.benchmark_results_path):
                try:
                    with open(server.benchmark_results_path, encoding="utf-8") as fh:
                        existing = json.load(fh)
                except (json.JSONDecodeError, OSError):
                    existing = []
            if not isinstance(existing, list):
                existing = []
            existing.append(record)
            with open(server.benchmark_results_path, "w", encoding="utf-8") as fh:
                json.dump(existing, fh, indent=2)
        self._send_json(200, record)

    def _handle_benchmark_results(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin", "validator"):
            return
        results: list = []
        with server.benchmark_lock:
            if os.path.exists(server.benchmark_results_path):
                try:
                    with open(server.benchmark_results_path, encoding="utf-8") as fh:
                        results = json.load(fh)
                except (json.JSONDecodeError, OSError):
                    results = []
        self._send_json(200, {"results": results if isinstance(results, list) else []})

    def _handle_toploc_calibration_run(self):
        """Privileged: honest-noise TOPLOC calibration dispatch (issue 21).

        Fans the same fixed prompt through every currently registered node
        that can solo-serve the full model (one pinned-route hop, mirroring
        `_handle_benchmark_hop_penalty`'s dispatch pattern), then verifies
        each node's own on-demand TOPLOC commitment against a teacher-forced
        replay on the reference node — same audit contract the validator
        uses (`packages/validator/README.md` "TOPLOC audit contract"). Each
        node's raw divergence (not just pass/fail) is recorded into the
        calibration corpus, keyed by wallet + GPU model + dtype, so
        thresholds can eventually be derived instead of guessed.

        Nodes that only hold a partial shard (need a multi-hop route) are
        skipped for this pass — solo dispatch isolates one node's hardware
        noise without a route composition confound — and nodes that don't
        answer the on-demand commitment fetch (endpoint down, or node-side
        TOPLOC serving not yet wired) are skipped and reported, not treated
        as a pass or a fail.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin", "validator"):
            return
        if server.toploc_calibration is None:
            self._send_json(503, {"error": "toploc calibration store is not enabled on this tracker"})
            return
        if not server.toploc_reference_node_url:
            self._send_json(503, {"error": "toploc_reference_node_url is not configured on this tracker"})
            return
        auth = self.headers.get("Authorization")
        body = self._read_json_body()
        if body is None:
            return
        model = body.get("model", "")
        if not model:
            self._send_json(400, {"error": "model is required"})
            return
        prompt = body.get("prompt") or "Calibration: say OK."
        max_new_tokens = int(body.get("max_new_tokens", 32))
        seed = body.get("seed", 0)

        with server.lock:
            self._purge_expired_nodes()
            resolved = _nodes_and_bounds_for_model(server, model)
        if resolved is None or not resolved[0]:
            self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
            return
        all_nodes, rs, re = resolved
        if server.contracts is not None:
            all_nodes = [
                node for node in all_nodes
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]
        solo_nodes = [
            node for node in all_nodes
            if node.shard_start is not None and node.shard_end is not None
            and node.shard_start <= rs and node.shard_end >= re
        ]

        self_url = f"http://127.0.0.1:{self.server.server_address[1]}"
        messages = [{"role": "user", "content": prompt}]
        node_results: list[dict] = []
        for node in solo_nodes:
            request_id = f"cal-{uuid.uuid4().hex}"
            request_body = json.dumps({
                "id": request_id,
                "model": model,
                "messages": messages,
                "max_tokens": max_new_tokens,
                "temperature": 0,
                "seed": seed,
                "route": [node.node_id],
            }).encode()
            req = urllib.request.Request(
                f"{self_url}/v1/chat/completions",
                data=request_body,
                headers={"Content-Type": "application/json", "Authorization": auth},
                method="POST",
            )
            try:
                with urllib.request.urlopen(req, timeout=300.0) as resp:
                    json.loads(resp.read())
            except Exception as exc:
                node_results.append({"node_id": node.node_id, "wallet_address": node.wallet_address, "error": str(exc)})
                continue

            outcome = self._verify_node_toploc_calibration(
                server, node, request_id=request_id, model=model, messages=messages,
            )
            node_results.append(outcome)

        skipped_partial_shard = [
            node.node_id for node in all_nodes if node not in solo_nodes
        ]
        record = {
            "timestamp": time.time(),
            "model": model,
            "prompt_hash": hashlib.sha256(prompt.encode()).hexdigest()[:16],
            "nodes": node_results,
            "skipped_partial_shard_node_ids": skipped_partial_shard,
            "gate_status": server.toploc_calibration.gate_status(
                min_hardware_profiles=self._toploc_calibration_gate_min_hardware_profiles(),
            ),
        }
        self._send_json(200, record)

    def _toploc_calibration_gate_min_hardware_profiles(self) -> int:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        return server.toploc_calibration_gate_min_hardware_profiles

    def _verify_node_toploc_calibration(
        self,
        server: "_TrackerHTTPServer",
        node: "_NodeEntry",
        *,
        request_id: str,
        model: str,
        messages: list[dict],
    ) -> dict:
        """One node's calibration outcome: fetch its on-demand commitment,
        teacher-force the claimed tokens on the reference node, verify, and
        persist the raw divergence into the corpus."""
        from meshnet_validator.audit import ToplocProofClaim, verify_activation_proofs_detailed

        gpu_model = (node.hardware_profile or {}).get("gpu_name") or (node.hardware_profile or {}).get("device") or "unknown"
        dtype = node.quantization or "unknown"
        base_result = {
            "node_id": node.node_id,
            "wallet_address": node.wallet_address,
            "gpu_model": gpu_model,
            "dtype": dtype,
        }
        commitment = _fetch_toploc_commitment(
            node, session_id=request_id, model=model, messages=messages,
        )
        if commitment is None:
            return {**base_result, "skipped": "no on-demand toploc commitment available"}

        try:
            claim = ToplocProofClaim.from_mapping(commitment["toploc_proof"])
        except (KeyError, TypeError, ValueError):
            return {**base_result, "skipped": "malformed toploc commitment"}

        reference_activations = _fetch_toploc_reference_activations(
            server.toploc_reference_node_url,
            model=model,
            messages=messages,
            claimed_token_ids=commitment["claimed_token_ids"],
            claim=claim,
        )
        if reference_activations is None:
            return {**base_result, "skipped": "reference node teacher-forced replay failed"}

        result = verify_activation_proofs_detailed(reference_activations, claim, backend=server.toploc_backend)
        if node.wallet_address:
            server.toploc_calibration.record_run(
                node_wallet=node.wallet_address,
                gpu_model=gpu_model,
                dtype=dtype,
                model=model,
                passed=result.passed,
                exp_intersections=result.exp_intersections,
                mant_err_mean=result.mant_err_mean,
                mant_err_median=result.mant_err_median,
            )
        return {
            **base_result,
            "passed": result.passed,
            "exp_intersections": result.exp_intersections,
            "mant_err_mean": result.mant_err_mean,
            "mant_err_median": result.mant_err_median,
            "chunk_count": result.chunk_count,
        }

    def _handle_toploc_calibration_results(self):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin", "validator"):
            return
        if server.toploc_calibration is None:
            self._send_json(503, {"error": "toploc calibration store is not enabled on this tracker"})
            return
        min_profiles = self._toploc_calibration_gate_min_hardware_profiles()
        self._send_json(200, {
            "runs": server.toploc_calibration.runs(),
            "envelope": server.toploc_calibration.envelope(),
            "gate_status": server.toploc_calibration.gate_status(min_hardware_profiles=min_profiles),
        })

    def _handle_hf_pricing_history(self, parsed: urllib.parse.ParseResult):
        """Dispute-auditability log for the dynamic HF-benchmarked pricing (issue 23)."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if not self._require_role("admin", "validator"):
            return
        if server.hf_pricing_log is None:
            self._send_json(503, {"error": "hf pricing log is not enabled on this tracker"})
            return
        params = urllib.parse.parse_qs(parsed.query)
        model = params.get("model", [None])[0]
        self._send_json(200, {"changes": server.hf_pricing_log.history(model)})

    def _handle_assign(self, parsed: urllib.parse.ParseResult):
        """Return an optimal shard assignment for a node given its hardware profile.

        Query params:
          model   — model preset name (default: first preset)
          device  — "cuda" | "cpu"
          vram_mb — integer VRAM in MB (0 for CPU)
          ram_mb  — integer system RAM in MB, used when vram_mb=0

        The greedy strategy: find the first gap in current layer coverage
        and assign it.  If no gap exists, assign the full model range so the
        node provides redundant coverage.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        params = urllib.parse.parse_qs(parsed.query)

        model_list = params.get("model")
        if not model_list:
            model = next(iter(server.model_presets), None)
            if model is None:
                self._send_json(503, {"error": "no model presets configured"})
                return
        else:
            model = model_list[0]

        resolved_name, preset = _resolve_model_preset(server.model_presets, model)
        if preset is None:
            self._send_json(404, {"error": f"unknown model preset: {model!r}"})
            return

        required_start, required_end = _preset_layer_bounds(preset)

        with server.lock:
            self._purge_expired_nodes()
            alive = [
                node for node in server.registry.values()
                if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
            ]
        if server.contracts is not None:
            alive = [
                node for node in alive
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]

        device = params.get("device", ["cpu"])[0]
        try:
            vram_mb = int(params.get("vram_mb", ["0"])[0])
        except ValueError:
            vram_mb = 0
        try:
            ram_mb = int(params.get("ram_mb", ["0"])[0])
        except ValueError:
            ram_mb = 0
        max_layers = required_end - required_start + 1
        # VRAM only bounds the shard for CUDA nodes; a CPU node may still report
        # a detected-but-unusable GPU, and must be sized by system RAM.
        memory_mb = vram_mb if (device == "cuda" and vram_mb > 0) else ram_mb
        if memory_mb > 0:
            layer_bytes = _preset_bytes_per_layer(preset).get("bfloat16", 30 * 1024 * 1024)
            max_layers = min(max_layers, max(1, int(((memory_mb * 1024 * 1024) * 0.8) // layer_bytes)))
        elif device != "cuda" or vram_mb < 8192:
            max_layers = min(max_layers, 16)

        # Collect covered intervals sorted by start layer.
        covered = sorted(
            [
                (n.shard_start, n.shard_end)
                for n in alive
                if n.shard_start is not None and n.shard_end is not None
            ],
            key=lambda t: t[0],
        )

        # Walk from required_start to find the first uncovered layer.
        gap_start = required_start
        for s, e in covered:
            if s <= gap_start:
                gap_start = max(gap_start, e + 1)
            else:
                break  # gap found before this node

        if gap_start > required_end:
            # Full coverage exists — assign the full range for redundancy.
            shard_start = required_start
            shard_end = min(required_end, shard_start + max_layers - 1)
        else:
            shard_start = gap_start
            shard_end = min(required_end, shard_start + max_layers - 1)

        peers = [
            {"endpoint": node.endpoint, "checksum": node.shard_checksum}
            for node in alive
            if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
            and node.shard_start == shard_start
            and node.shard_end == shard_end
            and node.shard_checksum
        ]

        self._send_json(200, {
            "shard_start": shard_start,
            "shard_end": shard_end,
            "model": resolved_name,
            "model_layers_end": required_end,
            "peers": peers,
            "bytes_per_layer": _preset_bytes_per_layer(preset),
            "model_sources": self._model_sources(
                resolved_name,
                preset,
                shard_start,
                shard_end,
            ),
            **({"hf_repo": preset["hf_repo"]} if "hf_repo" in preset else {}),
        })

    def _model_sources(self, model: str, preset: dict, shard_start: int, shard_end: int) -> list[dict]:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        hf_repo = preset.get("hf_repo")
        if not server.models_dir or not isinstance(hf_repo, str) or not hf_repo:
            return []
        snapshot_dir = snapshot_dir_for_repo(server.models_dir, hf_repo)
        if snapshot_dir is None:
            return []
        files = files_for_layer_range(snapshot_dir, shard_start, shard_end)
        if not files:
            return []
        host = self.headers.get("Host") or f"{self.server.server_address[0]}:{self.server.server_address[1]}"
        base_url = f"http://{host}"
        query = urllib.parse.urlencode({
            "model": model,
            "shard_start": shard_start,
            "shard_end": shard_end,
        })
        full_query = urllib.parse.urlencode({"model": model, "full": 1})
        full_files = _snapshot_regular_files(snapshot_dir)
        file_sizes: dict[str, int] = {}
        for rel in set(files) | set(full_files):
            try:
                file_sizes[rel] = (snapshot_dir / rel).resolve().stat().st_size
            except OSError:
                continue
        return [{
            "type": "tracker",
            "endpoint": base_url,
            "url": f"{base_url}/v1/model-files/download?{query}",
            "full_url": f"{base_url}/v1/model-files/download?{full_query}",
            "files": files,
            "full_files": full_files,
            "file_sizes": file_sizes,
        }]

    def _handle_model_files_download(self, parsed: urllib.parse.ParseResult) -> None:
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        if server.models_dir is None:
            self._send_json(404, {"error": "tracker model-file source is not enabled"})
            return
        params = urllib.parse.parse_qs(parsed.query)
        model = params.get("model", [""])[0]
        resolved_name, preset = _resolve_model_preset(server.model_presets, model)
        if preset is None:
            self._send_json(404, {"error": f"unknown model preset: {model!r}"})
            return
        hf_repo = preset.get("hf_repo")
        if not isinstance(hf_repo, str) or not hf_repo:
            self._send_json(404, {"error": f"model preset has no hf_repo: {resolved_name!r}"})
            return
        snapshot_dir = snapshot_dir_for_repo(server.models_dir, hf_repo)
        if snapshot_dir is None:
            self._send_json(404, {"error": f"local snapshot not found for {hf_repo}"})
            return
        full_download = params.get("full", ["0"])[0] in {"1", "true", "yes"}
        if full_download:
            rel_files = _snapshot_regular_files(snapshot_dir)
        else:
            try:
                shard_start = int(params.get("shard_start", [""])[0])
                shard_end = int(params.get("shard_end", [""])[0])
            except ValueError:
                self._send_json(400, {"error": "shard_start and shard_end must be integers"})
                return
            rel_files = files_for_layer_range(snapshot_dir, shard_start, shard_end)
        if not rel_files:
            self._send_json(404, {"error": "no local files matched the assigned shard"})
            return

        # Single-file mode: ?file=<rel> streams one file with Content-Length so
        # clients can verify completeness and retry per file instead of
        # restarting one giant tar stream after a network hiccup.
        single = params.get("file", [""])[0]
        if single:
            if single not in set(rel_files):
                self._send_json(404, {"error": f"file not in requested shard: {single!r}"})
                return
            file_path = snapshot_dir / single
            try:
                # resolve() dereferences HF snapshot symlinks into blobs/.
                size = file_path.resolve().stat().st_size
            except OSError:
                self._send_json(404, {"error": f"file unreadable: {single!r}"})
                return
            self.send_response(200)
            self.send_header("Content-Type", "application/octet-stream")
            self.send_header("Content-Length", str(size))
            self.send_header("X-Meshnet-Model-Source", "tracker")
            self.end_headers()
            try:
                with file_path.open("rb") as f:
                    while True:
                        chunk = f.read(1024 * 1024)
                        if not chunk:
                            break
                        self.wfile.write(chunk)
            except (BrokenPipeError, ConnectionResetError):
                print(
                    f"model-file download aborted by client "
                    f"({self.client_address[0]}, model={resolved_name}, file={single})",
                    flush=True,
                )
            return

        self.send_response(200)
        self.send_header("Content-Type", "application/x-tar")
        self.send_header("X-Meshnet-Model-Source", "tracker")
        self.end_headers()
        try:
            # dereference: HF cache snapshots are symlinks into blobs/ — ship contents.
            with tarfile.open(fileobj=self.wfile, mode="w|", dereference=True) as archive:
                for rel in rel_files:
                    archive.add(snapshot_dir / rel, arcname=rel)
        except (BrokenPipeError, ConnectionResetError):
            print(
                f"model-file download aborted by client "
                f"({self.client_address[0]}, model={resolved_name})",
                flush=True,
            )

    def _handle_network_assign(self, parsed: urllib.parse.ParseResult):
        """Assign a new node to fill the biggest uncovered shard gap across HF-model nodes.

        Query params:
          vram_mb  — integer VRAM in MB (0 = CPU-only node)
          ram_mb   — integer system RAM in MB, used when vram_mb=0
          device   — "cuda" | "cpu"
          hf_repo  — optional; if set, restrict search to this repo only

        Returns:
          {hf_repo, shard_start, shard_end, num_layers, gap_found}
          gap_found=true means a real uncovered gap was assigned; false means redundancy.
        """
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        params = urllib.parse.parse_qs(parsed.query)
        try:
            vram_mb = int(params.get("vram_mb", ["0"])[0])
        except ValueError:
            vram_mb = 0
        try:
            ram_mb = int(params.get("ram_mb", ["0"])[0])
        except ValueError:
            ram_mb = 0
        device = params.get("device", ["cpu"])[0]
        filter_repo = params.get("hf_repo", [None])[0]  # optional repo filter

        with server.lock:
            self._purge_expired_nodes()
            all_nodes = list(server.registry.values())

        # Collect only nodes that registered a real HF model (have hf_repo + shard bounds).
        hf_nodes = [
            n for n in all_nodes
            if n.hf_repo
            and n.shard_start is not None
            and n.shard_end is not None
            and n.num_layers is not None
            and (filter_repo is None or n.hf_repo == filter_repo)
        ]

        if not hf_nodes:
            # The caller is not registered yet — count its memory toward
            # deployability, or the first node can never bootstrap the network.
            caller = _NodeEntry(
                node_id="assign-candidate",
                endpoint="",
                shard_start=None,
                shard_end=None,
                model=None,
                shard_checksum=None,
                hardware_profile={},
                wallet_address=None,
                score=1.0,
                vram_bytes=vram_mb * 1024 * 1024,
                ram_bytes=ram_mb * 1024 * 1024,
            )
            pool_with_caller = all_nodes + [caller]
            resolved_name = None
            preset = None
            if filter_repo:
                resolved_name, preset = _resolve_model_preset(server.model_presets, filter_repo)
            else:
                deployable = [
                    (name, preset)
                    for name, preset in server.model_presets.items()
                    if preset.get("recommended") and _deployment_summary(pool_with_caller, preset)["deployable"]
                ]
                if deployable:
                    resolved_name, preset = deployable[0]
            if preset is not None and preset.get("hf_repo"):
                required_start, required_end = _preset_layer_bounds(preset)
                total_l = required_end - required_start + 1
                memory_mb = vram_mb if (device == "cuda" and vram_mb > 0) else ram_mb
                max_layers = _max_layers_for_memory(memory_mb, total_l, preset, device=device)
                shard_start = required_start
                shard_end = min(required_end, shard_start + max_layers - 1)
                self._send_json(200, {
                    "hf_repo": preset["hf_repo"],
                    "model": resolved_name,
                    "shard_start": shard_start,
                    "shard_end": shard_end,
                    "num_layers": total_l,
                    "gap_found": True,
                    "price_per_token": 0.0,
                    "bytes_per_layer": _preset_bytes_per_layer(preset),
                    "deployment": _deployment_summary(pool_with_caller, preset),
                    "model_sources": self._model_sources(
                        str(resolved_name),
                        preset,
                        shard_start,
                        shard_end,
                    ),
                })
                return

            msg = (
                f"no HF-model nodes registered for {filter_repo!r}"
                if filter_repo
                else "no HF-model nodes registered; cannot assign shards"
            )
            self._send_json(503, {"error": msg})
            return

        # Group by hf_repo; pick the one with the largest total_layers and biggest gap.
        from collections import defaultdict
        repo_groups: dict = defaultdict(list)
        repo_layers: dict = {}
        for n in hf_nodes:
            repo_groups[n.hf_repo].append(n)
            # Use the largest num_layers seen for this repo.
            if n.hf_repo not in repo_layers or n.num_layers > repo_layers[n.hf_repo]:
                repo_layers[n.hf_repo] = n.num_layers

        # Smart scoring: demand_rpm × coverage_deficit
        # coverage_deficit = uncovered_layers / total_layers (0 = fully covered)
        # demand_rpm comes from the stats collector; defaults to 0.0 when no traffic yet.
        demand_rpms: dict[str, float] = {}
        if server.stats is not None:
            local_rpms = server.stats.get_local_rpms()
            for repo in repo_groups:
                demand_rpms[repo] = _repo_demand_rpm(server, repo, local_rpms)

        best_repo = None
        best_score = -1.0
        best_gap_size = -1
        best_gap_start = 0
        best_num_layers = 0

        for repo, nodes in repo_groups.items():
            total = repo_layers[repo]
            served_copies = _served_model_copies(nodes, 0, total - 1)
            covered = sorted(
                [(n.shard_start, n.shard_end) for n in nodes],
                key=lambda t: t[0],
            )
            # Walk from 0 to find first uncovered layer.
            gap_start = 0
            for s, e in covered:
                if s <= gap_start:
                    gap_start = max(gap_start, e + 1)
                else:
                    break
            gap_size = max(0, (total - 1) - gap_start + 1)
            coverage_deficit = gap_size / max(total, 1)
            demand = demand_rpms.get(repo, 0.0)
            # Prefer demanded models already being served; coverage gaps still dominate.
            score = (demand + 1.0) * (coverage_deficit + 0.01) + served_copies * 0.01
            if score > best_score or (score == best_score and gap_size > best_gap_size):
                best_score = score
                best_gap_size = gap_size
                best_gap_start = gap_start
                best_repo = repo
                best_num_layers = total

        gap_found = best_gap_size > 0
        if not gap_found:
            # All shards covered — add redundancy on the most demanded/served model.
            best_repo = max(
                repo_groups,
                key=lambda repo: (
                    demand_rpms.get(repo, 0.0),
                    _served_model_copies(repo_groups[repo], 0, repo_layers[repo] - 1),
                    len(repo_groups[repo]),
                ),
            )
            best_gap_start = 0
            best_num_layers = repo_layers[best_repo]

        # Capacity: use the same 80%-of-memory rule as registered node planning.
        total_l = best_num_layers
        memory_mb = vram_mb if (device == "cuda" and vram_mb > 0) else ram_mb
        resolved_name, best_preset = _resolve_model_preset(server.model_presets, str(best_repo))
        if memory_mb > 0:
            max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset, device=device)
        elif device == "cuda" and vram_mb >= 8192:
            max_layers = total_l
        else:
            max_layers = _max_layers_for_memory(memory_mb, total_l, best_preset, device=device)

        shard_start = best_gap_start
        shard_end = min(total_l - 1, shard_start + max_layers - 1)

        self._send_json(200, {
            "hf_repo": best_repo,
            "model": resolved_name,
            "shard_start": shard_start,
            "shard_end": shard_end,
            "num_layers": total_l,
            "gap_found": gap_found,
            "price_per_token": 0.0,
            "bytes_per_layer": _preset_bytes_per_layer(best_preset) if best_preset is not None else None,
            "model_sources": self._model_sources(
                str(resolved_name or best_repo),
                best_preset or {"hf_repo": best_repo},
                shard_start,
                shard_end,
            ),
        })

    def _handle_route(self, parsed: urllib.parse.ParseResult):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        params = urllib.parse.parse_qs(parsed.query)
        model_list = params.get("model")
        if not model_list:
            self._send_json(400, {"error": "missing 'model' query parameter"})
            return

        model = model_list[0]
        resolved_name, preset = _resolve_model_preset(server.model_presets, model)

        with server.lock:
            self._purge_expired_nodes()
            if preset is not None:
                # Preset-based routing (stub-model system).
                alive = [
                    node for node in server.registry.values()
                    if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
                ]
                required_start, required_end = _preset_layer_bounds(preset)
            else:
                # HF model routing: match by hf_repo (full) or model short name.
                alive = [
                    node for node in server.registry.values()
                    if _node_matches_model(node, model)
                    and node.shard_start is not None
                    and node.shard_end is not None
                    and node.num_layers is not None
                ]
                if not alive:
                    self._send_json(404, {"error": f"no nodes registered for model {model!r}"})
                    return
                required_start = 0
                required_end = max(n.num_layers for n in alive) - 1  # type: ignore[type-var]

        if server.contracts is not None:
            alive = [
                node for node in alive
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]

        route_model = resolved_name or model
        candidates = _enumerate_routes(
            alive,
            required_start,
            required_end,
            model=route_model,
            contracts=server.contracts,
            max_candidates=server.route_stats.config.max_candidate_routes,
            policy=server.capability_policy,
        )
        if candidates:
            # Prefer a distributed multi-hop route when available. Greedy
            # _select_route alone would pick a single full-shard node and omit
            # partial head shards that share the same port on another host.
            route = max(candidates, key=lambda cand: (len(cand.nodes), cand.prior_tps)).nodes
            error = ""
        else:
            route, error = _select_route(
                alive, required_start, required_end,
                contracts=server.contracts,
                policy=server.capability_policy,
            )
        if error:
            _tracker_log(
                server,
                "warn",
                "route unavailable",
                model=model,
                route_model=resolved_name or model,
                error=error,
                candidate_count=len(alive),
                candidates=_node_route_summary(alive),
            )
            self._send_json(503, {"error": error})
            return

        covered_up_to = required_start - 1
        route_with_start: list[tuple] = []
        for rn in route:
            route_with_start.append((rn, covered_up_to + 1))
            covered_up_to = rn.shard_end if rn.shard_end is not None else covered_up_to

        self._send_json(200, {
            "route": [e.endpoint for e, _ in route_with_start],
            "nodes": [
                {
                    "node_id": e.node_id,
                    "endpoint": e.endpoint,
                    "start_layer": start,
                    "relay_addr": e.relay_addr,
                    "peer_id": e.peer_id,
                    "wallet_address": e.wallet_address,
                    "shard_start": e.shard_start,
                    "shard_end": e.shard_end,
                    "model": e.model,
                    "hf_repo": e.hf_repo,
                    "num_layers": e.num_layers,
                    "model_metadata": dict(e.model_metadata),
                    "downloaded_models": [dict(item) for item in e.downloaded_models],
                    "shard_checksum": e.shard_checksum,
                    "score": e.score,
                }
                for e, start in route_with_start
            ],
        })

    def _handle_routing(self, parsed: urllib.parse.ParseResult):
        """Learned route table (ADR-0021): per-model candidates with observed
        tps, coefficient vs the best proven route, and expected traffic share."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        params = urllib.parse.parse_qs(parsed.query)
        only_model = (params.get("model") or [None])[0]
        with server.lock:
            self._purge_expired_nodes()
            alive = list(server.registry.values())
        model_keys: dict[str, list] = {}
        for node in alive:
            if node.shard_start is None or node.shard_end is None:
                continue
            key = node.hf_repo or node.model
            if not key:
                continue
            model_keys.setdefault(key, []).append(node)
        cfg = server.route_stats.config
        out: dict[str, dict] = {}
        for key, nodes in model_keys.items():
            if only_model and key != only_model and not any(
                key == alias for alias in _model_aliases(only_model)
            ):
                continue
            resolved_name, preset = _resolve_model_preset(server.model_presets, key)
            # Stats are recorded under the proxy's resolved route_model —
            # use the same key here or lookups always miss.
            stats_key = resolved_name or key
            if preset is not None:
                rs, re = _preset_layer_bounds(preset)
            else:
                layer_counts = [n.num_layers for n in nodes if n.num_layers is not None]
                if not layer_counts:
                    continue
                rs, re = 0, max(layer_counts) - 1
            candidates = _enumerate_routes(
                nodes, rs, re,
                model=stats_key,
                contracts=server.contracts,
                max_candidates=cfg.max_candidate_routes,
                policy=server.capability_policy,
            )
            out[stats_key] = {
                "epoch": server.route_stats.epoch(stats_key),
                "routes": route_table(candidates, server.route_stats, stats_key),
            }
        self._send_json(200, {
            "config": {
                "explore_share": cfg.explore_share,
                "weight_alpha": cfg.weight_alpha,
                "stats_half_life_seconds": cfg.stats_half_life_seconds,
                "min_sample_tokens": cfg.min_sample_tokens,
            },
            "models": out,
        })

    def _handle_model_speed(self, parsed: urllib.parse.ParseResult):
        """Drill into observed model speed and the latency cost of each route."""
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        model = (urllib.parse.parse_qs(parsed.query).get("model") or [None])[0]
        if not model:
            self._send_json(400, {"error": "missing 'model' query parameter"})
            return
        resolved_name, preset = _resolve_model_preset(server.model_presets, model)
        stats_key = resolved_name or model
        with server.lock:
            self._purge_expired_nodes()
            nodes_by_id = dict(server.registry)
        routes = server.route_stats.model_rows(stats_key)
        one_hop = [r["latency_ms"] for r in routes if r["hop_count"] == 1 and r["latency_ms"] is not None]
        baseline_ms = min(one_hop) if one_hop else None
        for route in routes:
            node_ids = [part.split("[", 1)[0] for part in route["signature"].split("|", 1)[1].split("->")]
            devices = []
            for node_id in node_ids:
                node = nodes_by_id.get(node_id)
                profile = node.hardware_profile if node is not None else {}
                devices.append("gpu" if node is not None and (node.vram_bytes > 0 or profile.get("device") == "cuda") else "cpu")
            route["node_ids"] = node_ids
            route["device_mix"] = "-".join(dict.fromkeys(devices))
            route["latency_penalty_ms"] = round(route["latency_ms"] - baseline_ms, 3) if baseline_ms is not None else None
        node_rows = []
        for node in nodes_by_id.values():
            if not _node_matches_model(node, model) and not (preset and _node_matches_preset(node, resolved_name or model, preset)):
                continue
            sample = server.stats.get_node_model_stats(node.node_id, stats_key) if server.stats else {}
            node_rows.append({
                "node_id": node.node_id,
                "device": "gpu" if node.vram_bytes > 0 or node.hardware_profile.get("device") == "cuda" else "cpu",
                "benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
                **sample,
            })
        self._send_json(200, {"model": stats_key, "baseline_one_hop_latency_ms": baseline_ms, "routes": routes, "nodes": node_rows})

    def _handle_routes(self, parsed: urllib.parse.ParseResult):
        server: _TrackerHTTPServer = self.server  # type: ignore[assignment]
        params = urllib.parse.parse_qs(parsed.query)
        model_list = params.get("model")
        if not model_list:
            self._send_json(400, {"error": "missing 'model' query parameter"})
            return

        try:
            redundancy = int(params.get("redundancy", ["1"])[0])
        except ValueError:
            self._send_json(400, {"error": "redundancy must be an integer"})
            return
        if redundancy < 1:
            self._send_json(400, {"error": "redundancy must be at least 1"})
            return

        model = model_list[0]
        resolved_name, preset = _resolve_model_preset(server.model_presets, model)
        if preset is None:
            self._send_json(404, {"error": f"unknown model preset: {model!r}"})
            return

        required_start, required_end = _preset_layer_bounds(preset)

        with server.lock:
            self._purge_expired_nodes()
            candidates = [
                node for node in server.registry.values()
                if _node_matches_preset(node, resolved_name, preset)  # type: ignore[arg-type]
            ]
        if server.contracts is not None:
            candidates = [
                node for node in candidates
                if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
            ]

        routes = []
        remaining = list(candidates)
        for _ in range(redundancy):
            route, error = _select_route(
                remaining, required_start, required_end,
                contracts=server.contracts,
                policy=server.capability_policy,
            )
            if error:
                self._send_json(503, {"error": error})
                return
            route_endpoints = {node.endpoint for node in route}
            remaining = [node for node in remaining if node.endpoint not in route_endpoints]
            routes.append({
                "route": [e.endpoint for e in route],
                "nodes": [
                    {
                        "node_id": e.node_id,
                        "endpoint": e.endpoint,
                        "relay_addr": e.relay_addr,
                        "peer_id": e.peer_id,
                        "wallet_address": e.wallet_address,
                        "shard_start": e.shard_start,
                        "shard_end": e.shard_end,
                        "model": e.model,
                        "downloaded_models": [dict(item) for item in e.downloaded_models],
                        "shard_checksum": e.shard_checksum,
                        "score": e.score,
                    }
                    for e in route
                ],
            })

        self._send_json(200, {"routes": routes})


class TrackerServer:
    """HTTP tracker that manages node registration and resolves inference routes.

    Nodes register via POST /v1/nodes/register and keep themselves alive with
    POST /v1/nodes/<id>/heartbeat.  The gateway queries GET /v1/route?model=<name>
    to obtain an ordered list of node endpoints whose shards cover all layers.
    """

    def __init__(
        self,
        host: str = "127.0.0.1",
        port: int = 0,
        heartbeat_timeout: float = 90.0,
        rebalance_interval: float = 30.0,
        model_presets: dict | None = None,
        contracts: Any | None = None,
        minimum_stake: int = 0,
        cluster_peers: list[str] | None = None,
        cluster_self_url: str | None = None,
        stats_db: str | None = None,
        relay_url: str | None = None,
        embedded_relay: bool = False,
        embedded_relay_host: str = "127.0.0.1",
        embedded_relay_port: int = 8765,
        embedded_relay_max_peers: int = 500,
        billing: BillingLedger | None = None,
        enable_billing: bool = True,
        billing_db: str | None = None,
        accounts: AccountStore | None = None,
        accounts_db: str | None = None,
        benchmark_results_path: str | None = None,
        treasury: Any | None = None,
        deposit_poll_interval: float = 15.0,
        settle_period: float = 86400.0,
        payout_threshold: float = 5.0,
        payout_dust_floor: float = 0.01,
        settlement_check_interval: float | None = None,
        validator_service_token: str | None = None,
        hive_secret: str | None = None,
        max_charge_per_request: float | None = None,
        starting_credit: float = DEFAULT_CALLER_CREDIT_USDT,
        devnet_topup_amount: float = DEFAULT_DEVNET_TOPUP_USDT,
        toploc_calibration: ToplocCalibrationStore | None = None,
        toploc_calibration_db: str | None = None,
        toploc_reference_node_url: str | None = None,
        toploc_calibration_gate_min_hardware_profiles: int = 1,
        toploc_backend: Any | None = None,
        enable_hf_pricing: bool = False,
        hf_pricing_log: HfPricingLog | None = None,
        hf_pricing_log_db: str | None = None,
        hf_pricing_refresh_interval: float = 86400.0,
        hf_pricing_fetch_html: Any | None = None,
        models_dir: str | Path | None = None,
        routing_config: RoutingConfig | None = None,
        enable_test_runner: bool = False,
        test_runner: TestRunManager | None = None,
        capability_policy: str | None = None,
    ) -> None:
        self._capability_policy = normalize_policy(
            capability_policy if capability_policy is not None else policy_from_env()
        )
        self._host = host
        self._requested_port = port
        self._heartbeat_timeout = heartbeat_timeout
        self._rebalance_interval = rebalance_interval
        self._model_presets: dict = (
            model_presets if model_presets is not None else _clone_model_presets(DEFAULT_MODEL_PRESETS)
        )
        self._contracts = contracts
        self._minimum_stake = minimum_stake
        self._cluster_peers: list[str] = list(cluster_peers) if cluster_peers else []
        self._cluster_self_url: str | None = cluster_self_url
        self._relay_url = relay_url
        self._embedded_relay_enabled = bool(embedded_relay)
        self._embedded_relay_host = embedded_relay_host
        self._embedded_relay_port = int(embedded_relay_port)
        self._embedded_relay_max_peers = int(embedded_relay_max_peers)
        self._embedded_relay: Any | None = None
        self._embedded_relay_actual_port: int | None = None
        self._registry: dict[str, _NodeEntry] = {}
        self._lock = threading.Lock()
        self._server: _TrackerHTTPServer | None = None
        self._thread: threading.Thread | None = None
        self._rebalance_stop = threading.Event()
        self._rebalance_thread: threading.Thread | None = None
        self._raft: RaftNode | None = None
        self._gossip: NodeGossip | None = None
        self._stats: _StatsCollector | None = _StatsCollector(db_path=stats_db) if stats_db else _StatsCollector()
        self._stats_stop = threading.Event()
        self._stats_thread: threading.Thread | None = None
        if billing is None:
            db_path = billing_db
            if db_path is None and enable_billing:
                db_path = DEFAULT_BILLING_DB_PATH
            if db_path:
                preset_prices: dict[str, tuple[float, float]] = {}
                for name, preset in self._model_presets.items():
                    if not isinstance(preset, dict):
                        continue
                    base = preset.get("price_per_1k_tokens")
                    input_rate = preset.get("input_price_per_1k_tokens", base)
                    output_rate = preset.get("output_price_per_1k_tokens", base)
                    if input_rate is None or output_rate is None:
                        continue
                    for key in _preset_price_keys(name, preset):
                        preset_prices[key] = (float(input_rate), float(output_rate))
                billing = BillingLedger(db_path=db_path, prices=preset_prices)
        self._billing: BillingLedger | None = billing
        self._billing_gossip_cursor = 0
        if accounts is None:
            accounts_path = accounts_db
            if accounts_path is None and enable_billing:
                accounts_path = DEFAULT_ACCOUNTS_DB_PATH
            if accounts_path:
                accounts = AccountStore(db_path=accounts_path)
        self._accounts: AccountStore | None = accounts
        self._accounts_gossip_cursor = 0
        self._registry_gossip_cursor = 0
        self._benchmark_results_path = benchmark_results_path
        self._treasury = treasury
        self._deposit_poll_interval = deposit_poll_interval
        self._deposit_stop = threading.Event()
        self._deposit_thread: threading.Thread | None = None
        self._settle_period = settle_period
        self._payout_threshold = payout_threshold
        self._payout_dust_floor = payout_dust_floor
        self._settlement_check_interval = settlement_check_interval or max(
            1.0, min(settle_period / 4.0, 15.0)
        )
        self._settlement_stop = threading.Event()
        self._settlement_thread: threading.Thread | None = None
        if max_charge_per_request is not None and max_charge_per_request <= 0.0:
            raise ValueError("max_charge_per_request must be positive")
        self._max_charge_per_request = max_charge_per_request
        self._starting_credit = max(0.0, starting_credit)
        self._devnet_topup_amount = max(0.0, devnet_topup_amount)
        self._validator_service_token = (
            validator_service_token
            if validator_service_token is not None
            else os.environ.get("MESHNET_VALIDATOR_SERVICE_TOKEN") or None
        )
        self._hive_secret = (
            hive_secret
            if hive_secret is not None
            else os.environ.get("MESHNET_HIVE_SECRET") or None
        )
        if toploc_calibration is None and toploc_calibration_db:
            toploc_calibration = ToplocCalibrationStore(db_path=toploc_calibration_db)
        self._toploc_calibration: ToplocCalibrationStore | None = toploc_calibration
        self._toploc_reference_node_url = toploc_reference_node_url
        self._toploc_calibration_gate_min_hardware_profiles = toploc_calibration_gate_min_hardware_profiles
        self._toploc_backend = toploc_backend
        if hf_pricing_log is None and (enable_hf_pricing or hf_pricing_log_db):
            hf_pricing_log = HfPricingLog(db_path=hf_pricing_log_db or DEFAULT_HF_PRICING_LOG_DB_PATH)
        self._hf_pricing_log: HfPricingLog | None = hf_pricing_log
        self._enable_hf_pricing = enable_hf_pricing
        self._hf_pricing_refresh_interval = hf_pricing_refresh_interval
        self._hf_pricing_fetch_html = hf_pricing_fetch_html
        # MESHNET_DOWNLOAD_DIR is the node-side model store; on a box running
        # both tracker and node it doubles as the tracker's snapshot source.
        raw_models_dir = (
            models_dir
            if models_dir is not None
            else os.environ.get("MESHNET_MODELS_DIR") or os.environ.get("MESHNET_DOWNLOAD_DIR")
        )
        self._models_dir = Path(raw_models_dir).expanduser() if raw_models_dir else None
        self._hf_pricing_stop = threading.Event()
        self._hf_pricing_thread: threading.Thread | None = None
        if routing_config is None:
            routing_config = RoutingConfig(
                explore_share=float(os.environ.get("MESHNET_ROUTE_EXPLORE_SHARE", RoutingConfig.explore_share)),
                weight_alpha=float(os.environ.get("MESHNET_ROUTE_WEIGHT_ALPHA", RoutingConfig.weight_alpha)),
                stats_half_life_seconds=float(
                    os.environ.get("MESHNET_ROUTE_STATS_HALF_LIFE", RoutingConfig.stats_half_life_seconds)
                ),
                min_sample_tokens=int(
                    os.environ.get("MESHNET_ROUTE_MIN_SAMPLE_TOKENS", RoutingConfig.min_sample_tokens)
                ),
            )
        self._route_stats = RouteStatsStore(routing_config, db_path=stats_db)
        # Opt-in test runner: an injected manager implies enabled; otherwise
        # only the explicit flag or env var constructs one (fails closed).
        if test_runner is None and (
            enable_test_runner or os.environ.get(TEST_RUNNER_ENABLE_ENV_VAR) == "1"
        ):
            test_runner = TestRunManager()
        self._test_runner: TestRunManager | None = test_runner
        self.port: int | None = None

    def _start_embedded_relay(self) -> dict:
        """Start the shared RelayServer class in-process for tracker+relay deployments."""
        if not self._embedded_relay_enabled:
            return {"mode": "external" if self._relay_url else "off"}
        if self._embedded_relay is not None:
            return self._embedded_relay_status()
        try:
            from meshnet_relay.server import RelayServer  # type: ignore[import-not-found]
        except Exception as exc:  # pragma: no cover - only hit in broken installs
            raise RuntimeError(
                "embedded relay requires the meshnet-relay package to be installed"
            ) from exc
        relay = RelayServer(
            host=self._embedded_relay_host,
            port=self._embedded_relay_port,
            max_peers=self._embedded_relay_max_peers,
        )
        actual_port = relay.start()
        self._embedded_relay = relay
        self._embedded_relay_actual_port = actual_port
        return self._embedded_relay_status()

    def _embedded_relay_status(self) -> dict:
        advertised_url = (
            self._relay_url
            or derive_relay_url_from_public_tracker_url(self._cluster_self_url)
            or _local_relay_url(self._embedded_relay_host, self._embedded_relay_actual_port or self._embedded_relay_port)
        )
        return {
            "mode": "embedded",
            "url": advertised_url,
            "bind_host": self._embedded_relay_host,
            "bind_port": self._embedded_relay_actual_port or self._embedded_relay_port,
            "max_peers": self._embedded_relay_max_peers,
        }

    def start(self) -> int:
        if self._server is not None:
            raise RuntimeError("TrackerServer is already running")

        relay_status = self._start_embedded_relay()
        effective_relay_url = (
            self._relay_url
            or derive_relay_url_from_public_tracker_url(self._cluster_self_url)
            or (relay_status.get("url") if relay_status.get("mode") == "embedded" else None)
        )

        http_relay_status = (
            {**relay_status, "url": effective_relay_url}
            if relay_status.get("mode") == "embedded"
            else {"mode": "external", "url": effective_relay_url}
            if effective_relay_url
            else {"mode": "off"}
        )

        # Start HTTP server first so we know our port
        self._server = _TrackerHTTPServer(
            (self._host, self._requested_port),
            _TrackerHandler,
            self._registry,
            self._lock,
            self._heartbeat_timeout,
            self._model_presets,
            self._contracts,
            self._minimum_stake,
            relay_url=effective_relay_url,
            stats=self._stats,
            billing=self._billing,
            accounts=self._accounts,
            benchmark_results_path=self._benchmark_results_path,
            validator_service_token=self._validator_service_token,
            hive_secret=self._hive_secret,
            max_charge_per_request=self._max_charge_per_request,
            starting_credit=self._starting_credit,
            devnet_topup_amount=self._devnet_topup_amount,
            toploc_calibration=self._toploc_calibration,
            toploc_reference_node_url=self._toploc_reference_node_url,
            toploc_calibration_gate_min_hardware_profiles=self._toploc_calibration_gate_min_hardware_profiles,
            toploc_backend=self._toploc_backend,
            hf_pricing_log=self._hf_pricing_log,
            models_dir=self._models_dir,
            route_stats=self._route_stats,
            relay_status=http_relay_status,
            test_runner=self._test_runner,
            capability_policy=self._capability_policy,
        )
        self.port = self._server.server_address[1]

        # Start Raft + gossip if cluster peers are configured
        if self._cluster_peers:
            self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
            self._raft = RaftNode(
                self_url=self_url,
                peers=self._cluster_peers,
                apply_fn=self._raft_apply,
            )
            self._gossip = NodeGossip(peers=self._cluster_peers)
            self._server.raft = self._raft
            self._server.gossip = self._gossip
            self._raft.start()
            self._gossip.start()

        self._rebalance_stop.clear()
        self._stats_stop.clear()
        self._thread = threading.Thread(target=self._server.serve_forever, daemon=True)
        self._thread.start()
        self._rebalance_thread = threading.Thread(target=self._rebalance_loop, daemon=True)
        self._rebalance_thread.start()
        self._stats_thread = threading.Thread(target=self._stats_loop, daemon=True)
        self._stats_thread.start()
        if self._treasury is not None and self._billing is not None:
            self._deposit_stop.clear()
            self._deposit_thread = threading.Thread(target=self._deposit_loop, daemon=True)
            self._deposit_thread.start()
            self._settlement_stop.clear()
            self._settlement_thread = threading.Thread(target=self._settlement_loop, daemon=True)
            self._settlement_thread.start()
        if self._enable_hf_pricing and self._billing is not None:
            self._hf_pricing_stop.clear()
            self._hf_pricing_thread = threading.Thread(target=self._hf_pricing_loop, daemon=True)
            self._hf_pricing_thread.start()
        return self.port

    def _settlement_loop(self) -> None:
        """Leader-only on-chain settlement (US-033, ADR-0015).

        Pay a node when pending ≥ threshold OR its pending age ≥ max period,
        with a dust floor. Pending is debited (payout events, replicated)
        before the transaction is sent; unconfirmed batches are resent with
        the same settlement id, so retries never double-pay.
        """
        billing = self._billing
        treasury = self._treasury
        assert billing is not None and treasury is not None
        while not self._settlement_stop.wait(self._settlement_check_interval):
            if self._raft is not None and not self._raft.is_leader:
                continue  # followers replicate the ledger but never sign
            # resend batches whose transaction never confirmed
            for settlement_id, payouts in billing.unconfirmed_settlements().items():
                self._send_settlement(settlement_id, payouts)
            banned: set[str] = set()
            if self._server is not None and self._server.contracts is not None:
                registry = self._server.contracts.registry
                banned = {
                    wallet for wallet, _ in billing.payables(
                        threshold=0.0, max_period=0.0, dust_floor=0.0,
                    )
                    if registry.get_wallet(wallet).banned
                }
            due = billing.payables(
                threshold=self._payout_threshold,
                max_period=self._settle_period,
                dust_floor=self._payout_dust_floor,
                exclude=banned,
            )
            if not due:
                continue
            settlement_id = uuid.uuid4().hex
            registry = (
                self._server.contracts.registry
                if self._server is not None and self._server.contracts is not None
                else None
            )
            settled: list[tuple[str, float]] = []
            for wallet, amount in due:
                # ADR-0015: a fraud forfeiture landing between the payables()
                # snapshot above and this debit must never be paid out on top
                # of — recheck the ban and let settle_node_payout clamp to
                # whatever is still actually pending.
                if registry is not None and registry.get_wallet(wallet).banned:
                    continue
                event = billing.settle_node_payout(wallet, amount, reference=settlement_id)
                if event["amount"] > 0:
                    settled.append((wallet, event["amount"]))
            if settled:
                self._send_settlement(settlement_id, settled)

    def _send_settlement(self, settlement_id: str, payouts: list) -> None:
        billing = self._billing
        treasury = self._treasury
        assert billing is not None and treasury is not None
        try:
            signature = treasury.send_payouts([(w, a) for w, a in payouts])
        except Exception as exc:
            print(
                f"[tracker] settlement {settlement_id} failed (will retry): {exc}",
                flush=True,
            )
            return
        billing.confirm_settlement(settlement_id, signature)
        total = sum(a for _, a in payouts)
        print(
            f"[tracker] settled {settlement_id}: {len(payouts)} payout(s), "
            f"{total:.6f} USDT total (tx {signature})",
            flush=True,
        )

    def _deposit_loop(self) -> None:
        """Poll the treasury token account and credit confirmed deposits (US-032)."""
        billing = self._billing
        treasury = self._treasury
        assert billing is not None and treasury is not None
        while not self._deposit_stop.wait(self._deposit_poll_interval):
            try:
                deposits = treasury.list_new_deposits(
                    lambda sig: billing.has_event(f"deposit-{sig}")
                )
            except Exception as exc:
                print(f"[tracker] deposit poll failed: {exc}", flush=True)
                continue
            for deposit in deposits:
                api_key = billing.api_key_for_wallet(deposit.sender_wallet)
                if api_key is None:
                    print(
                        f"[tracker] unattributed deposit {deposit.signature}: "
                        f"{deposit.amount_usdt} USDT from unbound wallet "
                        f"{deposit.sender_wallet} — register via /v1/wallet/register",
                        flush=True,
                    )
                    continue
                credited = billing.credit_deposit(
                    api_key, deposit.amount_usdt, deposit.signature
                )
                if credited is not None:
                    print(
                        f"[tracker] deposit credited: {deposit.amount_usdt} USDT "
                        f"-> api_key …{api_key[-6:]} (tx {deposit.signature})",
                        flush=True,
                    )

    def _hf_pricing_loop(self) -> None:
        """Daily dynamic pricing refresh benchmarked against HF inference rates (issue 23).

        For every preset with a curated, human-verified ``hf_aliases`` list,
        fetch current HF marketplace pricing and set the client price to 80%
        of the cheapest matching alias. Presets with no (or an empty)
        ``hf_aliases`` are left entirely alone — they keep the static
        default price. Any single preset's fetch/parse failure is logged and
        skipped; it never raises into this loop or the request path.
        """
        billing = self._billing
        assert billing is not None
        while not self._hf_pricing_stop.wait(self._hf_pricing_refresh_interval):
            for name, preset in list(self._model_presets.items()):
                if not isinstance(preset, dict) or not preset.get("hf_aliases"):
                    continue
                try:
                    result = refresh_preset_price(
                        model_name=name,
                        preset=preset,
                        current_price=billing.price_for(name),
                        fetch_html=self._hf_pricing_fetch_html,
                    )
                except Exception as exc:
                    print(f"[tracker] hf pricing refresh failed for {name!r}: {exc}", flush=True)
                    continue
                if result is None:
                    continue
                new_input = result.get("new_input_price_per_1k", result["new_price_per_1k"])
                new_output = result.get("new_output_price_per_1k", result["new_price_per_1k"])
                for key in _preset_price_keys(name, preset):
                    billing.set_prices(key, new_input, new_output)
                preset["hf_last_price_per_1k"] = result["new_price_per_1k"]
                preset["hf_last_updated"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
                if self._hf_pricing_log is not None:
                    self._hf_pricing_log.record_change(
                        model=name,
                        old_price_per_1k=result["old_price_per_1k"],
                        new_price_per_1k=result["new_price_per_1k"],
                        source_repo_id=result["source_repo_id"],
                        source_provider=result["source_provider"],
                    )
                print(
                    f"[tracker] hf pricing: {name} {result['old_price_per_1k']:.6f} -> "
                    f"{result['new_price_per_1k']:.6f} USDT/1k tokens "
                    f"(80% of {result['source_repo_id']}::{result['source_provider']})",
                    flush=True,
                )

    def _raft_apply(self, command: str, payload: dict) -> None:
        """Called by RaftNode when a log entry is committed — replicate to local registry."""
        if command != "register":
            return
        # Re-apply the registration to our local registry (follower path).
        # On the leader this is a no-op since it already registered locally.
        node_id = payload.get("node_id")
        if not node_id or node_id in self._registry:
            return  # already present (leader case) or malformed
        endpoint = payload.get("endpoint", "")
        try:
            shard_start = int(payload["shard_start"]) if payload.get("shard_start") is not None else None
            shard_end   = int(payload["shard_end"])   if payload.get("shard_end")   is not None else None
        except (TypeError, ValueError):
            return
        try:
            friendly_name = _normalize_friendly_name(payload.get("friendly_name"))
        except ValueError:
            friendly_name = None
        # The replicated payload is the raw registration body, so the follower can
        # resolve precision exactly as the leader did -- including telling a legacy
        # absent `quantization` from a declared one. Dropping these fields here
        # would rebuild every node at the default precision and diverge follower
        # routing from leader routing.
        quantizations_payload = payload.get("quantizations")
        quantizations = (
            list(quantizations_payload)
            if (
                isinstance(quantizations_payload, list)
                and quantizations_payload
                and all(isinstance(item, str) and item for item in quantizations_payload)
            )
            else list(DEFAULT_QUANTIZATIONS)
        )
        entry = _NodeEntry(
            node_id=node_id,
            endpoint=endpoint.rstrip("/"),
            shard_start=shard_start,
            shard_end=shard_end,
            model=payload.get("model", "stub-model"),
            shard_checksum=payload.get("shard_checksum"),
            hardware_profile=payload.get("hardware_profile", {}),
            wallet_address=payload.get("wallet_address"),
            score=float(payload.get("score", 1.0)),
            quantizations=quantizations,
            quantization=_registration_quantization(payload, quantizations),
            tracker_mode=bool(payload.get("tracker_mode", False)),
            hf_repo=payload.get("hf_repo"),
            num_layers=int(payload["num_layers"]) if payload.get("num_layers") is not None else None,
            model_metadata=payload.get("model_metadata") if isinstance(payload.get("model_metadata"), dict) else None,
            downloaded_models=(
                payload.get("downloaded_models")
                if isinstance(payload.get("downloaded_models"), list)
                else None
            ),
            friendly_name=_normalize_friendly_name(payload.get("friendly_name")),
            # A replicated registration carries its proof: without this, a proven
            # node would be routable on the leader and dark on every follower.
            capability=_capability_from_registration(
                payload,
                model=payload.get("model", "stub-model"),
                hf_repo=payload.get("hf_repo"),
                shard_start=shard_start,
                shard_end=shard_end,
            ),
        )
        with self._lock:
            self._registry[node_id] = entry

    def _rebalance_loop(self) -> None:
        while not self._rebalance_stop.wait(self._rebalance_interval):
            server = self._server
            if server is None:
                return
            with self._lock:
                _purge_expired_nodes_locked(server)
                _rebalance_all_locked(server)
                _scale_demanded_models_locked(server)

    def _push_to_peers(self, path: str, body: bytes) -> bool:
        """POST a hive-signed payload to every cluster peer; True if all succeeded."""
        headers = {"Content-Type": "application/json"}
        if self._hive_secret:
            headers.update(sign_hive_request(self._hive_secret, body))
        delivered_all = True
        for peer in self._cluster_peers:
            try:
                req = urllib.request.Request(
                    f"{peer}{path}", data=body, headers=headers, method="POST",
                )
                with urllib.request.urlopen(req, timeout=2.0) as r:
                    r.read()
            except Exception:
                delivered_all = False
        return delivered_all

    def _save_stats_once(self) -> None:
        """Persist each store independently so one DB failure cannot kill the loop."""
        registry_log = getattr(self._contracts, "registry_log", None)
        stores = (
            ("request stats", self._stats),
            ("route stats", self._route_stats),
            ("billing", self._billing),
            ("accounts", self._accounts),
            ("registry", registry_log),
        )
        for label, store in stores:
            if store is None:
                continue
            try:
                store.save_to_db()
            except Exception as exc:
                print(
                    f"[tracker] warn: {label} persistence failed: {exc}",
                    file=sys.stderr,
                    flush=True,
                )

    def _stats_loop(self) -> None:
        """Periodically save stats/billing to DB and push local slices to cluster peers."""
        while not self._stats_stop.wait(_StatsCollector.SAVE_INTERVAL):
            self._save_stats_once()
            if self._cluster_peers and not self._hive_secret:
                print(
                    "[tracker] WARNING: cluster peers configured without --hive-secret — "
                    "gossip pushes will be rejected (ADR-0017)",
                    flush=True,
                )
            if self._stats is not None and self._cluster_peers:
                self_url = self._cluster_self_url or f"http://{self._host}:{self.port}"
                local_rpms = self._stats.get_local_rpms()
                body = json.dumps({"tracker_url": self_url, "stats": local_rpms}).encode()
                self._push_to_peers("/v1/stats/gossip", body)
            if self._billing is not None and self._cluster_peers:
                events, cursor = self._billing.events_since(self._billing_gossip_cursor)
                if events:
                    body = json.dumps({"events": events}).encode()
                    # Only advance past events every peer has seen; unreachable
                    # peers get the full backlog on the next tick (idempotent
                    # via event-id dedupe on the receiving side).
                    if self._push_to_peers("/v1/billing/gossip", body):
                        self._billing_gossip_cursor = cursor
            if self._accounts is not None and self._cluster_peers:
                events, cursor = self._accounts.events_since(self._accounts_gossip_cursor)
                if events:
                    body = json.dumps({"events": events}).encode()
                    if self._push_to_peers("/v1/accounts/gossip", body):
                        self._accounts_gossip_cursor = cursor
            registry_log = getattr(self._contracts, "registry_log", None)
            if registry_log is not None and self._cluster_peers:
                events, cursor = registry_log.events_since(self._registry_gossip_cursor)
                if events:
                    body = json.dumps({"events": events}).encode()
                    if self._push_to_peers("/v1/registry/gossip", body):
                        self._registry_gossip_cursor = cursor

    def stop(self) -> None:
        if self._raft is not None:
            self._raft.stop()
        if self._gossip is not None:
            self._gossip.stop()
        embedded_relay = self._embedded_relay
        self._embedded_relay = None
        self._embedded_relay_actual_port = None
        if embedded_relay is not None:
            embedded_relay.stop()
        if self._test_runner is not None:
            self._test_runner.shutdown()
        if self._server is None:
            return
        self._rebalance_stop.set()
        self._stats_stop.set()
        self._deposit_stop.set()
        self._settlement_stop.set()
        self._hf_pricing_stop.set()
        if self._stats is not None:
            self._stats.save_to_db()
        self._route_stats.save_to_db()
        if self._billing is not None:
            self._billing.save_to_db()
        if self._accounts is not None:
            self._accounts.save_to_db()
        registry_log = getattr(self._contracts, "registry_log", None)
        if registry_log is not None:
            registry_log.save_to_db()
        self._server.shutdown()
        self._server.server_close()
        if self._thread is not None:
            self._thread.join(timeout=1)
        if self._rebalance_thread is not None:
            self._rebalance_thread.join(timeout=1)
        if self._stats_thread is not None:
            self._stats_thread.join(timeout=1)
        if self._deposit_thread is not None:
            self._deposit_thread.join(timeout=1)
            self._deposit_thread = None
        if self._settlement_thread is not None:
            self._settlement_thread.join(timeout=1)
            self._settlement_thread = None
        if self._hf_pricing_thread is not None:
            self._hf_pricing_thread.join(timeout=1)
            self._hf_pricing_thread = None
        self._server = None
        self._thread = None
        self._rebalance_thread = None
        self._stats_thread = None
        self._raft = None
        self._gossip = None
        self.port = None