popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(alpha): complete hardening backlog

Browse Source 
Complete the alpha-hardening Ralph task set, including tracker billing/accounting guards, validator fraud-audit primitives, wallet binding proof support, documentation runbooks, and updated tests.

Verification: .venv/bin/python -m compileall -q packages tests; .venv/bin/python -m pytest -q --tb=short (313 passed, 3 skipped, 1 failed: tests/test_mining_cli.py::test_legacy_start_without_port_uses_next_available_port because meshnet-node pid 1263451 is already listening on port 7000).
This commit is contained in:
Dobromir Popov
2026-07-05 21:47:23 +03:00
parent c967e5cfc4
commit 9abe83b5f4
45 changed files with 4095 additions and 774 deletions
Download Patch File Download Diff File Expand all files Collapse all files

300
packages/tracker/meshnet_tracker/server.py
View File

@@ -38,6 +38,7 @@ from typing import Any
from .accounts import DEFAULT_ACCOUNTS_DB_PATH, AccountStore
from .auth import is_validator_token, sign_hive_request, verify_hive_request
from .wallet_proof import binding_message, verify_wallet_signature
from .billing import DEFAULT_BILLING_DB_PATH, BillingLedger
from .gossip import NodeGossip
from .raft import RaftNode
@@ -574,17 +575,30 @@ def _effective_throughput(node: "_NodeEntry", model: str | None = None) -> float
return base / (node.queue_depth + 1)
def _reputation_multiplier(node: "_NodeEntry", contracts: Any | None) -> float:
"""ADR-0018 §6: veteran, good-standing nodes route more -- earnings scale
with tenure/reputation. Maps reputation [0, 1] to a [0.5, 1.0] routing
weight so a damaged-but-not-banned wallet loses traffic gradually rather
than being cut off outright (banning is handled separately)."""
if contracts is None or not node.wallet_address:
return 1.0
reputation = contracts.registry.get_wallet(node.wallet_address).reputation
return 0.5 + 0.5 * reputation
def _select_route(
nodes: list[_NodeEntry],
required_start: int,
required_end: int,
model: str | None = None,
contracts: Any | None = None,
) -> tuple[list[_NodeEntry], str]:
"""Greedy interval-cover biased toward fast, lightly-loaded nodes.
"""Greedy interval-cover biased toward fast, lightly-loaded, reputable nodes.
Among nodes that equally advance coverage, prefer the one with higher
effective throughput: observed per-model tokens/sec / (queue_depth + 1),
falling back to startup benchmark_tokens_per_sec until observations exist.
falling back to startup benchmark_tokens_per_sec until observations exist,
weighted by the node's reputation multiplier (see `_reputation_multiplier`).
Tiebreak: higher shard_end (fewer hops).
"""
candidates = sorted(
@@ -594,6 +608,9 @@ def _select_route(
route: list[_NodeEntry] = []
covered_up_to = required_start - 1
def _routing_score(node: "_NodeEntry") -> float:
return _effective_throughput(node, model) * _reputation_multiplier(node, contracts)
while covered_up_to < required_end:
best: _NodeEntry | None = None
for node in candidates:
@@ -602,7 +619,7 @@ def _select_route(
best = node
elif node.shard_end > best.shard_end:
best = node
elif node.shard_end == best.shard_end and _effective_throughput(node, model) > _effective_throughput(best, model):
elif node.shard_end == best.shard_end and _routing_score(node) > _routing_score(best):
best = node
if best is None:
missing = covered_up_to + 1
@@ -1216,6 +1233,101 @@ def _usage_total_tokens(payload: dict) -> int | None:
return None
def _estimate_text_tokens(value: Any) -> int | None:
if isinstance(value, str):
text = value.strip()
if not text:
return 0
return len(text.split())
if isinstance(value, list):
total = 0
found = False
for item in value:
if isinstance(item, str):
estimated = _estimate_text_tokens(item)
elif isinstance(item, dict):
estimated = _estimate_text_tokens(item.get("text"))
else:
estimated = None
if estimated is not None:
total += estimated
found = True
return total if found else None
return None
def _estimate_prompt_tokens(body: dict) -> int | None:
messages = body.get("messages")
if isinstance(messages, list):
total = 0
found = False
for message in messages:
if not isinstance(message, dict):
continue
estimated = _estimate_text_tokens(message.get("content"))
if estimated is not None:
total += estimated
found = True
return total if found else None
prompt = body.get("prompt")
return _estimate_text_tokens(prompt)
def _requested_completion_token_limit(body: dict) -> int | None:
for field in ("max_completion_tokens", "max_tokens"):
value = body.get(field)
if isinstance(value, bool):
return None
if isinstance(value, (int, float)):
return max(0, int(value))
return None
def _request_total_token_upper_bound(body: dict) -> int | None:
completion_limit = _requested_completion_token_limit(body)
if completion_limit is None:
return None
prompt_estimate = _estimate_prompt_tokens(body)
return completion_limit + (prompt_estimate or 0)
def _billable_non_stream_tokens(payload: dict, request_body: dict) -> int:
reported = _usage_total_tokens(payload)
if reported is None:
return 0
billable = max(0, reported)
upper_bound = _request_total_token_upper_bound(request_body)
if upper_bound is not None:
billable = min(billable, upper_bound)
return billable
def _observed_stream_tokens(payload: dict) -> int:
choices = payload.get("choices")
if not isinstance(choices, list):
return 0
observed = 0
for choice in choices:
if not isinstance(choice, dict):
continue
delta = choice.get("delta")
if isinstance(delta, dict):
estimated = _estimate_text_tokens(delta.get("content"))
if estimated:
observed += estimated
continue
if choice:
observed += 1
return observed
def _billable_stream_tokens(observed_tokens: int, reported_tokens: int | None) -> int:
observed = max(0, observed_tokens)
if reported_tokens is None:
return observed
return min(observed, max(0, reported_tokens))
def _registration_ban_error(contracts: Any | None, wallet_address: str | None) -> str | None:
if contracts is None or not wallet_address:
return None
@@ -1267,6 +1379,7 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
benchmark_results_path: str | None = None,
validator_service_token: str | None = None,
hive_secret: str | None = None,
max_charge_per_request: float | None = None,
) -> None:
super().__init__(addr, handler)
self.registry = registry
@@ -1287,6 +1400,7 @@ class _TrackerHTTPServer(socketserver.ThreadingMixIn, http.server.HTTPServer):
self.benchmark_lock = threading.Lock()
self.validator_service_token = validator_service_token
self.hive_secret = hive_secret
self.max_charge_per_request = max_charge_per_request
class _TrackerHandler(http.server.BaseHTTPRequestHandler):
@@ -1422,6 +1536,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if self.path == "/v1/accounts/gossip":
self._handle_accounts_gossip()
return
if self.path == "/v1/registry/gossip":
self._handle_registry_gossip()
return
if self.path == "/v1/benchmark/hop-penalty":
self._handle_benchmark_hop_penalty()
return
@@ -1752,6 +1869,30 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"code": "insufficient_balance",
}})
return
if server.max_charge_per_request is not None:
token_limit = _requested_completion_token_limit(body)
if token_limit is None:
self._send_json(400, {"error": {
"message": (
"max_charge_per_request is enabled; include max_tokens "
"or max_completion_tokens so this request can be capped"
),
"type": "invalid_request_error",
"code": "missing_token_limit",
}})
return
estimated_charge = server.billing.price_for(model) * token_limit / 1000.0
if estimated_charge > server.max_charge_per_request:
self._send_json(402, {"error": {
"message": (
f"request exceeds max_charge_per_request "
f"({estimated_charge:.6f} USDT estimated > "
f"{server.max_charge_per_request:.6f} USDT cap)"
),
"type": "insufficient_quota",
"code": "spend_cap_exceeded",
}})
return
# US-030: optional pinned route — "route": [node_id, ...] uses those
# nodes in order instead of auto-selection. Absent field: unchanged.
@@ -1831,7 +1972,14 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if pinned_nodes is not None:
route_nodes = pinned_nodes
else:
route_nodes, _ = _select_route(all_nodes, rs, re, model=route_model)
route_nodes, route_error = _select_route(all_nodes, rs, re, model=route_model, contracts=server.contracts)
if route_error:
self._send_json(503, {"error": {
"message": route_error,
"type": "service_unavailable",
"code": "route_not_available",
}})
return
# Compute start_layer for each hop: each node begins where the previous ended + 1.
# This allows overlapping shard registrations without double-computation.
covered_up_to = rs - 1
@@ -1898,7 +2046,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if int(relayed.get("status", 503)) < 400:
body_text = relayed.get("body") or ""
try:
tokens = _usage_total_tokens(json.loads(body_text)) or 0
tokens = _billable_non_stream_tokens(json.loads(body_text), body)
except (json.JSONDecodeError, TypeError):
tokens = 0
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
@@ -1950,8 +2098,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.send_header("Content-Type", "text/event-stream; charset=utf-8")
self.send_header("Cache-Control", "no-cache")
self.end_headers()
stream_tokens: int | None = None
chunk_count = 0
reported_stream_tokens: int | None = None
observed_stream_tokens = 0
try:
while True:
line = upstream.readline()
@@ -1962,21 +2110,22 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if line.startswith(b"data:"):
payload = line[5:].strip()
if payload and payload != b"[DONE]":
chunk_count += 1
if b'"usage"' in payload:
try:
found = _usage_total_tokens(json.loads(payload))
if found:
stream_tokens = found
except json.JSONDecodeError:
pass
try:
chunk_payload = json.loads(payload)
except json.JSONDecodeError:
observed_stream_tokens += 1
continue
observed_stream_tokens += _observed_stream_tokens(chunk_payload)
found = _usage_total_tokens(chunk_payload)
if found is not None:
reported_stream_tokens = found
except BrokenPipeError:
pass
elapsed = time.monotonic() - started
# Bill even on client disconnect — the nodes did the work.
# Chunk count approximates generated tokens when the stream
# carries no usage record.
observed_tokens = stream_tokens if stream_tokens is not None else chunk_count
# Observed stream chunks are authoritative for the upper bound;
# upstream usage may only lower that count.
observed_tokens = _billable_stream_tokens(observed_stream_tokens, reported_stream_tokens)
self._record_observed_throughput(model, route_model, observed_tokens, elapsed, route_nodes)
self._bill_completed(
api_key, model,
@@ -1991,6 +2140,12 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
f"[tracker] proxy complete {request_id}: {target_url} bytes={len(resp_body)}",
flush=True,
)
try:
tokens = _billable_non_stream_tokens(json.loads(resp_body), body)
except json.JSONDecodeError:
tokens = 0
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
self._bill_completed(api_key, model, tokens, node_work)
self.send_response(200)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(resp_body)))
@@ -1999,12 +2154,6 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(resp_body)
except BrokenPipeError:
pass
try:
tokens = _usage_total_tokens(json.loads(resp_body)) or 0
except json.JSONDecodeError:
tokens = 0
self._record_observed_throughput(model, route_model, tokens, elapsed, route_nodes)
self._bill_completed(api_key, model, tokens, node_work)
def _record_observed_throughput(
self,
@@ -2468,6 +2617,11 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
"strike_count": state.strike_count,
"banned": state.banned,
"completed_jobs": state.completed_job_count,
"reputation": state.reputation,
"last_audit_ts": state.last_audit_ts,
"probationary_jobs_remaining": (
server.contracts.registry.probationary_jobs_remaining(wallet)
),
}
for wallet, state in wallets.items()
}})
@@ -2568,7 +2722,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
api_key = accounts.create_api_key(account["account_id"])
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
if server.billing is not None:
server.billing.ensure_client(api_key) # grant Caller Credit up front
server.billing.ensure_client(api_key)
print(
f"[tracker] account registered: {account.get('email') or account.get('wallet')} "
f"role={account['role']}", flush=True,
@@ -2691,20 +2845,39 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
applied = server.accounts.apply_events([e for e in events if isinstance(e, dict)])
self._send_json(200, {"applied": applied})
def _handle_registry_gossip(self):
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
body = self._read_hive_authenticated_body()
if body is None:
return
registry_log = getattr(server.contracts, "registry_log", None)
if registry_log is None:
self._send_json(200, {"applied": 0})
return
events = body.get("events")
if not isinstance(events, list):
self._send_json(400, {"error": "events must be a list"})
return
applied = registry_log.apply_events([e for e in events if isinstance(e, dict)])
self._send_json(200, {"applied": applied})
def _handle_wallet_register(self):
"""Bind the caller's client wallet pubkey to their API key (US-032).
"""Bind a client wallet pubkey to an API key (US-032, C6).
Deposits from that wallet into the treasury are then credited to the
API key's ledger balance by the deposit watcher.
Requires a signature proving ownership of the wallet's private key —
the caller signs ``binding_message(api_key, wallet)`` with the
wallet's ed25519 key. An admin session may force-rebind a wallet
already bound to a different API key (documented override; no
signed-release flow is implemented).
"""
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
api_key = _api_key_from_headers(self.headers)
if not api_key:
self._send_json(401, {"error": "Authorization header required"})
return
if server.billing is None:
self._send_json(404, {"error": "billing is not enabled on this tracker"})
return
role, _account = self._resolve_identity()
body = self._read_json_body()
if body is None:
return
@@ -2712,7 +2885,30 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not wallet or not isinstance(wallet, str):
self._send_json(400, {"error": "wallet is required"})
return
server.billing.bind_wallet(api_key, wallet)
if role == "admin":
api_key = body.get("api_key")
if not api_key or not isinstance(api_key, str):
self._send_json(400, {"error": "api_key is required for admin override"})
return
event = server.billing.bind_wallet(api_key, wallet, admin_override=True)
else:
api_key = _api_key_from_headers(self.headers)
if not api_key:
self._send_json(401, {"error": "Authorization header required"})
return
signature = body.get("signature")
if not signature or not isinstance(signature, str):
self._send_json(400, {"error": "signature is required to prove wallet ownership"})
return
if not verify_wallet_signature(wallet, binding_message(api_key, wallet), signature):
self._send_json(401, {"error": "invalid wallet signature"})
return
event = server.billing.bind_wallet(api_key, wallet)
if event.get("rejected"):
self._send_json(409, {"error": "wallet already bound to a different API key"})
return
print(f"[tracker] wallet bound: {wallet} -> api_key …{api_key[-6:]}", flush=True)
self._send_json(200, {"wallet": wallet, "bound": True})
@@ -3130,7 +3326,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
]
route, error = _select_route(alive, required_start, required_end)
route, error = _select_route(alive, required_start, required_end, contracts=server.contracts)
if error:
self._send_json(503, {"error": error})
return
@@ -3204,7 +3400,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
routes = []
remaining = list(candidates)
for _ in range(redundancy):
route, error = _select_route(remaining, required_start, required_end)
route, error = _select_route(remaining, required_start, required_end, contracts=server.contracts)
if error:
self._send_json(503, {"error": error})
return
@@ -3267,6 +3463,7 @@ class TrackerServer:
settlement_check_interval: float | None = None,
validator_service_token: str | None = None,
hive_secret: str | None = None,
max_charge_per_request: float | None = None,
) -> None:
self._host = host
self._requested_port = port
@@ -3312,6 +3509,7 @@ class TrackerServer:
accounts = AccountStore(db_path=accounts_path)
self._accounts: AccountStore | None = accounts
self._accounts_gossip_cursor = 0
self._registry_gossip_cursor = 0
self._benchmark_results_path = benchmark_results_path
self._treasury = treasury
self._deposit_poll_interval = deposit_poll_interval
@@ -3325,6 +3523,9 @@ class TrackerServer:
)
self._settlement_stop = threading.Event()
self._settlement_thread: threading.Thread | None = None
if max_charge_per_request is not None and max_charge_per_request <= 0.0:
raise ValueError("max_charge_per_request must be positive")
self._max_charge_per_request = max_charge_per_request
self._validator_service_token = (
validator_service_token
if validator_service_token is not None
@@ -3363,6 +3564,7 @@ class TrackerServer:
benchmark_results_path=self._benchmark_results_path,
validator_service_token=self._validator_service_token,
hive_secret=self._hive_secret,
max_charge_per_request=self._max_charge_per_request,
)
self.port = self._server.server_address[1]
@@ -3432,9 +3634,24 @@ class TrackerServer:
if not due:
continue
settlement_id = uuid.uuid4().hex
registry = (
self._server.contracts.registry
if self._server is not None and self._server.contracts is not None
else None
)
settled: list[tuple[str, float]] = []
for wallet, amount in due:
billing.settle_node_payout(wallet, amount, reference=settlement_id)
self._send_settlement(settlement_id, due)
# ADR-0015: a fraud forfeiture landing between the payables()
# snapshot above and this debit must never be paid out on top
# of — recheck the ban and let settle_node_payout clamp to
# whatever is still actually pending.
if registry is not None and registry.get_wallet(wallet).banned:
continue
event = billing.settle_node_payout(wallet, amount, reference=settlement_id)
if event["amount"] > 0:
settled.append((wallet, event["amount"]))
if settled:
self._send_settlement(settlement_id, settled)
def _send_settlement(self, settlement_id: str, payouts: list) -> None:
billing = self._billing
@@ -3557,6 +3774,9 @@ class TrackerServer:
self._billing.save_to_db()
if self._accounts is not None:
self._accounts.save_to_db()
registry_log = getattr(self._contracts, "registry_log", None)
if registry_log is not None:
registry_log.save_to_db()
if self._cluster_peers and not self._hive_secret:
print(
"[tracker] WARNING: cluster peers configured without --hive-secret — "
@@ -3583,6 +3803,13 @@ class TrackerServer:
body = json.dumps({"events": events}).encode()
if self._push_to_peers("/v1/accounts/gossip", body):
self._accounts_gossip_cursor = cursor
registry_log = getattr(self._contracts, "registry_log", None)
if registry_log is not None and self._cluster_peers:
events, cursor = registry_log.events_since(self._registry_gossip_cursor)
if events:
body = json.dumps({"events": events}).encode()
if self._push_to_peers("/v1/registry/gossip", body):
self._registry_gossip_cursor = cursor
def stop(self) -> None:
if self._raft is not None:
@@ -3601,6 +3828,9 @@ class TrackerServer:
self._billing.save_to_db()
if self._accounts is not None:
self._accounts.save_to_db()
registry_log = getattr(self._contracts, "registry_log", None)
if registry_log is not None:
registry_log.save_to_db()
self._server.shutdown()
self._server.server_close()
if self._thread is not None: