feat: tracker-as-first-layer-node inference entry point (US-014)
- Tracker: add GET /v1/tracker-nodes/<model> returning nodes registered with tracker_mode=true whose shard_start matches the model's first layer - Node: StubNodeServer and TorchNodeServer accept tracker_mode/tracker_url; when tracker_mode=True (or auto-detected via shard_start==0 for Torch), /v1/chat/completions is served alongside /forward - TorchNodeServer: full pipeline implementation — encode_prompt → route selection via tracker → binary forward through remaining hops → decode - Gateway: _handle_chat_completions checks _get_tracker_nodes() first and proxies round-robin to tracker-nodes; falls back to existing direct pipeline when none found (preserves all US-005 backward compat) - CLI: --tracker-mode and --tracker-url flags added to meshnet-node start - Test: two stub tracker-nodes + two mid-shard nodes for gpt2; 10 requests; round-robin 5/5 split verified; all OpenAI-format responses validated - All 78 tests pass Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -55,7 +55,7 @@ class _NodeEntry:
|
||||
"model", "shard_checksum", "hardware_profile", "wallet_address",
|
||||
"score", "vram_bytes", "ram_bytes", "quantizations",
|
||||
"benchmark_tokens_per_sec", "quantization", "managed_assignment",
|
||||
"pending_directives", "last_heartbeat",
|
||||
"pending_directives", "last_heartbeat", "tracker_mode",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
@@ -75,6 +75,7 @@ class _NodeEntry:
|
||||
benchmark_tokens_per_sec: float = DEFAULT_BENCHMARK_TOKENS_PER_SEC,
|
||||
quantization: str | None = None,
|
||||
managed_assignment: bool = False,
|
||||
tracker_mode: bool = False,
|
||||
) -> None:
|
||||
self.node_id = node_id
|
||||
self.endpoint = endpoint
|
||||
@@ -91,6 +92,7 @@ class _NodeEntry:
|
||||
self.benchmark_tokens_per_sec = benchmark_tokens_per_sec
|
||||
self.quantization = quantization
|
||||
self.managed_assignment = managed_assignment
|
||||
self.tracker_mode = tracker_mode
|
||||
self.pending_directives: list[dict] = []
|
||||
self.last_heartbeat: float = time.monotonic()
|
||||
|
||||
@@ -429,6 +431,9 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
elif parsed.path.startswith("/v1/coverage/"):
|
||||
model = urllib.parse.unquote(parsed.path.removeprefix("/v1/coverage/"))
|
||||
self._handle_coverage(model)
|
||||
elif parsed.path.startswith("/v1/tracker-nodes/"):
|
||||
model = urllib.parse.unquote(parsed.path.removeprefix("/v1/tracker-nodes/"))
|
||||
self._handle_tracker_nodes(model)
|
||||
else:
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
@@ -482,6 +487,40 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
coverage = _coverage_map(alive, required_start, required_end)
|
||||
self._send_json(200, {"model": model, "coverage": coverage})
|
||||
|
||||
def _handle_tracker_nodes(self, model: str):
|
||||
"""Return nodes registered with tracker_mode=True whose shard starts at layer 0."""
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
preset = server.model_presets.get(model)
|
||||
if preset is None:
|
||||
self._send_json(404, {"error": f"unknown model preset: {model!r}"})
|
||||
return
|
||||
required_start, _ = _preset_layer_bounds(preset)
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
alive = [node for node in server.registry.values() if node.model == model]
|
||||
if server.contracts is not None:
|
||||
alive = [
|
||||
node for node in alive
|
||||
if not node.wallet_address or not server.contracts.registry.get_wallet(node.wallet_address).banned
|
||||
]
|
||||
tracker_nodes = [
|
||||
node for node in alive
|
||||
if node.shard_start is not None
|
||||
and node.shard_start == required_start
|
||||
and node.tracker_mode
|
||||
]
|
||||
self._send_json(200, {
|
||||
"model": model,
|
||||
"tracker_nodes": [
|
||||
{
|
||||
"node_id": node.node_id,
|
||||
"endpoint": node.endpoint,
|
||||
"benchmark_tokens_per_sec": node.benchmark_tokens_per_sec,
|
||||
}
|
||||
for node in tracker_nodes
|
||||
],
|
||||
})
|
||||
|
||||
def _handle_register(self):
|
||||
server: _TrackerHTTPServer = self.server # type: ignore[assignment]
|
||||
body = self._read_json_body()
|
||||
@@ -567,6 +606,8 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._send_json(403, {"error": ban_error})
|
||||
return
|
||||
|
||||
tracker_mode = bool(body.get("tracker_mode", False))
|
||||
|
||||
node_id = str(uuid.uuid4())
|
||||
entry = _NodeEntry(
|
||||
node_id=node_id,
|
||||
@@ -584,6 +625,7 @@ class _TrackerHandler(http.server.BaseHTTPRequestHandler):
|
||||
benchmark_tokens_per_sec=benchmark_tokens_per_sec,
|
||||
quantization=quantization,
|
||||
managed_assignment=not explicit_shard,
|
||||
tracker_mode=tracker_mode,
|
||||
)
|
||||
with server.lock:
|
||||
self._purge_expired_nodes()
|
||||
|
||||
Reference in New Issue
Block a user