popov/neuron-tai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dash works !!! good data. billing seems to work

Browse Source
This commit is contained in:
Dobromir Popov
2026-07-02 23:07:41 +02:00
parent 1e0aa6ea8f
commit a938c19a82
14 changed files with 617 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
packages/gateway/meshnet_gateway/server.py
View File

@@ -159,7 +159,9 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
"shard_coverage_percentage": float(model.get("shard_coverage_percentage", 0.0)),
}
for model in models_resp.get("data", [])
if isinstance(model, dict) and isinstance(model.get("id"), str)
if isinstance(model, dict)
and isinstance(model.get("id"), str)
and float(model.get("shard_coverage_percentage", 0.0)) > 0.0
]
self._send_json(200, {"data": data})
return
@@ -244,7 +246,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
def _handle_chat_completions(self):
server: _GatewayHTTPServer = self.server # type: ignore[assignment]
# Read raw bytes first so we can proxy them if tracker-nodes are available
# Read raw bytes first so we can proxy them if head workers are available.
length = int(self.headers.get("Content-Length", 0))
raw_body = self.rfile.read(length)
try:
@@ -257,12 +259,11 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
return
model = str(body.get("model", "stub-model"))
tracker_nodes = _get_tracker_nodes(server, model)
if tracker_nodes:
# Proxy to a tracker-node (round-robin by request count)
target = tracker_nodes[server.request_count % len(tracker_nodes)]
head_workers = _get_head_workers(server, model)
if head_workers:
target = head_workers[server.request_count % len(head_workers)]
server.request_count += 1
return self._proxy_to_tracker_node(target, raw_body)
return self._proxy_to_head_worker(target, raw_body)
# Fallback: use existing direct pipeline (backward compat)
streaming = bool(body.get("stream", False))
@@ -284,8 +285,8 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
else:
self._send_json(200, completion)
def _proxy_to_tracker_node(self, url: str, body_bytes: bytes) -> None:
"""Forward a raw request body to a tracker-node and stream the response back."""
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
"""Forward a raw request body to a head worker and stream the response back."""
target_url = f"{url}/v1/chat/completions"
req = urllib.request.Request(
target_url,
@@ -307,7 +308,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(resp_body)
return
except Exception as exc:
self._send_json_error(503, f"tracker-node unreachable: {exc}")
self._send_json_error(503, f"head worker unreachable: {exc}")
return
self.send_response(status)
self.send_header("Content-Type", content_type)
@@ -816,13 +817,13 @@ def _get_json(url: str, timeout: float = 5.0) -> dict:
return json.loads(r.read())
def _get_tracker_nodes(server: _GatewayHTTPServer, model: str) -> list[str]:
"""Return tracker-node endpoint URLs for this model, or empty list on any failure."""
def _get_head_workers(server: _GatewayHTTPServer, model: str) -> list[str]:
"""Return head-worker endpoint URLs for this model, or empty list on failure."""
if server.tracker_url is None:
return []
try:
resp = _get_json(f"{server.tracker_url}/v1/tracker-nodes/{urllib.parse.quote(model)}")
return [n["endpoint"] for n in resp.get("tracker_nodes", [])]
resp = _get_json(f"{server.tracker_url}/v1/head-workers/{urllib.parse.quote(model)}")
return [n["endpoint"] for n in resp.get("head_workers", [])]
except Exception:
return []