inference fixes

This commit is contained in:
Dobromir Popov
2026-07-09 20:46:29 +02:00
parent 2b000ce9c3
commit 3d264a500a
6 changed files with 120 additions and 4 deletions

View File

@@ -1508,6 +1508,23 @@ def _assign_redundant_managed_nodes(
_emit_shard_change_directives(node, model, previous_range, preset)
# Activation tensors ride the relay as base64 inside one JSON frame, so the
# websockets default of 1 MiB rejects any real prefill (close code 1009).
DEFAULT_WS_MAX_BYTES = 256 * 1024 * 1024
def _ws_max_size() -> int | None:
"""Max inbound WebSocket frame size; MESHNET_WS_MAX_BYTES<=0 means unlimited."""
raw = os.environ.get("MESHNET_WS_MAX_BYTES", "").strip()
if not raw:
return DEFAULT_WS_MAX_BYTES
try:
value = int(raw)
except ValueError:
return DEFAULT_WS_MAX_BYTES
return None if value <= 0 else value
def _relay_http_request_frames(
relay_addr: str,
path: str,
@@ -1537,7 +1554,7 @@ def _relay_http_request_frames(
request_id = str(uuid.uuid4())
deadline = time.monotonic() + timeout
try:
with wsc.connect(relay_addr, open_timeout=10, close_timeout=5) as ws:
with wsc.connect(relay_addr, open_timeout=10, close_timeout=5, max_size=_ws_max_size()) as ws:
if ws_holder is not None:
if ws_lock is not None:
with ws_lock: