inference fixes
This commit is contained in:
@@ -1508,6 +1508,23 @@ def _assign_redundant_managed_nodes(
|
||||
_emit_shard_change_directives(node, model, previous_range, preset)
|
||||
|
||||
|
||||
# Activation tensors ride the relay as base64 inside one JSON frame, so the
|
||||
# websockets default of 1 MiB rejects any real prefill (close code 1009).
|
||||
DEFAULT_WS_MAX_BYTES = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def _ws_max_size() -> int | None:
|
||||
"""Max inbound WebSocket frame size; MESHNET_WS_MAX_BYTES<=0 means unlimited."""
|
||||
raw = os.environ.get("MESHNET_WS_MAX_BYTES", "").strip()
|
||||
if not raw:
|
||||
return DEFAULT_WS_MAX_BYTES
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
return DEFAULT_WS_MAX_BYTES
|
||||
return None if value <= 0 else value
|
||||
|
||||
|
||||
def _relay_http_request_frames(
|
||||
relay_addr: str,
|
||||
path: str,
|
||||
@@ -1537,7 +1554,7 @@ def _relay_http_request_frames(
|
||||
request_id = str(uuid.uuid4())
|
||||
deadline = time.monotonic() + timeout
|
||||
try:
|
||||
with wsc.connect(relay_addr, open_timeout=10, close_timeout=5) as ws:
|
||||
with wsc.connect(relay_addr, open_timeout=10, close_timeout=5, max_size=_ws_max_size()) as ws:
|
||||
if ws_holder is not None:
|
||||
if ws_lock is not None:
|
||||
with ws_lock:
|
||||
|
||||
Reference in New Issue
Block a user