try to fix streaming responses

This commit is contained in:
Dobromir Popov
2026-07-07 22:19:22 +03:00
parent a0b37ad1b9
commit f0dc3bd93f
4 changed files with 140 additions and 26 deletions

View File

@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
self._send_json(200, completion)
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
"""Forward a raw request body to a head worker and stream the response back."""
"""Forward a raw request body to a head worker and relay SSE without buffering."""
target_url = f"{url}/v1/chat/completions"
req = urllib.request.Request(
target_url,
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
try:
with urllib.request.urlopen(req, timeout=30.0) as r:
content_type = r.headers.get("Content-Type", "application/json")
if "text/event-stream" in content_type:
self.send_response(r.status)
self.send_header("Content-Type", content_type)
self.send_header("Cache-Control", "no-cache")
self.send_header("X-Accel-Buffering", "no")
self.end_headers()
while True:
line = r.readline()
if not line:
break
self.wfile.write(line)
self.wfile.flush()
return
resp_body = r.read()
status = r.status
except urllib.error.HTTPError as exc: