try to fix streaming responses
This commit is contained in:
@@ -286,7 +286,7 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
||||
self._send_json(200, completion)
|
||||
|
||||
def _proxy_to_head_worker(self, url: str, body_bytes: bytes) -> None:
|
||||
"""Forward a raw request body to a head worker and stream the response back."""
|
||||
"""Forward a raw request body to a head worker and relay SSE without buffering."""
|
||||
target_url = f"{url}/v1/chat/completions"
|
||||
req = urllib.request.Request(
|
||||
target_url,
|
||||
@@ -297,6 +297,19 @@ class _GatewayHandler(http.server.BaseHTTPRequestHandler):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30.0) as r:
|
||||
content_type = r.headers.get("Content-Type", "application/json")
|
||||
if "text/event-stream" in content_type:
|
||||
self.send_response(r.status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("X-Accel-Buffering", "no")
|
||||
self.end_headers()
|
||||
while True:
|
||||
line = r.readline()
|
||||
if not line:
|
||||
break
|
||||
self.wfile.write(line)
|
||||
self.wfile.flush()
|
||||
return
|
||||
resp_body = r.read()
|
||||
status = r.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
|
||||
Reference in New Issue
Block a user