more inference fixes

packages/node/meshnet_node/torch_server.py

@@ -686,6 +686,7 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
         last_gen_log = gen_started
         progress_line = [False]
         last_token_id: int | None = None
+        failure_reason: str | None = None
 
         def _prefill_step() -> tuple[str, int | None]:
             """Full-sequence prefill: initial step and cache-miss recovery."""
@@ -721,12 +722,15 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
                     token_str, token_id = _prefill_step()
             except _PipelineCacheMiss as exc:
                 print(f"  [node] unexpected cache miss on prefill: {exc}", flush=True)
+                failure_reason = f"cache miss on prefill: {exc}"
                 break
             except Exception as exc:
                 print(f"  [node] distributed encode error: {exc}", flush=True)
+                failure_reason = f"distributed encode error: {exc}"
                 break
             # Stop on error responses or EOS.
             if token_str.startswith(("pipeline error", "decode error", "no downstream", "error:")):
+                failure_reason = token_str
                 break
             if token_id is not None and token_id in eos_ids:
                 break
@@ -778,6 +782,16 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
             )
 
         result_text = "".join(generated)
+        # A failure before the first token is an upstream error, not an empty
+        # completion — tell the client instead of returning a blank 200.
+        if failure_reason and not generated:
+            if stream_emit is not None:
+                stream_emit(None, error=failure_reason)
+                return
+            self._send_json(502, {
+                "error": {"message": failure_reason, "type": "upstream_error"},
+            })
+            return
         if stream_emit is not None:
             stream_emit(None)
             return
@@ -1037,7 +1051,17 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
             "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}],
         }))
 
-        def emit_token(token_text: str | None) -> None:
+        def emit_token(token_text: str | None, *, error: str | None = None) -> None:
+            if error is not None:
+                # OpenAI-style mid-stream error frame; clients surface it
+                # instead of showing an empty completion.
+                _emit(json.dumps({"error": {"message": error, "type": "upstream_error"}}))
+                try:
+                    self.wfile.write(b"data: [DONE]\n\n")
+                    self.wfile.flush()
+                except (BrokenPipeError, ConnectionResetError):
+                    pass
+                return
             if token_text is None:
                 _emit(json.dumps({
                     "id": chunk_id, "object": "chat.completion.chunk", "created": created,