fix: proper autoregressive inference with streaming support

Single-node mode now uses HF model.generate() instead of one-shot
decode_tail(), giving correct multi-token output with KV cache.

model_backend.py:
- generate_text(messages, max_new_tokens, temperature, top_p) — full
  autoregressive generation via model.generate() with chat template
- generate_text_streaming() — yields token strings via TextIteratorStreamer
- _encode_messages() — applies chat template (tokenize=False then tokenize),
  falls back to joining user messages; avoids BatchEncoding issues

torch_server.py:
- _handle_chat_completions: fast path when backend is head+tail — calls
  generate_text() or generate_text_streaming() directly instead of the
  single-token encode_prompt+decode_tail pipeline
- _stream_openai_response: new SSE streaming handler for token iterators
- Parses max_tokens, temperature, top_p from request body
- Distributed path (partial shards) unchanged

Verified: streaming and non-streaming both work with Qwen2.5-0.5B-Instruct.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

packages/node/meshnet_node/torch_server.py

@@ -213,8 +213,31 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
         if body is None:
             return
         messages = body.get("messages", [])
+        if not isinstance(messages, list):
+            messages = []
         stream = bool(body.get("stream", False))
-        model = str(body.get("model", ""))
+        model_name = str(body.get("model", ""))
+        max_tokens = int(body.get("max_tokens") or body.get("max_new_tokens") or 256)
+        temperature = float(body.get("temperature") or 1.0)
+        top_p = float(body.get("top_p") or 1.0)
+
+        # Fast path: this node owns the complete model — use HF generate() with KV cache.
+        # Avoids the single-token-per-forward-pass limitation of the distributed path.
+        if server.backend.is_head and server.backend.is_tail:
+            try:
+                if stream:
+                    self._stream_openai_response(
+                        server.backend.generate_text_streaming(messages, max_tokens, temperature, top_p),
+                        model_name,
+                    )
+                else:
+                    text = server.backend.generate_text(messages, max_tokens, temperature, top_p)
+                    self._send_openai_response(text, model_name, False)
+            except Exception as exc:
+                self._send_json(500, {"error": f"generation failed: {exc}"})
+            return
+
+        # Distributed path: encode prompt at the head, forward activations along the route.
         prompt = " ".join(
             str(m.get("content", ""))
             for m in messages
@@ -225,9 +248,9 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
         except Exception as exc:
             self._send_json(500, {"error": f"encode_prompt failed: {exc}"})
             return
-        remaining_route = self._get_remaining_route(model)
+        remaining_route = self._get_remaining_route(model_name)
         result_text = self._run_downstream_pipeline(payload, remaining_route)
-        self._send_openai_response(result_text, model, stream)
+        self._send_openai_response(result_text, model_name, stream)
 
     def _get_remaining_route(self, model: str) -> list[str]:
         server: _TorchHTTPServer = self.server  # type: ignore[assignment]
@@ -246,7 +269,8 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
     def _run_downstream_pipeline(self, payload: object, route: list[str]) -> str:
         server: _TorchHTTPServer = self.server  # type: ignore[assignment]
         if not route:
-            # Single-node mode: decode tail locally if we're the tail
+            # Partial shard at tail: decode the activation from the previous node.
+            # Full single-node (head+tail) is handled before entering this method.
             if server.backend.is_tail:
                 try:
                     tensor = server.backend.torch.frombuffer(
@@ -256,7 +280,7 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
                     return server.backend.decode_tail(tensor)
                 except Exception as exc:
                     return f"decode error: {exc}"
-            return ""
+            return "no downstream route available for non-tail shard"
 
         session = str(uuid.uuid4())
         shape = payload.shape  # type: ignore[union-attr]
@@ -309,6 +333,40 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler):
             current_pos = resp_headers.get("x-meshnet-position-ids")
         return ""
 
+    def _stream_openai_response(self, token_iter, model: str) -> None:
+        """Stream tokens from an iterator as SSE chunks."""
+        chunk_id = "chatcmpl-node"
+        created = int(time.time())
+        self.send_response(200)
+        self.send_header("Content-Type", "text/event-stream; charset=utf-8")
+        self.send_header("Cache-Control", "no-cache")
+        self.end_headers()
+
+        def _emit(data: str) -> None:
+            self.wfile.write(f"data: {data}\n\n".encode())
+            self.wfile.flush()
+
+        _emit(json.dumps({
+            "id": chunk_id, "object": "chat.completion.chunk", "created": created,
+            "model": model,
+            "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "finish_reason": None}],
+        }))
+        for token_text in token_iter:
+            if not token_text:
+                continue
+            _emit(json.dumps({
+                "id": chunk_id, "object": "chat.completion.chunk", "created": created,
+                "model": model,
+                "choices": [{"index": 0, "delta": {"content": token_text}, "finish_reason": None}],
+            }))
+        _emit(json.dumps({
+            "id": chunk_id, "object": "chat.completion.chunk", "created": created,
+            "model": model,
+            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
+        }))
+        self.wfile.write(b"data: [DONE]\n\n")
+        self.wfile.flush()
+
     def _send_openai_response(self, text: str, model: str, stream: bool) -> None:
         chunk_id = "chatcmpl-node"
         created = int(time.time())