fix: proper autoregressive inference with streaming support

Single-node mode now uses HF model.generate() instead of one-shot
decode_tail(), giving correct multi-token output with KV cache.

model_backend.py:
- generate_text(messages, max_new_tokens, temperature, top_p) — full
  autoregressive generation via model.generate() with chat template
- generate_text_streaming() — yields token strings via TextIteratorStreamer
- _encode_messages() — applies chat template (tokenize=False then tokenize),
  falls back to joining user messages; avoids BatchEncoding issues

torch_server.py:
- _handle_chat_completions: fast path when backend is head+tail — calls
  generate_text() or generate_text_streaming() directly instead of the
  single-token encode_prompt+decode_tail pipeline
- _stream_openai_response: new SSE streaming handler for token iterators
- Parses max_tokens, temperature, top_p from request body
- Distributed path (partial shards) unchanged

Verified: streaming and non-streaming both work with Qwen2.5-0.5B-Instruct.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

packages/node/meshnet_node/model_backend.py

@@ -169,40 +169,97 @@ class TorchModelShard:
         token_id = int(self.torch.argmax(logits[:, -1, :], dim=-1)[0].item())
         return self.tokenizer.decode([token_id], skip_special_tokens=True)
 
-    def generate_text(self, prompt: str, max_new_tokens: int = 16) -> str:
-        """Generate text locally when this process owns the full model."""
+    def generate_text(
+        self,
+        messages: list[dict],
+        max_new_tokens: int = 256,
+        temperature: float = 1.0,
+        top_p: float = 1.0,
+    ) -> str:
+        """Autoregressive generation using HF generate() — single-node (head+tail) mode."""
         if not self.is_head or not self.is_tail:
             raise ModelBackendError("local generation requires a full head+tail shard")
-        if hasattr(self.tokenizer, "apply_chat_template"):
-            try:
-                encoded = self.tokenizer.apply_chat_template(
-                    [{"role": "user", "content": prompt}],
-                    add_generation_prompt=True,
-                    return_tensors="pt",
-                    return_dict=True,
-                )
-            except Exception:
-                encoded = self.tokenizer(prompt, return_tensors="pt")
-        else:
-            encoded = self.tokenizer(prompt, return_tensors="pt")
+        encoded = self._encode_messages(messages)
         input_ids = encoded["input_ids"].to(self.device)
         attention_mask = encoded.get("attention_mask")
         if attention_mask is not None:
             attention_mask = attention_mask.to(self.device)
-        pad_token_id = getattr(self.tokenizer, "pad_token_id", None)
-        if pad_token_id is None:
-            pad_token_id = getattr(self.tokenizer, "eos_token_id", None)
+        pad_token_id = getattr(self.tokenizer, "pad_token_id", None) or getattr(self.tokenizer, "eos_token_id", None)
+        do_sample = temperature != 1.0 or top_p != 1.0
         with self.torch.inference_mode():
             generated = self.model.generate(
                 input_ids=input_ids,
                 attention_mask=attention_mask,
                 max_new_tokens=max(1, int(max_new_tokens)),
-                do_sample=False,
+                do_sample=do_sample,
+                temperature=temperature if do_sample else None,
+                top_p=top_p if do_sample else None,
                 pad_token_id=pad_token_id,
             )
         new_tokens = generated[0, input_ids.shape[-1]:]
         return self.tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
 
+    def generate_text_streaming(
+        self,
+        messages: list[dict],
+        max_new_tokens: int = 256,
+        temperature: float = 1.0,
+        top_p: float = 1.0,
+    ):
+        """Yield decoded token strings one at a time using HF TextIteratorStreamer."""
+        if not self.is_head or not self.is_tail:
+            raise ModelBackendError("streaming generation requires a full head+tail shard")
+        import threading
+        try:
+            from transformers import TextIteratorStreamer  # type: ignore[import]
+        except ImportError:
+            yield self.generate_text(messages, max_new_tokens, temperature, top_p)
+            return
+
+        encoded = self._encode_messages(messages)
+        input_ids = encoded["input_ids"].to(self.device)
+        attention_mask = encoded.get("attention_mask")
+        if attention_mask is not None:
+            attention_mask = attention_mask.to(self.device)
+        pad_token_id = getattr(self.tokenizer, "pad_token_id", None) or getattr(self.tokenizer, "eos_token_id", None)
+        do_sample = temperature != 1.0 or top_p != 1.0
+
+        streamer = TextIteratorStreamer(self.tokenizer, skip_prompt=True, skip_special_tokens=True)
+        gen_kwargs = dict(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            max_new_tokens=max(1, int(max_new_tokens)),
+            do_sample=do_sample,
+            temperature=temperature if do_sample else None,
+            top_p=top_p if do_sample else None,
+            pad_token_id=pad_token_id,
+            streamer=streamer,
+        )
+        t = threading.Thread(target=self.model.generate, kwargs=gen_kwargs, daemon=True)
+        t.start()
+        for token_text in streamer:
+            yield token_text
+        t.join()
+
+    def _encode_messages(self, messages: list[dict]) -> dict:
+        """Format messages with chat template (if available) and tokenize."""
+        if hasattr(self.tokenizer, "apply_chat_template"):
+            try:
+                prompt_str = self.tokenizer.apply_chat_template(
+                    messages,
+                    add_generation_prompt=True,
+                    tokenize=False,
+                )
+                return dict(self.tokenizer(prompt_str, return_tensors="pt"))
+            except Exception:
+                pass
+        prompt = " ".join(
+            str(m.get("content", ""))
+            for m in messages
+            if isinstance(m, dict) and m.get("role") == "user"
+        )
+        return dict(self.tokenizer(prompt, return_tensors="pt"))
+
     def _run_layers(self, hidden_states: Any, attention_mask: Any, position_ids: Any) -> Any:
         position_embeddings = _rotary_position_embeddings(
             self.model,