diff --git a/packages/node/meshnet_node/model_backend.py b/packages/node/meshnet_node/model_backend.py index 008cf32..00d04f6 100644 --- a/packages/node/meshnet_node/model_backend.py +++ b/packages/node/meshnet_node/model_backend.py @@ -215,7 +215,7 @@ class TorchModelShard: def generate_text( self, messages: list[dict], - max_new_tokens: int = 256, + max_new_tokens: int = 5120, temperature: float = 1.0, top_p: float = 1.0, ) -> str: @@ -245,7 +245,7 @@ class TorchModelShard: def generate_text_streaming( self, messages: list[dict], - max_new_tokens: int = 256, + max_new_tokens: int = 5000, temperature: float = 1.0, top_p: float = 1.0, ): diff --git a/packages/node/meshnet_node/torch_server.py b/packages/node/meshnet_node/torch_server.py index 7d615ec..3e9f5ba 100644 --- a/packages/node/meshnet_node/torch_server.py +++ b/packages/node/meshnet_node/torch_server.py @@ -321,7 +321,7 @@ class _TorchHandler(http.server.BaseHTTPRequestHandler): if backend is None or not backend.is_head: self._send_json(400, {"error": "model not loaded on this node"}) return - max_tokens = int(body.get("max_tokens") or body.get("max_new_tokens") or 256) + max_tokens = int(body.get("max_tokens") or body.get("max_new_tokens") or 5120) temperature = float(body.get("temperature") or 1.0) top_p = float(body.get("top_p") or 1.0) diff --git a/packages/tracker/meshnet_tracker/dashboard.html b/packages/tracker/meshnet_tracker/dashboard.html index 18297ee..1c16306 100644 --- a/packages/tracker/meshnet_tracker/dashboard.html +++ b/packages/tracker/meshnet_tracker/dashboard.html @@ -930,7 +930,7 @@ async function sendChat() { { role: "user", content: prompt }, ], stream: false, - max_tokens: 256, + max_tokens: 15120, }; chatBusy = true; $("chat-send").disabled = true;