diff --git a/.env b/.env index 62d0e53..da5e0df 100644 --- a/.env +++ b/.env @@ -14,7 +14,7 @@ AIDER_MODEL= AIDER_4=false #AIDER_35TURBO= -# OPENAI_API_KEY=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 +# OPENAI_API_KEY=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 # OPENAI_API_BASE=https://api.deepseek.com/v1 # OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a # AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat diff --git a/.env.development b/.env.development index 920ae09..d065202 100644 --- a/.env.development +++ b/.env.development @@ -1,7 +1,16 @@ ENV_NAME=development TTS_API_URL=https://api.tts.d-popov.com/asr -LNN_API_URL=https://ollama.d-popov.com + +# LLN_MODEL=qwen2 +# LNN_API_URL=https://ollama.d-popov.com/api/generate + +LLN_MODEL=qwen2 +LNN_API_URL=https://ollama.d-popov.com/api/generate + +GROQ_API_KEY=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 +OPENAI_API_KEY=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 + WS_URL=ws://localhost:8081 SERVER_PORT_WS=8081 SERVER_PORT_HTTP=8080 diff --git a/package-lock.json b/package-lock.json index dd35903..3cd4954 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,12 +13,55 @@ "dotenv": "^16.4.5", "express": "^4.18.2", "git": "^0.1.5", + "groq-sdk": "^0.4.0", "node-persist": "^3.1.3", "ollama": "^0.5.1", + "openai": "^4.50.0", "request": "^2.88.2", "ws": "^8.12.1" } }, + "node_modules/@types/node": { + "version": "18.19.34", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz", + "integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/node-fetch/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -31,6 +74,17 @@ "node": ">= 0.6" } }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -316,6 +370,14 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -474,6 +536,31 @@ "node": ">= 0.12" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -532,6 +619,21 @@ "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz", "integrity": "sha512-WiLgbHTIq5AYUvU/Luli4mZ1bUcHpGNHyCsbl+KPMg4zt+XUDpQehWjuBjdLaEvDTinvKj/FgfQt3fPoT7j08g==" }, + "node_modules/groq-sdk": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.4.0.tgz", + "integrity": "sha512-h79q9sv4hcOBESR05N5eqHlGhAug9H9lr3EIiB+37ysWWekeG+KYQDK2lIIHYCm6O9LzgZzO/VdLdPP298+T0w==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + } + }, "node_modules/har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -604,6 +706,14 @@ "npm": ">=1.3.7" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -736,6 +846,43 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp-build": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", @@ -791,6 +938,24 @@ "node": ">= 0.8" } }, + "node_modules/openai": { + "version": "4.50.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.50.0.tgz", + "integrity": "sha512-2ADkNIU6Q589oYHr5pn9k7SbUcrBTK9X0rIXrYqwMVSoqOj1yK9/1OO0ExaWsqOOpD7o58UmRjeKlx9gKAcuKQ==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1050,6 +1215,11 @@ "node": ">=0.8" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -1078,6 +1248,11 @@ "node": ">= 0.6" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1146,11 +1321,33 @@ "extsprintf": "^1.2.0" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, "node_modules/whatwg-fetch": { "version": "3.6.20", "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/ws": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz", diff --git a/package.json b/package.json index fa13107..2b5bd06 100644 --- a/package.json +++ b/package.json @@ -13,8 +13,10 @@ "dotenv": "^16.4.5", "express": "^4.18.2", "git": "^0.1.5", + "groq-sdk": "^0.4.0", "node-persist": "^3.1.3", "ollama": "^0.5.1", + "openai": "^4.50.0", "request": "^2.88.2", "ws": "^8.12.1" } diff --git a/web/chat-client.html b/web/chat-client.html index 5e369c0..b515172 100644 --- a/web/chat-client.html +++ b/web/chat-client.html @@ -183,7 +183,7 @@ break; case "text": case "transcriptionResult": - transcription.innerHTML += "\r\n" + json.text; + transcription.innerHTML += "
" + json.text; let latency = Date.now() - serverTime; if (autosend.checked) { // const arr = event.data.split(/[(\)]/); @@ -197,6 +197,13 @@ //transcription.innerHTML = event.data; } break; + case 'audio': + const audioBuffer = Uint8Array.from(atob(json.audio), char => char.charCodeAt(0)); + const audioBlob = new Blob([audioBuffer], { type: 'audio/mp3' }); + const audioUrl = URL.createObjectURL(audioBlob); + const audio = new Audio(audioUrl); + audio.play(); + break; case "userList": users = json.users; @@ -235,14 +242,14 @@ } function userJoin(sessionId, username, language) { - socket.send(JSON.stringify({ type: 'join', username , language})); + socket.send(JSON.stringify({ type: 'join', username, language })); document.cookie = `sessionId=${sessionId}; path=/;`; document.cookie = `username=${username}; path=/;`; showClearSessionOption(); } - + function clearSession() { document.cookie = "sessionId=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;"; document.cookie = "username=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;"; @@ -303,7 +310,7 @@ users.forEach(user => { const option = document.createElement('option'); option.value = user.sessionId; - option.innerText = "["+user.language+"] " +user.username; + option.innerText = "[" + user.language + "] " + user.username; if (user.username === username) { option.innerText += " (me)"; } diff --git a/web/chat-server.js b/web/chat-server.js index 28f2e3e..381799d 100644 --- a/web/chat-server.js +++ b/web/chat-server.js @@ -8,6 +8,14 @@ const path = require('path'); const dotenv = require('dotenv'); const ollama = require('ollama'); const axios = require('axios'); +// import OpenAI from "openai"; +const OpenAI = require('openai'); +const openai = new OpenAI({ apiKey: "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" }); + +const Groq = require('groq-sdk'); +//const LLM = require("@themaximalist/llm.js"); //https://www.npmjs.com/package/@themaximalist/llm.js +const groq = new Groq({ apiKey: process.env.GROQ_API_KEY }); + if (dotenv) { const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env'; @@ -21,6 +29,7 @@ const PORT_HTTP = process.env.SERVER_PORT_HTTP || 3000; const PORT_WS = process.env.SERVER_PORT_WS || 8080; const TTS_API_URL = process.env.TTS_API_URL; const LNN_API_URL = process.env.LNN_API_URL; +const LLN_MODEL = process.env.LLN_MODEL; let language = "en"; let storeRecordings = false; @@ -221,17 +230,50 @@ function detectLanguage(ws, formData) { } async function translateText(originalText, originalLanguage, targetLanguage) { - return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText) - .then(response => { - console.log('Translation response:', response); - return response; + const prompt = "Translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText; + + + // const llm = new LLM(); + // llm.system("Translate voice transcriptions. some words may be omonymous, so please provide the most likely translation."); + + // let result = await llm.chat(prompt, { service: "groq", model: "mixtral-8x7b-32768" }); + // return result; + + + return groq.chat.completions + .create({ + messages: [ + { + role: "system", + content: "You are translating voice transcriptions from '" + originalLanguage + "' to '" + targetLanguage + "'. Reply with just the translation. It will be converted to speech using TTS - you can add more context if needed.", + }, + { + role: "user", + content: originalText, + }, + ], + model: "llama3-8b-8192", + }) + .then((chatCompletion) => { + let result = chatCompletion.choices[0]?.message?.content || ""; + console.log(result); + return { response: result }; }); + + + + + // return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText) + // .then(response => { + // console.log('Translation response:', response); + // return response; + // }); } async function queryLLM(prompt) { const requestData = { - model: 'qwen2', // ollama3 + model: LLN_MODEL || 'qwen2', // ollama3 prompt: prompt, - system: "you provide translations to the text transcribed from audio. The text is in a language you understand, and you can provide translations to any language you know.", + system: "Translate voice transcriptions. some words may be omonymous, so please provide the most likely translation.", //format: "json" }; const ola = new ollama.Ollama({ host: LNN_API_URL }) @@ -241,14 +283,14 @@ async function queryLLM(prompt) { ///obsolete function async function queryLLMAxios(prompt) { const requestData = { - model: 'qwen2', + model: LLN_MODEL || 'qwen2', prompt: prompt, - "system": "talk like a pirate", + "system": "Translate voice transcriptions. some words may be omonymous, so please provide the most likely translation.", "stream": false }; try { - const response = await axios.post(LNN_API_URL + "/api/generate", requestData, { + const response = await axios.post(LNN_API_URL, requestData, { headers: { // 'Authorization': `Bearer ${OLLAMA_API_KEY}`, 'Content-Type': 'application/json' @@ -261,7 +303,7 @@ async function queryLLMAxios(prompt) { } } -function transcribeAudio(ws, formData, sessionData) { +async function transcribeAudio(ws, formData, sessionData) { const start = new Date().getTime(); queueCounter++; @@ -289,16 +331,36 @@ function transcribeAudio(ws, formData, sessionData) { chat.participants.forEach(sessionId => { if (sessionId !== ws.sessionId) { let targetLang = sessions.get(sessionId)?.language || 'en'; - targetLang = "bg"; + //targetLang = "bg"; if (targetLang !== sessionData.language) { - console.log('Translating message "'+body+'" from ' + sessionData.language + ' to ' + targetLang); + console.log('Translating message "' + body + '" from ' + sessionData.language + ' to ' + targetLang); translateText(body, sessionData.language, targetLang) .then(translation => { - const jsonResp = JSON.parse(translation); - msg.translations.push({ language: targetLang, text: jsonResp.response }); + let jsonResp; + if (typeof translation === 'string') { + try { + jsonResp = JSON.parse(translation); + } catch (e) { + console.error('Failed to parse translation response:', e); + ws.send(JSON.stringify({ type: 'error', message: 'Invalid translation response' })); + return; + } + } else { + jsonResp = translation; + } + const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId); if (participantSocket && participantSocket.readyState === WebSocket.OPEN) { participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + jsonResp.response + "\n" })); + + // Generate and send the speech audio + generateSpeech(jsonResp.response) + .then(audioBuffer => { + console.log('Generated audio for translation:', audioBuffer.length); + msg.translations.push({ language: targetLang, text: jsonResp.response, audio: audioBuffer.toString('base64') }); + participantSocket.send(JSON.stringify({ type: 'audio', audio: audioBuffer.toString('base64') })); + }); + } }); } @@ -306,6 +368,7 @@ function transcribeAudio(ws, formData, sessionData) { const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId); if (participantSocket && participantSocket.readyState === WebSocket.OPEN) { participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + body + "\n" })); + participantSocket.send(JSON.stringify({ type: 'audio', audio: formData.toString('base64') })); } } } @@ -336,6 +399,16 @@ function broadcastUserList() { }); } +async function generateSpeech(text) { + const mp3 = await openai.audio.speech.create({ + model: "tts-1", + voice: "alloy", + input: text, + }); + const buffer = Buffer.from(await mp3.arrayBuffer()); + return buffer; +} + // HTTP Server app.get('/', (req, res) => { res.sendFile(path.join(__dirname, 'chat-client.html'));