main idea is now working :)
Using openai for tts and groq for ollama fast inference
This commit is contained in:
Dobromir Popov
parent
8bafcd9dbe
commit
4627f16284
@ -1,7 +1,16 @@
|
||||
|
||||
ENV_NAME=development
|
||||
TTS_API_URL=https://api.tts.d-popov.com/asr
|
||||
LNN_API_URL=https://ollama.d-popov.com
|
||||
|
||||
# LLN_MODEL=qwen2
|
||||
# LNN_API_URL=https://ollama.d-popov.com/api/generate
|
||||
|
||||
LLN_MODEL=qwen2
|
||||
LNN_API_URL=https://ollama.d-popov.com/api/generate
|
||||
|
||||
GROQ_API_KEY=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
|
||||
OPENAI_API_KEY=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
|
||||
|
||||
WS_URL=ws://localhost:8081
|
||||
SERVER_PORT_WS=8081
|
||||
SERVER_PORT_HTTP=8080
|
||||
|
||||
197
package-lock.json
generated
197
package-lock.json
generated
@ -13,12 +13,55 @@
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.18.2",
|
||||
"git": "^0.1.5",
|
||||
"groq-sdk": "^0.4.0",
|
||||
"node-persist": "^3.1.3",
|
||||
"ollama": "^0.5.1",
|
||||
"openai": "^4.50.0",
|
||||
"request": "^2.88.2",
|
||||
"ws": "^8.12.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "18.19.34",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz",
|
||||
"integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==",
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.6.11",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz",
|
||||
"integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==",
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/node-fetch/node_modules/form-data": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
|
||||
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/abort-controller": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
||||
"integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
|
||||
"dependencies": {
|
||||
"event-target-shim": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=6.5"
|
||||
}
|
||||
},
|
||||
"node_modules/accepts": {
|
||||
"version": "1.3.8",
|
||||
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
|
||||
@ -31,6 +74,17 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/agentkeepalive": {
|
||||
"version": "4.5.0",
|
||||
"resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz",
|
||||
"integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==",
|
||||
"dependencies": {
|
||||
"humanize-ms": "^1.2.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ajv": {
|
||||
"version": "6.12.6",
|
||||
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
|
||||
@ -316,6 +370,14 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/event-target-shim": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
||||
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
|
||||
"engines": {
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/express": {
|
||||
"version": "4.18.2",
|
||||
"resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz",
|
||||
@ -474,6 +536,31 @@
|
||||
"node": ">= 0.12"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data-encoder": {
|
||||
"version": "1.7.2",
|
||||
"resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz",
|
||||
"integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="
|
||||
},
|
||||
"node_modules/formdata-node": {
|
||||
"version": "4.4.1",
|
||||
"resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz",
|
||||
"integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==",
|
||||
"dependencies": {
|
||||
"node-domexception": "1.0.0",
|
||||
"web-streams-polyfill": "4.0.0-beta.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 12.20"
|
||||
}
|
||||
},
|
||||
"node_modules/formdata-node/node_modules/web-streams-polyfill": {
|
||||
"version": "4.0.0-beta.3",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz",
|
||||
"integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==",
|
||||
"engines": {
|
||||
"node": ">= 14"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded": {
|
||||
"version": "0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
|
||||
@ -532,6 +619,21 @@
|
||||
"resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz",
|
||||
"integrity": "sha512-WiLgbHTIq5AYUvU/Luli4mZ1bUcHpGNHyCsbl+KPMg4zt+XUDpQehWjuBjdLaEvDTinvKj/FgfQt3fPoT7j08g=="
|
||||
},
|
||||
"node_modules/groq-sdk": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.4.0.tgz",
|
||||
"integrity": "sha512-h79q9sv4hcOBESR05N5eqHlGhAug9H9lr3EIiB+37ysWWekeG+KYQDK2lIIHYCm6O9LzgZzO/VdLdPP298+T0w==",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18",
|
||||
"@types/node-fetch": "^2.6.4",
|
||||
"abort-controller": "^3.0.0",
|
||||
"agentkeepalive": "^4.2.1",
|
||||
"form-data-encoder": "1.7.2",
|
||||
"formdata-node": "^4.3.2",
|
||||
"node-fetch": "^2.6.7",
|
||||
"web-streams-polyfill": "^3.2.1"
|
||||
}
|
||||
},
|
||||
"node_modules/har-schema": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
|
||||
@ -604,6 +706,14 @@
|
||||
"npm": ">=1.3.7"
|
||||
}
|
||||
},
|
||||
"node_modules/humanize-ms": {
|
||||
"version": "1.2.1",
|
||||
"resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",
|
||||
"integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==",
|
||||
"dependencies": {
|
||||
"ms": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/iconv-lite": {
|
||||
"version": "0.4.24",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
|
||||
@ -736,6 +846,43 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/node-domexception": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz",
|
||||
"integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/jimmywarting"
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"url": "https://paypal.me/jimmywarting"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=10.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
|
||||
"integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"encoding": "^0.1.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"encoding": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build": {
|
||||
"version": "4.6.0",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz",
|
||||
@ -791,6 +938,24 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/openai": {
|
||||
"version": "4.50.0",
|
||||
"resolved": "https://registry.npmjs.org/openai/-/openai-4.50.0.tgz",
|
||||
"integrity": "sha512-2ADkNIU6Q589oYHr5pn9k7SbUcrBTK9X0rIXrYqwMVSoqOj1yK9/1OO0ExaWsqOOpD7o58UmRjeKlx9gKAcuKQ==",
|
||||
"dependencies": {
|
||||
"@types/node": "^18.11.18",
|
||||
"@types/node-fetch": "^2.6.4",
|
||||
"abort-controller": "^3.0.0",
|
||||
"agentkeepalive": "^4.2.1",
|
||||
"form-data-encoder": "1.7.2",
|
||||
"formdata-node": "^4.3.2",
|
||||
"node-fetch": "^2.6.7",
|
||||
"web-streams-polyfill": "^3.2.1"
|
||||
},
|
||||
"bin": {
|
||||
"openai": "bin/cli"
|
||||
}
|
||||
},
|
||||
"node_modules/parseurl": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
|
||||
@ -1050,6 +1215,11 @@
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
|
||||
},
|
||||
"node_modules/tunnel-agent": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
|
||||
@ -1078,6 +1248,11 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "5.26.5",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
|
||||
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="
|
||||
},
|
||||
"node_modules/unpipe": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
|
||||
@ -1146,11 +1321,33 @@
|
||||
"extsprintf": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/web-streams-polyfill": {
|
||||
"version": "3.3.3",
|
||||
"resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz",
|
||||
"integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==",
|
||||
"engines": {
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
|
||||
},
|
||||
"node_modules/whatwg-fetch": {
|
||||
"version": "3.6.20",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
|
||||
"integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.12.1",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz",
|
||||
|
||||
@ -13,8 +13,10 @@
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.18.2",
|
||||
"git": "^0.1.5",
|
||||
"groq-sdk": "^0.4.0",
|
||||
"node-persist": "^3.1.3",
|
||||
"ollama": "^0.5.1",
|
||||
"openai": "^4.50.0",
|
||||
"request": "^2.88.2",
|
||||
"ws": "^8.12.1"
|
||||
}
|
||||
|
||||
@ -183,7 +183,7 @@
|
||||
break;
|
||||
case "text":
|
||||
case "transcriptionResult":
|
||||
transcription.innerHTML += "\r\n" + json.text;
|
||||
transcription.innerHTML += "<br />" + json.text;
|
||||
let latency = Date.now() - serverTime;
|
||||
if (autosend.checked) {
|
||||
// const arr = event.data.split(/[(\)]/);
|
||||
@ -197,6 +197,13 @@
|
||||
//transcription.innerHTML = event.data;
|
||||
}
|
||||
break;
|
||||
case 'audio':
|
||||
const audioBuffer = Uint8Array.from(atob(json.audio), char => char.charCodeAt(0));
|
||||
const audioBlob = new Blob([audioBuffer], { type: 'audio/mp3' });
|
||||
const audioUrl = URL.createObjectURL(audioBlob);
|
||||
const audio = new Audio(audioUrl);
|
||||
audio.play();
|
||||
break;
|
||||
|
||||
case "userList":
|
||||
users = json.users;
|
||||
@ -235,7 +242,7 @@
|
||||
}
|
||||
|
||||
function userJoin(sessionId, username, language) {
|
||||
socket.send(JSON.stringify({ type: 'join', username , language}));
|
||||
socket.send(JSON.stringify({ type: 'join', username, language }));
|
||||
document.cookie = `sessionId=${sessionId}; path=/;`;
|
||||
document.cookie = `username=${username}; path=/;`;
|
||||
|
||||
@ -303,7 +310,7 @@
|
||||
users.forEach(user => {
|
||||
const option = document.createElement('option');
|
||||
option.value = user.sessionId;
|
||||
option.innerText = "["+user.language+"] " +user.username;
|
||||
option.innerText = "[" + user.language + "] " + user.username;
|
||||
if (user.username === username) {
|
||||
option.innerText += " (me)";
|
||||
}
|
||||
|
||||
@ -8,6 +8,14 @@ const path = require('path');
|
||||
const dotenv = require('dotenv');
|
||||
const ollama = require('ollama');
|
||||
const axios = require('axios');
|
||||
// import OpenAI from "openai";
|
||||
const OpenAI = require('openai');
|
||||
const openai = new OpenAI({ apiKey: "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" });
|
||||
|
||||
const Groq = require('groq-sdk');
|
||||
//const LLM = require("@themaximalist/llm.js"); //https://www.npmjs.com/package/@themaximalist/llm.js
|
||||
const groq = new Groq({ apiKey: process.env.GROQ_API_KEY });
|
||||
|
||||
|
||||
if (dotenv) {
|
||||
const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
|
||||
@ -21,6 +29,7 @@ const PORT_HTTP = process.env.SERVER_PORT_HTTP || 3000;
|
||||
const PORT_WS = process.env.SERVER_PORT_WS || 8080;
|
||||
const TTS_API_URL = process.env.TTS_API_URL;
|
||||
const LNN_API_URL = process.env.LNN_API_URL;
|
||||
const LLN_MODEL = process.env.LLN_MODEL;
|
||||
|
||||
let language = "en";
|
||||
let storeRecordings = false;
|
||||
@ -221,17 +230,50 @@ function detectLanguage(ws, formData) {
|
||||
}
|
||||
|
||||
async function translateText(originalText, originalLanguage, targetLanguage) {
|
||||
return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText)
|
||||
.then(response => {
|
||||
console.log('Translation response:', response);
|
||||
return response;
|
||||
const prompt = "Translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText;
|
||||
|
||||
|
||||
// const llm = new LLM();
|
||||
// llm.system("Translate voice transcriptions. some words may be omonymous, so please provide the most likely translation.");
|
||||
|
||||
// let result = await llm.chat(prompt, { service: "groq", model: "mixtral-8x7b-32768" });
|
||||
// return result;
|
||||
|
||||
|
||||
return groq.chat.completions
|
||||
.create({
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are translating voice transcriptions from '" + originalLanguage + "' to '" + targetLanguage + "'. Reply with just the translation. It will be converted to speech using TTS - you can add more context if needed.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: originalText,
|
||||
},
|
||||
],
|
||||
model: "llama3-8b-8192",
|
||||
})
|
||||
.then((chatCompletion) => {
|
||||
let result = chatCompletion.choices[0]?.message?.content || "";
|
||||
console.log(result);
|
||||
return { response: result };
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
// return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText)
|
||||
// .then(response => {
|
||||
// console.log('Translation response:', response);
|
||||
// return response;
|
||||
// });
|
||||
}
|
||||
async function queryLLM(prompt) {
|
||||
const requestData = {
|
||||
model: 'qwen2', // ollama3
|
||||
model: LLN_MODEL || 'qwen2', // ollama3
|
||||
prompt: prompt,
|
||||
system: "you provide translations to the text transcribed from audio. The text is in a language you understand, and you can provide translations to any language you know.",
|
||||
system: "Translate voice transcriptions. some words may be omonymous, so please provide the most likely translation.",
|
||||
//format: "json"
|
||||
};
|
||||
const ola = new ollama.Ollama({ host: LNN_API_URL })
|
||||
@ -241,14 +283,14 @@ async function queryLLM(prompt) {
|
||||
///obsolete function
|
||||
async function queryLLMAxios(prompt) {
|
||||
const requestData = {
|
||||
model: 'qwen2',
|
||||
model: LLN_MODEL || 'qwen2',
|
||||
prompt: prompt,
|
||||
"system": "talk like a pirate",
|
||||
"system": "Translate voice transcriptions. some words may be omonymous, so please provide the most likely translation.",
|
||||
"stream": false
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await axios.post(LNN_API_URL + "/api/generate", requestData, {
|
||||
const response = await axios.post(LNN_API_URL, requestData, {
|
||||
headers: {
|
||||
// 'Authorization': `Bearer ${OLLAMA_API_KEY}`,
|
||||
'Content-Type': 'application/json'
|
||||
@ -261,7 +303,7 @@ async function queryLLMAxios(prompt) {
|
||||
}
|
||||
}
|
||||
|
||||
function transcribeAudio(ws, formData, sessionData) {
|
||||
async function transcribeAudio(ws, formData, sessionData) {
|
||||
const start = new Date().getTime();
|
||||
queueCounter++;
|
||||
|
||||
@ -289,16 +331,36 @@ function transcribeAudio(ws, formData, sessionData) {
|
||||
chat.participants.forEach(sessionId => {
|
||||
if (sessionId !== ws.sessionId) {
|
||||
let targetLang = sessions.get(sessionId)?.language || 'en';
|
||||
targetLang = "bg";
|
||||
//targetLang = "bg";
|
||||
if (targetLang !== sessionData.language) {
|
||||
console.log('Translating message "'+body+'" from ' + sessionData.language + ' to ' + targetLang);
|
||||
console.log('Translating message "' + body + '" from ' + sessionData.language + ' to ' + targetLang);
|
||||
translateText(body, sessionData.language, targetLang)
|
||||
.then(translation => {
|
||||
const jsonResp = JSON.parse(translation);
|
||||
msg.translations.push({ language: targetLang, text: jsonResp.response });
|
||||
let jsonResp;
|
||||
if (typeof translation === 'string') {
|
||||
try {
|
||||
jsonResp = JSON.parse(translation);
|
||||
} catch (e) {
|
||||
console.error('Failed to parse translation response:', e);
|
||||
ws.send(JSON.stringify({ type: 'error', message: 'Invalid translation response' }));
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
jsonResp = translation;
|
||||
}
|
||||
|
||||
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
|
||||
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
|
||||
participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + jsonResp.response + "\n" }));
|
||||
|
||||
// Generate and send the speech audio
|
||||
generateSpeech(jsonResp.response)
|
||||
.then(audioBuffer => {
|
||||
console.log('Generated audio for translation:', audioBuffer.length);
|
||||
msg.translations.push({ language: targetLang, text: jsonResp.response, audio: audioBuffer.toString('base64') });
|
||||
participantSocket.send(JSON.stringify({ type: 'audio', audio: audioBuffer.toString('base64') }));
|
||||
});
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
@ -306,6 +368,7 @@ function transcribeAudio(ws, formData, sessionData) {
|
||||
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
|
||||
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
|
||||
participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + body + "\n" }));
|
||||
participantSocket.send(JSON.stringify({ type: 'audio', audio: formData.toString('base64') }));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -336,6 +399,16 @@ function broadcastUserList() {
|
||||
});
|
||||
}
|
||||
|
||||
async function generateSpeech(text) {
|
||||
const mp3 = await openai.audio.speech.create({
|
||||
model: "tts-1",
|
||||
voice: "alloy",
|
||||
input: text,
|
||||
});
|
||||
const buffer = Buffer.from(await mp3.arrayBuffer());
|
||||
return buffer;
|
||||
}
|
||||
|
||||
// HTTP Server
|
||||
app.get('/', (req, res) => {
|
||||
res.sendFile(path.join(__dirname, 'chat-client.html'));
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user