popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

doing translations

Browse Source
This commit is contained in:
Dobromir Popov
2024-06-11 22:53:39 +03:00
parent 364df3d891
commit 8bafcd9dbe
7 changed files with 610 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
web/chat-server.js
View File

@ -1,8 +1,3 @@
// server.js
if (require('dotenv')) {
const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
require('dotenv').config({ path: envFile });
}
const express = require('express');
const bodyParser = require('body-parser');
const WebSocket = require('ws');
@ -10,6 +5,14 @@ const storage = require('node-persist');
const request = require('request');
const fs = require('fs');
const path = require('path');
const dotenv = require('dotenv');
const ollama = require('ollama');
const axios = require('axios');
if (dotenv) {
const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
dotenv.config({ path: envFile });
}
const app = express();
app.use(bodyParser.json());
@ -17,6 +20,7 @@ app.use(bodyParser.json());
const PORT_HTTP = process.env.SERVER_PORT_HTTP || 3000;
const PORT_WS = process.env.SERVER_PORT_WS || 8080;
const TTS_API_URL = process.env.TTS_API_URL;
const LNN_API_URL = process.env.LNN_API_URL;
let language = "en";
let storeRecordings = false;
@ -34,78 +38,146 @@ storage.init().then(() => {
if (value !== undefined) storeRecordings = value;
else storage.setItem('storeRecordings', storeRecordings);
});
// Load existing chats, sessions from storage
storage.getItem('chats').then((value) => {
if (value !== undefined) {
value.forEach(chat => chats.set(chat.id, chat));
}
});
storage.getItem('sessions').then((value) => {
if (value !== undefined) {
value.forEach(session => sessions.set(session.sessionId, session));
}
});
});
// WebSocket Server
const wss = new WebSocket.Server({ port: PORT_WS });
wss.on('connection', (ws) => {
ws.sessionId = Math.random().toString(36).substring(2);
sessions.set(ws.sessionId, { language: 'en' });
ws.send(JSON.stringify({ sessionId: ws.sessionId, language, storeRecordings }));
ws.on('message', (message) => {
let isJson = false;
let data = null;
// Check if message is JSON
try {
const data = JSON.parse(message);
console.log('Received message:', data.type);
data = JSON.parse(message);
isJson = true;
}
catch (e) {
}
switch (data.type) {
case 'join':
const { username } = data;
sessions.set(ws.sessionId, { username, sessionId: ws.sessionId });
broadcastUserList();
break;
case 'startChat':
const { users: chatUsers } = data;
const chatId = Math.random().toString(36).substring(2);
let participants = [ws.sessionId, ...chatUsers];
// Deduplicate participants
participants = [...new Set(participants)];
chats.set(chatId, { participants, messages: [] });
const userNames = participants.map(sessionId => {
const user = sessions.get(sessionId);
return user ? `${user.username}(${user.sessionId})` : 'Unknown User';
});
console.log('Creating chat room. Users:', userNames);
participants.forEach(sessionId => {
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
participantSocket.send(JSON.stringify({ newChatRoom: { id: chatId, participants: userNames } }));
if (isJson) {
// Handle JSON messages
try {
console.log('Received message:', data.type);
switch (data.type) {
case 'sessionId':
ws.sessionId = Math.random().toString(36).substring(2);
sessions.set(ws.sessionId, { language: 'en' });
//store new session in storage sessions
storage.getItem('sessions').then((value) => {
if (value !== undefined) {
value.push({ sessionId: ws.sessionId, language: 'en' });
storage.setItem('sessions', value);
} else {
storage.setItem('sessions', [{ sessionId: ws.sessionId, language: 'en' }]);
}
}
});
broadcastUserList();
break;
case 'audio':
console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client. Crrent language: ' + language, 'task: ' + data.task);
handleAudioData(ws, data.audio);
break;
case 'reconnect':
const userSession = sessions.get(data.sessionId);
if (userSession) {
sessions.set(ws.sessionId, userSession);
ws.sessionId = data.sessionId;
broadcastUserList();
// Send existing chats
);
break;
case 'join':
const { username, language } = data;
sessions.set(ws.sessionId, { username, sessionId: ws.sessionId, language });
ws.send(JSON.stringify({ type: 'sessionId', sessionId: ws.sessionId, language, storeRecordings }));
const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId));
ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
ws.send(JSON.stringify({ type: 'userList', users: userList }));
}
break;
default:
console.log('Unknown message type:', data.type);
broadcastUserList();
break;
case 'startChat':
const { users: chatUsers } = data;
const chatId = Math.random().toString(36).substring(2);
let participants = [ws.sessionId, ...chatUsers];
// Deduplicate participants
participants = [...new Set(participants)];
chats.set(chatId, { participants, messages: [] });
//store new chat in storage chats
storage.getItem('chats').then((value) => {
if (value !== undefined) {
value.push({ id: chatId, participants });
storage.setItem('chats', value);
} else {
storage.setItem('chats', [{ id: chatId, participants }]);
}
});
const userNames = participants.map(sessionId => {
const user = sessions.get(sessionId);
return user ? `${user.username}(${user.sessionId})` : 'Unknown User';
});
console.log('Creating chat room. Users:', userNames);
participants.forEach(sessionId => {
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
const userChats = Array.from(chats.entries())
.filter(([id, chat]) => chat.participants.includes(sessionId))
.map(([id, chat]) => ({ id, participants: chat.participants }));
participantSocket.send(JSON.stringify({ type: 'chats', chats: userChats }));
}
});
broadcastUserList();
break;
// case 'audio':
// console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client.');
// handleAudioData(ws, data.audiobase64);
// break;
case 'enterChat':
const { chatId: Id } = data;
const enteredChat = chats.get(Id);
const currentSession = sessions.get(ws.sessionId);
currentSession.currentChat = Id;
if (enteredChat && enteredChat.participants.includes(ws.sessionId)) {
ws.send(JSON.stringify({ type: 'chat', chat: enteredChat }));
}
break;
case 'reconnect':
const userSession = sessions.get(data.sessionId);
if (userSession) {
sessions.set(ws.sessionId, userSession);
ws.sessionId = data.sessionId;
const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId));
ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
}
else {
console.log('Session not found:', data.sessionId);
}
broadcastUserList();
break;
default:
console.log('Unknown message type:', data.type);
}
} catch (err) {
console.error('Failed to parse message', err);
}
} catch (err) {
console.error('Failed to parse message', err);
} else {
// Handle binary data
handleAudioData(ws, message);
}
});
ws.on('close', () => {
// allUsers.delete(ws.sessionId);
sessions.delete(ws.sessionId);
broadcastUserList();
});
@ -142,12 +214,53 @@ function detectLanguage(ws, formData) {
const language = result.language_code;
const sessionData = sessions.get(ws.sessionId);
sessionData.language = language;
ws.send(JSON.stringify({ languageDetected: result.detected_language }));
ws.send(JSON.stringify({ type: 'languageDetected', languageDetected: result.detected_language }));
transcribeAudio(ws, formData, sessionData);
}
});
}
async function translateText(originalText, originalLanguage, targetLanguage) {
return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText)
.then(response => {
console.log('Translation response:', response);
return response;
});
}
async function queryLLM(prompt) {
const requestData = {
model: 'qwen2', // ollama3
prompt: prompt,
system: "you provide translations to the text transcribed from audio. The text is in a language you understand, and you can provide translations to any language you know.",
//format: "json"
};
const ola = new ollama.Ollama({ host: LNN_API_URL })
await ola.generate(requestData)
}
///obsolete function
async function queryLLMAxios(prompt) {
const requestData = {
model: 'qwen2',
prompt: prompt,
"system": "talk like a pirate",
"stream": false
};
try {
const response = await axios.post(LNN_API_URL + "/api/generate", requestData, {
headers: {
// 'Authorization': `Bearer ${OLLAMA_API_KEY}`,
'Content-Type': 'application/json'
}
});
return response.data;
} catch (error) {
console.error('Error calling Ollama API:', error.response ? error.response.data : error.message);
throw error;
}
}
function transcribeAudio(ws, formData, sessionData) {
const start = new Date().getTime();
queueCounter++;
@ -158,11 +271,48 @@ function transcribeAudio(ws, formData, sessionData) {
const duration = new Date().getTime() - start;
ws.send(JSON.stringify({
type: 'text',
queueCounter,
duration,
language: sessionData.language,
text: body
}));
if (sessionData.currentChat) {
const chat = chats.get(sessionData.currentChat);
if (chat) {
let msg = { sender: sessionData.username, text: body, translations: [] };
// init messages array if it doesn't exist
if (!chat.messages) chat.messages = [];
chat.messages.push(msg);
chat.participants.forEach(sessionId => {
if (sessionId !== ws.sessionId) {
let targetLang = sessions.get(sessionId)?.language || 'en';
targetLang = "bg";
if (targetLang !== sessionData.language) {
console.log('Translating message "'+body+'" from ' + sessionData.language + ' to ' + targetLang);
translateText(body, sessionData.language, targetLang)
.then(translation => {
const jsonResp = JSON.parse(translation);
msg.translations.push({ language: targetLang, text: jsonResp.response });
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + jsonResp.response + "\n" }));
}
});
}
else {
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + body + "\n" }));
}
}
}
});
}
}
});
if (storeRecordings) {
@ -178,7 +328,7 @@ function transcribeAudio(ws, formData, sessionData) {
}
function broadcastUserList() {
const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId }));
const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId, currentChat: user.currentChat, language: user.language }));
wss.clients.forEach(client => {
if (client.readyState === WebSocket.OPEN) {
client.send(JSON.stringify({ type: 'userList', users: userList }));
@ -195,7 +345,6 @@ app.get('/audio.js', (req, res) => {
res.sendFile(path.join(__dirname, 'audio.js'));
});
app.post('/log', (req, res) => {
console.log(`[LOG ${new Date().toISOString()}] ${req.body.message}`);
res.status(200).send('OK');