doing translations

.env.development

@@ -1,6 +1,7 @@
 
 ENV_NAME=development
 TTS_API_URL=https://api.tts.d-popov.com/asr
+LNN_API_URL=https://ollama.d-popov.com
 WS_URL=ws://localhost:8081
 SERVER_PORT_WS=8081
 SERVER_PORT_HTTP=8080

package-lock.json

@@ -8,11 +8,13 @@
             "name": "kevin-ai",
             "version": "1.0.0",
             "dependencies": {
+                "axios": "^1.7.2",
                 "body-parser": "^1.20.2",
                 "dotenv": "^16.4.5",
                 "express": "^4.18.2",
                 "git": "^0.1.5",
                 "node-persist": "^3.1.3",
+                "ollama": "^0.5.1",
                 "request": "^2.88.2",
                 "ws": "^8.12.1"
             }
@@ -83,6 +85,29 @@
             "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz",
             "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg=="
         },
+        "node_modules/axios": {
+            "version": "1.7.2",
+            "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz",
+            "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==",
+            "dependencies": {
+                "follow-redirects": "^1.15.6",
+                "form-data": "^4.0.0",
+                "proxy-from-env": "^1.1.0"
+            }
+        },
+        "node_modules/axios/node_modules/form-data": {
+            "version": "4.0.0",
+            "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+            "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+            "dependencies": {
+                "asynckit": "^0.4.0",
+                "combined-stream": "^1.0.8",
+                "mime-types": "^2.1.12"
+            },
+            "engines": {
+                "node": ">= 6"
+            }
+        },
         "node_modules/bcrypt-pbkdf": {
             "version": "1.0.2",
             "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
@@ -409,6 +434,25 @@
                 "node": ">= 0.8"
             }
         },
+        "node_modules/follow-redirects": {
+            "version": "1.15.6",
+            "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+            "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
+            "funding": [
+                {
+                    "type": "individual",
+                    "url": "https://github.com/sponsors/RubenVerborgh"
+                }
+            ],
+            "engines": {
+                "node": ">=4.0"
+            },
+            "peerDependenciesMeta": {
+                "debug": {
+                    "optional": true
+                }
+            }
+        },
         "node_modules/forever-agent": {
             "version": "0.6.1",
             "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@@ -728,6 +772,14 @@
                 "url": "https://github.com/sponsors/ljharb"
             }
         },
+        "node_modules/ollama": {
+            "version": "0.5.1",
+            "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.1.tgz",
+            "integrity": "sha512-mAiCHxdvu63E8EFopz0y82QG7rGfYmKAWgmjG2C7soiRuz/Sj3r/ebvCOp+jasiCubqUPE0ZThKT5LR6wrrPtA==",
+            "dependencies": {
+                "whatwg-fetch": "^3.6.20"
+            }
+        },
         "node_modules/on-finished": {
             "version": "2.4.1",
             "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
@@ -769,6 +821,11 @@
                 "node": ">= 0.10"
             }
         },
+        "node_modules/proxy-from-env": {
+            "version": "1.1.0",
+            "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+            "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+        },
         "node_modules/psl": {
             "version": "1.9.0",
             "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
@@ -1089,6 +1146,11 @@
                 "extsprintf": "^1.2.0"
             }
         },
+        "node_modules/whatwg-fetch": {
+            "version": "3.6.20",
+            "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
+            "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="
+        },
         "node_modules/ws": {
             "version": "8.12.1",
             "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz",

package.json

@@ -8,11 +8,13 @@
         "start:tele": "python agent-py-bot/agent.py"
     },
     "dependencies": {
+        "axios": "^1.7.2",
         "body-parser": "^1.20.2",
         "dotenv": "^16.4.5",
         "express": "^4.18.2",
         "git": "^0.1.5",
         "node-persist": "^3.1.3",
+        "ollama": "^0.5.1",
         "request": "^2.88.2",
         "ws": "^8.12.1"
     }

web/audio.js

@@ -1,7 +1,7 @@
 let selectedDeviceId = "default";
 export let serverTime;
 export let recordButton;
-let socket;
+export let socket;
 let audioRecorder;
 let audioStream;
 let recording = false;
@@ -42,7 +42,6 @@ export function InitAudioAnalyser(stream) {
 }
 
 export function startListening() {
-    //canvasCtx.fillStyle = "green";
     recording = true;
     navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
         .then((stream) => {
@@ -75,7 +74,6 @@ export function startListening() {
                 }
                 if (event.data.size > 0) {
                     let data = event.data;
-                    console.log("audio data size: " + data.size);
                     if (preBuffer.length > 0) {
                         sendAudioToServerPost(preBuffer);
                     }
@@ -117,14 +115,25 @@ export function sendAudioToServerPost(data) {
     });
 }
 
-export function sendAudioToServer(data) {
+export function sendAudioToServerJson(data) {
-    //if (connected) {
+    if (socket && socket.readyState === WebSocket.OPEN) {
-        socket.send(JSON.stringify({ type: 'audio', task:"transcribe", audio: data }));
+
+    const binaryData = Buffer.from(base64AudioData, 'base64');
+        socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData }));
         serverTime = Date.now();
         if (!autosend.checked) {
-            transcription.innerHTML = "Processing audio...";
+            transcription.placeholder = "Processing audio...";
+        }
+    }
+}
+export function sendAudioToServer(data) {
+    if (socket && socket.readyState === WebSocket.OPEN) {
+        socket.send(data);
+        serverTime = Date.now();
+        if (!autosend.checked) {
+            transcription.placeholder  = "Processing audio...";
+        }
     }
-    //}
 }
 
 export function toggleListening() {

web/chat-client.html

@@ -11,12 +11,20 @@
     <div class="container mx-auto px-4 py-8">
         <h1 class="text-2xl font-bold mb-4 text-center">Real-time Voice Chat</h1>
 
-
         <div class="flex justify-center items-center mb-4">
             <!-- Username Input -->
             <input type="text" id="username" class="border rounded p-2 mr-4" placeholder="Enter your username">
-            <button id="btn-join" onclick="joinChat()"
+            <div id="join-container" class="hidden">
-                class="hidden bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
+                <button id="btn-join" onclick="logInAndStoreSession()"
+                    class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
+                <select id="language-select">
+                    <option value="auto">Auto</option>
+                    <option value="en">English</option>
+                    <option value="bg">Български</option>
+                    <option value="fr">Français</option>
+                </select>
+            </div>
+
 
             <!-- Clear Session Option -->
             <button id="btn-disconnect" onclick="clearSession()"
@@ -48,6 +56,11 @@
             <h2 class="text-xl font-bold mb-2">Chat Room</h2>
             <div id="chat-room" class="bg-white p-4 rounded shadow mb-4">
                 <!-- Chat room content -->
+                <div>
+                    <div id="chat-room-users" class="flex flex-wrap mb-4">
+                        <!-- Participants list -->
+                    </div>
+                </div>
                 <div class="mb-4">
                     <label class="flex items-center space-x-2">
                         <input type="checkbox" id="autosend" class="mr-2">
@@ -59,6 +72,9 @@
                         class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">Push to
                         Talk</button>
                 </div>
+                <div id="status-recording" class="flex justify-center items-center mb-4">
+                </div>
+
                 <div id="transcription" class="border rounded p-4 h-48 overflow-y-scroll mb-4">
                     <!-- Transcription content -->
                 </div>
@@ -81,11 +97,11 @@
         <div class="flex justify-center items-center mb-4">
             <div id="info"></div>
         </div>
-        <div id="status-recording" class="flex justify-center items-center mb-4"></div>
+        <div id="status-recording" class="flex justify-center items-center mb-4"> status</div>
     </div>
 
     <script type="module">
-        import { startListening, stopListening, toggleListening, InitAudioAnalyser, sendAudioToServerPost, sendAudioToServer, initializeVolumeChecker, serverTime, setSocket, setRecordButton } from './audio.js';
+        // import * as audio from './audio.js';
 
         let socket;
         let sessionId;
@@ -120,11 +136,20 @@
                     })
                     .then(() => {
                         socket = new WebSocket(wsurl);
+                        // audio.setSocket(socket); // Set the socket in the audio module
                         socket.onopen = () => {
                             connectionStatus.innerHTML = "Connected to " + wsurl;
                             recordButton.disabled = false;
                             connected = true;
-                            resolve();
+                            //if we stored a session id in a cookie, reconnect
+                            const sessionId = getCookie("sessionId");
+                            if (sessionId) {
+                                socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
+                            }
+                            else {
+                                socket.send(JSON.stringify({ type: 'sessionId' }));
+                            }
+                            resolve(socket);
                         };
                         socket.onmessage = onmessage;
                         socket.onclose = () => {
@@ -135,7 +160,6 @@
                                 connect().then(resolve).catch(reject);
                             }, 5000);
                         };
-                        setSocket(socket);
                     })
                     .catch((error) => {
                         connectionStatus.innerHTML = "Error getting ws url: " + error;
@@ -150,14 +174,30 @@
                 switch (json.type) {
                     case "sessionId":
                         sessionId = json.sessionId;
+                        //set the session id in the cookie
+                        document.cookie = `sessionId=${sessionId}; path=/;`;
                         console.log("Got session id: " + sessionId);
                         break;
                     case "languageDetected":
                         statusRecording.innerHTML = "Detected language: " + json.languageDetected;
                         break;
                     case "text":
+                    case "transcriptionResult":
                         transcription.innerHTML += "\r\n" + json.text;
+                        let latency = Date.now() - serverTime;
+                        if (autosend.checked) {
+                            // const arr = event.data.split(/[(\)]/);
+                            // let queue = arr[1];
+                            // let text = arr[2].trim();
+                            // info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
+                            //transcription.value += event.data + " ";
+                            statusRecording.innerHTML = "Listening...";
+                            statusRecording.style.color = "black";
+                        } else {
+                            //transcription.innerHTML = event.data;
+                        }
                         break;
+
                     case "userList":
                         users = json.users;
                         updateUserList();
@@ -166,9 +206,8 @@
                         chats = json.chats;
                         updateChatList();
                         break;
-                    case "newChatRoom":
+                    case "chat":
-                        chats.push(json.newChatRoom);
+                        displayChatParticipants(json.chat.id, json.chat.participants);
-                        updateChatList();
                         break;
                     default:
                         console.log("Unknown message type:", json.type);
@@ -177,36 +216,33 @@
                 console.error("Failed to parse message", e);
             }
 
-            let latency = Date.now() - serverTime;
+
-            if (autosend.checked) {
-                const arr = event.data.split(/[(\)]/);
-                let queue = arr[1];
-                let text = arr[2].trim();
-                info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
-                transcription.value += text + " ";
-                statusRecording.innerHTML = "Listening...";
-                statusRecording.style.color = "black";
-            } else {
-                transcription.innerHTML = event.data;
-            }
         }
 
-        function joinChat() {
+        function logInAndStoreSession() {
             username = document.getElementById('username').value;
             if (username.trim() === "") {
                 alert("Please enter a username");
                 return;
             }
             if (!socket || socket.readyState !== WebSocket.OPEN) {
-                alert("WebSocket connection is not open. Please wait and try again.");
+                connect().then(() => {
-                return;
+                    userJoin(sessionId, username, document.getElementById('language-select').value);
+                });
+            } else {
+                userJoin(sessionId, username, document.getElementById('language-select').value);
             }
-            socket.send(JSON.stringify({ type: 'join', username }));
+        }
+
+        function userJoin(sessionId, username, language) {
+            socket.send(JSON.stringify({ type: 'join', username , language}));
             document.cookie = `sessionId=${sessionId}; path=/;`;
             document.cookie = `username=${username}; path=/;`;
+
             showClearSessionOption();
         }
 
+        
         function clearSession() {
             document.cookie = "sessionId=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
             document.cookie = "username=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
@@ -217,21 +253,19 @@
             const sessionId = getCookie("sessionId");
             if (sessionId) {
                 if (!socket || socket.readyState !== WebSocket.OPEN) {
-                    connect().then(() => {
+                    connect().then((s) => {
-                        socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
+                        s.send(JSON.stringify({ type: 'reconnect', sessionId }));
-                        //initializeVolumeChecker();
-
                     });
                 }
                 document.getElementById('btn-disconnect').classList.remove('hidden');
-                document.getElementById('btn-join').classList.add('hidden');
+                document.getElementById('join-container').classList.add('hidden');
 
                 document.getElementById('active-users-container').classList.remove('hidden');
                 document.getElementById('previous-chats-container').classList.remove('hidden');
 
             } else {
                 document.getElementById('btn-disconnect').classList.add('hidden');
-                document.getElementById('btn-join').classList.remove('hidden');
+                document.getElementById('join-container').classList.remove('hidden');
             }
         }
 
@@ -249,7 +283,7 @@
 
             showClearSessionOption();
             connect().then(() => {
-                initializeVolumeChecker();
+                // audio.initializeVolumeChecker();
             });
         };
 
@@ -269,7 +303,7 @@
             users.forEach(user => {
                 const option = document.createElement('option');
                 option.value = user.sessionId;
-                option.innerText = user.username;
+                option.innerText = "["+user.language+"] " +user.username;
                 if (user.username === username) {
                     option.innerText += " (me)";
                 }
@@ -287,11 +321,63 @@
             selectedUsers.push(sessionId); // Add self to the selected users list for self-chat
             socket.send(JSON.stringify({ type: 'startChat', users: selectedUsers }));
             document.getElementById('chat-room-container').classList.remove('hidden');
-            displayChatParticipants(selectedUsers);
         }
 
-        function displayChatParticipants(participants) {
+
-            const chatRoom = document.getElementById('chat-room');
+
+        function fetchPreviousChats(username) {
+            fetch(`/chats?username=${username}`)
+                .then(response => response.json())
+                .then(data => {
+                    chats = data.chats;
+                    updateChatList();
+                });
+        }
+
+        function updateChatList() {
+            const previousChats = document.getElementById('previous-chats');
+            previousChats.innerHTML = '';
+            chats.forEach(chat => {
+                const chatDiv = document.createElement('div');
+                chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2', 'cursor-pointer');
+                chatDiv.setAttribute('data-chat-id', chat.id); // Store chat ID in data attribute
+
+                const participants = chat.participants.join(', ');
+                const status = chat.participants.map(participant => {
+                    const user = users.find(u => u.username === participant);
+                    return user ? `${participant} (online)` : `${participant} (offline)`;
+                }).join(', ');
+
+                chatDiv.innerHTML = `${status}`;
+
+                chatDiv.addEventListener('click', () => {
+                    // Remove highlight from all chat divs
+                    document.querySelectorAll('#previous-chats > div').forEach(div => {
+                        div.classList.remove('bg-blue-100');
+                    });
+                    // Highlight selected chat div
+                    chatDiv.classList.add('bg-blue-100');
+                    selectChatRoom(chat.id);
+                });
+
+                previousChats.appendChild(chatDiv);
+            });
+        }
+
+        function selectChatRoom(chatId) {
+            const chat = chats.find(c => c.id === chatId);
+            if (!chat) return;
+
+            const chatRoomUsers = document.getElementById('chat-room-users');
+            chatRoomUsers.innerHTML = '';  // Clear existing content
+
+            socket.send(JSON.stringify({ type: 'enterChat', chatId }));
+            document.getElementById('chat-room-container').classList.remove('hidden');
+            // displayChatParticipants(chatId, chat.participants);
+        }
+
+        function displayChatParticipants(chatId, participants) {
+            const chatRoomUsers = document.getElementById('chat-room-users');
             let participantsHtml = '<div class="flex flex-wrap mb-4">';
             participants.forEach(participantId => {
                 const user = users.find(u => u.sessionId === participantId);
@@ -302,41 +388,214 @@
                              </span>`;
             });
             participantsHtml += '</div>';
-            chatRoom.insertAdjacentHTML('afterbegin', participantsHtml);
+            chatRoomUsers.innerHTML = participantsHtml;
         }
 
-        function fetchPreviousChats(username) {
+        // REGION AUDIO RECORDING
-            fetch(`/chats?username=${username}`)
+        let selectedDeviceId = "default";
-                .then(response => response.json())
+        export let serverTime;
-                .then(data => {
+        // export let recordButton;
-                    chats = data.chats;
+        // export let socket;
-                    updateChatList();
+        // let connectionStatus;
+        // let statusRecording;
+        let audioRecorder;
+        let audioStream;
+        let recording = false;
+        let audioContext;
+        let volumeChecker;
+        let lastVolumes = new Array(5);
+        let averageVolume;
+        let silenceCount = 0;
+        let isSpeaking = false;
+        let soundDetected = false;
+        let speakingCount = 0;
+        let analyser = null;
+
+        let SILENCE_DELAY_MS = 50;
+        let preDetect_IncludedAudio = 400; //ms
+        let soundCount_Threshold = 10;
+        let silenceCount_Threshold = 10;
+
+        const volumeHistory = [];
+
+        function setSocket(newSocket) {
+            socket = newSocket;
+        }
+        export function setRecordButton(newRecordButton) {
+            recordButton = newRecordButton;
+            recordButton.addEventListener("click", toggleListening);
+        }
+
+        export function InitAudioAnalyser(stream) {
+            audioContext = new AudioContext();
+            const source = audioContext.createMediaStreamSource(stream);
+            analyser = audioContext.createAnalyser();
+            analyser.fftSize = 2048;
+            analyser.smoothingTimeConstant = 0.8;
+            source.connect(analyser);
+        }
+
+        export function startListening() {
+            recording = true;
+            navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
+                .then((stream) => {
+                    audioStream = stream;
+
+                    const audioContext = new AudioContext();
+                    const sourceNode = audioContext.createMediaStreamSource(audioStream);
+                    const audioSampleRate = sourceNode.context.sampleRate;
+
+                    info.innerHTML = "Sample rate: " + audioSampleRate + " Hz";
+                    var preBuffer = [];
+
+                    const channelSplitter = audioContext.createChannelSplitter(2);
+                    const channelMerger = audioContext.createChannelMerger(1);
+                    sourceNode.connect(channelSplitter);
+                    channelSplitter.connect(channelMerger, 0, 0);
+                    const outputNode = channelMerger;
+
+                    const mediaStreamDestination = audioContext.createMediaStreamDestination();
+                    outputNode.connect(mediaStreamDestination);
+                    const singleChannelStream = mediaStreamDestination.stream;
+
+                    audioRecorder = new MediaRecorder(singleChannelStream);
+                    audioRecorder.start();
+                    audioRecorder.addEventListener("dataavailable", (event) => {
+                        if (!soundDetected && autosend.checked) {
+                            preBuffer = [];
+                            preBuffer.push(event.data);
+                            return;
+                        }
+                        if (event.data.size > 0) {
+                            let data = event.data;
+                            if (preBuffer.length > 0) {
+                                sendAudioToServerPost(preBuffer);
+                            }
+                            sendAudioToServer(data);
+                            soundDetected = false;
+                        }
+                    });
+
+                    InitAudioAnalyser(stream);
+                });
+
+            recordButton.innerHTML = "Stop Talking";
+            recordButton.classList.toggle('bg-red-500');
+            recordButton.classList.toggle('bg-blue-500');
+            recordButton.classList.toggle('hover:bg-blue-700');
+        }
+
+        export function stopListening() {
+            recording = false;
+            audioRecorder.stop();
+            recordButton.innerHTML = "Push to Talk";
+            recordButton.classList.toggle('bg-blue-500');
+            recordButton.classList.toggle('bg-red-500');
+            recordButton.classList.toggle('hover:bg-blue-700');
+            clearInterval(volumeChecker);
+            if (audioStream) {
+                audioStream.getTracks().forEach(track => track.stop());
+                audioStream = null;
+            }
+        }
+
+        export function sendAudioToServerPost(data) {
+            const blob = new Blob(data, { type: "audio/ogg; codecs=opus" });
+            var formData = new FormData();
+            formData.append('file', data);
+            fetch('/upload', {
+                method: 'POST',
+                body: formData
             });
         }
-        function sendSocketMessage(message) {
+
+        export function sendAudioToServerJson(data) {
             if (socket && socket.readyState === WebSocket.OPEN) {
-                socket.send(JSON.stringify(message));
+
+                const binaryData = Buffer.from(base64AudioData, 'base64');
+                socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData }));
+                serverTime = Date.now();
+                if (!autosend.checked) {
+                    transcription.placeholder = "Processing audio...";
+                }
+            }
+        }
+        export function sendAudioToServer(data) {
+            if (socket && socket.readyState === WebSocket.OPEN) {
+                socket.send(data);
+                serverTime = Date.now();
+                if (!autosend.checked) {
+                    transcription.placeholder = "Processing audio...";
+                }
             }
         }
 
-        function updateChatList() {
+        export function toggleListening() {
-            const previousChats = document.getElementById('previous-chats');
+            if (socket.readyState === WebSocket.OPEN) {
-            previousChats.innerHTML = '';
+                if (recording) {
-            chats.forEach(chat => {
+                    stopListening();
-                const chatDiv = document.createElement('div');
+                } else {
-                chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2');
+                    startListening();
-                const participants = chat.participants.join(', ');
+                }
-                const status = chat.participants.map(participant => {
+            }
-                    const user = users.find(u => u.username === participant);
+        }
-                    return user ? `${participant} (online)` : `${participant} (offline)`;
+
-                }).join(', ');
+        export function initializeVolumeChecker() {
-                chatDiv.innerHTML = `<strong>Participants:</strong> ${participants}<br><strong>Status:</strong> ${status}`;
+            volumeChecker = setInterval(() => {
-                previousChats.appendChild(chatDiv);
+                if (!audioContext) {
-            });
+                    console.log("No audio context");
+                    return;
+                }
+                const frequencyData = new Uint8Array(analyser.frequencyBinCount);
+                analyser.getByteFrequencyData(frequencyData);
+
+                let totalVolume = 0;
+                for (let i = 0; i < frequencyData.length; i++) {
+                    totalVolume += frequencyData[i];
+                }
+                averageVolume = totalVolume / frequencyData.length;
+
+                volumeHistory.push(averageVolume);
+                if (volumeHistory.length > 100) {
+                    volumeHistory.shift();
+                }
+
+                const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5;
+                const isSilent = averageVolume < threshold;
+
+                if (averageVolume > threshold) {
+                    if (autosend.checked && speakingCount == 0 && audioRecorder) {
+                        soundDetected = false;
+                        audioRecorder.stop();
+                        audioRecorder.start();
+                    }
+                    speakingCount++;
+                    if (speakingCount > soundCount_Threshold) {
+                        statusRecording.innerHTML = "Listening...";
+                        statusRecording.style.color = "green";
+                        isSpeaking = true;
+                    }
+                } else if (averageVolume - 5 < threshold) {
+                    speakingCount = 0;
+                    if (isSpeaking) {
+                        silenceCount++;
+                        if (silenceCount > silenceCount_Threshold) {
+                            if (autosend.checked) {
+                                soundDetected = true;
+                                audioRecorder.stop();
+                                audioRecorder.start();
+                            }
+                            isSpeaking = false;
+                            statusRecording.innerHTML = "Silence detected...";
+                            statusRecording.style.color = "orange";
+                        }
+                    }
+                }
+            }, SILENCE_DELAY_MS);
         }
 
         // Expose functions to global scope
-        window.joinChat = joinChat;
+        window.logInAndStoreSession = logInAndStoreSession;
         window.clearSession = clearSession;
         window.copyToClipboard = copyToClipboard;
         window.clearTranscription = clearTranscription;

web/chat-server.js

@@ -1,8 +1,3 @@
-// server.js
-if (require('dotenv')) {
-    const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
-    require('dotenv').config({ path: envFile });
-}
 const express = require('express');
 const bodyParser = require('body-parser');
 const WebSocket = require('ws');
@@ -10,6 +5,14 @@ const storage = require('node-persist');
 const request = require('request');
 const fs = require('fs');
 const path = require('path');
+const dotenv = require('dotenv');
+const ollama = require('ollama');
+const axios = require('axios');
+
+if (dotenv) {
+    const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
+    dotenv.config({ path: envFile });
+}
 
 const app = express();
 app.use(bodyParser.json());
@@ -17,6 +20,7 @@ app.use(bodyParser.json());
 const PORT_HTTP = process.env.SERVER_PORT_HTTP || 3000;
 const PORT_WS = process.env.SERVER_PORT_WS || 8080;
 const TTS_API_URL = process.env.TTS_API_URL;
+const LNN_API_URL = process.env.LNN_API_URL;
 
 let language = "en";
 let storeRecordings = false;
@@ -34,25 +38,65 @@ storage.init().then(() => {
         if (value !== undefined) storeRecordings = value;
         else storage.setItem('storeRecordings', storeRecordings);
     });
+
+    // Load existing chats, sessions from storage
+    storage.getItem('chats').then((value) => {
+        if (value !== undefined) {
+            value.forEach(chat => chats.set(chat.id, chat));
+        }
+    });
+    storage.getItem('sessions').then((value) => {
+        if (value !== undefined) {
+            value.forEach(session => sessions.set(session.sessionId, session));
+        }
+    });
 });
 
 // WebSocket Server
 const wss = new WebSocket.Server({ port: PORT_WS });
 wss.on('connection', (ws) => {
-    ws.sessionId = Math.random().toString(36).substring(2);
-    sessions.set(ws.sessionId, { language: 'en' });
 
-    ws.send(JSON.stringify({ sessionId: ws.sessionId, language, storeRecordings }));
+
 
     ws.on('message', (message) => {
+        let isJson = false;
+        let data = null;
+        // Check if message is JSON
+        try {
+            data = JSON.parse(message);
+            isJson = true;
+        }
+        catch (e) {
+        }
+
+        if (isJson) {
+            // Handle JSON messages
             try {
-            const data = JSON.parse(message);
                 console.log('Received message:', data.type);
 
                 switch (data.type) {
+                    case 'sessionId':
+                        ws.sessionId = Math.random().toString(36).substring(2);
+                        sessions.set(ws.sessionId, { language: 'en' });
+                        //store new session in storage sessions
+                        storage.getItem('sessions').then((value) => {
+                            if (value !== undefined) {
+                                value.push({ sessionId: ws.sessionId, language: 'en' });
+                                storage.setItem('sessions', value);
+                            } else {
+                                storage.setItem('sessions', [{ sessionId: ws.sessionId, language: 'en' }]);
+                            }
+                        }
+                        );
+                        break;
                     case 'join':
-                    const { username } = data;
+                        const { username, language } = data;
-                    sessions.set(ws.sessionId, { username, sessionId: ws.sessionId });
+                        sessions.set(ws.sessionId, { username, sessionId: ws.sessionId, language });
+                        ws.send(JSON.stringify({ type: 'sessionId', sessionId: ws.sessionId, language, storeRecordings }));
+
+                        const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId));
+                        ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
+
                         broadcastUserList();
                         break;
                     case 'startChat':
@@ -64,6 +108,16 @@ wss.on('connection', (ws) => {
 
                         chats.set(chatId, { participants, messages: [] });
 
+                        //store new chat in storage chats
+                        storage.getItem('chats').then((value) => {
+                            if (value !== undefined) {
+                                value.push({ id: chatId, participants });
+                                storage.setItem('chats', value);
+                            } else {
+                                storage.setItem('chats', [{ id: chatId, participants }]);
+                            }
+                        });
+
                         const userNames = participants.map(sessionId => {
                             const user = sessions.get(sessionId);
                             return user ? `${user.username}(${user.sessionId})` : 'Unknown User';
@@ -74,27 +128,40 @@ wss.on('connection', (ws) => {
                         participants.forEach(sessionId => {
                             const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
                             if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
-                            participantSocket.send(JSON.stringify({ newChatRoom: { id: chatId, participants: userNames } }));
+                                const userChats = Array.from(chats.entries())
+                                    .filter(([id, chat]) => chat.participants.includes(sessionId))
+                                    .map(([id, chat]) => ({ id, participants: chat.participants }));
+                                participantSocket.send(JSON.stringify({ type: 'chats', chats: userChats }));
                             }
                         });
-            
                         broadcastUserList();
                         break;
-                case 'audio':
+                    // case 'audio':
-                    console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client. Crrent language: ' + language, 'task: ' + data.task);
+                    //     console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client.');
-                    handleAudioData(ws, data.audio);
+                    //     handleAudioData(ws, data.audiobase64);
+                    //     break;
+                    case 'enterChat':
+                        const { chatId: Id } = data;
+                        const enteredChat = chats.get(Id);
+                        const currentSession = sessions.get(ws.sessionId);
+                        currentSession.currentChat = Id;
+                        if (enteredChat && enteredChat.participants.includes(ws.sessionId)) {
+                            ws.send(JSON.stringify({ type: 'chat', chat: enteredChat }));
+                        }
                         break;
                     case 'reconnect':
                         const userSession = sessions.get(data.sessionId);
                         if (userSession) {
                             sessions.set(ws.sessionId, userSession);
                             ws.sessionId = data.sessionId;
-                        broadcastUserList();
-                        // Send existing chats
                             const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId));
                             ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
-                        ws.send(JSON.stringify({ type: 'userList', users: userList }));
                         }
+                        else {
+                            console.log('Session not found:', data.sessionId);
+                        }
+
+                        broadcastUserList();
                         break;
                     default:
                         console.log('Unknown message type:', data.type);
@@ -102,10 +169,15 @@ wss.on('connection', (ws) => {
             } catch (err) {
                 console.error('Failed to parse message', err);
             }
+
+        } else {
+            // Handle binary data
+            handleAudioData(ws, message);
+        }
+
     });
 
     ws.on('close', () => {
-        // allUsers.delete(ws.sessionId);
         sessions.delete(ws.sessionId);
         broadcastUserList();
     });
@@ -142,12 +214,53 @@ function detectLanguage(ws, formData) {
             const language = result.language_code;
             const sessionData = sessions.get(ws.sessionId);
             sessionData.language = language;
-            ws.send(JSON.stringify({ languageDetected: result.detected_language }));
+            ws.send(JSON.stringify({ type: 'languageDetected', languageDetected: result.detected_language }));
             transcribeAudio(ws, formData, sessionData);
         }
     });
 }
 
+async function translateText(originalText, originalLanguage, targetLanguage) {
+    return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText)
+        .then(response => {
+            console.log('Translation response:', response);
+            return response;
+        });
+}
+async function queryLLM(prompt) {
+    const requestData = {
+        model: 'qwen2', // ollama3
+        prompt: prompt,
+        system: "you provide translations to the text transcribed from audio. The text is in a language you understand, and you can provide translations to any language you know.",
+        //format: "json"
+    };
+    const ola = new ollama.Ollama({ host: LNN_API_URL })
+    await ola.generate(requestData)
+}
+
+///obsolete function
+async function queryLLMAxios(prompt) {
+    const requestData = {
+        model: 'qwen2',
+        prompt: prompt,
+        "system": "talk like a pirate",
+        "stream": false
+    };
+
+    try {
+        const response = await axios.post(LNN_API_URL + "/api/generate", requestData, {
+            headers: {
+                // 'Authorization': `Bearer ${OLLAMA_API_KEY}`,
+                'Content-Type': 'application/json'
+            }
+        });
+        return response.data;
+    } catch (error) {
+        console.error('Error calling Ollama API:', error.response ? error.response.data : error.message);
+        throw error;
+    }
+}
+
 function transcribeAudio(ws, formData, sessionData) {
     const start = new Date().getTime();
     queueCounter++;
@@ -158,11 +271,48 @@ function transcribeAudio(ws, formData, sessionData) {
 
         const duration = new Date().getTime() - start;
         ws.send(JSON.stringify({
+            type: 'text',
             queueCounter,
             duration,
             language: sessionData.language,
             text: body
         }));
+
+        if (sessionData.currentChat) {
+            const chat = chats.get(sessionData.currentChat);
+            if (chat) {
+                let msg = { sender: sessionData.username, text: body, translations: [] };
+                // init messages array if it doesn't exist
+                if (!chat.messages) chat.messages = [];
+                chat.messages.push(msg);
+
+                chat.participants.forEach(sessionId => {
+                    if (sessionId !== ws.sessionId) {
+                        let targetLang = sessions.get(sessionId)?.language || 'en';
+                        targetLang = "bg";
+                        if (targetLang !== sessionData.language) {
+                            console.log('Translating message "'+body+'" from ' + sessionData.language + ' to ' + targetLang);
+                            translateText(body, sessionData.language, targetLang)
+                                .then(translation => {
+                                    const jsonResp = JSON.parse(translation);
+                                    msg.translations.push({ language: targetLang, text: jsonResp.response });
+                                    const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
+                                    if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
+                                        participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + jsonResp.response + "\n" }));
+                                    }
+                                });
+                        }
+                        else {
+                            const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
+                            if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
+                                participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + body + "\n" }));
+                            }
+                        }
+                    }
+                });
+
+            }
+        }
     });
 
     if (storeRecordings) {
@@ -178,7 +328,7 @@ function transcribeAudio(ws, formData, sessionData) {
 }
 
 function broadcastUserList() {
-    const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId }));
+    const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId, currentChat: user.currentChat, language: user.language }));
     wss.clients.forEach(client => {
         if (client.readyState === WebSocket.OPEN) {
             client.send(JSON.stringify({ type: 'userList', users: userList }));
@@ -195,7 +345,6 @@ app.get('/audio.js', (req, res) => {
     res.sendFile(path.join(__dirname, 'audio.js'));
 });
 
-
 app.post('/log', (req, res) => {
     console.log(`[LOG ${new Date().toISOString()}] ${req.body.message}`);
     res.status(200).send('OK');

web/client.html

@@ -432,7 +432,7 @@
                     serverTime = Date.now();
                     console.log("Sent some audio data to server.");
                     if (!autosend.checked) {
-                        transcription.innerHTML = "Processing audio...";
+                        transcription.placeholder = "Processing audio...";
                     }
                 } else {
                     console.log("Not connected, not sending audio data to server.");