doing translations
This commit is contained in:
Dobromir Popov
parent
364df3d891
commit
8bafcd9dbe
@ -1,6 +1,7 @@
|
||||
|
||||
ENV_NAME=development
|
||||
TTS_API_URL=https://api.tts.d-popov.com/asr
|
||||
LNN_API_URL=https://ollama.d-popov.com
|
||||
WS_URL=ws://localhost:8081
|
||||
SERVER_PORT_WS=8081
|
||||
SERVER_PORT_HTTP=8080
|
||||
|
||||
62
package-lock.json
generated
62
package-lock.json
generated
@ -8,11 +8,13 @@
|
||||
"name": "kevin-ai",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"axios": "^1.7.2",
|
||||
"body-parser": "^1.20.2",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.18.2",
|
||||
"git": "^0.1.5",
|
||||
"node-persist": "^3.1.3",
|
||||
"ollama": "^0.5.1",
|
||||
"request": "^2.88.2",
|
||||
"ws": "^8.12.1"
|
||||
}
|
||||
@ -83,6 +85,29 @@
|
||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz",
|
||||
"integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg=="
|
||||
},
|
||||
"node_modules/axios": {
|
||||
"version": "1.7.2",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz",
|
||||
"integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==",
|
||||
"dependencies": {
|
||||
"follow-redirects": "^1.15.6",
|
||||
"form-data": "^4.0.0",
|
||||
"proxy-from-env": "^1.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/axios/node_modules/form-data": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
|
||||
"integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/bcrypt-pbkdf": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz",
|
||||
@ -409,6 +434,25 @@
|
||||
"node": ">= 0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/follow-redirects": {
|
||||
"version": "1.15.6",
|
||||
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
|
||||
"integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "individual",
|
||||
"url": "https://github.com/sponsors/RubenVerborgh"
|
||||
}
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=4.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"debug": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/forever-agent": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
|
||||
@ -728,6 +772,14 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/ollama": {
|
||||
"version": "0.5.1",
|
||||
"resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.1.tgz",
|
||||
"integrity": "sha512-mAiCHxdvu63E8EFopz0y82QG7rGfYmKAWgmjG2C7soiRuz/Sj3r/ebvCOp+jasiCubqUPE0ZThKT5LR6wrrPtA==",
|
||||
"dependencies": {
|
||||
"whatwg-fetch": "^3.6.20"
|
||||
}
|
||||
},
|
||||
"node_modules/on-finished": {
|
||||
"version": "2.4.1",
|
||||
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz",
|
||||
@ -769,6 +821,11 @@
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/proxy-from-env": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
|
||||
},
|
||||
"node_modules/psl": {
|
||||
"version": "1.9.0",
|
||||
"resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
|
||||
@ -1089,6 +1146,11 @@
|
||||
"extsprintf": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-fetch": {
|
||||
"version": "3.6.20",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
|
||||
"integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg=="
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.12.1",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz",
|
||||
|
||||
@ -8,11 +8,13 @@
|
||||
"start:tele": "python agent-py-bot/agent.py"
|
||||
},
|
||||
"dependencies": {
|
||||
"axios": "^1.7.2",
|
||||
"body-parser": "^1.20.2",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.18.2",
|
||||
"git": "^0.1.5",
|
||||
"node-persist": "^3.1.3",
|
||||
"ollama": "^0.5.1",
|
||||
"request": "^2.88.2",
|
||||
"ws": "^8.12.1"
|
||||
}
|
||||
|
||||
25
web/audio.js
25
web/audio.js
@ -1,7 +1,7 @@
|
||||
let selectedDeviceId = "default";
|
||||
export let serverTime;
|
||||
export let recordButton;
|
||||
let socket;
|
||||
export let socket;
|
||||
let audioRecorder;
|
||||
let audioStream;
|
||||
let recording = false;
|
||||
@ -42,7 +42,6 @@ export function InitAudioAnalyser(stream) {
|
||||
}
|
||||
|
||||
export function startListening() {
|
||||
//canvasCtx.fillStyle = "green";
|
||||
recording = true;
|
||||
navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
|
||||
.then((stream) => {
|
||||
@ -75,7 +74,6 @@ export function startListening() {
|
||||
}
|
||||
if (event.data.size > 0) {
|
||||
let data = event.data;
|
||||
console.log("audio data size: " + data.size);
|
||||
if (preBuffer.length > 0) {
|
||||
sendAudioToServerPost(preBuffer);
|
||||
}
|
||||
@ -117,14 +115,25 @@ export function sendAudioToServerPost(data) {
|
||||
});
|
||||
}
|
||||
|
||||
export function sendAudioToServer(data) {
|
||||
//if (connected) {
|
||||
socket.send(JSON.stringify({ type: 'audio', task:"transcribe", audio: data }));
|
||||
export function sendAudioToServerJson(data) {
|
||||
if (socket && socket.readyState === WebSocket.OPEN) {
|
||||
|
||||
const binaryData = Buffer.from(base64AudioData, 'base64');
|
||||
socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData }));
|
||||
serverTime = Date.now();
|
||||
if (!autosend.checked) {
|
||||
transcription.innerHTML = "Processing audio...";
|
||||
transcription.placeholder = "Processing audio...";
|
||||
}
|
||||
}
|
||||
}
|
||||
export function sendAudioToServer(data) {
|
||||
if (socket && socket.readyState === WebSocket.OPEN) {
|
||||
socket.send(data);
|
||||
serverTime = Date.now();
|
||||
if (!autosend.checked) {
|
||||
transcription.placeholder = "Processing audio...";
|
||||
}
|
||||
}
|
||||
//}
|
||||
}
|
||||
|
||||
export function toggleListening() {
|
||||
|
||||
@ -11,12 +11,20 @@
|
||||
<div class="container mx-auto px-4 py-8">
|
||||
<h1 class="text-2xl font-bold mb-4 text-center">Real-time Voice Chat</h1>
|
||||
|
||||
|
||||
<div class="flex justify-center items-center mb-4">
|
||||
<!-- Username Input -->
|
||||
<input type="text" id="username" class="border rounded p-2 mr-4" placeholder="Enter your username">
|
||||
<button id="btn-join" onclick="joinChat()"
|
||||
class="hidden bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
|
||||
<div id="join-container" class="hidden">
|
||||
<button id="btn-join" onclick="logInAndStoreSession()"
|
||||
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
|
||||
<select id="language-select">
|
||||
<option value="auto">Auto</option>
|
||||
<option value="en">English</option>
|
||||
<option value="bg">Български</option>
|
||||
<option value="fr">Français</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- Clear Session Option -->
|
||||
<button id="btn-disconnect" onclick="clearSession()"
|
||||
@ -48,6 +56,11 @@
|
||||
<h2 class="text-xl font-bold mb-2">Chat Room</h2>
|
||||
<div id="chat-room" class="bg-white p-4 rounded shadow mb-4">
|
||||
<!-- Chat room content -->
|
||||
<div>
|
||||
<div id="chat-room-users" class="flex flex-wrap mb-4">
|
||||
<!-- Participants list -->
|
||||
</div>
|
||||
</div>
|
||||
<div class="mb-4">
|
||||
<label class="flex items-center space-x-2">
|
||||
<input type="checkbox" id="autosend" class="mr-2">
|
||||
@ -59,6 +72,9 @@
|
||||
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">Push to
|
||||
Talk</button>
|
||||
</div>
|
||||
<div id="status-recording" class="flex justify-center items-center mb-4">
|
||||
</div>
|
||||
|
||||
<div id="transcription" class="border rounded p-4 h-48 overflow-y-scroll mb-4">
|
||||
<!-- Transcription content -->
|
||||
</div>
|
||||
@ -81,11 +97,11 @@
|
||||
<div class="flex justify-center items-center mb-4">
|
||||
<div id="info"></div>
|
||||
</div>
|
||||
<div id="status-recording" class="flex justify-center items-center mb-4"></div>
|
||||
<div id="status-recording" class="flex justify-center items-center mb-4"> status</div>
|
||||
</div>
|
||||
|
||||
<script type="module">
|
||||
import { startListening, stopListening, toggleListening, InitAudioAnalyser, sendAudioToServerPost, sendAudioToServer, initializeVolumeChecker, serverTime, setSocket, setRecordButton } from './audio.js';
|
||||
// import * as audio from './audio.js';
|
||||
|
||||
let socket;
|
||||
let sessionId;
|
||||
@ -120,11 +136,20 @@
|
||||
})
|
||||
.then(() => {
|
||||
socket = new WebSocket(wsurl);
|
||||
// audio.setSocket(socket); // Set the socket in the audio module
|
||||
socket.onopen = () => {
|
||||
connectionStatus.innerHTML = "Connected to " + wsurl;
|
||||
recordButton.disabled = false;
|
||||
connected = true;
|
||||
resolve();
|
||||
//if we stored a session id in a cookie, reconnect
|
||||
const sessionId = getCookie("sessionId");
|
||||
if (sessionId) {
|
||||
socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
|
||||
}
|
||||
else {
|
||||
socket.send(JSON.stringify({ type: 'sessionId' }));
|
||||
}
|
||||
resolve(socket);
|
||||
};
|
||||
socket.onmessage = onmessage;
|
||||
socket.onclose = () => {
|
||||
@ -135,7 +160,6 @@
|
||||
connect().then(resolve).catch(reject);
|
||||
}, 5000);
|
||||
};
|
||||
setSocket(socket);
|
||||
})
|
||||
.catch((error) => {
|
||||
connectionStatus.innerHTML = "Error getting ws url: " + error;
|
||||
@ -150,14 +174,30 @@
|
||||
switch (json.type) {
|
||||
case "sessionId":
|
||||
sessionId = json.sessionId;
|
||||
//set the session id in the cookie
|
||||
document.cookie = `sessionId=${sessionId}; path=/;`;
|
||||
console.log("Got session id: " + sessionId);
|
||||
break;
|
||||
case "languageDetected":
|
||||
statusRecording.innerHTML = "Detected language: " + json.languageDetected;
|
||||
break;
|
||||
case "text":
|
||||
case "transcriptionResult":
|
||||
transcription.innerHTML += "\r\n" + json.text;
|
||||
let latency = Date.now() - serverTime;
|
||||
if (autosend.checked) {
|
||||
// const arr = event.data.split(/[(\)]/);
|
||||
// let queue = arr[1];
|
||||
// let text = arr[2].trim();
|
||||
// info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
|
||||
//transcription.value += event.data + " ";
|
||||
statusRecording.innerHTML = "Listening...";
|
||||
statusRecording.style.color = "black";
|
||||
} else {
|
||||
//transcription.innerHTML = event.data;
|
||||
}
|
||||
break;
|
||||
|
||||
case "userList":
|
||||
users = json.users;
|
||||
updateUserList();
|
||||
@ -166,9 +206,8 @@
|
||||
chats = json.chats;
|
||||
updateChatList();
|
||||
break;
|
||||
case "newChatRoom":
|
||||
chats.push(json.newChatRoom);
|
||||
updateChatList();
|
||||
case "chat":
|
||||
displayChatParticipants(json.chat.id, json.chat.participants);
|
||||
break;
|
||||
default:
|
||||
console.log("Unknown message type:", json.type);
|
||||
@ -177,36 +216,33 @@
|
||||
console.error("Failed to parse message", e);
|
||||
}
|
||||
|
||||
let latency = Date.now() - serverTime;
|
||||
if (autosend.checked) {
|
||||
const arr = event.data.split(/[(\)]/);
|
||||
let queue = arr[1];
|
||||
let text = arr[2].trim();
|
||||
info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
|
||||
transcription.value += text + " ";
|
||||
statusRecording.innerHTML = "Listening...";
|
||||
statusRecording.style.color = "black";
|
||||
} else {
|
||||
transcription.innerHTML = event.data;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function joinChat() {
|
||||
function logInAndStoreSession() {
|
||||
username = document.getElementById('username').value;
|
||||
if (username.trim() === "") {
|
||||
alert("Please enter a username");
|
||||
return;
|
||||
}
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||
alert("WebSocket connection is not open. Please wait and try again.");
|
||||
return;
|
||||
connect().then(() => {
|
||||
userJoin(sessionId, username, document.getElementById('language-select').value);
|
||||
});
|
||||
} else {
|
||||
userJoin(sessionId, username, document.getElementById('language-select').value);
|
||||
}
|
||||
socket.send(JSON.stringify({ type: 'join', username }));
|
||||
}
|
||||
|
||||
function userJoin(sessionId, username, language) {
|
||||
socket.send(JSON.stringify({ type: 'join', username , language}));
|
||||
document.cookie = `sessionId=${sessionId}; path=/;`;
|
||||
document.cookie = `username=${username}; path=/;`;
|
||||
|
||||
showClearSessionOption();
|
||||
}
|
||||
|
||||
|
||||
function clearSession() {
|
||||
document.cookie = "sessionId=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
|
||||
document.cookie = "username=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
|
||||
@ -217,21 +253,19 @@
|
||||
const sessionId = getCookie("sessionId");
|
||||
if (sessionId) {
|
||||
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
||||
connect().then(() => {
|
||||
socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
|
||||
//initializeVolumeChecker();
|
||||
|
||||
connect().then((s) => {
|
||||
s.send(JSON.stringify({ type: 'reconnect', sessionId }));
|
||||
});
|
||||
}
|
||||
document.getElementById('btn-disconnect').classList.remove('hidden');
|
||||
document.getElementById('btn-join').classList.add('hidden');
|
||||
document.getElementById('join-container').classList.add('hidden');
|
||||
|
||||
document.getElementById('active-users-container').classList.remove('hidden');
|
||||
document.getElementById('previous-chats-container').classList.remove('hidden');
|
||||
|
||||
} else {
|
||||
document.getElementById('btn-disconnect').classList.add('hidden');
|
||||
document.getElementById('btn-join').classList.remove('hidden');
|
||||
document.getElementById('join-container').classList.remove('hidden');
|
||||
}
|
||||
}
|
||||
|
||||
@ -249,7 +283,7 @@
|
||||
|
||||
showClearSessionOption();
|
||||
connect().then(() => {
|
||||
initializeVolumeChecker();
|
||||
// audio.initializeVolumeChecker();
|
||||
});
|
||||
};
|
||||
|
||||
@ -269,7 +303,7 @@
|
||||
users.forEach(user => {
|
||||
const option = document.createElement('option');
|
||||
option.value = user.sessionId;
|
||||
option.innerText = user.username;
|
||||
option.innerText = "["+user.language+"] " +user.username;
|
||||
if (user.username === username) {
|
||||
option.innerText += " (me)";
|
||||
}
|
||||
@ -287,11 +321,63 @@
|
||||
selectedUsers.push(sessionId); // Add self to the selected users list for self-chat
|
||||
socket.send(JSON.stringify({ type: 'startChat', users: selectedUsers }));
|
||||
document.getElementById('chat-room-container').classList.remove('hidden');
|
||||
displayChatParticipants(selectedUsers);
|
||||
}
|
||||
|
||||
function displayChatParticipants(participants) {
|
||||
const chatRoom = document.getElementById('chat-room');
|
||||
|
||||
|
||||
function fetchPreviousChats(username) {
|
||||
fetch(`/chats?username=${username}`)
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
chats = data.chats;
|
||||
updateChatList();
|
||||
});
|
||||
}
|
||||
|
||||
function updateChatList() {
|
||||
const previousChats = document.getElementById('previous-chats');
|
||||
previousChats.innerHTML = '';
|
||||
chats.forEach(chat => {
|
||||
const chatDiv = document.createElement('div');
|
||||
chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2', 'cursor-pointer');
|
||||
chatDiv.setAttribute('data-chat-id', chat.id); // Store chat ID in data attribute
|
||||
|
||||
const participants = chat.participants.join(', ');
|
||||
const status = chat.participants.map(participant => {
|
||||
const user = users.find(u => u.username === participant);
|
||||
return user ? `${participant} (online)` : `${participant} (offline)`;
|
||||
}).join(', ');
|
||||
|
||||
chatDiv.innerHTML = `${status}`;
|
||||
|
||||
chatDiv.addEventListener('click', () => {
|
||||
// Remove highlight from all chat divs
|
||||
document.querySelectorAll('#previous-chats > div').forEach(div => {
|
||||
div.classList.remove('bg-blue-100');
|
||||
});
|
||||
// Highlight selected chat div
|
||||
chatDiv.classList.add('bg-blue-100');
|
||||
selectChatRoom(chat.id);
|
||||
});
|
||||
|
||||
previousChats.appendChild(chatDiv);
|
||||
});
|
||||
}
|
||||
|
||||
function selectChatRoom(chatId) {
|
||||
const chat = chats.find(c => c.id === chatId);
|
||||
if (!chat) return;
|
||||
|
||||
const chatRoomUsers = document.getElementById('chat-room-users');
|
||||
chatRoomUsers.innerHTML = ''; // Clear existing content
|
||||
|
||||
socket.send(JSON.stringify({ type: 'enterChat', chatId }));
|
||||
document.getElementById('chat-room-container').classList.remove('hidden');
|
||||
// displayChatParticipants(chatId, chat.participants);
|
||||
}
|
||||
|
||||
function displayChatParticipants(chatId, participants) {
|
||||
const chatRoomUsers = document.getElementById('chat-room-users');
|
||||
let participantsHtml = '<div class="flex flex-wrap mb-4">';
|
||||
participants.forEach(participantId => {
|
||||
const user = users.find(u => u.sessionId === participantId);
|
||||
@ -302,41 +388,214 @@
|
||||
</span>`;
|
||||
});
|
||||
participantsHtml += '</div>';
|
||||
chatRoom.insertAdjacentHTML('afterbegin', participantsHtml);
|
||||
chatRoomUsers.innerHTML = participantsHtml;
|
||||
}
|
||||
|
||||
function fetchPreviousChats(username) {
|
||||
fetch(`/chats?username=${username}`)
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
chats = data.chats;
|
||||
updateChatList();
|
||||
// REGION AUDIO RECORDING
|
||||
let selectedDeviceId = "default";
|
||||
export let serverTime;
|
||||
// export let recordButton;
|
||||
// export let socket;
|
||||
// let connectionStatus;
|
||||
// let statusRecording;
|
||||
let audioRecorder;
|
||||
let audioStream;
|
||||
let recording = false;
|
||||
let audioContext;
|
||||
let volumeChecker;
|
||||
let lastVolumes = new Array(5);
|
||||
let averageVolume;
|
||||
let silenceCount = 0;
|
||||
let isSpeaking = false;
|
||||
let soundDetected = false;
|
||||
let speakingCount = 0;
|
||||
let analyser = null;
|
||||
|
||||
let SILENCE_DELAY_MS = 50;
|
||||
let preDetect_IncludedAudio = 400; //ms
|
||||
let soundCount_Threshold = 10;
|
||||
let silenceCount_Threshold = 10;
|
||||
|
||||
const volumeHistory = [];
|
||||
|
||||
function setSocket(newSocket) {
|
||||
socket = newSocket;
|
||||
}
|
||||
export function setRecordButton(newRecordButton) {
|
||||
recordButton = newRecordButton;
|
||||
recordButton.addEventListener("click", toggleListening);
|
||||
}
|
||||
|
||||
export function InitAudioAnalyser(stream) {
|
||||
audioContext = new AudioContext();
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
analyser = audioContext.createAnalyser();
|
||||
analyser.fftSize = 2048;
|
||||
analyser.smoothingTimeConstant = 0.8;
|
||||
source.connect(analyser);
|
||||
}
|
||||
|
||||
export function startListening() {
|
||||
recording = true;
|
||||
navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
|
||||
.then((stream) => {
|
||||
audioStream = stream;
|
||||
|
||||
const audioContext = new AudioContext();
|
||||
const sourceNode = audioContext.createMediaStreamSource(audioStream);
|
||||
const audioSampleRate = sourceNode.context.sampleRate;
|
||||
|
||||
info.innerHTML = "Sample rate: " + audioSampleRate + " Hz";
|
||||
var preBuffer = [];
|
||||
|
||||
const channelSplitter = audioContext.createChannelSplitter(2);
|
||||
const channelMerger = audioContext.createChannelMerger(1);
|
||||
sourceNode.connect(channelSplitter);
|
||||
channelSplitter.connect(channelMerger, 0, 0);
|
||||
const outputNode = channelMerger;
|
||||
|
||||
const mediaStreamDestination = audioContext.createMediaStreamDestination();
|
||||
outputNode.connect(mediaStreamDestination);
|
||||
const singleChannelStream = mediaStreamDestination.stream;
|
||||
|
||||
audioRecorder = new MediaRecorder(singleChannelStream);
|
||||
audioRecorder.start();
|
||||
audioRecorder.addEventListener("dataavailable", (event) => {
|
||||
if (!soundDetected && autosend.checked) {
|
||||
preBuffer = [];
|
||||
preBuffer.push(event.data);
|
||||
return;
|
||||
}
|
||||
if (event.data.size > 0) {
|
||||
let data = event.data;
|
||||
if (preBuffer.length > 0) {
|
||||
sendAudioToServerPost(preBuffer);
|
||||
}
|
||||
sendAudioToServer(data);
|
||||
soundDetected = false;
|
||||
}
|
||||
});
|
||||
|
||||
InitAudioAnalyser(stream);
|
||||
});
|
||||
|
||||
recordButton.innerHTML = "Stop Talking";
|
||||
recordButton.classList.toggle('bg-red-500');
|
||||
recordButton.classList.toggle('bg-blue-500');
|
||||
recordButton.classList.toggle('hover:bg-blue-700');
|
||||
}
|
||||
|
||||
export function stopListening() {
|
||||
recording = false;
|
||||
audioRecorder.stop();
|
||||
recordButton.innerHTML = "Push to Talk";
|
||||
recordButton.classList.toggle('bg-blue-500');
|
||||
recordButton.classList.toggle('bg-red-500');
|
||||
recordButton.classList.toggle('hover:bg-blue-700');
|
||||
clearInterval(volumeChecker);
|
||||
if (audioStream) {
|
||||
audioStream.getTracks().forEach(track => track.stop());
|
||||
audioStream = null;
|
||||
}
|
||||
}
|
||||
|
||||
export function sendAudioToServerPost(data) {
|
||||
const blob = new Blob(data, { type: "audio/ogg; codecs=opus" });
|
||||
var formData = new FormData();
|
||||
formData.append('file', data);
|
||||
fetch('/upload', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
}
|
||||
function sendSocketMessage(message) {
|
||||
|
||||
export function sendAudioToServerJson(data) {
|
||||
if (socket && socket.readyState === WebSocket.OPEN) {
|
||||
socket.send(JSON.stringify(message));
|
||||
|
||||
const binaryData = Buffer.from(base64AudioData, 'base64');
|
||||
socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData }));
|
||||
serverTime = Date.now();
|
||||
if (!autosend.checked) {
|
||||
transcription.placeholder = "Processing audio...";
|
||||
}
|
||||
}
|
||||
}
|
||||
export function sendAudioToServer(data) {
|
||||
if (socket && socket.readyState === WebSocket.OPEN) {
|
||||
socket.send(data);
|
||||
serverTime = Date.now();
|
||||
if (!autosend.checked) {
|
||||
transcription.placeholder = "Processing audio...";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function updateChatList() {
|
||||
const previousChats = document.getElementById('previous-chats');
|
||||
previousChats.innerHTML = '';
|
||||
chats.forEach(chat => {
|
||||
const chatDiv = document.createElement('div');
|
||||
chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2');
|
||||
const participants = chat.participants.join(', ');
|
||||
const status = chat.participants.map(participant => {
|
||||
const user = users.find(u => u.username === participant);
|
||||
return user ? `${participant} (online)` : `${participant} (offline)`;
|
||||
}).join(', ');
|
||||
chatDiv.innerHTML = `<strong>Participants:</strong> ${participants}<br><strong>Status:</strong> ${status}`;
|
||||
previousChats.appendChild(chatDiv);
|
||||
});
|
||||
export function toggleListening() {
|
||||
if (socket.readyState === WebSocket.OPEN) {
|
||||
if (recording) {
|
||||
stopListening();
|
||||
} else {
|
||||
startListening();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function initializeVolumeChecker() {
|
||||
volumeChecker = setInterval(() => {
|
||||
if (!audioContext) {
|
||||
console.log("No audio context");
|
||||
return;
|
||||
}
|
||||
const frequencyData = new Uint8Array(analyser.frequencyBinCount);
|
||||
analyser.getByteFrequencyData(frequencyData);
|
||||
|
||||
let totalVolume = 0;
|
||||
for (let i = 0; i < frequencyData.length; i++) {
|
||||
totalVolume += frequencyData[i];
|
||||
}
|
||||
averageVolume = totalVolume / frequencyData.length;
|
||||
|
||||
volumeHistory.push(averageVolume);
|
||||
if (volumeHistory.length > 100) {
|
||||
volumeHistory.shift();
|
||||
}
|
||||
|
||||
const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5;
|
||||
const isSilent = averageVolume < threshold;
|
||||
|
||||
if (averageVolume > threshold) {
|
||||
if (autosend.checked && speakingCount == 0 && audioRecorder) {
|
||||
soundDetected = false;
|
||||
audioRecorder.stop();
|
||||
audioRecorder.start();
|
||||
}
|
||||
speakingCount++;
|
||||
if (speakingCount > soundCount_Threshold) {
|
||||
statusRecording.innerHTML = "Listening...";
|
||||
statusRecording.style.color = "green";
|
||||
isSpeaking = true;
|
||||
}
|
||||
} else if (averageVolume - 5 < threshold) {
|
||||
speakingCount = 0;
|
||||
if (isSpeaking) {
|
||||
silenceCount++;
|
||||
if (silenceCount > silenceCount_Threshold) {
|
||||
if (autosend.checked) {
|
||||
soundDetected = true;
|
||||
audioRecorder.stop();
|
||||
audioRecorder.start();
|
||||
}
|
||||
isSpeaking = false;
|
||||
statusRecording.innerHTML = "Silence detected...";
|
||||
statusRecording.style.color = "orange";
|
||||
}
|
||||
}
|
||||
}
|
||||
}, SILENCE_DELAY_MS);
|
||||
}
|
||||
|
||||
// Expose functions to global scope
|
||||
window.joinChat = joinChat;
|
||||
window.logInAndStoreSession = logInAndStoreSession;
|
||||
window.clearSession = clearSession;
|
||||
window.copyToClipboard = copyToClipboard;
|
||||
window.clearTranscription = clearTranscription;
|
||||
|
||||
@ -1,8 +1,3 @@
|
||||
// server.js
|
||||
if (require('dotenv')) {
|
||||
const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
|
||||
require('dotenv').config({ path: envFile });
|
||||
}
|
||||
const express = require('express');
|
||||
const bodyParser = require('body-parser');
|
||||
const WebSocket = require('ws');
|
||||
@ -10,6 +5,14 @@ const storage = require('node-persist');
|
||||
const request = require('request');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const dotenv = require('dotenv');
|
||||
const ollama = require('ollama');
|
||||
const axios = require('axios');
|
||||
|
||||
if (dotenv) {
|
||||
const envFile = process.env.NODE_ENV === 'development' ? '.env.development' : '.env';
|
||||
dotenv.config({ path: envFile });
|
||||
}
|
||||
|
||||
const app = express();
|
||||
app.use(bodyParser.json());
|
||||
@ -17,6 +20,7 @@ app.use(bodyParser.json());
|
||||
const PORT_HTTP = process.env.SERVER_PORT_HTTP || 3000;
|
||||
const PORT_WS = process.env.SERVER_PORT_WS || 8080;
|
||||
const TTS_API_URL = process.env.TTS_API_URL;
|
||||
const LNN_API_URL = process.env.LNN_API_URL;
|
||||
|
||||
let language = "en";
|
||||
let storeRecordings = false;
|
||||
@ -34,25 +38,65 @@ storage.init().then(() => {
|
||||
if (value !== undefined) storeRecordings = value;
|
||||
else storage.setItem('storeRecordings', storeRecordings);
|
||||
});
|
||||
|
||||
// Load existing chats, sessions from storage
|
||||
storage.getItem('chats').then((value) => {
|
||||
if (value !== undefined) {
|
||||
value.forEach(chat => chats.set(chat.id, chat));
|
||||
}
|
||||
});
|
||||
storage.getItem('sessions').then((value) => {
|
||||
if (value !== undefined) {
|
||||
value.forEach(session => sessions.set(session.sessionId, session));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// WebSocket Server
|
||||
const wss = new WebSocket.Server({ port: PORT_WS });
|
||||
wss.on('connection', (ws) => {
|
||||
ws.sessionId = Math.random().toString(36).substring(2);
|
||||
sessions.set(ws.sessionId, { language: 'en' });
|
||||
|
||||
ws.send(JSON.stringify({ sessionId: ws.sessionId, language, storeRecordings }));
|
||||
|
||||
|
||||
ws.on('message', (message) => {
|
||||
let isJson = false;
|
||||
let data = null;
|
||||
// Check if message is JSON
|
||||
try {
|
||||
data = JSON.parse(message);
|
||||
isJson = true;
|
||||
}
|
||||
catch (e) {
|
||||
}
|
||||
|
||||
if (isJson) {
|
||||
// Handle JSON messages
|
||||
try {
|
||||
const data = JSON.parse(message);
|
||||
console.log('Received message:', data.type);
|
||||
|
||||
switch (data.type) {
|
||||
case 'sessionId':
|
||||
ws.sessionId = Math.random().toString(36).substring(2);
|
||||
sessions.set(ws.sessionId, { language: 'en' });
|
||||
//store new session in storage sessions
|
||||
storage.getItem('sessions').then((value) => {
|
||||
if (value !== undefined) {
|
||||
value.push({ sessionId: ws.sessionId, language: 'en' });
|
||||
storage.setItem('sessions', value);
|
||||
} else {
|
||||
storage.setItem('sessions', [{ sessionId: ws.sessionId, language: 'en' }]);
|
||||
}
|
||||
}
|
||||
);
|
||||
break;
|
||||
case 'join':
|
||||
const { username } = data;
|
||||
sessions.set(ws.sessionId, { username, sessionId: ws.sessionId });
|
||||
const { username, language } = data;
|
||||
sessions.set(ws.sessionId, { username, sessionId: ws.sessionId, language });
|
||||
ws.send(JSON.stringify({ type: 'sessionId', sessionId: ws.sessionId, language, storeRecordings }));
|
||||
|
||||
const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId));
|
||||
ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
|
||||
|
||||
broadcastUserList();
|
||||
break;
|
||||
case 'startChat':
|
||||
@ -64,6 +108,16 @@ wss.on('connection', (ws) => {
|
||||
|
||||
chats.set(chatId, { participants, messages: [] });
|
||||
|
||||
//store new chat in storage chats
|
||||
storage.getItem('chats').then((value) => {
|
||||
if (value !== undefined) {
|
||||
value.push({ id: chatId, participants });
|
||||
storage.setItem('chats', value);
|
||||
} else {
|
||||
storage.setItem('chats', [{ id: chatId, participants }]);
|
||||
}
|
||||
});
|
||||
|
||||
const userNames = participants.map(sessionId => {
|
||||
const user = sessions.get(sessionId);
|
||||
return user ? `${user.username}(${user.sessionId})` : 'Unknown User';
|
||||
@ -74,27 +128,40 @@ wss.on('connection', (ws) => {
|
||||
participants.forEach(sessionId => {
|
||||
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
|
||||
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
|
||||
participantSocket.send(JSON.stringify({ newChatRoom: { id: chatId, participants: userNames } }));
|
||||
const userChats = Array.from(chats.entries())
|
||||
.filter(([id, chat]) => chat.participants.includes(sessionId))
|
||||
.map(([id, chat]) => ({ id, participants: chat.participants }));
|
||||
participantSocket.send(JSON.stringify({ type: 'chats', chats: userChats }));
|
||||
}
|
||||
});
|
||||
|
||||
broadcastUserList();
|
||||
break;
|
||||
case 'audio':
|
||||
console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client. Crrent language: ' + language, 'task: ' + data.task);
|
||||
handleAudioData(ws, data.audio);
|
||||
// case 'audio':
|
||||
// console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client.');
|
||||
// handleAudioData(ws, data.audiobase64);
|
||||
// break;
|
||||
case 'enterChat':
|
||||
const { chatId: Id } = data;
|
||||
const enteredChat = chats.get(Id);
|
||||
const currentSession = sessions.get(ws.sessionId);
|
||||
currentSession.currentChat = Id;
|
||||
if (enteredChat && enteredChat.participants.includes(ws.sessionId)) {
|
||||
ws.send(JSON.stringify({ type: 'chat', chat: enteredChat }));
|
||||
}
|
||||
break;
|
||||
case 'reconnect':
|
||||
const userSession = sessions.get(data.sessionId);
|
||||
if (userSession) {
|
||||
sessions.set(ws.sessionId, userSession);
|
||||
ws.sessionId = data.sessionId;
|
||||
broadcastUserList();
|
||||
// Send existing chats
|
||||
const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId));
|
||||
ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
|
||||
ws.send(JSON.stringify({ type: 'userList', users: userList }));
|
||||
}
|
||||
else {
|
||||
console.log('Session not found:', data.sessionId);
|
||||
}
|
||||
|
||||
broadcastUserList();
|
||||
break;
|
||||
default:
|
||||
console.log('Unknown message type:', data.type);
|
||||
@ -102,10 +169,15 @@ wss.on('connection', (ws) => {
|
||||
} catch (err) {
|
||||
console.error('Failed to parse message', err);
|
||||
}
|
||||
|
||||
} else {
|
||||
// Handle binary data
|
||||
handleAudioData(ws, message);
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
ws.on('close', () => {
|
||||
// allUsers.delete(ws.sessionId);
|
||||
sessions.delete(ws.sessionId);
|
||||
broadcastUserList();
|
||||
});
|
||||
@ -142,12 +214,53 @@ function detectLanguage(ws, formData) {
|
||||
const language = result.language_code;
|
||||
const sessionData = sessions.get(ws.sessionId);
|
||||
sessionData.language = language;
|
||||
ws.send(JSON.stringify({ languageDetected: result.detected_language }));
|
||||
ws.send(JSON.stringify({ type: 'languageDetected', languageDetected: result.detected_language }));
|
||||
transcribeAudio(ws, formData, sessionData);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function translateText(originalText, originalLanguage, targetLanguage) {
|
||||
return queryLLMAxios("translate this text from " + originalLanguage + " to " + targetLanguage + ": " + originalText)
|
||||
.then(response => {
|
||||
console.log('Translation response:', response);
|
||||
return response;
|
||||
});
|
||||
}
|
||||
async function queryLLM(prompt) {
|
||||
const requestData = {
|
||||
model: 'qwen2', // ollama3
|
||||
prompt: prompt,
|
||||
system: "you provide translations to the text transcribed from audio. The text is in a language you understand, and you can provide translations to any language you know.",
|
||||
//format: "json"
|
||||
};
|
||||
const ola = new ollama.Ollama({ host: LNN_API_URL })
|
||||
await ola.generate(requestData)
|
||||
}
|
||||
|
||||
///obsolete function
|
||||
async function queryLLMAxios(prompt) {
|
||||
const requestData = {
|
||||
model: 'qwen2',
|
||||
prompt: prompt,
|
||||
"system": "talk like a pirate",
|
||||
"stream": false
|
||||
};
|
||||
|
||||
try {
|
||||
const response = await axios.post(LNN_API_URL + "/api/generate", requestData, {
|
||||
headers: {
|
||||
// 'Authorization': `Bearer ${OLLAMA_API_KEY}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
});
|
||||
return response.data;
|
||||
} catch (error) {
|
||||
console.error('Error calling Ollama API:', error.response ? error.response.data : error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function transcribeAudio(ws, formData, sessionData) {
|
||||
const start = new Date().getTime();
|
||||
queueCounter++;
|
||||
@ -158,11 +271,48 @@ function transcribeAudio(ws, formData, sessionData) {
|
||||
|
||||
const duration = new Date().getTime() - start;
|
||||
ws.send(JSON.stringify({
|
||||
type: 'text',
|
||||
queueCounter,
|
||||
duration,
|
||||
language: sessionData.language,
|
||||
text: body
|
||||
}));
|
||||
|
||||
if (sessionData.currentChat) {
|
||||
const chat = chats.get(sessionData.currentChat);
|
||||
if (chat) {
|
||||
let msg = { sender: sessionData.username, text: body, translations: [] };
|
||||
// init messages array if it doesn't exist
|
||||
if (!chat.messages) chat.messages = [];
|
||||
chat.messages.push(msg);
|
||||
|
||||
chat.participants.forEach(sessionId => {
|
||||
if (sessionId !== ws.sessionId) {
|
||||
let targetLang = sessions.get(sessionId)?.language || 'en';
|
||||
targetLang = "bg";
|
||||
if (targetLang !== sessionData.language) {
|
||||
console.log('Translating message "'+body+'" from ' + sessionData.language + ' to ' + targetLang);
|
||||
translateText(body, sessionData.language, targetLang)
|
||||
.then(translation => {
|
||||
const jsonResp = JSON.parse(translation);
|
||||
msg.translations.push({ language: targetLang, text: jsonResp.response });
|
||||
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
|
||||
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
|
||||
participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + jsonResp.response + "\n" }));
|
||||
}
|
||||
});
|
||||
}
|
||||
else {
|
||||
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
|
||||
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
|
||||
participantSocket.send(JSON.stringify({ type: 'text', text: sessionData.username + ': ' + body + "\n" }));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (storeRecordings) {
|
||||
@ -178,7 +328,7 @@ function transcribeAudio(ws, formData, sessionData) {
|
||||
}
|
||||
|
||||
function broadcastUserList() {
|
||||
const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId }));
|
||||
const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId, currentChat: user.currentChat, language: user.language }));
|
||||
wss.clients.forEach(client => {
|
||||
if (client.readyState === WebSocket.OPEN) {
|
||||
client.send(JSON.stringify({ type: 'userList', users: userList }));
|
||||
@ -195,7 +345,6 @@ app.get('/audio.js', (req, res) => {
|
||||
res.sendFile(path.join(__dirname, 'audio.js'));
|
||||
});
|
||||
|
||||
|
||||
app.post('/log', (req, res) => {
|
||||
console.log(`[LOG ${new Date().toISOString()}] ${req.body.message}`);
|
||||
res.status(200).send('OK');
|
||||
|
||||
@ -432,7 +432,7 @@
|
||||
serverTime = Date.now();
|
||||
console.log("Sent some audio data to server.");
|
||||
if (!autosend.checked) {
|
||||
transcription.innerHTML = "Processing audio...";
|
||||
transcription.placeholder = "Processing audio...";
|
||||
}
|
||||
} else {
|
||||
console.log("Not connected, not sending audio data to server.");
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user