popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

doing translations

Browse Source
This commit is contained in:
Dobromir Popov
2024-06-11 22:53:39 +03:00
parent 364df3d891
commit 8bafcd9dbe
7 changed files with 610 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

373
web/chat-client.html
View File

@ -11,12 +11,20 @@
<div class="container mx-auto px-4 py-8">
<h1 class="text-2xl font-bold mb-4 text-center">Real-time Voice Chat</h1>
<div class="flex justify-center items-center mb-4">
<!-- Username Input -->
<input type="text" id="username" class="border rounded p-2 mr-4" placeholder="Enter your username">
<button id="btn-join" onclick="joinChat()"
class="hidden bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
<div id="join-container" class="hidden">
<button id="btn-join" onclick="logInAndStoreSession()"
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
<select id="language-select">
<option value="auto">Auto</option>
<option value="en">English</option>
<option value="bg">Български</option>
<option value="fr">Français</option>
</select>
</div>
<!-- Clear Session Option -->
<button id="btn-disconnect" onclick="clearSession()"
@ -48,6 +56,11 @@
<h2 class="text-xl font-bold mb-2">Chat Room</h2>
<div id="chat-room" class="bg-white p-4 rounded shadow mb-4">
<!-- Chat room content -->
<div>
<div id="chat-room-users" class="flex flex-wrap mb-4">
<!-- Participants list -->
</div>
</div>
<div class="mb-4">
<label class="flex items-center space-x-2">
<input type="checkbox" id="autosend" class="mr-2">
@ -59,6 +72,9 @@
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">Push to
Talk</button>
</div>
<div id="status-recording" class="flex justify-center items-center mb-4">
</div>
<div id="transcription" class="border rounded p-4 h-48 overflow-y-scroll mb-4">
<!-- Transcription content -->
</div>
@ -81,11 +97,11 @@
<div class="flex justify-center items-center mb-4">
<div id="info"></div>
</div>
<div id="status-recording" class="flex justify-center items-center mb-4"></div>
<div id="status-recording" class="flex justify-center items-center mb-4"> status</div>
</div>
<script type="module">
import { startListening, stopListening, toggleListening, InitAudioAnalyser, sendAudioToServerPost, sendAudioToServer, initializeVolumeChecker, serverTime, setSocket, setRecordButton } from './audio.js';
// import * as audio from './audio.js';
let socket;
let sessionId;
@ -120,11 +136,20 @@
})
.then(() => {
socket = new WebSocket(wsurl);
// audio.setSocket(socket); // Set the socket in the audio module
socket.onopen = () => {
connectionStatus.innerHTML = "Connected to " + wsurl;
recordButton.disabled = false;
connected = true;
resolve();
//if we stored a session id in a cookie, reconnect
const sessionId = getCookie("sessionId");
if (sessionId) {
socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
}
else {
socket.send(JSON.stringify({ type: 'sessionId' }));
}
resolve(socket);
};
socket.onmessage = onmessage;
socket.onclose = () => {
@ -135,7 +160,6 @@
connect().then(resolve).catch(reject);
}, 5000);
};
setSocket(socket);
})
.catch((error) => {
connectionStatus.innerHTML = "Error getting ws url: " + error;
@ -150,14 +174,30 @@
switch (json.type) {
case "sessionId":
sessionId = json.sessionId;
//set the session id in the cookie
document.cookie = `sessionId=${sessionId}; path=/;`;
console.log("Got session id: " + sessionId);
break;
case "languageDetected":
statusRecording.innerHTML = "Detected language: " + json.languageDetected;
break;
case "text":
case "transcriptionResult":
transcription.innerHTML += "\r\n" + json.text;
let latency = Date.now() - serverTime;
if (autosend.checked) {
// const arr = event.data.split(/[(\)]/);
// let queue = arr[1];
// let text = arr[2].trim();
// info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
//transcription.value += event.data + " ";
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "black";
} else {
//transcription.innerHTML = event.data;
}
break;
case "userList":
users = json.users;
updateUserList();
@ -166,9 +206,8 @@
chats = json.chats;
updateChatList();
break;
case "newChatRoom":
chats.push(json.newChatRoom);
updateChatList();
case "chat":
displayChatParticipants(json.chat.id, json.chat.participants);
break;
default:
console.log("Unknown message type:", json.type);
@ -177,36 +216,33 @@
console.error("Failed to parse message", e);
}
let latency = Date.now() - serverTime;
if (autosend.checked) {
const arr = event.data.split(/[(\)]/);
let queue = arr[1];
let text = arr[2].trim();
info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
transcription.value += text + " ";
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "black";
} else {
transcription.innerHTML = event.data;
}
}
function joinChat() {
function logInAndStoreSession() {
username = document.getElementById('username').value;
if (username.trim() === "") {
alert("Please enter a username");
return;
}
if (!socket || socket.readyState !== WebSocket.OPEN) {
alert("WebSocket connection is not open. Please wait and try again.");
return;
connect().then(() => {
userJoin(sessionId, username, document.getElementById('language-select').value);
});
} else {
userJoin(sessionId, username, document.getElementById('language-select').value);
}
socket.send(JSON.stringify({ type: 'join', username }));
}
function userJoin(sessionId, username, language) {
socket.send(JSON.stringify({ type: 'join', username , language}));
document.cookie = `sessionId=${sessionId}; path=/;`;
document.cookie = `username=${username}; path=/;`;
showClearSessionOption();
}
function clearSession() {
document.cookie = "sessionId=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
document.cookie = "username=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
@ -217,21 +253,19 @@
const sessionId = getCookie("sessionId");
if (sessionId) {
if (!socket || socket.readyState !== WebSocket.OPEN) {
connect().then(() => {
socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
//initializeVolumeChecker();
connect().then((s) => {
s.send(JSON.stringify({ type: 'reconnect', sessionId }));
});
}
document.getElementById('btn-disconnect').classList.remove('hidden');
document.getElementById('btn-join').classList.add('hidden');
document.getElementById('join-container').classList.add('hidden');
document.getElementById('active-users-container').classList.remove('hidden');
document.getElementById('previous-chats-container').classList.remove('hidden');
} else {
document.getElementById('btn-disconnect').classList.add('hidden');
document.getElementById('btn-join').classList.remove('hidden');
document.getElementById('join-container').classList.remove('hidden');
}
}
@ -249,7 +283,7 @@
showClearSessionOption();
connect().then(() => {
initializeVolumeChecker();
// audio.initializeVolumeChecker();
});
};
@ -269,7 +303,7 @@
users.forEach(user => {
const option = document.createElement('option');
option.value = user.sessionId;
option.innerText = user.username;
option.innerText = "["+user.language+"] " +user.username;
if (user.username === username) {
option.innerText += " (me)";
}
@ -287,23 +321,9 @@
selectedUsers.push(sessionId); // Add self to the selected users list for self-chat
socket.send(JSON.stringify({ type: 'startChat', users: selectedUsers }));
document.getElementById('chat-room-container').classList.remove('hidden');
displayChatParticipants(selectedUsers);
}
function displayChatParticipants(participants) {
const chatRoom = document.getElementById('chat-room');
let participantsHtml = '<div class="flex flex-wrap mb-4">';
participants.forEach(participantId => {
const user = users.find(u => u.sessionId === participantId);
const status = user ? "online" : "offline";
const username = user ? user.username : "Unknown User";
participantsHtml += `<span class="inline-flex items-center px-3 py-0.5 rounded-full text-sm font-medium bg-${status === 'online' ? 'green' : 'gray'}-100 text-${status === 'online' ? 'green' : 'gray'}-800 mr-2 mb-2">
${username} (${status})
</span>`;
});
participantsHtml += '</div>';
chatRoom.insertAdjacentHTML('afterbegin', participantsHtml);
}
function fetchPreviousChats(username) {
fetch(`/chats?username=${username}`)
@ -313,30 +333,269 @@
updateChatList();
});
}
function sendSocketMessage(message) {
if (socket && socket.readyState === WebSocket.OPEN) {
socket.send(JSON.stringify(message));
}
}
function updateChatList() {
const previousChats = document.getElementById('previous-chats');
previousChats.innerHTML = '';
chats.forEach(chat => {
const chatDiv = document.createElement('div');
chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2');
chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2', 'cursor-pointer');
chatDiv.setAttribute('data-chat-id', chat.id); // Store chat ID in data attribute
const participants = chat.participants.join(', ');
const status = chat.participants.map(participant => {
const user = users.find(u => u.username === participant);
return user ? `${participant} (online)` : `${participant} (offline)`;
}).join(', ');
chatDiv.innerHTML = `<strong>Participants:</strong> ${participants}<br><strong>Status:</strong> ${status}`;
chatDiv.innerHTML = `${status}`;
chatDiv.addEventListener('click', () => {
// Remove highlight from all chat divs
document.querySelectorAll('#previous-chats > div').forEach(div => {
div.classList.remove('bg-blue-100');
});
// Highlight selected chat div
chatDiv.classList.add('bg-blue-100');
selectChatRoom(chat.id);
});
previousChats.appendChild(chatDiv);
});
}
function selectChatRoom(chatId) {
const chat = chats.find(c => c.id === chatId);
if (!chat) return;
const chatRoomUsers = document.getElementById('chat-room-users');
chatRoomUsers.innerHTML = ''; // Clear existing content
socket.send(JSON.stringify({ type: 'enterChat', chatId }));
document.getElementById('chat-room-container').classList.remove('hidden');
// displayChatParticipants(chatId, chat.participants);
}
function displayChatParticipants(chatId, participants) {
const chatRoomUsers = document.getElementById('chat-room-users');
let participantsHtml = '<div class="flex flex-wrap mb-4">';
participants.forEach(participantId => {
const user = users.find(u => u.sessionId === participantId);
const status = user ? "online" : "offline";
const username = user ? user.username : "Unknown User";
participantsHtml += `<span class="inline-flex items-center px-3 py-0.5 rounded-full text-sm font-medium bg-${status === 'online' ? 'green' : 'gray'}-100 text-${status === 'online' ? 'green' : 'gray'}-800 mr-2 mb-2">
${username} (${status})
</span>`;
});
participantsHtml += '</div>';
chatRoomUsers.innerHTML = participantsHtml;
}
// REGION AUDIO RECORDING
let selectedDeviceId = "default";
export let serverTime;
// export let recordButton;
// export let socket;
// let connectionStatus;
// let statusRecording;
let audioRecorder;
let audioStream;
let recording = false;
let audioContext;
let volumeChecker;
let lastVolumes = new Array(5);
let averageVolume;
let silenceCount = 0;
let isSpeaking = false;
let soundDetected = false;
let speakingCount = 0;
let analyser = null;
let SILENCE_DELAY_MS = 50;
let preDetect_IncludedAudio = 400; //ms
let soundCount_Threshold = 10;
let silenceCount_Threshold = 10;
const volumeHistory = [];
function setSocket(newSocket) {
socket = newSocket;
}
export function setRecordButton(newRecordButton) {
recordButton = newRecordButton;
recordButton.addEventListener("click", toggleListening);
}
export function InitAudioAnalyser(stream) {
audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);
analyser = audioContext.createAnalyser();
analyser.fftSize = 2048;
analyser.smoothingTimeConstant = 0.8;
source.connect(analyser);
}
export function startListening() {
recording = true;
navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
.then((stream) => {
audioStream = stream;
const audioContext = new AudioContext();
const sourceNode = audioContext.createMediaStreamSource(audioStream);
const audioSampleRate = sourceNode.context.sampleRate;
info.innerHTML = "Sample rate: " + audioSampleRate + " Hz";
var preBuffer = [];
const channelSplitter = audioContext.createChannelSplitter(2);
const channelMerger = audioContext.createChannelMerger(1);
sourceNode.connect(channelSplitter);
channelSplitter.connect(channelMerger, 0, 0);
const outputNode = channelMerger;
const mediaStreamDestination = audioContext.createMediaStreamDestination();
outputNode.connect(mediaStreamDestination);
const singleChannelStream = mediaStreamDestination.stream;
audioRecorder = new MediaRecorder(singleChannelStream);
audioRecorder.start();
audioRecorder.addEventListener("dataavailable", (event) => {
if (!soundDetected && autosend.checked) {
preBuffer = [];
preBuffer.push(event.data);
return;
}
if (event.data.size > 0) {
let data = event.data;
if (preBuffer.length > 0) {
sendAudioToServerPost(preBuffer);
}
sendAudioToServer(data);
soundDetected = false;
}
});
InitAudioAnalyser(stream);
});
recordButton.innerHTML = "Stop Talking";
recordButton.classList.toggle('bg-red-500');
recordButton.classList.toggle('bg-blue-500');
recordButton.classList.toggle('hover:bg-blue-700');
}
export function stopListening() {
recording = false;
audioRecorder.stop();
recordButton.innerHTML = "Push to Talk";
recordButton.classList.toggle('bg-blue-500');
recordButton.classList.toggle('bg-red-500');
recordButton.classList.toggle('hover:bg-blue-700');
clearInterval(volumeChecker);
if (audioStream) {
audioStream.getTracks().forEach(track => track.stop());
audioStream = null;
}
}
export function sendAudioToServerPost(data) {
const blob = new Blob(data, { type: "audio/ogg; codecs=opus" });
var formData = new FormData();
formData.append('file', data);
fetch('/upload', {
method: 'POST',
body: formData
});
}
export function sendAudioToServerJson(data) {
if (socket && socket.readyState === WebSocket.OPEN) {
const binaryData = Buffer.from(base64AudioData, 'base64');
socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData }));
serverTime = Date.now();
if (!autosend.checked) {
transcription.placeholder = "Processing audio...";
}
}
}
export function sendAudioToServer(data) {
if (socket && socket.readyState === WebSocket.OPEN) {
socket.send(data);
serverTime = Date.now();
if (!autosend.checked) {
transcription.placeholder = "Processing audio...";
}
}
}
export function toggleListening() {
if (socket.readyState === WebSocket.OPEN) {
if (recording) {
stopListening();
} else {
startListening();
}
}
}
export function initializeVolumeChecker() {
volumeChecker = setInterval(() => {
if (!audioContext) {
console.log("No audio context");
return;
}
const frequencyData = new Uint8Array(analyser.frequencyBinCount);
analyser.getByteFrequencyData(frequencyData);
let totalVolume = 0;
for (let i = 0; i < frequencyData.length; i++) {
totalVolume += frequencyData[i];
}
averageVolume = totalVolume / frequencyData.length;
volumeHistory.push(averageVolume);
if (volumeHistory.length > 100) {
volumeHistory.shift();
}
const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5;
const isSilent = averageVolume < threshold;
if (averageVolume > threshold) {
if (autosend.checked && speakingCount == 0 && audioRecorder) {
soundDetected = false;
audioRecorder.stop();
audioRecorder.start();
}
speakingCount++;
if (speakingCount > soundCount_Threshold) {
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "green";
isSpeaking = true;
}
} else if (averageVolume - 5 < threshold) {
speakingCount = 0;
if (isSpeaking) {
silenceCount++;
if (silenceCount > silenceCount_Threshold) {
if (autosend.checked) {
soundDetected = true;
audioRecorder.stop();
audioRecorder.start();
}
isSpeaking = false;
statusRecording.innerHTML = "Silence detected...";
statusRecording.style.color = "orange";
}
}
}
}, SILENCE_DELAY_MS);
}
// Expose functions to global scope
window.joinChat = joinChat;
window.logInAndStoreSession = logInAndStoreSession;
window.clearSession = clearSession;
window.copyToClipboard = copyToClipboard;
window.clearTranscription = clearTranscription;