diff --git a/web/audio.js b/web/audio.js new file mode 100644 index 0000000..53b1e2b --- /dev/null +++ b/web/audio.js @@ -0,0 +1,192 @@ +let selectedDeviceId = "default"; +export let serverTime; +export let recordButton; +let socket; +let audioRecorder; +let audioStream; +let recording = false; +let connectionStatus; +let statusRecording; +let audioContext; +let volumeChecker; +let lastVolumes = new Array(5); +let averageVolume; +let silenceCount = 0; +let isSpeaking = false; +let soundDetected = false; +let speakingCount = 0; +let analyser = null; + +let SILENCE_DELAY_MS = 50; +let preDetect_IncludedAudio = 400; //ms +let soundCount_Threshold = 10; +let silenceCount_Threshold = 10; + +const volumeHistory = []; + +export function setSocket(newSocket) { + socket = newSocket; +} +export function setRecordButton(newRecordButton) { + recordButton = newRecordButton; + recordButton.addEventListener("click", toggleListening); +} + +export function InitAudioAnalyser(stream) { + audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + analyser = audioContext.createAnalyser(); + analyser.fftSize = 2048; + analyser.smoothingTimeConstant = 0.8; + source.connect(analyser); +} + +export function startListening() { + //canvasCtx.fillStyle = "green"; + recording = true; + navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } }) + .then((stream) => { + audioStream = stream; + + const audioContext = new AudioContext(); + const sourceNode = audioContext.createMediaStreamSource(audioStream); + const audioSampleRate = sourceNode.context.sampleRate; + + info.innerHTML = "Sample rate: " + audioSampleRate + " Hz"; + var preBuffer = []; + + const channelSplitter = audioContext.createChannelSplitter(2); + const channelMerger = audioContext.createChannelMerger(1); + sourceNode.connect(channelSplitter); + channelSplitter.connect(channelMerger, 0, 0); + const outputNode = channelMerger; + + const mediaStreamDestination = audioContext.createMediaStreamDestination(); + outputNode.connect(mediaStreamDestination); + const singleChannelStream = mediaStreamDestination.stream; + + audioRecorder = new MediaRecorder(singleChannelStream); + audioRecorder.start(); + audioRecorder.addEventListener("dataavailable", (event) => { + if (!soundDetected && autosend.checked) { + preBuffer = []; + preBuffer.push(event.data); + return; + } + if (event.data.size > 0) { + let data = event.data; + console.log("audio data size: " + data.size); + if (preBuffer.length > 0) { + sendAudioToServerPost(preBuffer); + } + sendAudioToServer(data); + soundDetected = false; + } + }); + + InitAudioAnalyser(stream); + }); + + recordButton.innerHTML = "Stop Talking"; + recordButton.classList.toggle('bg-red-500'); + recordButton.classList.toggle('bg-blue-500'); + recordButton.classList.toggle('hover:bg-blue-700'); +} + +export function stopListening() { + recording = false; + audioRecorder.stop(); + recordButton.innerHTML = "Push to Talk"; + recordButton.classList.toggle('bg-blue-500'); + recordButton.classList.toggle('bg-red-500'); + recordButton.classList.toggle('hover:bg-blue-700'); + clearInterval(volumeChecker); + if (audioStream) { + audioStream.getTracks().forEach(track => track.stop()); + audioStream = null; + } +} + +export function sendAudioToServerPost(data) { + const blob = new Blob(data, { type: "audio/ogg; codecs=opus" }); + var formData = new FormData(); + formData.append('file', data); + fetch('/upload', { + method: 'POST', + body: formData + }); +} + +export function sendAudioToServer(data) { + //if (connected) { + socket.send(JSON.stringify({ type: 'audio', task:"transcribe", audio: data })); + serverTime = Date.now(); + if (!autosend.checked) { + transcription.innerHTML = "Processing audio..."; + } + //} +} + +export function toggleListening() { + if (socket.readyState === WebSocket.OPEN) { + if (recording) { + stopListening(); + } else { + startListening(); + } + } +} + +export function initializeVolumeChecker() { + volumeChecker = setInterval(() => { + if (!audioContext) { + console.log("No audio context"); + return; + } + const frequencyData = new Uint8Array(analyser.frequencyBinCount); + analyser.getByteFrequencyData(frequencyData); + + let totalVolume = 0; + for (let i = 0; i < frequencyData.length; i++) { + totalVolume += frequencyData[i]; + } + averageVolume = totalVolume / frequencyData.length; + + volumeHistory.push(averageVolume); + if (volumeHistory.length > 100) { + volumeHistory.shift(); + } + + const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5; + const isSilent = averageVolume < threshold; + + if (averageVolume > threshold) { + if (autosend.checked && speakingCount == 0 && audioRecorder) { + soundDetected = false; + audioRecorder.stop(); + audioRecorder.start(); + } + speakingCount++; + if (speakingCount > soundCount_Threshold) { + statusRecording.innerHTML = "Listening..."; + statusRecording.style.color = "green"; + isSpeaking = true; + } + } else if (averageVolume - 5 < threshold) { + speakingCount = 0; + if (isSpeaking) { + silenceCount++; + if (silenceCount > silenceCount_Threshold) { + if (autosend.checked) { + soundDetected = true; + audioRecorder.stop(); + audioRecorder.start(); + } + isSpeaking = false; + statusRecording.innerHTML = "Silence detected..."; + statusRecording.style.color = "orange"; + } + } + } + }, SILENCE_DELAY_MS); +} diff --git a/web/chat-client.html b/web/chat-client.html index 1527051..c8cc410 100644 --- a/web/chat-client.html +++ b/web/chat-client.html @@ -11,10 +11,16 @@

Real-time Voice Chat

- +
+ - + + + +
@@ -24,7 +30,16 @@ - + +
+ + + @@ -41,21 +56,22 @@
+ class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">Push to + Talk
- - + +
-

Previous Chats

-
- -
@@ -68,42 +84,19 @@
- + - + \ No newline at end of file diff --git a/web/chat-server.js b/web/chat-server.js index e060dab..98af4c4 100644 --- a/web/chat-server.js +++ b/web/chat-server.js @@ -23,7 +23,6 @@ let storeRecordings = false; let queueCounter = 0; const sessions = new Map(); -const users = new Map(); const chats = new Map(); // Store chat rooms storage.init().then(() => { @@ -48,18 +47,57 @@ wss.on('connection', (ws) => { ws.on('message', (message) => { try { const data = JSON.parse(message); + console.log('Received message:', data.type); - if (data.type === 'join') { - const { username } = data; - users.set(ws.sessionId, { username, sessionId: ws.sessionId }); - broadcastUserList(); - } else if (data.type === 'startChat') { - const { users: chatUsers } = data; - const chatId = Math.random().toString(36).substring(2); - chats.set(chatId, { participants: [ws.sessionId, ...chatUsers], messages: [] }); - broadcastUserList(); - } else if (data.type === 'audio') { - handleAudioData(ws, data.audio); + switch (data.type) { + case 'join': + const { username } = data; + sessions.set(ws.sessionId, { username, sessionId: ws.sessionId }); + broadcastUserList(); + break; + case 'startChat': + const { users: chatUsers } = data; + const chatId = Math.random().toString(36).substring(2); + let participants = [ws.sessionId, ...chatUsers]; + // Deduplicate participants + participants = [...new Set(participants)]; + + chats.set(chatId, { participants, messages: [] }); + + const userNames = participants.map(sessionId => { + const user = sessions.get(sessionId); + return user ? `${user.username}(${user.sessionId})` : 'Unknown User'; + }); + + console.log('Creating chat room. Users:', userNames); + + participants.forEach(sessionId => { + const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId); + if (participantSocket && participantSocket.readyState === WebSocket.OPEN) { + participantSocket.send(JSON.stringify({ newChatRoom: { id: chatId, participants: userNames } })); + } + }); + + broadcastUserList(); + break; + case 'audio': + console.log('(queue ' + queueCounter + ') Received ' + (data.audio.length / 1024 / 1024).toFixed(3) + ' MB audio from client. Crrent language: ' + language, 'task: ' + data.task); + handleAudioData(ws, data.audio); + break; + case 'reconnect': + const userSession = sessions.get(data.sessionId); + if (userSession) { + sessions.set(ws.sessionId, userSession); + ws.sessionId = data.sessionId; + broadcastUserList(); + // Send existing chats + const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId)); + ws.send(JSON.stringify({ type: 'chats', chats: userChats })); + ws.send(JSON.stringify({ type: 'userList', users: userList })); + } + break; + default: + console.log('Unknown message type:', data.type); } } catch (err) { console.error('Failed to parse message', err); @@ -67,7 +105,7 @@ wss.on('connection', (ws) => { }); ws.on('close', () => { - users.delete(ws.sessionId); + // allUsers.delete(ws.sessionId); sessions.delete(ws.sessionId); broadcastUserList(); }); @@ -140,7 +178,7 @@ function transcribeAudio(ws, formData, sessionData) { } function broadcastUserList() { - const userList = Array.from(users.values()).map(user => ({ username: user.username, sessionId: user.sessionId })); + const userList = Array.from(sessions.values()).map(user => ({ username: user.username, sessionId: user.sessionId })); wss.clients.forEach(client => { if (client.readyState === WebSocket.OPEN) { client.send(JSON.stringify({ type: 'userList', users: userList })); @@ -153,6 +191,11 @@ app.get('/', (req, res) => { res.sendFile(path.join(__dirname, 'chat-client.html')); }); +app.get('/audio.js', (req, res) => { + res.sendFile(path.join(__dirname, 'audio.js')); +}); + + app.post('/log', (req, res) => { console.log(`[LOG ${new Date().toISOString()}] ${req.body.message}`); res.status(200).send('OK'); @@ -185,6 +228,7 @@ app.post('/settings', (req, res) => { app.post('/upload', (req, res) => { const timestamp = Date.now(); + console.log('Received audio data:', timestamp); fs.mkdir('rec', { recursive: true }, (err) => { if (err) return res.status(500).send('ERROR'); const file = fs.createWriteStream(`rec/audio_slice_${timestamp}.ogg`);