popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

various improvements:

Browse Source 
using mono audio;
options to store recordings;
multple backends;
new audio file name;
sessions;
This commit is contained in:
popov 2023-03-15 14:20:35 +00:00
parent 5354d8c328
commit 1c15463b21
6 changed files with 588 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
node_modules/* node_modules/*
package-lock.json package-lock.json
rec/*

1
.node-persist/storage/8512ae7d57b1396273f76fe6ed341a23 Normal file
View File

@ -0,0 +1 @@
{"key":"language","value":"bg"}

1
.node-persist/storage/dfe9cbcde628e8a86855f6d2cd16dd2b Normal file
View File

@ -0,0 +1 @@
{"key":"storeRecordings","value":"true"}

3
Dockerfile
View File

@ -37,7 +37,7 @@ COPY package*.json ./
# Install dependencies # Install dependencies
RUN npm install ws express request #--only=production RUN npm install ws express request node-persist body-parser dotenv #--only=production
# Copy the rest of the application files # Copy the rest of the application files
COPY . . COPY . .
@ -45,6 +45,7 @@ COPY . .
# Start the application # Start the application
#CMD ["npm", "start"] #CMD ["npm", "start"]
CMD npm start CMD npm start
# portainer: '-c' 'echo Container started; trap "exit 0" 15; exec npm start'
EXPOSE 8080 8081 EXPOSE 8080 8081

801
web/client.html
View File

@ -3,322 +3,459 @@
<head> <head>
<title>Real-time Speech-to-Text</title> <title>Real-time Speech-to-Text</title>
<style> <meta name="viewport"
.recording { content="width=device-width, initial-scale=1">
background-color: red; <!-- Add the Tailwind CSS library -->
color: white; <link rel="stylesheet"
} href="https://cdnjs.cloudflare.com/ajax/libs/tailwindcss/2.2.19/tailwind.min.css">
</style>
</head> </head>
<body> <body class="bg-gray-100">
<h1>Rt STT</h1> <div class="container mx-auto px-4 py-8">
<label class="toggle"> <h1 class="text-2xl font-bold mb-4 text-center">Rt STT</h1>
<input type="checkbox" <div class="flex justify-center items-center mb-4">
id="autosend" /> <label class="toggle flex items-center">
<span class="slider">Continious</span> <input type="checkbox"
</label> id="autosend"
<select id="input-devices"> class="mr-2">
<option value="default">Default</option> <span class="slider"></span>
</select> <span class="ml-2">Continuous</span>
</label>
<select id="input-devices"
class="ml-4">
<option value="default">Default</option>
</select>
<select id="language-select">
<option value="en">English</option>
<option value="bg">Български</option>
</select>
<label class="toggle flex items-center ml-4">
<input type="checkbox"
id="store-recordings"
class="mr-2">
<span class="slider"></span>
<span class="ml-2">Store Recordings</span>
</div>
<div class="flex justify-center items-center mb-4">
<span id="record-actions">
<button id="record-button"
disabled
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">
Start Recording</button>
<button id="record-button-speakers"
disabled
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">
Stream from speakers</button>
</span>
</div>
<div class="flex justify-center items-center mb-4">
<div id="connection-status"
style="margin-right: 5px;"></div>
</div>
<div class="flex justify-center items-center mb-4">
<div id="info"></div>
</div>
<div id="status-recording"
class="flex justify-center items-center mb-4">
</div>
<div class="relative rounded-lg border border-gray-300 shadow-sm">
<textarea id="transcription"
class="block w-full h-48 p-4 resize-none"
placeholder="Whisper something into the microphone..."></textarea>
<button id="copyButton"
class="absolute top-0 right-0 px-4 py-2 text-sm font-medium text-gray-700 bg-gray-200 hover:bg-gray-300 rounded-bl-lg focus:outline-none"
onclick="copyToClipboard('transcription')">
Copy
</button>
</div>
<canvas id="canvas"
class="w-full"></canvas>
<script>
let sessionId;
<button id="record-button" let selectedDeviceId = "default";
disabled>Start Recording</button> let socket;
<span id="connection-status"></span> let audioRecorder;
let audioStream;
let recording = false;
let recordButton;
let connected = false;
let connectionStatus; //HTML auto generated
let statusRecording; //HTML auto generated
let audioContext;
let audioSampleRate;
let serverTime;
<div id="status-recording"></div> let volumeChecker;
<p id="transcription"></p> let lastVolumes = new Array(5);
let averageVolume;
let silenceCount = 0;
let isSpeaking = false;
let soundDetected = false;
let speakingCount = 0;
<canvas id="canvas" let SILENCE_DELAY_MS = 50;
width="500" let preDetect_IncludedAudio = 400; //ms
height="500"></canvas> let soundCount_Threshold = 10;
<script> let silenceCount_Threshold = 10;
let selectedDeviceId = "default";
let socket;
let audioRecorder;
let recording = false;
let recordButton;
let connected = false;
let connectionStatus; //HTML auto generated
let statusRecording; //HTML auto generated
let audioContext;
let serverTime;
let volumeChecker; const volumeHistory = [];
let lastVolumes = new Array(5);
let averageVolume;
let silenceCount = 0;
let isSpeaking = false;
let soundDetected = false;
let speakingCount = 0;
let SILENCE_DELAY_MS = 50; //was 100 with good results
const volumeHistory = [];
let canvas = document.getElementById("canvas"); let canvas = document.getElementById("canvas");
let canvasCtx = canvas.getContext("2d"); let canvasCtx = canvas.getContext("2d");
let barWidth = 10; let barWidth = 10;
let barSpacing = 5; let barSpacing = 5;
// Draw sliding bar graph // Handle language select change
function drawSlidingBarGraph(lastVolumes) { document.getElementById('language-select').addEventListener('change', (event) => {
canvasCtx.clearRect(0, 0, canvas.width, canvas.height); const language = event.target.value;
// Draw bars fetch('/settings', {
for (let i = 0; i < lastVolumes.length; i++) { method: 'POST',
let value = lastVolumes[i]; body: JSON.stringify({ language, sessionId }),
let barHeight = (value / 255) * canvas.height; headers: { 'Content-Type': 'application/json' },
let x = i * (barWidth + barSpacing); credentials: 'same-origin'
let y = canvas.height - barHeight;
canvasCtx.fillRect(x, y, barWidth, barHeight);
}
}
// Check the audio level every SILENCE_DELAY_MS milliseconds
volumeChecker = setInterval(() => {
if (!audioContext) {
console.log("No audio context");
return;
}
const frequencyData = new Uint8Array(analyser.frequencyBinCount);
//analyser.getByteTimeDomainData(dataArray);//history
analyser.getByteFrequencyData(frequencyData); //current
let totalVolume = 0;
for (let i = 0; i < frequencyData.length; i++) {
totalVolume += frequencyData[i];
}
averageVolume = totalVolume / frequencyData.length;
volumeHistory.push(averageVolume);
if (volumeHistory.length > 100) {
volumeHistory.shift();
}
const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5;
const isSilent = averageVolume < threshold;
// count speaking and silence
if (averageVolume > threshold) {
if (autosend.checked && speakingCount == 0 && audioRecorder) {
console.log("startint new recording");
soundDetected = false;
audioRecorder.stop();
audioRecorder.start();
}
speakingCount++;
if (speakingCount > 7) {
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "green";
isSpeaking = true;
console.log("Was silent and is now speaking. (" + averageVolume + " averageVolume).");
}
} else if (averageVolume - 5 < threshold) {
speakingCount = 0;
if (isSpeaking) {
silenceCount++;
if (silenceCount > 3) {
if (autosend.checked) {
console.log("Was speakng and is now silence. (" + averageVolume + " averageVolume). Sending audio to server.");
soundDetected = true;
audioRecorder.stop();
audioRecorder.start();
}
isSpeaking = false;
statusRecording.innerHTML = "Silence detected...";
statusRecording.style.color = "orange";
}
}
}
//console.log(`Average volume: ${averageVolume}, isSilent: ${isSilent}, threshold: ${threshold}`);
//drawSlidingBarGraph(lastVolumes);
}, SILENCE_DELAY_MS);
function InitAudioAnalyser(stream) {
// Initialize the Web Audio API
audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);
analyser = audioContext.createAnalyser();
//analyser.fftSize = 32;
analyser.fftSize = 2048;
analyser.smoothingTimeConstant = 0.8;
source.connect(analyser);
console.log("Audio context initialized. analyser sampling: " + audioContext.sampleRate + "Hz, recorder sampling: " + audioRecorder.sampleRate + "Hz");
}
function connect() {
connectionStatus.innerHTML = "Connecting to WS...";
let wsurl = "ws://localhost:8081";
//get crrent ws url from the server
fetch("/wsurl")
.then((response) => response.text())
.then((data) => {
wsurl = data;
console.log("Got ws url: '" + wsurl + "'");
})
.then(() => {
console.log("connecting to '" + wsurl + "'...");
socket = new WebSocket(wsurl);
socket.onopen = () => {
console.log("WebSocket connection opened.");
connectionStatus.innerHTML = "Connected to " + wsurl;
transcription.innerHTML = "Whisper something into the microphone...";
recordButton.disabled = false;
connected = true;
};
socket.onmessage = onmessage;
socket.onclose = () => {
console.log("WebSocket connection closed");
connectionStatus.innerHTML = "Disconnected";
recordButton.disabled = true;
connected = false;
setTimeout(() => {
connect();
}, 5000);
};
})
.catch((error) => {
console.log("Error getting ws url: " + error);
connectionStatus.innerHTML = "Error getting ws url: " + error;
}); });
}; });
document.getElementById('store-recordings').addEventListener('change', (event) => {
function onmessage(event) { const storeRecordings = event.target.checked;
let latency = Date.now() - serverTime; fetch('/settings', {
console.log("Received message from server: " + event.data + " (latency: " + latency + "ms)"); method: 'POST',
if (autosend.checked) { body: JSON.stringify({ storeRecordings, sessionId }),
//append to the text on new line headers: { 'Content-Type': 'application/json' },
transcription.innerHTML += "<br>>" + event.data; credentials: 'same-origin'
statusRecording.innerHTML = "waiting...";
statusRecording.style.color = "black";
} else {
//replace the text
transcription.innerHTML = event.data;
}
}
const preBufferDuration = 500 ; // duration of pre-buffer in ms
function startListening() {
// Initialize canvas
canvasCtx.fillStyle = "green";
recording = true;
navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, echoCancellation: true } }).then((stream) => {
audioRecorder = new MediaRecorder(stream);
audioRecorder.start();
console.log("Started listening to microphone (sample rate: " + stream.getAudioTracks()[0].getSettings().sampleRate + " Hz, echoCancellation: " + stream.getAudioTracks()[0].getSettings().echoCancellation + ", " +audioRecorder.mimeType.audioChannels + " channels)");
//const preBufferLength = Math.ceil(preBufferDuration * audioRecorder.mimeType.audioSampleRate * audioRecorder.mimeType.audioChannels);
const preBufferLength = Math.ceil(preBufferDuration * 48000 * 2 );
var preBuffer = [];
audioRecorder.addEventListener("dataavailable", (event) => {
console.log("Audio data available: " + event.data.size + " bytes");
if (!soundDetected && autosend.checked) {
console.log("discarding audio data because not speaking");
//store last 100ms of audio data
preBuffer = [];
const audioData = event.data;
//const start = audioBlob.size - preBufferLength; // calculate offset to trim pre-buffer
//const audioBlob = new Blob([...preBuffer, audioData], { type: "audio/wav" });
//const end = audioBlob.size;
//const slicedAudio = audioBlob.slice(start, end);
const audioBlob = new Blob([...preBuffer, audioData], { type: "audio/ogg; codecs=opus" });
const start = audioBlob.size - 500 * 48 * 2 / 8; // Assuming 48 kHz sampling rate and 16-bit PCM audio
const end = audioBlob.size;
const slicedAudio = audioBlob.slice(start, end);
preBuffer = slicedAudio;
//sendAudioToServer(event.data);
return;
}
if (event.data.size > 0) {
let data = event.data;
if (preBuffer.length > 0) {
const audioBlob = new Blob([...preBuffer, audioData], { type: "audio/ogg; codecs=opus" });
const newEventChunk = new Blob([audioBlob, preBuffer], { type: "audio/ogg; codecs=opus" });
data = newEventChunk;
}
sendAudioToServer(data);
soundDetected = false;
}
}); });
InitAudioAnalyser(stream);
}); });
recordButton.innerHTML = "Stop Recording"; // Draw sliding bar graph
recordButton.classList.add("recording"); function drawSlidingBarGraph(lastVolumes) {
} canvasCtx.clearRect(0, 0, canvas.width, canvas.height);
// Draw bars
function getlast500ms(audioData, preBuffer) { for (let i = 0; i < lastVolumes.length; i++) {
const audioBlob = new Blob([...preBuffer, audioData], { type: "audio/ogg; codecs=opus" }); let value = lastVolumes[i];
const start = audioBlob.size - 500 * 48000 * 2 / 8; // Assuming 48 kHz sampling rate and 16-bit PCM audio let barHeight = (value / 255) * canvas.height;
const end = audioBlob.size; let x = i * (barWidth + barSpacing);
const slicedAudio = audioBlob.slice(start, end); let y = canvas.height - barHeight;
canvasCtx.fillRect(x, y, barWidth, barHeight);
// Create a new Blob object with the remaining audio data
const remainingAudio = audioBlob.slice(0, start);
const newEvent = new Blob([remainingAudio], { type: "audio/ogg; codecs=opus" });
// Replace the original event data with the new event data
const newEventChunk = new Blob([newEvent, slicedAudio], { type: "audio/ogg; codecs=opus" });
return newEventChunk;
}
function stopListening() {
recording = false;
audioRecorder.stop();
recordButton.innerHTML = "Start Recording";
recordButton.classList.remove("recording");
clearInterval(volumeChecker);
}
function sendAudioToServer(data) {
if (connected) {
//const blob = new Blob(data, { type: 'audio/webm' });
socket.send(data);
serverTime = Date.now();
console.log("Sent some audio data to server.");
if (!autosend.checked) {
transcription.innerHTML = "Processing audio...";
}
} else {
console.log("Not connected, not sending audio data to server.");
}
}
function toggleListening() {
if (socket.readyState === WebSocket.OPEN) {
if (recording) {
stopListening();
} else {
startListening();
} }
} }
}
function enumerateDevices() { // Check the audio level every SILENCE_DELAY_MS milliseconds
// Enumerate the available audio input devices volumeChecker = setInterval(() => {
navigator.mediaDevices.enumerateDevices() if (!audioContext) {
.then(function (devices) { console.log("No audio context");
var audioInputDevices = devices.filter(function (device) { return;
return device.kind === 'audioinput'; }
const frequencyData = new Uint8Array(analyser.frequencyBinCount);
//analyser.getByteTimeDomainData(dataArray);//history
analyser.getByteFrequencyData(frequencyData); //current
let totalVolume = 0;
for (let i = 0; i < frequencyData.length; i++) {
totalVolume += frequencyData[i];
}
averageVolume = totalVolume / frequencyData.length;
volumeHistory.push(averageVolume);
if (volumeHistory.length > 100) {
volumeHistory.shift();
}
const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5;
const isSilent = averageVolume < threshold;
// count speaking and silence
if (averageVolume > threshold) {
if (autosend.checked && speakingCount == 0 && audioRecorder) {
console.log("startint new recording");
soundDetected = false;
audioRecorder.stop();
audioRecorder.start();
}
speakingCount++;
if (speakingCount > soundCount_Threshold) {
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "green";
isSpeaking = true;
console.log("Was silent and is now speaking. (" + averageVolume + " averageVolume).");
}
} else if (averageVolume - 5 < threshold) {
speakingCount = 0;
if (isSpeaking) {
silenceCount++;
if (silenceCount > silenceCount_Threshold) {
if (autosend.checked) {
console.log("Was speakng and is now silence. (" + averageVolume + " averageVolume). Sending audio to server.");
soundDetected = true;
audioRecorder.stop();
audioRecorder.start();
}
isSpeaking = false;
statusRecording.innerHTML = "Silence detected...";
statusRecording.style.color = "orange";
}
}
}
//console.log(`Average volume: ${averageVolume}, isSilent: ${isSilent}, threshold: ${threshold}`);
//drawSlidingBarGraph(lastVolumes);
}, SILENCE_DELAY_MS);
function InitAudioAnalyser(stream) {
// Initialize the Web Audio API
audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);
analyser = audioContext.createAnalyser();
//analyser.fftSize = 32;
analyser.fftSize = 2048;
analyser.smoothingTimeConstant = 0.8;
source.connect(analyser);
console.log("Audio context initialized. analyser sampling: " + audioContext.sampleRate + "Hz, recorder sampling: " + audioRecorder.sampleRate + "Hz");
}
function connect() {
connectionStatus.innerHTML = "Connecting to WS...";
let wsurl = "ws://localhost:8081";
//get crrent ws url from the server
fetch("/wsurl")
.then((response) => response.text())
.then((data) => {
wsurl = data;
console.log("Got ws url: '" + wsurl + "'");
})
.then(() => {
console.log("connecting to '" + wsurl + "'...");
socket = new WebSocket(wsurl);
socket.onopen = () => {
console.log("WebSocket connection opened.");
connectionStatus.innerHTML = "Connected to " + wsurl;
recordButton.disabled = false;
connected = true;
};
socket.onmessage = onmessage;
socket.onclose = () => {
console.log("WebSocket connection closed");
connectionStatus.innerHTML = "Disconnected";
recordButton.disabled = true;
connected = false;
setTimeout(() => {
connect();
}, 5000);
};
})
.catch((error) => {
console.log("Error getting ws url: " + error);
connectionStatus.innerHTML = "Error getting ws url: " + error;
}); });
console.log(audioInputDevices.length + ' audio input devices found'); };
// If more than one audio input device is available, populate the select list function onmessage(event) {
if (audioInputDevices.length > 1) { //check if the message is json
audioInputDevices.forEach(function (device) { try {
var option = document.createElement('option'); let json = JSON.parse(event.data);
option.value = device.deviceId; //store session id in cookies
option.text = device.label || 'Device ' + device.deviceId; if (json.hasOwnProperty("sessionId")) {
inputDevices.appendChild(option); sessionId = json.sessionId;
console.log("Got session id: " + sessionId);
}
if (json.hasOwnProperty("language")) {
languageSelect.value = json.language;
}
//storerecordings checkbox
if (json.hasOwnProperty("storeRecordings")) {
storeRecordings.checked = json.storeRecordings;
}
return;
} catch (e) {
//not json
}
let latency = Date.now() - serverTime;
console.log("Received message from server: " + event.data + " (latency: " + latency + "ms)");
info.innerHTML = "latency: " + latency + "ms";
if (autosend.checked) {
//data is in format (count)text. split it
const arr = event.data.split(/[(\)]/); // split the string at "(" or ")"
let queue = arr[1]
let text = arr[2].trim();
info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
transcription.value += text + " ";
statusRecording.innerHTML = "listening...";
statusRecording.style.color = "black";
} else {
//replace the text
transcription.innerHTML = event.data;
}
}
const preBufferDuration = 500; // duration of pre-buffer in ms
function startListening() {
// Initialize canvas
canvasCtx.fillStyle = "green";
recording = true;
navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
.then((stream) => {
audioStream = stream;
const audioContext = new AudioContext();
const sourceNode = audioContext.createMediaStreamSource(audioStream);
const audioSampleRate = sourceNode.context.sampleRate;
console.log("Started listening to microphone (sample rate: " + audioSampleRate + " Hz, echoCancellation: " + stream.getAudioTracks()[0].getSettings().echoCancellation + ", " + sourceNode.channelCount + " channels)");
info.innerHTML = "Sample rate: " + audioSampleRate + " Hz";
var preBuffer = [];
//merge audio channels
const channelSplitter = audioContext.createChannelSplitter(2);
const channelMerger = audioContext.createChannelMerger(1);
sourceNode.connect(channelSplitter);
channelSplitter.connect(channelMerger, 0, 0);
const outputNode = channelMerger;
//const singleChannelStream = new MediaStream();
//singleChannelStream.addTrack(outputNode.stream.getAudioTracks()[0]);
const mediaStreamDestination = audioContext.createMediaStreamDestination();
outputNode.connect(mediaStreamDestination);
const singleChannelStream = mediaStreamDestination.stream;
audioRecorder = new MediaRecorder(singleChannelStream);
audioRecorder.start();
audioRecorder.addEventListener("dataavailable", (event) => {
console.log("Audio data available: " + (event.data.size / 1024).toFixed(2) + " KB in " + event.data.type + " format. (" + audioContext.sampleRate + " Hz)");
if (!soundDetected && autosend.checked) {
console.log("discarding audio data because not speaking");
//store last 100ms of audio data
preBuffer = [];
preBuffer.push(event.data);
return;
}
if (event.data.size > 0) {
let data = event.data;
if (preBuffer.length > 0) {
// const audioBlob = new Blob([...preBuffer, audioData], { type: "audio/ogg; codecs=opus" });
// const newEventChunk = new Blob([audioBlob, preBuffer], { type: "audio/ogg; codecs=opus" });
/// data = newEventChunk;
sendAudioToServerPost(preBuffer);
}
sendAudioToServer(data);
soundDetected = false;
}
}); });
// Listen for changes to the select list and connect to the selected audio input device
inputDevices.addEventListener('change', function (event) { InitAudioAnalyser(stream);
var selectedDeviceId = event.target.value; });
var constraints = { audio: { deviceId: selectedDeviceId } };
recordButton.innerHTML = "Stop Recording";
recordButton.classList.toggle('bg-red-500');
recordButton.classList.toggle('bg-blue-500');
recordButton.classList.toggle('hover:bg-blue-700');
}
function getlast500ms(audioData, preBuffer) {
const audioBlob = new Blob([...preBuffer, audioData], { type: "audio/ogg; codecs=opus" });
const start = audioBlob.size - 500 * 48000 * 2 / 8; // Assuming 48 kHz sampling rate and 16-bit PCM audio
const end = audioBlob.size;
const slicedAudio = audioBlob.slice(start, end);
// Create a new Blob object with the remaining audio data
const remainingAudio = audioBlob.slice(0, start);
const newEvent = new Blob([remainingAudio], { type: "audio/ogg; codecs=opus" });
// Replace the original event data with the new event data
//const newEventChunk = new Blob([newEvent, slicedAudio], { type: "audio/ogg; codecs=opus" });
const newEventChunk = new Blob([slicedAudio], { type: "audio/ogg; codecs=opus" });
return audioData;
}
function stopListening() {
recording = false;
audioRecorder.stop();
recordButton.innerHTML = "Start Recording";
//recordButton.classList.remove("recording");
recordButton.classList.toggle('bg-blue-500');
recordButton.classList.toggle('bg-red-500');
recordButton.classList.toggle('hover:bg-blue-700');
clearInterval(volumeChecker);
//stop using microphone
if (audioStream) {
// stop all tracks in the stream
audioStream.getTracks().forEach(track => track.stop());
// set the stream variable to null or undefined to indicate it's no longer in use
audioStream = null;
}
}
function sendAudioToServerPost(data) {
const blob = new Blob(data, { type: "audio/ogg; codecs=opus" });
var formData = new FormData();
formData.append('file', data);
fetch('/upload', {
method: 'POST',
body: formData
});
}
function sendAudioToServer(data) {
if (connected) {
//const blob = new Blob(data, { type: 'audio/webm' });
socket.send(data);
serverTime = Date.now();
console.log("Sent some audio data to server.");
if (!autosend.checked) {
transcription.innerHTML = "Processing audio...";
}
} else {
console.log("Not connected, not sending audio data to server.");
}
}
function toggleListening() {
if (socket.readyState === WebSocket.OPEN) {
if (recording) {
stopListening();
} else {
startListening();
}
}
}
function enumerateDevices() {
// Enumerate the available audio input devices
navigator.mediaDevices.enumerateDevices()
.then(function (devices) {
var audioInputDevices = devices.filter(function (device) {
return device.kind === 'audioinput';
});
console.log(audioInputDevices.length + ' audio input devices found');
// If more than one audio input device is available, populate the select list
if (audioInputDevices.length > 1) {
audioInputDevices.forEach(function (device) {
var option = document.createElement('option');
option.value = device.deviceId;
option.text = device.label || 'Device ' + device.deviceId;
inputDevices.appendChild(option);
});
// Listen for changes to the select list and connect to the selected audio input device
inputDevices.addEventListener('change', function (event) {
var selectedDeviceId = event.target.value;
var constraints = { audio: { deviceId: selectedDeviceId } };
navigator.mediaDevices.getUserMedia(constraints)
.then(function (stream) {
// Handle the audio stream from the selected device here
})
.catch(function (error) {
console.log('Error accessing audio stream:', error);
});
});
}
// If only one audio input device is available, connect to it automatically
else if (audioInputDevices.length === 1) {
var constraints = { audio: { deviceId: audioInputDevices[0].deviceId } };
navigator.mediaDevices.getUserMedia(constraints) navigator.mediaDevices.getUserMedia(constraints)
.then(function (stream) { .then(function (stream) {
// Handle the audio stream from the selected device here // Handle the audio stream from the selected device here
@ -326,42 +463,42 @@
.catch(function (error) { .catch(function (error) {
console.log('Error accessing audio stream:', error); console.log('Error accessing audio stream:', error);
}); });
}); }
} // If no audio input devices are available, show an error message
// If only one audio input device is available, connect to it automatically else {
else if (audioInputDevices.length === 1) { console.log('No audio input devices available');
var constraints = { audio: { deviceId: audioInputDevices[0].deviceId } }; }
navigator.mediaDevices.getUserMedia(constraints) })
.then(function (stream) { .catch(function (error) {
// Handle the audio stream from the selected device here console.log('Error listing audio input devices:', error);
}) });
.catch(function (error) { }
console.log('Error accessing audio stream:', error);
});
}
// If no audio input devices are available, show an error message
else {
console.log('No audio input devices available');
}
})
.catch(function (error) {
console.log('Error listing audio input devices:', error);
});
}
window.onload = () => { window.onload = () => {
recordButton = document.getElementById("record-button"); recordButton = document.getElementById("record-button");
recordButton.addEventListener("click", toggleListening); recordButton.addEventListener("click", toggleListening);
connectionStatus = document.getElementById("connection-status"); connectionStatus = document.getElementById("connection-status");
//transcription = document.getElementById("transcription"); //transcription = document.getElementById("transcription");
//autosend = document.getElementById("autosend"); //autosend = document.getElementById("autosend");
statusRecording = document.getElementById("status-recording"); statusRecording = document.getElementById("status-recording");
languageSelect = document.getElementById("language-select");
inputDevices = document.getElementById("input-devices");
storeRecordings = document.getElementById("store-recordings");
enumerateDevices(); enumerateDevices();
connect(socket); connect(socket);
}; };
</script>
<script src="https://cdn.webrtc-experiment.com/MediaStreamRecorder.js"></script> function copyToClipboard(id) {
var textarea = document.getElementById(id);
textarea.select();
document.execCommand('copy');
}
</script>
<script src="https://cdn.webrtc-experiment.com/MediaStreamRecorder.js"></script>
</body> </body>
</html> </html>

136
web/server.js
View File

@ -13,18 +13,49 @@ console.log(process.env)
console.log(process.env.TTS_BACKEND_URL) console.log(process.env.TTS_BACKEND_URL)
console.log(process.env.WS_URL) console.log(process.env.WS_URL)
let language = "en";
let storeRecordings = false;
let queueCounter = 0;
const storage = require('node-persist');
storage.init().then(() => {
storage.getItem('language').then((value) => {
if (value != undefined) { language = value; console.log('language: ' + language); }
else { storage.setItem('language', language).then(() => { console.log('language set to ' + language + "(default)"); }); }
});
storage.getItem('storeRecordings').then((value) => {
if (value != undefined) { storeRecordings = value; console.log('storeRecordings: ' + storeRecordings); }
else { storage.setItem('storeRecordings', storeRecordings).then(() => { console.log('storeRecordings set to ' + storeRecordings + "(default)"); }); }
});
});
//we use https://hub.docker.com/r/onerahmet/openai-whisper-asr-webservice to transcribe the audio //we use https://hub.docker.com/r/onerahmet/openai-whisper-asr-webservice to transcribe the audio
//docker run -p 9009:9009 -d onerahmet/openai-whisper-asr-webservice //docker run -p 9009:9009 -d onerahmet/openai-whisper-asr-webservice
wss.on('connection', (ws) => { const sessions = new Map(); // Store session data
console.log('Client ' + ws._socket.remoteAddress + ' connected');
wss.on('connection', (ws, req) => {
ws.sessionId = Math.random().toString(36).slice(2);
sessions.set(ws.sessionId, { language: 'en' });
console.log('Client ' + ws._socket.remoteAddress + ' connected with session id ' + ws.sessionId);
//send cookie to client
ws.send(JSON.stringify({ sessionId: ws.sessionId, language: language, storeRecordings: storeRecordings }));
ws.on('message', (data) => { ws.on('message', (data) => {
let webSocket = ws;
const sessionData = sessions.get(webSocket.sessionId);
if (!sessionData) {
console.log('No session data found for session id ' + webSocket.sessionId);
}
let language = sessionData?.language || 'en';
//show the size of the audio data as 0.000 MB //show the size of the audio data as 0.000 MB
console.log('Received data from client: ' + (data.length / 1024 / 1024).toFixed(3) + ' MB'); console.log('(queue ' + queueCounter + ') Received ' + (data.length / 1024 / 1024).toFixed(3) + ' MB audio from client. Crrent language: ' + language);
var request = require('request'); var request = require('request');
var formData = { var formData = {
task: 'transcribe', task: 'transcribe',
language: 'en-US', //bg-BG|en-US language: sessionData.language,
output: 'json', output: 'json',
audio_file: { audio_file: {
value: data, value: data,
@ -35,30 +66,35 @@ wss.on('connection', (ws) => {
} }
}; };
//"yyyymmdd-hhMMss" storeRecordings = sessionData?.storeRecordings || storeRecordings;
var timestampfilename = Date.now("yyyymmdd-hhMMss"); if (storeRecordings) {
//"yyyymmdd-hhMMss"
//save the audio data to a file to /rec subfolder var timestampfilename = Date.now("yyyymmdd-hhMMss");
var fs = require('fs'); var fs = require('fs');
fs.mkdir('rec', { recursive: true }, (err) => { fs.mkdir('rec', { recursive: true }, (err) => {
if (err) throw err; if (err) throw err;
}); });
fs.writeFile('rec/audio' + timestampfilename + '.ogg', data, function (err) {
fs.writeFile('rec/audio' + timestampfilename + '.ogg', data, function (err) { if (err) {
if (err) { return console.log(err);
return console.log(err); }
} console.log('Audio data saved to audio.ogg');
console.log('Audio data saved to audio.ogg'); });
}); }
//record start time
var start = new Date().getTime();
queueCounter++;
request.post({ url: process.env.TTS_BACKEND_URL, formData: formData }, function optionalCallback(err, httpResponse, body) { request.post({ url: process.env.TTS_BACKEND_URL, formData: formData }, function optionalCallback(err, httpResponse, body) {
queueCounter--;
if (err) { if (err) {
return console.error('upload failed:', err); return console.error('upload failed:', err);
} }
console.log('Whisper decoded:', body); //duration of the transcribe in 0.00s
ws.send(body); var duration = new Date().getTime() - start;
//console.log('decoded (' + duration + 'ms):', body);
console.log('decoded (' + (duration / 1000).toFixed(2) + 's):', body);
webSocket.send("(" + queueCounter + ") " + body);
}); });
}); });
}); });
@ -72,7 +108,11 @@ function transcribeAudio(audioData) {
// --- web server that servers client.html // --- web server that servers client.html
const express = require('express'); const express = require('express');
const bodyParser = require('body-parser');
const app = express(); const app = express();
app.use(bodyParser.json());
// app.use(bodyParser.urlencoded({ extended: false })); // Parse request body as URL-encoded
const path = require('path'); const path = require('path');
app.get('/', (req, res) => { app.get('/', (req, res) => {
@ -84,6 +124,56 @@ app.get('/', (req, res) => {
app.get('/wsurl', (req, res) => { app.get('/wsurl', (req, res) => {
res.send(process.env.WS_URL, 200, { 'Content-Type': 'text/plain' }); res.send(process.env.WS_URL, 200, { 'Content-Type': 'text/plain' });
}); });
//GET used to store default settings for all clients
app.get('/settings', (req, res) => {
if (req.query.language != undefined) {
language = req.query.language;
storage.setItem('language', language).then(() => { console.log('language set to ' + language); });
}
if (req.query.storeRecordings != undefined) {
storeRecordings = req.query.storeRecordings;
storage.setItem('storeRecordings', storeRecordings).then(() => { console.log('storeRecordings set to ' + storeRecordings); });
}
//send back the current settings as json
res.send(JSON.stringify({ language: language, storeRecordings: storeRecordings }), 200, { 'Content-Type': 'text/plain' });
});
//POST used to store settings for a specific client
app.post('/settings', (req, res) => {
//get the language from the json body ( { language: language, sessionId: sessionId })
const body = req.body;
const sid = body.sessionId;
const sessionData = sessions.get(sid);
if (body.language != undefined) {
sessionData.language = body.language;
console.log(`Session ${sid}: language set to ${sessionData.language}`);
}
if(body.storeRecordings != undefined) {
sessionData.storeRecordings = body.storeRecordings;
console.log(`Session ${sid}: storeRecordings set to ${sessionData.storeRecordings}`);
}
res.send('OK', 200, { 'Content-Type': 'text/plain' });
});
//save the audio file
app.post('/upload', (req, res) => {
try {
//save the audio file
var timestampfilename = Date.now("yyyymmdd-hhMMss");
var fs = require('fs');
fs.mkdir('rec', { recursive: true }, (err) => {
if (err) throw err;
});
var file = fs.createWriteStream('rec/audio_slice_' + timestampfilename + '.ogg');
req.pipe(file);
res.send('OK', 200, { 'Content-Type': 'text/plain' });
} catch (err) {
console.log(err);
res.send('ERROR', 500, { 'Content-Type': 'text/plain' });
}
});
app.listen(8080, () => { app.listen(8080, () => {
console.log('Server listening on port 8080'); console.log('Server listening on port 8080');