diff --git a/.env.development b/.env.development index 392ad57..920ae09 100644 --- a/.env.development +++ b/.env.development @@ -1,6 +1,7 @@ ENV_NAME=development TTS_API_URL=https://api.tts.d-popov.com/asr +LNN_API_URL=https://ollama.d-popov.com WS_URL=ws://localhost:8081 SERVER_PORT_WS=8081 SERVER_PORT_HTTP=8080 diff --git a/package-lock.json b/package-lock.json index dc62bf8..dd35903 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,11 +8,13 @@ "name": "kevin-ai", "version": "1.0.0", "dependencies": { + "axios": "^1.7.2", "body-parser": "^1.20.2", "dotenv": "^16.4.5", "express": "^4.18.2", "git": "^0.1.5", "node-persist": "^3.1.3", + "ollama": "^0.5.1", "request": "^2.88.2", "ws": "^8.12.1" } @@ -83,6 +85,29 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, + "node_modules/axios": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -409,6 +434,25 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -728,6 +772,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ollama": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.1.tgz", + "integrity": "sha512-mAiCHxdvu63E8EFopz0y82QG7rGfYmKAWgmjG2C7soiRuz/Sj3r/ebvCOp+jasiCubqUPE0ZThKT5LR6wrrPtA==", + "dependencies": { + "whatwg-fetch": "^3.6.20" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -769,6 +821,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -1089,6 +1146,11 @@ "extsprintf": "^1.2.0" } }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" + }, "node_modules/ws": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz", diff --git a/package.json b/package.json index 4a57dac..fa13107 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,13 @@ "start:tele": "python agent-py-bot/agent.py" }, "dependencies": { + "axios": "^1.7.2", "body-parser": "^1.20.2", "dotenv": "^16.4.5", "express": "^4.18.2", "git": "^0.1.5", "node-persist": "^3.1.3", + "ollama": "^0.5.1", "request": "^2.88.2", "ws": "^8.12.1" } diff --git a/web/audio.js b/web/audio.js index 53b1e2b..0a1920d 100644 --- a/web/audio.js +++ b/web/audio.js @@ -1,7 +1,7 @@ let selectedDeviceId = "default"; export let serverTime; export let recordButton; -let socket; +export let socket; let audioRecorder; let audioStream; let recording = false; @@ -42,7 +42,6 @@ export function InitAudioAnalyser(stream) { } export function startListening() { - //canvasCtx.fillStyle = "green"; recording = true; navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } }) .then((stream) => { @@ -75,7 +74,6 @@ export function startListening() { } if (event.data.size > 0) { let data = event.data; - console.log("audio data size: " + data.size); if (preBuffer.length > 0) { sendAudioToServerPost(preBuffer); } @@ -117,14 +115,25 @@ export function sendAudioToServerPost(data) { }); } -export function sendAudioToServer(data) { - //if (connected) { - socket.send(JSON.stringify({ type: 'audio', task:"transcribe", audio: data })); +export function sendAudioToServerJson(data) { + if (socket && socket.readyState === WebSocket.OPEN) { + + const binaryData = Buffer.from(base64AudioData, 'base64'); + socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData })); serverTime = Date.now(); if (!autosend.checked) { - transcription.innerHTML = "Processing audio..."; + transcription.placeholder = "Processing audio..."; } - //} + } +} +export function sendAudioToServer(data) { + if (socket && socket.readyState === WebSocket.OPEN) { + socket.send(data); + serverTime = Date.now(); + if (!autosend.checked) { + transcription.placeholder = "Processing audio..."; + } + } } export function toggleListening() { diff --git a/web/chat-client.html b/web/chat-client.html index c8cc410..5e369c0 100644 --- a/web/chat-client.html +++ b/web/chat-client.html @@ -11,12 +11,20 @@

Real-time Voice Chat

-
- + +
+
+
+
@@ -81,11 +97,11 @@
-
+
status