This commit is contained in:
Dobromir Popov 2024-10-02 14:26:42 +03:00
commit 194b3cab34
22 changed files with 911 additions and 641 deletions

4
.gitignore vendored
View File

@ -12,4 +12,8 @@ agent-mobile/artimobile/supervisord.pid
agent-pyter/lag-llama
agent-pyter/google-chrome-stable_current_amd64.deb
web/.node-persist/*
agent-mAId/output.wav
agent-mAId/build/*
agent-mAId/dist/main.exe
agent-mAId/output.wav
.node-persist/storage/*

9
agent-mAId/config.json Normal file
View File

@ -0,0 +1,9 @@
{
"api_key": "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",
"kb_key": "ctrl",
"mouse_btn": "left",
"model": "distil-whisper-large-v3-en",
"language":"en", // whisper-large-v3 or distil-whisper-large-v3-en
"action": "type" //type,copy
}

267
agent-mAId/main.py Normal file
View File

@ -0,0 +1,267 @@
import os
import sys
import pyaudio
import wave
import pyautogui
import keyboard
import mouse
import threading
from groq import Groq
import pystray
from pystray import MenuItem as item
from PIL import Image
import ctypes
import io
import time
import json5
import wave
import pyperclip
import argparse
import atexit
# # Load configuration from config.json
DEFAULT_CONFIG = {
"api_key": "xxx",
"kb_key": "ctrl",
"mouse_btn": "left",
"model": "distil-whisper-large-v3-en",
"language": "en", # whisper-large-v3 or distil-whisper-large-v3-en
"action": "type" # type, copy
}
def parse_args():
"""Parse command line arguments for config file."""
parser = argparse.ArgumentParser(description='Run the AI transcription app.')
parser.add_argument(
'--config', type=str, help='Path to config file', default=None
)
return parser.parse_args()
def load_config(config_path=None):
"""Load the configuration file, adjusting for PyInstaller's temp path when bundled."""
config = DEFAULT_CONFIG.copy() # Start with default configuration
try:
if config_path is None:
# Determine if the script is running as a PyInstaller bundle
if getattr(sys, 'frozen', False):
# If running in a bundle, use the temp path where PyInstaller extracts files
config_path = os.path.join(sys._MEIPASS, 'config.json')
else:
# If running in development (normal execution), use the local directory
config_path = os.path.join(os.path.dirname(__file__), 'config.json')
print(f'Trying to load config from: {config_path}')
with open(config_path, 'r') as config_file:
loaded_config = json5.load(config_file)
# Update the default config with any values from config.json
config.update(loaded_config)
except FileNotFoundError as ex:
print("Config file not found, using defaults." + ex.strerror)
raise ex
except json5.JSONDecodeError as ex:
print("Error decoding config file, using defaults." + ex.msg)
except Exception as e:
print(f"Unexpected error while loading config: {e}, using defaults.")
return config
# Load the config
# config = load_config()
# Parse command line arguments
args = parse_args()
# Load the config from the specified path or default location
config = load_config(args.config)
# Extract API key and button from the config file
API_KEY = config['api_key']
KB_KEY = config['kb_key']
MOUSE_BTN = config['mouse_btn']
MODEL = config['model']
POST_TRANSCRIBE = config['action']
# Constants
AUTO_START_PATH = os.path.expanduser(r"~\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup") # For autostart
# Initialize the Groq client
client = Groq(api_key=API_KEY)
def save_audio_to_disk(filename, audio_data, audio_format, channels, rate):
"""Save the audio data to disk asynchronously."""
with wave.open(filename, 'wb') as wave_file:
wave_file.setnchannels(channels)
wave_file.setsampwidth(audio_format)
wave_file.setframerate(rate)
wave_file.writeframes(audio_data)
def record_audio():
"""Records audio when the key and mouse button is pressed, stores in memory."""
audio = pyaudio.PyAudio()
stream = audio.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024)
frames = []
print("Recording...")
# Record while both keyboard and mouse button are pressed
while keyboard.is_pressed(KB_KEY) and mouse.is_pressed(button=MOUSE_BTN):
data = stream.read(1024)
frames.append(data)
recording_duration = len(frames) * 1024 / 16000 # Calculate audio duration in seconds
print(f"Recording stopped. Duration: {recording_duration:.2f} seconds.")
stream.stop_stream()
stream.close()
audio.terminate()
# Store the recorded audio in an in-memory stream as a valid WAV file
memory_stream = io.BytesIO()
with wave.open(memory_stream, 'wb') as wave_file:
wave_file.setnchannels(1)
wave_file.setsampwidth(audio.get_sample_size(pyaudio.paInt16))
wave_file.setframerate(16000)
wave_file.writeframes(b''.join(frames))
memory_stream.seek(0) # Reset the stream position to the beginning for reading
# Save audio to disk asynchronously as a side task (optional)
threading.Thread(target=save_audio_to_disk, args=("output.wav", b''.join(frames), audio.get_sample_size(pyaudio.paInt16), 1, 16000)).start()
return memory_stream
def transcribe_audio(memory_stream):
"""Transcribes the recorded audio using the Groq Whisper model."""
memory_stream.seek(0) # Reset the stream position to the beginning
start_time = time.time()
transcription = client.audio.transcriptions.create(
file=('audio.wav', memory_stream),
model=MODEL,
prompt="Transcribe the following audio",
language=config['language'],
response_format="json",
temperature=0.0
)
end_time = time.time()
transcription_time = end_time - start_time
print(f"Transcription took: {transcription_time:.2f} seconds. Result: {transcription.text}")
log_transcription_time(transcription_time)
return transcription.text
def simulate_keypress(text):
"""Simulates typing of transcribed text quickly."""
pyautogui.typewrite(text, interval=0.01) # Reduce interval between characters for faster typing
# pyautogui.press('enter')
def add_to_autostart():
"""Registers the app to autostart on login."""
script_path = os.path.abspath(__file__)
shortcut_path = os.path.join(AUTO_START_PATH, "mAId.lnk")
# Use ctypes to create the shortcut (this is Windows specific)
shell = ctypes.windll.shell32
shell.ShellExecuteW(None, "runas", "cmd.exe", f'/C mklink "{shortcut_path}" "{script_path}"', None, 1)
print("App added to autostart.")
icon = None # Global variable to store the tray icon object
def cleanup_and_exit():
"""Clean up the tray icon and exit the application."""
global icon
if icon:
print("Stopping and removing tray icon...")
icon.stop() # Stop the tray icon to remove it from the tray
sys.exit()
def setup_tray_icon():
global icon
"""Setup system tray icon and menu."""
if getattr(sys, 'frozen', False):
# If running as a bundle, use the temp path where PyInstaller extracts files
icon_path = os.path.join(sys._MEIPASS, 'mic.webp')
else:
# If running in development (normal execution), use the local directory
icon_path = os.path.join(os.path.dirname(__file__), 'mic.webp')
try:
# Load the tray icon
icon_image = Image.open(icon_path)
except FileNotFoundError:
print(f"Icon file not found at {icon_path}")
icon_image = Image.new('RGB', (64, 64), color=(255, 0, 0)) # Red icon as an example
return
menu = (
item('Register to Autostart', add_to_autostart),
item('Exit', lambda: quit_app(icon))
)
icon = pystray.Icon("mAId", icon_image, menu=pystray.Menu(*menu))
icon.run()
# Ensure the tray icon is removed when the app exits
atexit.register(cleanup_and_exit)
response_times = []
ma_window_size = 10 # Moving average over the last 10 responses
def log_transcription_time(transcription_time):
"""Logs the transcription time and updates the moving average."""
global response_times
# Add the transcription time to the list
response_times.append(transcription_time)
# If the number of logged times exceeds the window size, remove the oldest entry
if len(response_times) > ma_window_size:
response_times.pop(0)
# Calculate and print the moving average
moving_average = sum(response_times) / len(response_times)
print(f"Moving Average of Transcription Time (last {ma_window_size} responses): {moving_average:.2f} seconds.")
def main_loop():
"""Continuously listen for key or mouse press and transcribe audio."""
filename = "output.wav"
while True:
print("Waiting for key and mouse press...")
# Wait for KB_KEY or mouse press
while not (keyboard.is_pressed(KB_KEY) and mouse.is_pressed(button=MOUSE_BTN)):
time.sleep(0.1) # Small sleep to avoid busy-waiting
# Record audio
memory_stream = record_audio()
# Transcribe audio
print("Transcribing audio...")
transcribed_text = transcribe_audio(memory_stream)
if POST_TRANSCRIBE == "type":
# Simulate typing the transcribed text
print("Typing transcribed text...")
simulate_keypress(transcribed_text)
elif POST_TRANSCRIBE == "copy":
# Copy the transcribed text to clipboard
pyperclip.copy(transcribed_text)
print("Transcribed text copied to clipboard.")
if __name__ == "__main__":
# Start the tray icon in a separate thread so it doesn't block the main functionality
tray_thread = threading.Thread(target=setup_tray_icon)
tray_thread.daemon = True
tray_thread.start()
# Run the main loop that listens for key or mouse presses in the background
main_loop()

39
agent-mAId/main.spec Normal file
View File

@ -0,0 +1,39 @@
# -*- mode: python ; coding: utf-8 -*-
a = Analysis(
['main.py'],
pathex=[],
binaries=[],
datas=[('config.json', '.'), ('mic.webp', '.')],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
a.binaries,
a.datas,
[],
name='main',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
upx_exclude=[],
runtime_tmpdir=None,
console=True,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)

BIN
agent-mAId/mic.webp Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 70 KiB

BIN
agent-mAId/output.wav Normal file

Binary file not shown.

10
agent-mAId/readme.md Normal file
View File

@ -0,0 +1,10 @@
<!-- # to install as an app: -->
pip install pyinstaller
pyinstaller --onefile main.py
pyinstaller main.spec
<!-- to generate reqs.txt -->
pipreqs .
<!-- or -->
pip freeze > requirements.txt

View File

@ -0,0 +1,9 @@
groq==0.11.0
json5==0.9.25
keyboard==0.13.5
mouse==0.7.1
Pillow==10.4.0
PyAudio==0.2.14
PyAutoGUI==0.9.54
pyperclip==1.9.0
pystray==0.19.5

View File

@ -10,4 +10,8 @@ def execute_python_code(code_block):
except Exception as e:
return f"Execution error: {str(e)}"
def execute_trading_action(action):
# Placeholder for executing trading actions
# This could be an API call to a trading platform
print(f"Executing trading action: {action}")

View File

@ -46,9 +46,15 @@ def parse_rss_feed(feed_url):
articles = [{'title': entry.title, 'link': entry.link} for entry in feed.entries]
return articles
import yfinance as yf
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def fetch_stock_data(ticker, interval='1d', period='1mo'):
stock = yf.Ticker(ticker)
hist = stock.history(interval=interval, period=period)
return hist
def search_google_news(topic):
options = Options()
options.headless = True
@ -148,6 +154,17 @@ def get_news_api_results(query, api_key, from_param):
except Exception as e:
return f"API Request Error: {e}"
def search_tavily(topic, api_key):
url = f"https://app.tavily.com/api/search?q={topic}"
headers = {
"Authorization": f"Bearer {api_key}"
}
response = requests.get(url, headers=headers)
if response.status_code == 200:
return response.json()
else:
return {"error": response.text}
def search_news(topic):
# DuckDuckGo Results
duck_results = search_duckduckgo(topic)
@ -205,4 +222,7 @@ def summarize_data(data):
def run_web_agent(topic, folder):
print(f"[{datetime.now()}] Running web agent for topic: {topic}")
news_data = search_news(topic)
tavily_api_key = "YOUR_TAVILY_API_KEY"
tavily_results = search_tavily(topic, tavily_api_key)
news_data["tavily"] = tavily_results
return news_data

View File

@ -1,4 +1,12 @@
//C:\Users\popov\.continue\config.json
{
"models": [ {
"title": "local ollama> yi-coder",
"provider": "ollama",
"model": "yi-coder:9b",
"apiBase": "http://localhost:11434"
}
],
"tabAutocompleteModel": {
"title": "Tab Autocomplete Model",
"provider": "ollama",
@ -8,6 +16,7 @@
}
// original: "tabAutocompleteModel": {
// "title": "Starcoder 3b",
// "provider": "ollama",

View File

@ -1,5 +1,3 @@
version: '3.4'
services:
# kevinai:
# image: kevinai
@ -28,6 +26,28 @@ services:
WS_URL: wss://tts.d-popov.com
SERVER_PORT_WS: 8081
SERVER_PORT_HTTP: 8080
SERVER_PORT_WS: 8082
ports:
- 28080:8080
- 28081:8081
chat-server:
image: node:20-alpine
container_name: ml-voice-chat-server
working_dir: /usr/src/app
volumes:
- /mnt/apps/DEV/REPOS/git.d-popov.com/ai-kevin:/usr/src/app
command: >
sh -c "npm install && node web/chat-server.js"
environment:
NODE_ENV: demo
#TTS_BACKEND_URL: https://api.tts.d-popov.com/asr
TTS_API_URL: http://192.168.0.11:9009/asr
WS_URL: wss://ws.tts.d-popov.com
SERVER_PORT_HTTP: 8080
SERVER_PORT_WS: 8082
ports:
- 28080:8080
- 28081:8082
dns:
- 8.8.8.8
- 8.8.4.4

BIN
output.wav Normal file

Binary file not shown.

18
package-lock.json generated
View File

@ -8,6 +8,7 @@
"name": "kevin-ai",
"version": "1.0.0",
"dependencies": {
"@prisma/client": "^5.16.1",
"axios": "^1.7.2",
"body-parser": "^1.20.2",
"dotenv": "^16.4.5",
@ -21,6 +22,23 @@
"ws": "^8.12.1"
}
},
"node_modules/@prisma/client": {
"version": "5.16.1",
"resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.16.1.tgz",
"integrity": "sha512-wM9SKQjF0qLxdnOZIVAIMKiz6Hu7vDt4FFAih85K1dk/Rr2mdahy6d3QP41K62N9O0DJJA//gUDA3Mp49xsKIg==",
"hasInstallScript": true,
"engines": {
"node": ">=16.13"
},
"peerDependencies": {
"prisma": "*"
},
"peerDependenciesMeta": {
"prisma": {
"optional": true
}
}
},
"node_modules/@types/node": {
"version": "18.19.34",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz",

View File

@ -7,12 +7,12 @@
"start:demo": "NODE_ENV=demo node web/server.js",
"start:demo-chat": "node web/chat-server.js",
"start:tele": "python agent-py-bot/agent.py"
},
"env": {
"NODE_ENV": "demo"
},
"dependencies": {
"@prisma/client": "^5.16.1",
"axios": "^1.7.2",
"body-parser": "^1.20.2",
"dotenv": "^16.4.5",

25
vision/notes.md Normal file
View File

@ -0,0 +1,25 @@
Visual options :
-- OD:
- object detction /w fine tuning: Yolo V5: https://learnopencv.com/custom-object-detection-training-using-yolov5/
-- V-aware
- visual LLM: LLAVA : https://llava.hliu.cc/
-- BOTH detection and comprehention:
-Phi
https://huggingface.co/microsoft/Phi-3-vision-128k-instruct
https://github.com/microsoft/Phi-3CookBook
- Lavva chat
https://github.com/LLaVA-VL/LLaVA-Interactive-Demo?tab=readme-ov-file
git clone https://github.com/LLaVA-VL/LLaVA-Interactive-Demo.git
conda create -n llava_int -c conda-forge -c pytorch python=3.10.8 pytorch=2.0.1 -y
conda activate llava_int
cd LLaVA-Interactive-Demo
pip install -r requirements.txt
source setup.sh
- decision making based on ENV, RL: https://github.com/OpenGenerativeAI/llm-colosseum

View File

@ -14,10 +14,19 @@ TTS_API_URL=https://api.tts.d-popov.com/asr
LLN_MODEL=qwen2
LNN_API_URL=https://ollama.d-popov.com/api/generate
GROQ_API_KEY=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
OPENAI_API_KEY=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
# GROQ_API_KEY=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
# OPENAI_API_KEY=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
WS_URL=wss://tts.d-popov.com
PUBLIC_HOSTNAME=tts.d-popov.com
SERVER_PORT_WS=8080
SERVER_PORT_HTTP=8080
# WS_URL=wss://tts.d-popov.com
# PUBLIC_HOSTNAME=tts.d-popov.com
# SERVER_PORT_WS=8080
# SERVER_PORT_HTTP=8080
# This was inserted by `prisma init`:
# Environment variables declared in this file are automatically made available to Prisma.
# See the documentation for more detail: https://pris.ly/d/prisma-schema#accessing-environment-variables-from-the-schema
# Prisma supports the native connection string format for PostgreSQL, MySQL, SQLite, SQL Server, MongoDB and CockroachDB.
# See the documentation for all the connection string options: https://pris.ly/d/connection-strings
DATABASE_URL="postgresql://johndoe:randompassword@localhost:5432/mydb?schema=public"

3
web/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
node_modules
# Keep environment variables out of version control
.env

View File

@ -1,615 +1,354 @@
<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Real-time Voice Chat</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/tailwindcss/2.2.19/tailwind.min.css">
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Voice Chat Messenger</title>
<link href="https://cdn.jsdelivr.net/npm/tailwindcss@2.2.19/dist/tailwind.min.css" rel="stylesheet">
</head>
<body class="bg-gray-100">
<div class="container mx-auto px-4 py-8">
<h1 class="text-2xl font-bold mb-4 text-center">Real-time Voice Chat</h1>
<h1 class="text-3xl font-bold mb-8 text-center text-blue-600">Voice Chat Messenger</h1>
<div class="flex justify-center items-center mb-4">
<!-- Username Input -->
<input type="text" id="username" class="border rounded p-2 mr-4" placeholder="Enter your username">
<div id="join-container" class="hidden">
<button id="btn-join" onclick="logInAndStoreSession()"
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Join Chat</button>
<select id="language-select">
<option value="auto">Auto</option>
<option value="en">English</option>
<option value="bg">Български</option>
<option value="fr">Français</option>
</select>
<!-- Login/Register Form -->
<div id="auth-container" class="max-w-md mx-auto bg-white p-6 rounded-lg shadow-md mb-8">
<h2 class="text-2xl font-semibold mb-4">Login or Register</h2>
<input type="text" id="username" class="w-full border rounded p-2 mb-4" placeholder="Username">
<input type="password" id="password" class="w-full border rounded p-2 mb-4" placeholder="Password">
<div class="flex justify-between">
<button onclick="login()"
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">Login</button>
<button onclick="register()"
class="bg-green-500 hover:bg-green-700 text-white font-bold py-2 px-4 rounded">Register</button>
</div>
</div>
<!-- Clear Session Option -->
<button id="btn-disconnect" onclick="clearSession()"
class="hidden bg-red-500 hover:bg-red-700 text-white font-bold py-2 px-4 rounded">Clear Session</button>
<!-- Main Chat Interface (initially hidden) -->
<div id="chat-interface" class="hidden">
<div class="flex">
<!-- Sidebar -->
<div class="w-1/4 bg-white rounded-lg shadow-md mr-4 p-4">
<h2 class="text-xl font-semibold mb-4">Chats</h2>
<div id="chat-list" class="space-y-2">
<!-- Chat list items will be inserted here -->
</div>
<!-- Active Users List -->
<div id="active-users-container" class="hidden flex justify-center items-center mb-4">
<div class="w-1/3">
<h2 class="text-xl font-bold mb-2">Active Users</h2>
<select id="users-list" class="w-full bg-white p-4 rounded shadow" multiple size="10">
<!-- Dynamic list of users -->
</select>
<button onclick="startChat()"
class="bg-green-500 hover:bg-green-700 text-white font-bold py-2 px-4 rounded mt-4">Start
<button onclick="showNewChatModal()"
class="mt-4 bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded w-full">New
Chat</button>
</div>
</div>
<div id="previous-chats-container" class="hidden w-2/3 mx-auto">
<h2 class="text-xl font-bold mb-2">Previous Chats</h2>
<div id="previous-chats" class="bg-white p-4 rounded shadow">
<!-- Previous chats content -->
<!-- Main Chat Area -->
<div class="w-3/4 bg-white rounded-lg shadow-md p-4">
<div id="current-chat-info" class="mb-4">
<h2 class="text-xl font-semibold">Current Chat</h2>
<div id="chat-participants" class="text-sm text-gray-600"></div>
</div>
<div id="messages" class="h-96 overflow-y-auto mb-4 p-2 border rounded">
<!-- Messages will be inserted here -->
</div>
<!-- Chat Room -->
<div id="chat-room-container" class="hidden w-2/3 mx-auto">
<h2 class="text-xl font-bold mb-2">Chat Room</h2>
<div id="chat-room" class="bg-white p-4 rounded shadow mb-4">
<!-- Chat room content -->
<div>
<div id="chat-room-users" class="flex flex-wrap mb-4">
<!-- Participants list -->
<div class="flex items-center">
<button id="record-button"
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-2">Push to
Talk</button>
<div id="status-recording" class="text-sm"></div>
</div>
</div>
<div class="mb-4">
<label class="flex items-center space-x-2">
<input type="checkbox" id="autosend" class="mr-2">
<span>Continuous</span>
<div class="mt-2">
<label class="inline-flex items-center">
<input type="checkbox" id="autosend" class="form-checkbox">
<span class="ml-2">Continuous Mode</span>
</label>
</div>
<div class="mb-4">
<button id="record-button" disabled
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-4">Push to
Talk</button>
</div>
<div id="status-recording" class="flex justify-center items-center mb-4">
</div>
<div id="transcription" class="border rounded p-4 h-48 overflow-y-scroll mb-4">
<!-- Transcription content -->
</div>
<canvas id="canvas" class="w-full mb-4"></canvas>
<div class="flex justify-between items-center">
<button id="copyButton"
class="bg-gray-200 hover:bg-gray-300 text-gray-700 font-medium py-2 px-4 rounded focus:outline-none"
onclick="copyToClipboard('transcription')">Copy</button>
<button id="clearButton"
class="bg-gray-200 hover:bg-gray-300 text-gray-700 font-medium py-2 px-4 rounded focus:outline-none"
onclick="clearTranscription()">Clear</button>
</div>
</div>
</div>
<!-- Connection Status and Info -->
<div class="flex justify-center items-center mb-4">
<div id="connection-status" class="mr-4"></div>
<!-- New Chat Modal (initially hidden) -->
<div id="new-chat-modal" class="hidden fixed inset-0 bg-gray-600 bg-opacity-50 overflow-y-auto h-full w-full">
<div class="relative top-20 mx-auto p-5 border w-96 shadow-lg rounded-md bg-white">
<h3 class="text-lg font-semibold mb-4">Start a New Chat</h3>
<select id="users-list" class="w-full p-2 border rounded mb-4" multiple>
<!-- User options will be inserted here -->
</select>
<div class="flex justify-end">
<button onclick="startNewChat()"
class="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded mr-2">Start
Chat</button>
<button onclick="hideNewChatModal()"
class="bg-gray-300 hover:bg-gray-400 text-black font-bold py-2 px-4 rounded">Cancel</button>
</div>
<div class="flex justify-center items-center mb-4">
<div id="info"></div>
</div>
<div id="status-recording" class="flex justify-center items-center mb-4"> status</div>
</div>
<!-- Connection Status -->
<div id="connection-status" class="fixed bottom-4 right-4 bg-gray-800 text-white p-2 rounded"></div>
</div>
<script>
// Declare these variables and functions in the global scope
var socket;
var sessionId;
var currentUser;
var users = [];
var chats = [];
var currentChatId;
<script type="module">
// import * as audio from './audio.js';
let socket;
let sessionId;
let username;
let users = [];
let selectedUsers = [];
let chats = [];
let recordButton;
let connectionStatus;
let statusRecording;
let connected = false;
document.getElementById('autosend').addEventListener('change', (event) => {
const autosend = event.target.checked;
fetch('/settings', {
// Make these functions global
window.login = function() {
const username = document.getElementById('username').value;
const password = document.getElementById('password').value;
fetch('/login', {
method: 'POST',
body: JSON.stringify({ autosend, sessionId }),
headers: { 'Content-Type': 'application/json' },
credentials: 'same-origin'
});
body: JSON.stringify({ username, password })
})
.then(response => response.json())
.then(data => {
if (data.sessionId) {
sessionId = data.sessionId;
currentUser = { id: data.userId, username };
showChatInterface();
connect();
} else {
alert('Login failed. Please try again.');
}
});
};
window.register = function() {
const username = document.getElementById('username').value;
const password = document.getElementById('password').value;
fetch('/register', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ username, password })
})
.then(response => response.json())
.then(data => {
if (data.userId) {
alert('Registration successful. Please login.');
} else {
alert('Registration failed. Please try again.');
}
});
};
window.showNewChatModal = function() {
document.getElementById('new-chat-modal').classList.remove('hidden');
};
window.hideNewChatModal = function() {
document.getElementById('new-chat-modal').classList.add('hidden');
};
window.startNewChat = function() {
const selectedUsers = Array.from(document.getElementById('users-list').selectedOptions).map(option => option.value);
if (selectedUsers.length > 0) {
socket.send(JSON.stringify({ type: 'startChat', users: selectedUsers }));
hideNewChatModal();
} else {
alert('Please select at least one user to start a chat.');
}
};
// Connect to WebSocket
function connect() {
return new Promise((resolve, reject) => {
connectionStatus.innerHTML = "Connecting to WS...";
let wsurl = "ws://localhost:8080";
const connectionStatus = document.getElementById("connection-status");
connectionStatus.textContent = "Connecting...";
fetch("/wsurl")
.then((response) => response.text())
.then((data) => {
wsurl = data;
console.log("Got ws url: '" + wsurl + "'");
})
.then(() => {
.then(response => response.text())
.then(wsurl => {
socket = new WebSocket(wsurl);
// audio.setSocket(socket); // Set the socket in the audio module
socket.onopen = () => {
connectionStatus.innerHTML = "Connected to " + wsurl;
recordButton.disabled = false;
connected = true;
//if we stored a session id in a cookie, reconnect
const sessionId = getCookie("sessionId");
if (sessionId) {
socket.send(JSON.stringify({ type: 'reconnect', sessionId }));
}
else {
socket.send(JSON.stringify({ type: 'sessionId' }));
}
connectionStatus.textContent = "Connected";
resolve(socket);
};
socket.onmessage = onmessage;
socket.onmessage = handleMessage;
socket.onclose = () => {
connectionStatus.innerHTML = "Disconnected";
recordButton.disabled = true;
connected = false;
setTimeout(() => {
connect().then(resolve).catch(reject);
}, 5000);
connectionStatus.textContent = "Disconnected";
setTimeout(() => connect().then(resolve).catch(reject), 5000);
};
})
.catch((error) => {
connectionStatus.innerHTML = "Error getting ws url: " + error;
.catch(error => {
connectionStatus.textContent = "Connection error";
reject(error);
});
});
};
function onmessage(event) {
try {
let json = JSON.parse(event.data);
switch (json.type) {
case "sessionId":
sessionId = json.sessionId;
//set the session id in the cookie
document.cookie = `sessionId=${sessionId}; path=/;`;
console.log("Got session id: " + sessionId);
break;
case "languageDetected":
statusRecording.innerHTML = "Detected language: " + json.languageDetected;
break;
case "text":
case "transcriptionResult":
transcription.innerHTML += "<br />" + json.text;
let latency = Date.now() - serverTime;
if (autosend.checked) {
// const arr = event.data.split(/[(\)]/);
// let queue = arr[1];
// let text = arr[2].trim();
// info.innerHTML = "latency: " + latency + "ms; server queue: " + queue + " requests";
//transcription.value += event.data + " ";
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "black";
} else {
//transcription.innerHTML = event.data;
}
break;
case 'audio':
const audioBuffer = Uint8Array.from(atob(json.audio), char => char.charCodeAt(0));
const audioBlob = new Blob([audioBuffer], { type: 'audio/mp3' });
const audioUrl = URL.createObjectURL(audioBlob);
const audio = new Audio(audioUrl);
audio.play();
break;
case "userList":
users = json.users;
function handleMessage(event) {
const data = JSON.parse(event.data);
switch (data.type) {
case 'sessionId':
sessionId = data.sessionId;
break;
case 'userList':
users = data.users;
updateUserList();
break;
case "chats":
chats = json.chats;
case 'chats':
chats = data.chats;
updateChatList();
break;
case "chat":
displayChatParticipants(json.chat.id, json.chat.participants);
case 'chat':
displayChat(data.chat);
break;
case 'text':
case 'transcriptionResult':
addMessage(data.text);
break;
case 'audio':
playAudio(data.audio);
break;
default:
console.log("Unknown message type:", json.type);
}
} catch (e) {
console.error("Failed to parse message", e);
}
}
function logInAndStoreSession() {
username = document.getElementById('username').value;
if (username.trim() === "") {
alert("Please enter a username");
return;
}
if (!socket || socket.readyState !== WebSocket.OPEN) {
connect().then(() => {
userJoin(sessionId, username, document.getElementById('language-select').value);
});
} else {
userJoin(sessionId, username, document.getElementById('language-select').value);
}
}
function userJoin(sessionId, username, language) {
socket.send(JSON.stringify({ type: 'join', username, language }));
document.cookie = `sessionId=${sessionId}; path=/;`;
document.cookie = `username=${username}; path=/;`;
showClearSessionOption();
function showChatInterface() {
document.getElementById('auth-container').classList.add('hidden');
document.getElementById('chat-interface').classList.remove('hidden');
}
function clearSession() {
document.cookie = "sessionId=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
document.cookie = "username=; expires=Thu, 01 Jan 1970 00:00:00 UTC; path=/;";
location.reload();
}
function showClearSessionOption() {
const sessionId = getCookie("sessionId");
if (sessionId) {
if (!socket || socket.readyState !== WebSocket.OPEN) {
connect().then((s) => {
s.send(JSON.stringify({ type: 'reconnect', sessionId }));
});
}
document.getElementById('btn-disconnect').classList.remove('hidden');
document.getElementById('join-container').classList.add('hidden');
document.getElementById('active-users-container').classList.remove('hidden');
document.getElementById('previous-chats-container').classList.remove('hidden');
} else {
document.getElementById('btn-disconnect').classList.add('hidden');
document.getElementById('join-container').classList.remove('hidden');
}
}
function getCookie(name) {
const value = `; ${document.cookie}`;
const parts = value.split(`; ${name}=`);
if (parts.length === 2) return parts.pop().split(';').shift();
}
window.onload = () => {
recordButton = document.getElementById("record-button");
setRecordButton(recordButton);
connectionStatus = document.getElementById("connection-status");
statusRecording = document.getElementById("status-recording");
showClearSessionOption();
connect().then(() => {
initializeVolumeChecker();
});
};
function copyToClipboard(id) {
var textarea = document.getElementById(id);
textarea.select();
document.execCommand('copy');
}
function clearTranscription() {
document.getElementById('transcription').innerText = '';
}
function updateUserList() {
const usersList = document.getElementById('users-list');
usersList.innerHTML = '';
users.forEach(user => {
if (user.id !== currentUser.id) {
const option = document.createElement('option');
option.value = user.sessionId;
option.innerText = "[" + user.language + "] " + user.username;
if (user.username === username) {
option.innerText += " (me)";
}
option.value = user.id;
option.textContent = user.username;
usersList.appendChild(option);
}
});
}
function startChat() {
const selectedOptions = Array.from(document.querySelectorAll('#users-list option:checked'));
selectedUsers = selectedOptions.map(option => option.value);
if (selectedUsers.length === 0) {
alert("Please select at least one user to start a chat.");
return;
}
selectedUsers.push(sessionId); // Add self to the selected users list for self-chat
socket.send(JSON.stringify({ type: 'startChat', users: selectedUsers }));
document.getElementById('chat-room-container').classList.remove('hidden');
}
function fetchPreviousChats(username) {
fetch(`/chats?username=${username}`)
.then(response => response.json())
.then(data => {
chats = data.chats;
updateChatList();
});
}
function updateChatList() {
const previousChats = document.getElementById('previous-chats');
previousChats.innerHTML = '';
const chatList = document.getElementById('chat-list');
chatList.innerHTML = '';
chats.forEach(chat => {
const chatDiv = document.createElement('div');
chatDiv.classList.add('border', 'rounded', 'p-2', 'mb-2', 'cursor-pointer');
chatDiv.setAttribute('data-chat-id', chat.id); // Store chat ID in data attribute
const participants = chat.participants.join(', ');
const status = chat.participants.map(participant => {
const user = users.find(u => u.username === participant);
return user ? `${participant} (online)` : `${participant} (offline)`;
}).join(', ');
chatDiv.innerHTML = `${status}`;
chatDiv.addEventListener('click', () => {
// Remove highlight from all chat divs
document.querySelectorAll('#previous-chats > div').forEach(div => {
div.classList.remove('bg-blue-100');
});
// Highlight selected chat div
chatDiv.classList.add('bg-blue-100');
selectChatRoom(chat.id);
});
previousChats.appendChild(chatDiv);
const chatItem = document.createElement('div');
chatItem.className = 'p-2 hover:bg-gray-100 cursor-pointer';
chatItem.textContent = chat.participants.map(p => p.username).join(', ');
chatItem.onclick = () => selectChat(chat.id);
chatList.appendChild(chatItem);
});
}
function selectChatRoom(chatId) {
const chat = chats.find(c => c.id === chatId);
if (!chat) return;
const chatRoomUsers = document.getElementById('chat-room-users');
chatRoomUsers.innerHTML = ''; // Clear existing content
function selectChat(chatId) {
currentChatId = chatId;
socket.send(JSON.stringify({ type: 'enterChat', chatId }));
document.getElementById('chat-room-container').classList.remove('hidden');
// displayChatParticipants(chatId, chat.participants);
}
function displayChatParticipants(chatId, participants) {
const chatRoomUsers = document.getElementById('chat-room-users');
let participantsHtml = '<div class="flex flex-wrap mb-4">';
participants.forEach(participantId => {
const user = users.find(u => u.sessionId === participantId);
const status = user ? "online" : "offline";
const username = user ? user.username : "Unknown User";
participantsHtml += `<span class="inline-flex items-center px-3 py-0.5 rounded-full text-sm font-medium bg-${status === 'online' ? 'green' : 'gray'}-100 text-${status === 'online' ? 'green' : 'gray'}-800 mr-2 mb-2">
${username} (${status})
</span>`;
});
participantsHtml += '</div>';
chatRoomUsers.innerHTML = participantsHtml;
function displayChat(chat) {
const messagesContainer = document.getElementById('messages');
messagesContainer.innerHTML = '';
chat.messages.forEach(addMessage);
const participantsContainer = document.getElementById('chat-participants');
participantsContainer.textContent = Participants: ${ chat.participants.map(p => p.username).join(', ') };
}
// REGION AUDIO RECORDING
let selectedDeviceId = "default";
export let serverTime;
// export let recordButton;
// export let socket;
// let connectionStatus;
// let statusRecording;
let audioRecorder;
let audioStream;
let recording = false;
let audioContext;
let volumeChecker;
let lastVolumes = new Array(5);
let averageVolume;
let silenceCount = 0;
let isSpeaking = false;
let soundDetected = false;
let speakingCount = 0;
let analyser = null;
let SILENCE_DELAY_MS = 50;
let preDetect_IncludedAudio = 400; //ms
let soundCount_Threshold = 10;
let silenceCount_Threshold = 10;
const volumeHistory = [];
function setSocket(newSocket) {
socket = newSocket;
function addMessage(message) {
const messagesContainer = document.getElementById('messages');
const messageElement = document.createElement('div');
messageElement.className = 'mb-2';
messageElement.textContent = ${ message.sender }: ${ message.text };
messagesContainer.appendChild(messageElement);
messagesContainer.scrollTop = messagesContainer.scrollHeight;
}
export function setRecordButton(newRecordButton) {
recordButton = newRecordButton;
recordButton.addEventListener("click", toggleListening);
function playAudio(audioBase64) {
const audio = new Audio(data: audio / mp3; base64, ${ audioBase64 });
audio.play();
}
// Audio recording logic
let mediaRecorder;
let audioChunks = [];
document.getElementById('record-button').addEventListener('mousedown', startRecording);
document.getElementById('record-button').addEventListener('mouseup', stopRecording);
document.getElementById('record-button').addEventListener('mouseleave', stopRecording);
function startRecording() {
navigator.mediaDevices.getUserMedia({ audio: true })
.then(stream => {
mediaRecorder = new MediaRecorder(stream);
export function InitAudioAnalyser(stream) {
audioContext = new AudioContext();
const source = audioContext.createMediaStreamSource(stream);
analyser = audioContext.createAnalyser();
analyser.fftSize = 2048;
analyser.smoothingTimeConstant = 0.8;
source.connect(analyser);
}
export function startListening() {
recording = true;
navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } })
.then((stream) => {
audioStream = stream;
const audioContext = new AudioContext();
const sourceNode = audioContext.createMediaStreamSource(audioStream);
const audioSampleRate = sourceNode.context.sampleRate;
info.innerHTML = "Sample rate: " + audioSampleRate + " Hz";
var preBuffer = [];
const channelSplitter = audioContext.createChannelSplitter(2);
const channelMerger = audioContext.createChannelMerger(1);
sourceNode.connect(channelSplitter);
channelSplitter.connect(channelMerger, 0, 0);
const outputNode = channelMerger;
const mediaStreamDestination = audioContext.createMediaStreamDestination();
outputNode.connect(mediaStreamDestination);
const singleChannelStream = mediaStreamDestination.stream;
audioRecorder = new MediaRecorder(singleChannelStream);
audioRecorder.start();
audioRecorder.addEventListener("dataavailable", (event) => {
if (!soundDetected && autosend.checked) {
preBuffer = [];
preBuffer.push(event.data);
return;
}
if (event.data.size > 0) {
let data = event.data;
if (preBuffer.length > 0) {
sendAudioToServerPost(preBuffer);
}
sendAudioToServer(data);
soundDetected = false;
}
});
InitAudioAnalyser(stream);
});
recordButton.innerHTML = "Stop Talking";
recordButton.classList.toggle('bg-red-500');
recordButton.classList.toggle('bg-blue-500');
recordButton.classList.toggle('hover:bg-blue-700');
}
export function stopListening() {
recording = false;
audioRecorder.stop();
recordButton.innerHTML = "Push to Talk";
recordButton.classList.toggle('bg-blue-500');
recordButton.classList.toggle('bg-red-500');
recordButton.classList.toggle('hover:bg-blue-700');
clearInterval(volumeChecker);
if (audioStream) {
audioStream.getTracks().forEach(track => track.stop());
audioStream = null;
}
}
export function sendAudioToServerPost(data) {
const blob = new Blob(data, { type: "audio/ogg; codecs=opus" });
var formData = new FormData();
formData.append('file', data);
fetch('/upload', {
method: 'POST',
body: formData
mediaRecorder.ondataavailable = event => {
audioChunks.push(event.data);
};
mediaRecorder.onstop = sendAudioToServer;
mediaRecorder.start();
document.getElementById('status-recording').textContent = 'Recording...';
});
}
export function sendAudioToServerJson(data) {
if (socket && socket.readyState === WebSocket.OPEN) {
const binaryData = Buffer.from(base64AudioData, 'base64');
socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData }));
serverTime = Date.now();
if (!autosend.checked) {
transcription.placeholder = "Processing audio...";
}
}
}
export function sendAudioToServer(data) {
if (socket && socket.readyState === WebSocket.OPEN) {
socket.send(data);
serverTime = Date.now();
if (!autosend.checked) {
transcription.placeholder = "Processing audio...";
}
function stopRecording() {
if (mediaRecorder && mediaRecorder.state !== 'inactive') {
mediaRecorder.stop();
document.getElementById('status-recording').textContent = 'Processing...';
}
}
export function toggleListening() {
if (socket.readyState === WebSocket.OPEN) {
if (recording) {
stopListening();
function sendAudioToServer() {
const audioBlob = new Blob(audioChunks, { type: 'audio/ogg; codecs=opus' });
audioChunks = [];
const reader = new FileReader();
reader.onloadend = () => {
const base64Audio = reader.result.split(',')[1];
socket.send(JSON.stringify({
type: 'audio',
chatId: currentChatId,
audio: base64Audio
}));
};
reader.readAsDataURL(audioBlob);
}
// Initialize the app
document.addEventListener('DOMContentLoaded', () => {
const storedSessionId = localStorage.getItem('sessionId');
if (storedSessionId) {
sessionId = storedSessionId;
showChatInterface();
connect();
}
});
// Continuous mode toggle
document.getElementById('autosend').addEventListener('change', (event) => {
const autosend = event.target.checked;
if (autosend) {
startContinuousRecording();
} else {
startListening();
stopContinuousRecording();
}
});
let continuousRecorder;
function startContinuousRecording() {
navigator.mediaDevices.getUserMedia({ audio: true })
.then(stream => {
continuousRecorder = new MediaRecorder(stream);
continuousRecorder.ondataavailable = event => {
sendAudioToServer();
};
continuousRecorder.start(1000); // Send audio every second
document.getElementById('status-recording').textContent = 'Continuous mode active';
});
}
function stopContinuousRecording() {
if (continuousRecorder && continuousRecorder.state !== 'inactive') {
continuousRecorder.stop();
document.getElementById('status-recording').textContent = '';
}
}
export function initializeVolumeChecker() {
volumeChecker = setInterval(() => {
if (!audioContext) {
//console.log("No audio context");
return;
}
const frequencyData = new Uint8Array(analyser.frequencyBinCount);
analyser.getByteFrequencyData(frequencyData);
let totalVolume = 0;
for (let i = 0; i < frequencyData.length; i++) {
totalVolume += frequencyData[i];
}
averageVolume = totalVolume / frequencyData.length;
volumeHistory.push(averageVolume);
if (volumeHistory.length > 100) {
volumeHistory.shift();
// Logout function
function logout() {
localStorage.removeItem('sessionId');
location.reload();
}
const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5;
const isSilent = averageVolume < threshold;
if (averageVolume > threshold) {
if (autosend.checked && speakingCount == 0 && audioRecorder) {
soundDetected = false;
audioRecorder.stop();
audioRecorder.start();
}
speakingCount++;
if (speakingCount > soundCount_Threshold) {
statusRecording.innerHTML = "Listening...";
statusRecording.style.color = "green";
isSpeaking = true;
}
} else if (averageVolume - 5 < threshold) {
speakingCount = 0;
if (isSpeaking) {
silenceCount++;
if (silenceCount > silenceCount_Threshold) {
if (autosend.checked) {
soundDetected = true;
audioRecorder.stop();
audioRecorder.start();
}
isSpeaking = false;
statusRecording.innerHTML = "Silence detected...";
statusRecording.style.color = "orange";
}
}
}
}, SILENCE_DELAY_MS);
}
// Expose functions to global scope
window.logInAndStoreSession = logInAndStoreSession;
window.clearSession = clearSession;
window.copyToClipboard = copyToClipboard;
window.clearTranscription = clearTranscription;
window.startChat = startChat;
// Add logout button to the UI
const logoutButton = document.createElement('button');
logoutButton.textContent = 'Logout';
logoutButton.className = 'bg-red-500 hover:bg-red-700 text-white font-bold py-2 px-4 rounded mt-4';
logoutButton.onclick = logout;
document.querySelector('.container').appendChild(logoutButton);
</script>
<script src="https://cdn.webrtc-experiment.com/MediaStreamRecorder.js"></script>
</body>
</html>

View File

@ -1,15 +1,18 @@
const express = require('express')
const bodyParser = require('body-parser')
const WebSocket = require('ws')
const storage = require('node-persist')
const request = require('request')
const fs = require('fs')
const path = require('path')
const dotenv = require('dotenv')
const ollama = require('ollama')
const axios = require('axios')
const OpenAI = require('openai')
const Groq = require('groq-sdk')
const express = require('express');
const bodyParser = require('body-parser');
const WebSocket = require('ws');
const storage = require('node-persist');
const request = require('request');
const fs = require('fs');
const path = require('path');
const dotenv = require('dotenv');
const ollama = require('ollama');
const axios = require('axios');
const OpenAI = require('openai');
const Groq = require('groq-sdk');
const { PrismaClient } = require('@prisma/client');
const prisma = new PrismaClient();
// Load environment variables
dotenv.config({
@ -44,21 +47,51 @@ let queueCounter = 0
const sessions = new Map()
const chats = new Map() // Store chat rooms
// Initialize storage and load initial values
async function initStorage () {
await storage.init()
language = (await storage.getItem('language')) || language
storeRecordings =
(await storage.getItem('storeRecordings')) || storeRecordings
const storedChats = (await storage.getItem('chats')) || []
storedChats.forEach(chat => chats.set(chat.id, chat))
const storedSessions = (await storage.getItem('sessions')) || []
storedSessions.forEach(session => sessions.set(session.sessionId, session))
}
// User registration
app.post('/register', async (req, res) => {
const { username, password } = req.body;
try {
const user = await prisma.user.create({
data: { username, password: hashPassword(password) },
});
res.status(201).json({ message: 'User registered successfully', userId: user.id });
} catch (error) {
res.status(400).json({ error: 'Username already exists' });
}
});
initStorage()
// User login
app.post('/login', async (req, res) => {
const { username, password } = req.body;
const user = await prisma.user.findUnique({ where: { username } });
if (user && verifyPassword(password, user.password)) {
const session = await prisma.session.create({
data: { userId: user.id, lastLogin: new Date() },
});
res.json({ sessionId: session.id, userId: user.id });
} else {
res.status(401).json({ error: 'Invalid credentials' });
}
});
// // Initialize storage and load initial values
// async function initStorage () {
// await storage.init()
// language = (await storage.getItem('language')) || language
// storeRecordings =
// (await storage.getItem('storeRecordings')) || storeRecordings
// const storedChats = (await storage.getItem('chats')) || []
// storedChats.forEach(chat => chats.set(chat.id, chat))
// const storedSessions = (await storage.getItem('sessions')) || []
// storedSessions.forEach(session => sessions.set(session.sessionId, session))
// }
// initStorage()
// WebSocket Server
const wss = new WebSocket.Server({ port: PORT_WS })
@ -113,43 +146,56 @@ async function handleSessionId (ws) {
await storage.setItem('sessions', Array.from(sessions.values()))
}
async function handleJoin (ws, { username, language }) {
sessions.set(ws.sessionId, { username, sessionId: ws.sessionId, language })
ws.send(
JSON.stringify({
type: 'sessionId',
sessionId: ws.sessionId,
language,
storeRecordings
})
)
// Modified handleJoin function
async function handleJoin(ws, { username, language, sessionId }) {
const session = await prisma.session.update({
where: { id: sessionId },
data: { language },
include: { user: true },
});
ws.sessionId = sessionId;
ws.userId = session.userId;
ws.send(JSON.stringify({ type: 'sessionId', sessionId, language, storeRecordings }));
const userChats = Array.from(chats.values()).filter(chat =>
chat.participants.includes(ws.sessionId)
)
ws.send(JSON.stringify({ type: 'chats', chats: userChats }))
const userChats = await prisma.chat.findMany({
where: { participants: { some: { userId: session.userId } } },
include: { participants: true },
});
ws.send(JSON.stringify({ type: 'chats', chats: userChats }));
broadcastUserList()
}
broadcastUserList();
}
async function handleStartChat (ws, { users }) {
const chatId = generateChatId()
let participants = [ws.sessionId, ...users]
participants = [...new Set(participants)]
async function handleStartChat(ws, { users }) {
const chatId = generateChatId();
let participants = [ws.userId, ...users];
participants = [...new Set(participants)];
chats.set(chatId, { participants, messages: [] })
await storage.setItem('chats', Array.from(chats.values()))
const chat = await prisma.chat.create({
data: {
id: chatId,
participants: {
connect: participants.map(userId => ({ id: userId })),
},
},
include: { participants: true },
});
notifyParticipants(participants)
broadcastUserList()
}
notifyParticipants(participants);
broadcastUserList();
}
async function handleEnterChat (ws, { chatId }) {
const enteredChat = chats.get(chatId)
const currentSession = sessions.get(ws.sessionId)
currentSession.currentChat = chatId
if (enteredChat && enteredChat.participants.includes(ws.sessionId)) {
ws.send(JSON.stringify({ type: 'chat', chat: enteredChat }))
async function handleEnterChat(ws, { chatId }) {
const enteredChat = await prisma.chat.findUnique({
where: { id: chatId },
include: { participants: true, messages: true },
});
if (enteredChat && enteredChat.participants.some(p => p.id === ws.userId)) {
await prisma.session.update({
where: { id: ws.sessionId },
data: { currentChatId: chatId },
});
ws.send(JSON.stringify({ type: 'chat', chat: enteredChat }));
}
}
@ -177,35 +223,34 @@ function generateChatId () {
return Math.random().toString(36).substring(2)
}
function broadcastUserList () {
const userList = Array.from(sessions.values()).map(user => ({
username: user.username,
sessionId: user.sessionId,
currentChat: user.currentChat,
language: user.language
}))
async function broadcastUserList() {
const users = await prisma.session.findMany({
include: { user: true },
});
const userList = users.map(session => ({
username: session.user.username,
sessionId: session.id,
currentChat: session.currentChatId,
language: session.language,
}));
wss.clients.forEach(client => {
if (client.readyState === WebSocket.OPEN) {
client.send(JSON.stringify({ type: 'userList', users: userList }))
client.send(JSON.stringify({ type: 'userList', users: userList }));
}
});
}
})
}
function notifyParticipants (participants) {
function notifyParticipants(participants) {
participants.forEach(sessionId => {
const participantSocket = Array.from(wss.clients).find(
client => client.sessionId === sessionId
)
const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId);
if (participantSocket && participantSocket.readyState === WebSocket.OPEN) {
const userChats = Array.from(chats.entries())
.filter(([id, chat]) => chat.participants.includes(sessionId))
.map(([id, chat]) => ({ id, participants: chat.participants }))
participantSocket.send(
JSON.stringify({ type: 'chats', chats: userChats })
)
.map(([id, chat]) => ({ id, participants: chat.participants }));
participantSocket.send(JSON.stringify({ type: 'chats', chats: userChats }));
}
})
});
}
async function handleAudioData (ws, data) {

View File

@ -173,7 +173,7 @@
// count speaking and silence
if (averageVolume > threshold) {
if (autosend.checked && speakingCount == 0 && audioRecorder) {
console.log("startint new recording");
console.log("starting new recording");
soundDetected = false;
audioRecorder.stop();
audioRecorder.start();

40
web/prisma/schema.prisma Normal file
View File

@ -0,0 +1,40 @@
datasource db {
provider = "sqlite"
url = "file:./dev.db"
}
generator client {
provider = "prisma-client-js"
}
model User {
id String @id @default(uuid())
username String @unique
password String
sessions Session[]
chats Chat[]
}
model Session {
id String @id @default(uuid())
userId String
user User @relation(fields: [userId], references: [id])
language String?
lastLogin DateTime
currentChatId String?
}
model Chat {
id String @id
participants User[]
messages Message[]
}
model Message {
id String @id @default(uuid())
chatId String
chat Chat @relation(fields: [chatId], references: [id])
senderId String
content String
timestamp DateTime @default(now())
}