diff --git a/.env b/.env index cab7084..7e9f218 100644 --- a/.env +++ b/.env @@ -4,9 +4,11 @@ TTS_BACKEND_URL=https://api.tts.d-popov.com/ #TTS_BACKEND_URL=http://localhost:9001/asr #gpu 9002-cpu TTS_BACKEND_URL2=http://localhost:9002/asr TTS_BACKEND_URL3=http://192.168.0.10:9008/asr #gpu -TTS_BACKEND_URL4=http://192.168.0.10:9009/asr #cpu 9008-gpu -WS_URL=ws://localhost:8081 -SERVER_PORT_WS=8081 +#! TTS_BACKEND_URL4=http://192.168.0.10:9009/asr #cpu 9008-gpu +# WS_URL=ws://localhost:8080 +PUBLIC_HOSTNAME=tts.d-popov.com +WS_URL=wss://tts.d-popov.com +SERVER_PORT_WS=8080 SERVER_PORT_HTTP=3005 # aider @@ -14,14 +16,15 @@ AIDER_MODEL= AIDER_4=false #AIDER_35TURBO= -# OPENAI_API_KEY=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 +# OPENAI_API_KEY=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 # OPENAI_API_BASE=https://api.deepseek.com/v1 # OPENAI_API_KEY=sk-99df7736351f4536bd72cd64a416318a # AIDER_MODEL=deepseek-coder #deepseek-coder, deepseek-chat GROQ_API_KEY=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 -aider --model groq/llama3-70b-8192 +OPENAI_API_KEY=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 +# aider --model groq/llama3-70b-8192 # List models available from Groq -aider --models groq/ \ No newline at end of file +# aider --models groq/ \ No newline at end of file diff --git a/.env.demo b/.env.demo index 19023d3..fea3dc7 100644 --- a/.env.demo +++ b/.env.demo @@ -1,6 +1,23 @@ -TTS_BACKEND_URL=http://192.168.0.10:9008/asr -WS_URL=ws://192.168.0.10:9008:8081 -SERVER_PORT_WS=8081 -SERVER_PORT_HTTP=8080 \ No newline at end of file +# TTS_BACKEND_URL=http://192.168.0.10:9008/asr +# WS_URL=ws://192.168.0.10:9008 +# SERVER_PORT_WS=8081 +# SERVER_PORT_HTTP=8080 + +ENV_NAME=demo +TTS_API_URL=https://api.tts.d-popov.com/asr + +# LLN_MODEL=qwen2 +# LNN_API_URL=https://ollama.d-popov.com/api/generate + +LLN_MODEL=qwen2 +LNN_API_URL=https://ollama.d-popov.com/api/generate + +GROQ_API_KEY=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 +OPENAI_API_KEY=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 + +WS_URL=wss://tts.d-popov.com +PUBLIC_HOSTNAME=tts.d-popov.com +SERVER_PORT_HTTP=8080 +SERVER_PORT_WS=8081 \ No newline at end of file diff --git a/.env.development b/.env.development new file mode 100644 index 0000000..81cfce8 --- /dev/null +++ b/.env.development @@ -0,0 +1,16 @@ + +ENV_NAME=development +TTS_API_URL=https://api.tts.d-popov.com/asr + +# LLN_MODEL=qwen2 +# LNN_API_URL=https://ollama.d-popov.com/api/generate + +LLN_MODEL=qwen2 +LNN_API_URL=https://ollama.d-popov.com/api/generate + +GROQ_API_KEY=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 +OPENAI_API_KEY=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 + +WS_URL=ws://localhost:8080 +SERVER_PORT_WS=8080 +SERVER_PORT_HTTP=8080 diff --git a/.env.production b/.env.production index f3f5ef2..decab52 100644 --- a/.env.production +++ b/.env.production @@ -2,7 +2,7 @@ TTS_BACKEND_URL=http://localhost:9001/asr #gpu 9002-cpu TTS_BACKEND_URL2=http://localhost:9002/asr TTS_BACKEND_URL3=http://192.168.0.10:9008/asr #gpu -TTS_BACKEND_URL4=http://192.168.0.10:9009/asr #cpu 9008-gpu +#! TTS_BACKEND_URL4=http://192.168.0.10:9009/asr #cpu 9008-gpu WS_URL=ws://localhost:8081 SERVER_PORT_WS=8081 SERVER_PORT_HTTP=8080 diff --git a/.gitignore b/.gitignore index 25b2b34..b6c8ffa 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ agent-mobile/jdk/* agent-mobile/artimobile/supervisord.pid agent-pyter/lag-llama agent-pyter/google-chrome-stable_current_amd64.deb +web/.node-persist/* diff --git a/.vscode/launch.json b/.vscode/launch.json index 2f20d3e..8906574 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -27,10 +27,37 @@ } }, { - "name": "node: Launch server.js", + "name": "start chat-server.js", "type": "node", "request": "launch", - "program": "conda activate node && node web/server.js", + // "program": "${workspaceFolder}/web/chat-server.js", + "runtimeExecutable": "npm", // Use npm to run the script + "runtimeArgs": [ + "run", + "start:demo-chat" // The script to run + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "env": { + "NODE_ENV": "demo" + "OPENAI_API_KEY": + }, + "skipFiles": [ + "/**" + ] + }, + { + "name": "Launch server.js", + "type": "node", + "request": "launch", + // "program": "conda activate node && ${workspaceFolder}/web/server.js", + "program": "${workspaceFolder}/web/server.js", + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + "env": { + "CONDA_ENV": "node", //? + "NODE_ENV": "development" + }, "skipFiles": [ "/**" ] diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 6a0c2a1..e3c705c 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -86,17 +86,19 @@ "panel": "new" }, }, - { - "label": "conda-activate", - "type": "shell", - "command": "source ~/miniconda3/etc/profile.d/conda.sh && conda activate py && echo 'Activated Conda Environment (py)!'", - "problemMatcher": [], - "options": { - "shell": { - "executable": "/bin/bash", - "args": ["-c"] - } - } - } - ] + { + "label": "conda-activate", + "type": "shell", + "command": "source ~/miniconda3/etc/profile.d/conda.sh && conda activate ${input:condaEnv} && echo 'Activated Conda Environment (${input:condaEnv})!'", + "problemMatcher": [], + } + ], + "inputs": [ + { + "id": "condaEnv", + "type": "promptString", + "description": "Enter the Conda environment name", + "default": "py" + } + ] } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a5ef686..3cb3638 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ ENV NODE_ENV=demo # RUN apk update && apk add bash RUN apk update && apk add git -RUN npm install -g npm@latest +#RUN npm install -g npm@latest WORKDIR /app @@ -69,6 +69,7 @@ COPY ["package.json", "package-lock.json*", "npm-shrinkwrap.json*", "./"] # && mv node_modules ../ COPY . . RUN npm install +#RUN mpm install nodemon EXPOSE 8080 8081 diff --git a/Niki/trader/test-NNFX/dealer.py b/Niki/trader/test-NNFX/dealer.py new file mode 100644 index 0000000..d67ab1f --- /dev/null +++ b/Niki/trader/test-NNFX/dealer.py @@ -0,0 +1,11 @@ +import ccxt +import pandas as pd +exchange = ccxt.coinbase() +symbol = 'BTC/USDT' +timeframe = '1m' +ohlcv = exchange.fetch_ohlcv(symbol, timeframe) +df = pd.DataFrame(ohlcv, columns=['timestamp', 'open', 'high', 'low', 'close', 'volume']) +df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms') +df.set_index('timestamp', inplace=True) +# print(df.head()) +print(df) \ No newline at end of file diff --git a/Niki/trader/test-NNFX/examples/exchanges.py b/Niki/trader/test-NNFX/examples/exchanges.py new file mode 100644 index 0000000..7a150b5 --- /dev/null +++ b/Niki/trader/test-NNFX/examples/exchanges.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +import os +import sys +from pprint import pprint + +root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.append(root + '/python') + +import ccxt # noqa: E402 + + +print('CCXT Version:', ccxt.__version__) + +for exchange_id in ccxt.exchanges: + try: + exchange = getattr(ccxt, exchange_id)() + print(exchange_id) + # do what you want with this exchange + # pprint(dir(exchange)) + except Exception as e: + print(e) \ No newline at end of file diff --git a/Niki/trader/test-NNFX/strategy.pine b/Niki/trader/test-NNFX/strategy.pine new file mode 100644 index 0000000..9132d2f --- /dev/null +++ b/Niki/trader/test-NNFX/strategy.pine @@ -0,0 +1,99 @@ +//@version=5 +// https://www.youtube.com/watch?v=3fBLZgWSsy4 +strategy("NNFX Style Strategy with ADX, EMA, ATR SL and TP", overlay=true) + +// SSL Channel +period = input.int(title="SSL Period", defval=140) +smaHigh = ta.sma(high, period) +smaLow = ta.sma(low, period) +var float Hlv = na +Hlv := close > smaHigh ? 1 : close < smaLow ? -1 : nz(Hlv[1]) +sslDown = Hlv < 0 ? smaHigh : smaLow +sslUp = Hlv < 0 ? smaLow : smaHigh + +plot(sslDown, linewidth=2, color=color.red) +plot(sslUp, linewidth=2, color=color.lime) + +// T3 Indicator +length_fast = input.int(40, minval=1, title="Fast T3 Length") +length_slow = input.int(90, minval=1, title="Slow T3 Length") +b = 0.7 + +t3(x, length) => + e1 = ta.ema(x, length) + e2 = ta.ema(e1, length) + e3 = ta.ema(e2, length) + e4 = ta.ema(e3, length) + e5 = ta.ema(e4, length) + e6 = ta.ema(e5, length) + c1 = -b * b * b + c2 = 3 * b * b + 3 * b * b * b + c3 = -6 * b * b - 3 * b - 3 * b * b * b + c4 = 1 + 3 * b + b * b * b + 3 * b * b + c1 * e6 + c2 * e5 + c3 * e4 + c4 * e3 + +t3_fast = t3(close, length_fast) +t3_slow = t3(close, length_slow) + +plot(t3_fast, color=color.blue, title="T3 Fast") +plot(t3_slow, color=color.red, title="T3 Slow") + +// ADX Calculation +adxlen = input.int(100, title="ADX Smoothing") +dilen = input.int(110, title="DI Length") + +dirmov(len) => + up = ta.change(high) + down = -ta.change(low) + plusDM = na(up) ? na : (up > down and up > 0 ? up : 0) + minusDM = na(down) ? na : (down > up and down > 0 ? down : 0) + truerange = ta.rma(ta.tr(true), len) + plus = nz(100 * ta.rma(plusDM, len) / truerange) + minus = nz(100 * ta.rma(minusDM, len) / truerange) + [plus, minus] + +adx(dilen, adxlen) => + [plus, minus] = dirmov(dilen) + sum = plus + minus + adx = 100 * ta.rma(math.abs(plus - minus) / (sum == 0 ? 1 : sum), adxlen) + adx + +adx_value = adx(dilen, adxlen) +adx_ema_length = input.int(80, title="ADX EMA Length") +adx_ema = ta.ema(adx_value, adx_ema_length) + +plot(adx_value, title="ADX", color=color.orange) +plot(adx_ema, title="ADX EMA", color=color.purple) + +// ATR-based Stop Loss and Take Profit +atr_length = input.int(120, title="ATR Length") +atr_stop_loss_multiplier = input.float(10, title="ATR Stop Loss Multiplier") +atr_take_profit_multiplier = input.float(20, title="ATR Take Profit Multiplier") +atr = ta.atr(atr_length) + +// Strategy Logic +longCondition = ta.crossover(t3_fast, t3_slow) and adx_value > adx_ema and Hlv > 0 +shortCondition = ta.crossunder(t3_fast, t3_slow) and adx_value > adx_ema and Hlv < 0 + +exitLongCondition = ta.crossunder(t3_fast, t3_slow) or Hlv < 0 +exitShortCondition = ta.crossover(t3_fast, t3_slow) or Hlv > 0 + +// Debug plots +plotshape(series=longCondition, location=location.belowbar, color=color.green, style=shape.labelup, text="LONG") +plotshape(series=shortCondition, location=location.abovebar, color=color.red, style=shape.labeldown, text="SHORT") + +if (longCondition) + stopLoss = close - atr_stop_loss_multiplier * atr + takeProfit = close + atr_take_profit_multiplier * atr + strategy.entry("Long", strategy.long) + strategy.exit("Long TP/SL", from_entry="Long", stop=stopLoss, limit=takeProfit) +if (shortCondition) + stopLoss = close + atr_stop_loss_multiplier * atr + takeProfit = close - atr_take_profit_multiplier * atr + strategy.entry("Short", strategy.short) + strategy.exit("Short TP/SL", from_entry="Short", stop=stopLoss, limit=takeProfit) + +if (exitLongCondition) + strategy.close("Long") +if (exitShortCondition) + strategy.close("Short") \ No newline at end of file diff --git a/Niki/trader/test-NNFX/strategy.py b/Niki/trader/test-NNFX/strategy.py new file mode 100644 index 0000000..f47f90c --- /dev/null +++ b/Niki/trader/test-NNFX/strategy.py @@ -0,0 +1,89 @@ +import numpy as np +import pandas as pd + +class NNFXStrategy: + def __init__(self, ssl_period=140, t3_fast_length=40, t3_slow_length=90, + adx_len=100, di_len=110, adx_ema_length=80, + atr_length=120, atr_stop_loss_multiplier=10, atr_take_profit_multiplier=20): + self.ssl_period = ssl_period + self.t3_fast_length = t3_fast_length + self.t3_slow_length = t3_slow_length + self.adx_len = adx_len + self.di_len = di_len + self.adx_ema_length = adx_ema_length + self.atr_length = atr_length + self.atr_stop_loss_multiplier = atr_stop_loss_multiplier + self.atr_take_profit_multiplier = atr_take_profit_multiplier + + def sma(self, series, period): + return series.rolling(window=period).mean() + + def t3(self, series, length, b=0.7): + e1 = series.ewm(span=length).mean() + e2 = e1.ewm(span=length).mean() + e3 = e2.ewm(span=length).mean() + e4 = e3.ewm(span=length).mean() + e5 = e4.ewm(span=length).mean() + e6 = e5.ewm(span=length).mean() + c1 = -b * b * b + c2 = 3 * b * b + 3 * b * b * b + c3 = -6 * b * b - 3 * b - 3 * b * b * b + c4 = 1 + 3 * b + b * b * b + 3 * b * b + return c1 * e6 + c2 * e5 + c3 * e4 + c4 * e3 + + def adx(self, high, low, close, di_len, adx_len): + plus_dm = high.diff().clip(lower=0) + minus_dm = low.diff().clip(upper=0).abs() + tr = np.maximum.reduce([high - low, (high - close.shift()).abs(), (low - close.shift()).abs()]) + atr = tr.rolling(window=di_len).mean() + plus_di = 100 * (plus_dm.rolling(window=di_len).mean() / atr) + minus_di = 100 * (minus_dm.rolling(window=di_len).mean() / atr) + dx = 100 * (plus_di - minus_di).abs() / (plus_di + minus_di) + adx = dx.rolling(window=adx_len).mean() + adx_ema = adx.ewm(span=self.adx_ema_length).mean() + return adx, adx_ema + + def atr(self, high, low, close, atr_length): + tr = np.maximum.reduce([high - low, (high - close.shift()).abs(), (low - close.shift()).abs()]) + return tr.rolling(window=atr_length).mean() + + def generate_signals(self, data): + data['sma_high'] = self.sma(data['high'], self.ssl_period) + data['sma_low'] = self.sma(data['low'], self.ssl_period) + data['hlv'] = np.where(data['close'] > data['sma_high'], 1, np.where(data['close'] < data['sma_low'], -1, np.nan)) + data['hlv'] = data['hlv'].ffill().fillna(0) + data['ssl_down'] = np.where(data['hlv'] < 0, data['sma_high'], data['sma_low']) + data['ssl_up'] = np.where(data['hlv'] < 0, data['sma_low'], data['sma_high']) + + data['t3_fast'] = self.t3(data['close'], self.t3_fast_length) + data['t3_slow'] = self.t3(data['close'], self.t3_slow_length) + + data['adx'], data['adx_ema'] = self.adx(data['high'], data['low'], data['close'], self.di_len, self.adx_len) + + data['atr'] = self.atr(data['high'], data['low'], data['close'], self.atr_length) + + data['long_condition'] = (data['t3_fast'] > data['t3_slow']) & (data['adx'] > data['adx_ema']) & (data['hlv'] > 0) + data['short_condition'] = (data['t3_fast'] < data['t3_slow']) & (data['adx'] > data['adx_ema']) & (data['hlv'] < 0) + + data['exit_long_condition'] = (data['t3_fast'] < data['t3_slow']) | (data['hlv'] < 0) + data['exit_short_condition'] = (data['t3_fast'] > data['t3_slow']) | (data['hlv'] > 0) + + return data + + def apply_strategy(self, data): + data = self.generate_signals(data) + trades = [] + for i in range(1, len(data)): + if data['long_condition'].iloc[i]: + stop_loss = data['close'].iloc[i] - self.atr_stop_loss_multiplier * data['atr'].iloc[i] + take_profit = data['close'].iloc[i] + self.atr_take_profit_multiplier * data['atr'].iloc[i] + trades.append(('long', data.index[i], stop_loss, take_profit)) + elif data['short_condition'].iloc[i]: + stop_loss = data['close'].iloc[i] + self.atr_stop_loss_multiplier * data['atr'].iloc[i] + take_profit = data['close'].iloc[i] - self.atr_take_profit_multiplier * data['atr'].iloc[i] + trades.append(('short', data.index[i], stop_loss, take_profit)) + elif data['exit_long_condition'].iloc[i]: + trades.append(('exit_long', data.index[i])) + elif data['exit_short_condition'].iloc[i]: + trades.append(('exit_short', data.index[i])) + return trades diff --git a/_doc/_notes/readme.md b/_doc/_notes/readme.md new file mode 100644 index 0000000..25e9ed8 --- /dev/null +++ b/_doc/_notes/readme.md @@ -0,0 +1,9 @@ +# build: +docker build -t kevin-ai . +# start the project in container: +docker-compose up +# for debugging: +docker-compose -f docker-compose.debug.yml up + +# demo (only node) +# docker-compose -f docker-compose.demo.yml up diff --git a/docker-compose.debug.yml b/docker-compose.debug.yml index a3107c4..0f229c0 100644 --- a/docker-compose.debug.yml +++ b/docker-compose.debug.yml @@ -8,7 +8,13 @@ services: dockerfile: ./Dockerfile environment: NODE_ENV: development + # TTS_BACKEND_URL: https://tts.d-popov.com/asr + TTS_BACKEND_URL: http://192.168.0.10:9009/asr + WS_URL: wss://ws.ai.d-popov.com + SERVER_PORT_WS: 3001 + SERVER_PORT_HTTP: 3000 ports: - - 3000:3000 + - 23000:3000 + - 23001:3001 - 9229:9229 command: ["node", "--inspect=0.0.0.0:9229", "web/server.js"] diff --git a/docker-compose.demo.yml b/docker-compose.demo.yml new file mode 100644 index 0000000..0cc78d3 --- /dev/null +++ b/docker-compose.demo.yml @@ -0,0 +1,15 @@ +version: '3.8' + +services: + node-app: + container_name: node-voice-chat + build: + context: . + dockerfile: web/deploy/demo.Dockerfile + ports: + - "8880:8080" # Exposes port 3000 on the host and maps it to port 3000 on the container + volumes: + - .:/usr/src/app # Mounts the current directory to /usr/src/app in the container + environment: + NODE_ENV: demo # Sets the environment variable NODE_ENV to development + command: npm run start:demo-chat # Runs npm start when the container starts diff --git a/docker-compose.yml b/docker-compose.yml index 31c1a7e..64b4241 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,18 +1,33 @@ version: '3.4' services: - kevinai: + # kevinai: + # image: kevinai + # container_name: kevinai-dev + # build: + # context: . + # dockerfile: ./Dockerfile + # environment: + # NODE_ENV: production + # # TTS_BACKEND_URL: http://192.168.0.10:9009/asr + # WS_URL: ws://192.168.0.10:28081 + # SERVER_PORT_WS: 8081 + # SERVER_PORT_HTTP: 8080 + # ports: + # - 28081:8081 + # - 28080:8080 + mlchat: image: kevinai - container_name: kevinai-dev + container_name: kevinai-chat build: context: . dockerfile: ./Dockerfile environment: - NODE_ENV: production - TTS_BACKEND_URL: http://192.168.0.10:9009/asr - WS_URL: ws://192.168.0.10:28081 + NODE_ENV: demo + # TTS_BACKEND_URL: http://192.168.0.10:9009/asr + WS_URL: wss://tts.d-popov.com SERVER_PORT_WS: 8081 SERVER_PORT_HTTP: 8080 ports: - - 28081:8081 - 28080:8080 + - 28081:8081 \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 572fe31..3cd4954 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,15 +8,60 @@ "name": "kevin-ai", "version": "1.0.0", "dependencies": { + "axios": "^1.7.2", "body-parser": "^1.20.2", - "dotenv": "^16.0.3", + "dotenv": "^16.4.5", "express": "^4.18.2", "git": "^0.1.5", + "groq-sdk": "^0.4.0", "node-persist": "^3.1.3", + "ollama": "^0.5.1", + "openai": "^4.50.0", "request": "^2.88.2", "ws": "^8.12.1" } }, + "node_modules/@types/node": { + "version": "18.19.34", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.34.tgz", + "integrity": "sha512-eXF4pfBNV5DAMKGbI02NnDtWrQ40hAN558/2vvS4gMpMIxaf6JmD7YjnZbq0Q9TDSSkKBamime8ewRoomHdt4g==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/node-fetch/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -29,6 +74,17 @@ "node": ">= 0.6" } }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -83,6 +139,29 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, + "node_modules/axios": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -246,11 +325,14 @@ } }, "node_modules/dotenv": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", - "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", "engines": { "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, "node_modules/ecc-jsbn": { @@ -288,6 +370,14 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/express": { "version": "4.18.2", "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", @@ -406,6 +496,25 @@ "node": ">= 0.8" } }, + "node_modules/follow-redirects": { + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -427,6 +536,31 @@ "node": ">= 0.12" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -485,6 +619,21 @@ "resolved": "https://registry.npmjs.org/mime/-/mime-1.2.9.tgz", "integrity": "sha512-WiLgbHTIq5AYUvU/Luli4mZ1bUcHpGNHyCsbl+KPMg4zt+XUDpQehWjuBjdLaEvDTinvKj/FgfQt3fPoT7j08g==" }, + "node_modules/groq-sdk": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.4.0.tgz", + "integrity": "sha512-h79q9sv4hcOBESR05N5eqHlGhAug9H9lr3EIiB+37ysWWekeG+KYQDK2lIIHYCm6O9LzgZzO/VdLdPP298+T0w==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + } + }, "node_modules/har-schema": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", @@ -557,6 +706,14 @@ "npm": ">=1.3.7" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -689,6 +846,43 @@ "node": ">= 0.6" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-gyp-build": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", @@ -725,6 +919,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/ollama": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/ollama/-/ollama-0.5.1.tgz", + "integrity": "sha512-mAiCHxdvu63E8EFopz0y82QG7rGfYmKAWgmjG2C7soiRuz/Sj3r/ebvCOp+jasiCubqUPE0ZThKT5LR6wrrPtA==", + "dependencies": { + "whatwg-fetch": "^3.6.20" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -736,6 +938,24 @@ "node": ">= 0.8" } }, + "node_modules/openai": { + "version": "4.50.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.50.0.tgz", + "integrity": "sha512-2ADkNIU6Q589oYHr5pn9k7SbUcrBTK9X0rIXrYqwMVSoqOj1yK9/1OO0ExaWsqOOpD7o58UmRjeKlx9gKAcuKQ==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -766,6 +986,11 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "node_modules/psl": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", @@ -990,6 +1215,11 @@ "node": ">=0.8" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -1018,6 +1248,11 @@ "node": ">= 0.6" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -1086,6 +1321,33 @@ "extsprintf": "^1.2.0" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/ws": { "version": "8.12.1", "resolved": "https://registry.npmjs.org/ws/-/ws-8.12.1.tgz", diff --git a/package.json b/package.json index e86a2c5..79e272c 100644 --- a/package.json +++ b/package.json @@ -5,14 +5,23 @@ "scripts": { "start": "node web/server.js", "start:demo": "NODE_ENV=demo node web/server.js", + "start:demo-chat": "node web/chat-server.js", "start:tele": "python agent-py-bot/agent.py" - }, + + }, + "env": { + "NODE_ENV": "demo" + }, "dependencies": { + "axios": "^1.7.2", "body-parser": "^1.20.2", - "dotenv": "^16.0.3", + "dotenv": "^16.4.5", "express": "^4.18.2", "git": "^0.1.5", + "groq-sdk": "^0.4.0", "node-persist": "^3.1.3", + "ollama": "^0.5.1", + "openai": "^4.50.0", "request": "^2.88.2", "ws": "^8.12.1" } diff --git a/web/.env b/web/.env new file mode 100644 index 0000000..38be4ef --- /dev/null +++ b/web/.env @@ -0,0 +1,23 @@ + + +# TTS_BACKEND_URL=http://192.168.0.10:9008/asr +# WS_URL=ws://192.168.0.10:9008 +# SERVER_PORT_WS=8081 +# SERVER_PORT_HTTP=8080 + +ENV_NAME=development +TTS_API_URL=https://api.tts.d-popov.com/asr + +# LLN_MODEL=qwen2 +# LNN_API_URL=https://ollama.d-popov.com/api/generate + +LLN_MODEL=qwen2 +LNN_API_URL=https://ollama.d-popov.com/api/generate + +GROQ_API_KEY=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 +OPENAI_API_KEY=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 + +WS_URL=wss://tts.d-popov.com +PUBLIC_HOSTNAME=tts.d-popov.com +SERVER_PORT_WS=8080 +SERVER_PORT_HTTP=8080 \ No newline at end of file diff --git a/web/audio.js b/web/audio.js new file mode 100644 index 0000000..0a1920d --- /dev/null +++ b/web/audio.js @@ -0,0 +1,201 @@ +let selectedDeviceId = "default"; +export let serverTime; +export let recordButton; +export let socket; +let audioRecorder; +let audioStream; +let recording = false; +let connectionStatus; +let statusRecording; +let audioContext; +let volumeChecker; +let lastVolumes = new Array(5); +let averageVolume; +let silenceCount = 0; +let isSpeaking = false; +let soundDetected = false; +let speakingCount = 0; +let analyser = null; + +let SILENCE_DELAY_MS = 50; +let preDetect_IncludedAudio = 400; //ms +let soundCount_Threshold = 10; +let silenceCount_Threshold = 10; + +const volumeHistory = []; + +export function setSocket(newSocket) { + socket = newSocket; +} +export function setRecordButton(newRecordButton) { + recordButton = newRecordButton; + recordButton.addEventListener("click", toggleListening); +} + +export function InitAudioAnalyser(stream) { + audioContext = new AudioContext(); + const source = audioContext.createMediaStreamSource(stream); + analyser = audioContext.createAnalyser(); + analyser.fftSize = 2048; + analyser.smoothingTimeConstant = 0.8; + source.connect(analyser); +} + +export function startListening() { + recording = true; + navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000 } }) + .then((stream) => { + audioStream = stream; + + const audioContext = new AudioContext(); + const sourceNode = audioContext.createMediaStreamSource(audioStream); + const audioSampleRate = sourceNode.context.sampleRate; + + info.innerHTML = "Sample rate: " + audioSampleRate + " Hz"; + var preBuffer = []; + + const channelSplitter = audioContext.createChannelSplitter(2); + const channelMerger = audioContext.createChannelMerger(1); + sourceNode.connect(channelSplitter); + channelSplitter.connect(channelMerger, 0, 0); + const outputNode = channelMerger; + + const mediaStreamDestination = audioContext.createMediaStreamDestination(); + outputNode.connect(mediaStreamDestination); + const singleChannelStream = mediaStreamDestination.stream; + + audioRecorder = new MediaRecorder(singleChannelStream); + audioRecorder.start(); + audioRecorder.addEventListener("dataavailable", (event) => { + if (!soundDetected && autosend.checked) { + preBuffer = []; + preBuffer.push(event.data); + return; + } + if (event.data.size > 0) { + let data = event.data; + if (preBuffer.length > 0) { + sendAudioToServerPost(preBuffer); + } + sendAudioToServer(data); + soundDetected = false; + } + }); + + InitAudioAnalyser(stream); + }); + + recordButton.innerHTML = "Stop Talking"; + recordButton.classList.toggle('bg-red-500'); + recordButton.classList.toggle('bg-blue-500'); + recordButton.classList.toggle('hover:bg-blue-700'); +} + +export function stopListening() { + recording = false; + audioRecorder.stop(); + recordButton.innerHTML = "Push to Talk"; + recordButton.classList.toggle('bg-blue-500'); + recordButton.classList.toggle('bg-red-500'); + recordButton.classList.toggle('hover:bg-blue-700'); + clearInterval(volumeChecker); + if (audioStream) { + audioStream.getTracks().forEach(track => track.stop()); + audioStream = null; + } +} + +export function sendAudioToServerPost(data) { + const blob = new Blob(data, { type: "audio/ogg; codecs=opus" }); + var formData = new FormData(); + formData.append('file', data); + fetch('/upload', { + method: 'POST', + body: formData + }); +} + +export function sendAudioToServerJson(data) { + if (socket && socket.readyState === WebSocket.OPEN) { + + const binaryData = Buffer.from(base64AudioData, 'base64'); + socket.send(JSON.stringify({ type: 'audio', audiobase64: binaryData })); + serverTime = Date.now(); + if (!autosend.checked) { + transcription.placeholder = "Processing audio..."; + } + } +} +export function sendAudioToServer(data) { + if (socket && socket.readyState === WebSocket.OPEN) { + socket.send(data); + serverTime = Date.now(); + if (!autosend.checked) { + transcription.placeholder = "Processing audio..."; + } + } +} + +export function toggleListening() { + if (socket.readyState === WebSocket.OPEN) { + if (recording) { + stopListening(); + } else { + startListening(); + } + } +} + +export function initializeVolumeChecker() { + volumeChecker = setInterval(() => { + if (!audioContext) { + console.log("No audio context"); + return; + } + const frequencyData = new Uint8Array(analyser.frequencyBinCount); + analyser.getByteFrequencyData(frequencyData); + + let totalVolume = 0; + for (let i = 0; i < frequencyData.length; i++) { + totalVolume += frequencyData[i]; + } + averageVolume = totalVolume / frequencyData.length; + + volumeHistory.push(averageVolume); + if (volumeHistory.length > 100) { + volumeHistory.shift(); + } + + const threshold = volumeHistory.reduce((acc, curr) => acc + curr) / volumeHistory.length + 5; + const isSilent = averageVolume < threshold; + + if (averageVolume > threshold) { + if (autosend.checked && speakingCount == 0 && audioRecorder) { + soundDetected = false; + audioRecorder.stop(); + audioRecorder.start(); + } + speakingCount++; + if (speakingCount > soundCount_Threshold) { + statusRecording.innerHTML = "Listening..."; + statusRecording.style.color = "green"; + isSpeaking = true; + } + } else if (averageVolume - 5 < threshold) { + speakingCount = 0; + if (isSpeaking) { + silenceCount++; + if (silenceCount > silenceCount_Threshold) { + if (autosend.checked) { + soundDetected = true; + audioRecorder.stop(); + audioRecorder.start(); + } + isSpeaking = false; + statusRecording.innerHTML = "Silence detected..."; + statusRecording.style.color = "orange"; + } + } + } + }, SILENCE_DELAY_MS); +} diff --git a/web/chat-client.html b/web/chat-client.html new file mode 100644 index 0000000..d51d6a4 --- /dev/null +++ b/web/chat-client.html @@ -0,0 +1,615 @@ + + + + + Real-time Voice Chat + + + + + +
+

Real-time Voice Chat

+ +
+ + + + + + + +
+ + + + + + + + + + +
+
+
+
+
+
+
status
+
+ + + + + + + \ No newline at end of file diff --git a/web/chat-server.js b/web/chat-server.js new file mode 100644 index 0000000..6585318 --- /dev/null +++ b/web/chat-server.js @@ -0,0 +1,403 @@ +const express = require('express'); +const bodyParser = require('body-parser'); +const WebSocket = require('ws'); +const storage = require('node-persist'); +const request = require('request'); +const fs = require('fs'); +const path = require('path'); +const dotenv = require('dotenv'); +const ollama = require('ollama'); +const axios = require('axios'); +const OpenAI = require('openai'); +const Groq = require('groq-sdk'); + +// Load environment variables +dotenv.config({ path: `.env${process.env.NODE_ENV === 'development' ? '.development' :'.'+ process.env.NODE_ENV }` }); + +// Initialize services +const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); +const groq = new Groq({ apiKey: process.env.GROQ_API_KEY }); + +// Express setup +const app = express(); +app.use(bodyParser.json()); + +// Configuration constants +const PORT_HTTP = process.env.SERVER_PORT_HTTP || 3000; +const PORT_WS = process.env.SERVER_PORT_WS || 8080; +const TTS_API_URL = process.env.TTS_API_URL; +const LNN_API_URL = process.env.LNN_API_URL; +const LLN_MODEL = process.env.LLN_MODEL; + +let language = "en"; +let storeRecordings = false; +let queueCounter = 0; + +const sessions = new Map(); +const chats = new Map(); // Store chat rooms + +// Initialize storage and load initial values +async function initStorage() { + await storage.init(); + language = await storage.getItem('language') || language; + storeRecordings = await storage.getItem('storeRecordings') || storeRecordings; + + const storedChats = await storage.getItem('chats') || []; + storedChats.forEach(chat => chats.set(chat.id, chat)); + + const storedSessions = await storage.getItem('sessions') || []; + storedSessions.forEach(session => sessions.set(session.sessionId, session)); +} + +initStorage(); + +// WebSocket Server +const wss = new WebSocket.Server({ port: PORT_WS }); +wss.on('connection', ws => { + ws.on('message', async message => handleMessage(ws, message)); + ws.on('close', () => handleClose(ws)); +}); + +// Handle WebSocket messages +async function handleMessage(ws, message) { + let data; + try { + data = JSON.parse(message); + } catch { + return handleAudioData(ws, message); + } + + try { + switch (data.type) { + case 'sessionId': + await handleSessionId(ws); + break; + case 'join': + await handleJoin(ws, data); + break; + case 'startChat': + await handleStartChat(ws, data); + break; + case 'enterChat': + await handleEnterChat(ws, data); + break; + case 'reconnect': + await handleReconnect(ws, data); + break; + default: + console.log('Unknown message type:', data.type); + } + } catch (err) { + console.error('Failed to handle message', err); + } +} + +function handleClose(ws) { + sessions.delete(ws.sessionId); + broadcastUserList(); +} + +// Handlers for specific message types +async function handleSessionId(ws) { + ws.sessionId = generateSessionId(); + sessions.set(ws.sessionId, { language: 'en' }); + await storage.setItem('sessions', Array.from(sessions.values())); +} + +async function handleJoin(ws, { username, language }) { + sessions.set(ws.sessionId, { username, sessionId: ws.sessionId, language }); + ws.send(JSON.stringify({ type: 'sessionId', sessionId: ws.sessionId, language, storeRecordings })); + + const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId)); + ws.send(JSON.stringify({ type: 'chats', chats: userChats })); + + broadcastUserList(); +} + +async function handleStartChat(ws, { users }) { + const chatId = generateChatId(); + let participants = [ws.sessionId, ...users]; + participants = [...new Set(participants)]; + + chats.set(chatId, { participants, messages: [] }); + await storage.setItem('chats', Array.from(chats.values())); + + notifyParticipants(participants); + broadcastUserList(); +} + +async function handleEnterChat(ws, { chatId }) { + const enteredChat = chats.get(chatId); + const currentSession = sessions.get(ws.sessionId); + currentSession.currentChat = chatId; + if (enteredChat && enteredChat.participants.includes(ws.sessionId)) { + ws.send(JSON.stringify({ type: 'chat', chat: enteredChat })); + } +} + +async function handleReconnect(ws, { sessionId }) { + const userSession = sessions.get(sessionId); + if (userSession) { + sessions.set(ws.sessionId, userSession); + ws.sessionId = sessionId; + const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(ws.sessionId)); + ws.send(JSON.stringify({ type: 'chats', chats: userChats })); + } else { + console.log('Session not found:', sessionId); + } + broadcastUserList(); +} + +// Utility functions +function generateSessionId() { + return Math.random().toString(36).substring(2); +} + +function generateChatId() { + return Math.random().toString(36).substring(2); +} + +function broadcastUserList() { + const userList = Array.from(sessions.values()).map(user => ({ + username: user.username, + sessionId: user.sessionId, + currentChat: user.currentChat, + language: user.language + })); + + wss.clients.forEach(client => { + if (client.readyState === WebSocket.OPEN) { + client.send(JSON.stringify({ type: 'userList', users: userList })); + } + }); +} + +function notifyParticipants(participants) { + participants.forEach(sessionId => { + const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId); + if (participantSocket && participantSocket.readyState === WebSocket.OPEN) { + const userChats = Array.from(chats.entries()) + .filter(([id, chat]) => chat.participants.includes(sessionId)) + .map(([id, chat]) => ({ id, participants: chat.participants })); + participantSocket.send(JSON.stringify({ type: 'chats', chats: userChats })); + } + }); +} + +async function handleAudioData(ws, data) { + const sessionData = sessions.get(ws.sessionId); + let { language, task } = sessionData; + + const formData = { + task: task || 'transcribe', + language, + vad_filter: 'true', + output: 'json', + audio_file: { + value: data, + options: { filename: 'audio.ogg', contentType: 'audio/ogg' } + } + }; + + if (!language || language === 'auto') { + await detectLanguage(ws, formData); + } else { + await transcribeAudio(ws, formData, sessionData); + } +} + +async function detectLanguage(ws, formData) { + try { + const result = await requestPromise({ + method: 'POST', + url: TTS_API_URL.replace('/asr', '/detect-language'), + formData + }); + const { language_code } = JSON.parse(result); + if (language_code) { + const sessionData = sessions.get(ws.sessionId); + sessionData.language = language_code; + ws.send(JSON.stringify({ type: 'languageDetected', languageDetected: language_code })); + await transcribeAudio(ws, formData, sessionData); + } + } catch (err) { + console.error('Language detection failed:', err); + } +} + +async function transcribeAudio(ws, formData, sessionData) { + const start = new Date().getTime(); + queueCounter++; + + try { + if(sessionData.language) { + formData.language = sessionData.language; + } + formData.vad_filter = 'true'; + const body = await requestPromise({ method: 'POST', url: TTS_API_URL, formData }); + queueCounter--; + + const duration = new Date().getTime() - start; + ws.send(JSON.stringify({ + type: 'text', + queueCounter, + duration, + language: sessionData.language, + text: body + })); + + await handleChatTranscription(ws, body, sessionData); + + } catch (err) { + console.error('Transcription failed:', err); + } + + if (storeRecordings) { + const timestamp = Date.now(); + fs.mkdir('rec', { recursive: true }, err => { + if (err) console.error(err); + else { + fs.writeFile(`rec/audio${timestamp}.ogg`, formData.audio_file.value, err => { + if (err) console.error(err); + else console.log(`Audio data saved to rec/audio${timestamp}.ogg`); + }); + } + }); + } +} + +async function handleChatTranscription(ws, body, sessionData) { + if (sessionData.currentChat) { + const chat = chats.get(sessionData.currentChat); + if (chat) { + let msg = { sender: sessionData.username, text: body, translations: [] }; + chat.messages.push(msg); + + for (let sessionId of chat.participants) { + if (sessionId !== ws.sessionId) { + const targetLang = sessions.get(sessionId)?.language || 'en'; + if (targetLang !== sessionData.language) { + const translation = await translateText(body, sessionData.language, targetLang); + msg.translations.push({ language: targetLang, text: translation }); + + const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId); + if (participantSocket && participantSocket.readyState === WebSocket.OPEN) { + participantSocket.send(JSON.stringify({ type: 'text', text: `${sessionData.username}: ${translation}` })); + const audioBuffer = await generateSpeech(translation); + participantSocket.send(JSON.stringify({ type: 'audio', audio: audioBuffer.toString('base64') })); + } + } else { + const participantSocket = Array.from(wss.clients).find(client => client.sessionId === sessionId); + if (participantSocket && participantSocket.readyState === WebSocket.OPEN) { + participantSocket.send(JSON.stringify({ type: 'text', text: `${sessionData.username}: ${body}` })); + participantSocket.send(JSON.stringify({ type: 'audio', audio: formData.toString('base64') })); + } + } + } + } + } + } +} + +async function translateText(originalText, originalLanguage, targetLanguage) { + const prompt = `Translate this text from ${originalLanguage} to ${targetLanguage}: ${originalText}`; + + const response = await groq.chat.completions.create({ + messages: [ + { + role: "system", + content: `You are translating voice transcriptions from '${originalLanguage}' to '${targetLanguage}'. Reply with just the translation.`, + }, + { + role: "user", + content: originalText, + }, + ], + model: "llama3-8b-8192", + }); + + return response.choices[0]?.message?.content || ""; +} + +async function generateSpeech(text) { + const mp3 = await openai.audio.speech.create({ + model: "tts-1", + voice: "alloy", + input: text, + }); + return Buffer.from(await mp3.arrayBuffer()); +} + +// HTTP Server +app.get('/', (req, res) => { + res.sendFile(path.join(__dirname, 'chat-client.html')); +}); + +app.get('/audio.js', (req, res) => { + res.sendFile(path.join(__dirname, 'audio.js')); +}); + +app.post('/log', (req, res) => { + console.log(`[LOG ${new Date().toISOString()}] ${req.body.message}`); + res.status(200).send('OK'); +}); + +app.get('/wsurl', (req, res) => { + if(process.env.PUBLIC_HOSTNAME){ + process.env.WS_URL = `wss://${process.env.PUBLIC_HOSTNAME}` + } + console.log('Request for WS URL resolved with:', process.env.WS_URL ); + res.status(200).send(process.env.WS_URL); +}); + +app.get('/settings', async (req, res) => { + if (req.query.language) { + language = req.query.language; + await storage.setItem('language', language); + } + if (req.query.storeRecordings) { + storeRecordings = req.query.storeRecordings; + await storage.setItem('storeRecordings', storeRecordings); + } + res.status(200).send({ language, storeRecordings }); +}); + +app.post('/settings', async (req, res) => { + const { sessionId, language, storeRecordings, task } = req.body; + const sessionData = sessions.get(sessionId); + if (language) sessionData.language = language; + if (storeRecordings !== undefined) sessionData.storeRecordings = storeRecordings; + if (task) sessionData.task = task; + res.status(200).send('OK'); +}); + +app.post('/upload', (req, res) => { + const timestamp = Date.now(); + console.log('Received audio data:', timestamp); + fs.mkdir('rec', { recursive: true }, err => { + if (err) return res.status(500).send('ERROR'); + const file = fs.createWriteStream(`rec/audio_slice_${timestamp}.ogg`); + req.pipe(file); + file.on('finish', () => res.status(200).send('OK')); + }); +}); + +app.get('/chats', (req, res) => { + const { username } = req.query; + const userChats = Array.from(chats.values()).filter(chat => chat.participants.includes(username)); + res.status(200).send({ chats: userChats }); +}); + +app.listen(PORT_HTTP, () => { + console.log(`Server listening on port ${PORT_HTTP}`); +}); + +// Helper to wrap request in a promise +function requestPromise(options) { + return new Promise((resolve, reject) => { + request(options, (error, response, body) => { + if (error) return reject(error); + resolve(body); + }); + }); +} diff --git a/web/client.html b/web/client.html index 1b3801d..10eb79c 100644 --- a/web/client.html +++ b/web/client.html @@ -3,11 +3,9 @@ Real-time Speech-to-Text - + - + @@ -15,65 +13,61 @@

Rt STT

-
- -
-
+
-
+
- +
- +