diff --git a/.env b/.env index f0ffaa4..aa301cf 100644 --- a/.env +++ b/.env @@ -7,4 +7,4 @@ TTS_BACKEND_URL3=http://192.168.0.10:9008/asr #gpu TTS_BACKEND_URL4=http://192.168.0.10:9009/asr #cpu 9008-gpu WS_URL=ws://localhost:8081 SERVER_PORT_WS=8081 -SERVER_PORT_HTTP=8080 \ No newline at end of file +SERVER_PORT_HTTP=3005 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2c73ccf..8e77a12 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,7 @@ rec/* */__pycache__/* __pycache__ agent-py-bot/scrape/raw/summary_log.txt -agent-py-bot/scrape/raw/* \ No newline at end of file +agent-py-bot/scrape/raw/* +.aider* +tts/*.m4a +agent-mobile/jdk/* diff --git a/.vscode/launch.json b/.vscode/launch.json index cc04cbc..d1eed27 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -25,7 +25,7 @@ }, { "name": "Docker Python Launch with venv", - "type": "python", + "type": "debugpy", "request": "launch", "program": "${workspaceFolder}/agent-py-bot/agent.py", "console": "integratedTerminal", @@ -33,7 +33,16 @@ "env": { "PYTHONUNBUFFERED": "1" } + }, + { + "name": "node: Launch server.js", + "type": "node", + "request": "launch", + "program": "conda activate node && node web/server.js", + "skipFiles": [ + "/**" + ] + } - ] } \ No newline at end of file diff --git a/agent-mobile/artimobile/.bundle/config b/agent-mobile/artimobile/.bundle/config new file mode 100644 index 0000000..848943b --- /dev/null +++ b/agent-mobile/artimobile/.bundle/config @@ -0,0 +1,2 @@ +BUNDLE_PATH: "vendor/bundle" +BUNDLE_FORCE_RUBY_PLATFORM: 1 diff --git a/agent-mobile/artimobile/.env b/agent-mobile/artimobile/.env new file mode 100644 index 0000000..dc8dba7 --- /dev/null +++ b/agent-mobile/artimobile/.env @@ -0,0 +1 @@ +TTS_BACKEND_URL=http://192.168.0.10:9008/asr diff --git a/agent-mobile/artimobile/.eslintrc.js b/agent-mobile/artimobile/.eslintrc.js new file mode 100644 index 0000000..187894b --- /dev/null +++ b/agent-mobile/artimobile/.eslintrc.js @@ -0,0 +1,4 @@ +module.exports = { + root: true, + extends: '@react-native', +}; diff --git a/agent-mobile/artimobile/.gitignore b/agent-mobile/artimobile/.gitignore new file mode 100644 index 0000000..0cab2ac --- /dev/null +++ b/agent-mobile/artimobile/.gitignore @@ -0,0 +1,66 @@ +# OSX +# +.DS_Store + +# Xcode +# +build/ +*.pbxuser +!default.pbxuser +*.mode1v3 +!default.mode1v3 +*.mode2v3 +!default.mode2v3 +*.perspectivev3 +!default.perspectivev3 +xcuserdata +*.xccheckout +*.moved-aside +DerivedData +*.hmap +*.ipa +*.xcuserstate +ios/.xcode.env.local + +# Android/IntelliJ +# +build/ +.idea +.gradle +local.properties +*.iml +*.hprof +.cxx/ +*.keystore +!debug.keystore + +# node.js +# +node_modules/ +npm-debug.log +yarn-error.log + +# fastlane +# +# It is recommended to not store the screenshots in the git repo. Instead, use fastlane to re-generate the +# screenshots whenever they are needed. +# For more information about the recommended setup visit: +# https://docs.fastlane.tools/best-practices/source-control/ + +**/fastlane/report.xml +**/fastlane/Preview.html +**/fastlane/screenshots +**/fastlane/test_output + +# Bundle artifact +*.jsbundle + +# Ruby / CocoaPods +/ios/Pods/ +/vendor/bundle/ + +# Temporary files created by Metro to check the health of the file watcher +.metro-health-check* + +# testing +/coverage diff --git a/agent-mobile/artimobile/.prettierrc.js b/agent-mobile/artimobile/.prettierrc.js new file mode 100644 index 0000000..2b54074 --- /dev/null +++ b/agent-mobile/artimobile/.prettierrc.js @@ -0,0 +1,7 @@ +module.exports = { + arrowParens: 'avoid', + bracketSameLine: true, + bracketSpacing: false, + singleQuote: true, + trailingComma: 'all', +}; diff --git a/agent-mobile/artimobile/.watchmanconfig b/agent-mobile/artimobile/.watchmanconfig new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/agent-mobile/artimobile/.watchmanconfig @@ -0,0 +1 @@ +{} diff --git a/agent-mobile/artimobile/App.tsx b/agent-mobile/artimobile/App.tsx new file mode 100644 index 0000000..125fe1b --- /dev/null +++ b/agent-mobile/artimobile/App.tsx @@ -0,0 +1,118 @@ +/** + * Sample React Native App + * https://github.com/facebook/react-native + * + * @format + */ + +import React from 'react'; +import type {PropsWithChildren} from 'react'; +import { + SafeAreaView, + ScrollView, + StatusBar, + StyleSheet, + Text, + useColorScheme, + View, +} from 'react-native'; + +import { + Colors, + DebugInstructions, + Header, + LearnMoreLinks, + ReloadInstructions, +} from 'react-native/Libraries/NewAppScreen'; + +type SectionProps = PropsWithChildren<{ + title: string; +}>; + +function Section({children, title}: SectionProps): React.JSX.Element { + const isDarkMode = useColorScheme() === 'dark'; + return ( + + + {title} + + + {children} + + + ); +} + +function App(): React.JSX.Element { + const isDarkMode = useColorScheme() === 'dark'; + + const backgroundStyle = { + backgroundColor: isDarkMode ? Colors.darker : Colors.lighter, + }; + + return ( + + + +
+ +
+ Edit App.tsx to change this + screen and then come back to see your edits. +
+
+ +
+
+ +
+
+ Read the docs to discover what to do next: +
+ + + + + ); +} + +const styles = StyleSheet.create({ + sectionContainer: { + marginTop: 32, + paddingHorizontal: 24, + }, + sectionTitle: { + fontSize: 24, + fontWeight: '600', + }, + sectionDescription: { + marginTop: 8, + fontSize: 18, + fontWeight: '400', + }, + highlight: { + fontWeight: '700', + }, +}); + +export default App; diff --git a/agent-mobile/artimobile/Gemfile b/agent-mobile/artimobile/Gemfile new file mode 100644 index 0000000..8d72c37 --- /dev/null +++ b/agent-mobile/artimobile/Gemfile @@ -0,0 +1,9 @@ +source 'https://rubygems.org' + +# You may use http://rbenv.org/ or https://rvm.io/ to install and use this version +ruby ">= 2.6.10" + +# Cocoapods 1.15 introduced a bug which break the build. We will remove the upper +# bound in the template on Cocoapods with next React Native release. +gem 'cocoapods', '>= 1.13', '< 1.15' +gem 'activesupport', '>= 6.1.7.5', '< 7.1.0' diff --git a/agent-mobile/artimobile/README.md b/agent-mobile/artimobile/README.md new file mode 100644 index 0000000..12470c3 --- /dev/null +++ b/agent-mobile/artimobile/README.md @@ -0,0 +1,79 @@ +This is a new [**React Native**](https://reactnative.dev) project, bootstrapped using [`@react-native-community/cli`](https://github.com/react-native-community/cli). + +# Getting Started + +>**Note**: Make sure you have completed the [React Native - Environment Setup](https://reactnative.dev/docs/environment-setup) instructions till "Creating a new application" step, before proceeding. + +## Step 1: Start the Metro Server + +First, you will need to start **Metro**, the JavaScript _bundler_ that ships _with_ React Native. + +To start Metro, run the following command from the _root_ of your React Native project: + +```bash +# using npm +npm start + +# OR using Yarn +yarn start +``` + +## Step 2: Start your Application + +Let Metro Bundler run in its _own_ terminal. Open a _new_ terminal from the _root_ of your React Native project. Run the following command to start your _Android_ or _iOS_ app: + +### For Android + +```bash +# using npm +npm run android + +# OR using Yarn +yarn android +``` + +### For iOS + +```bash +# using npm +npm run ios + +# OR using Yarn +yarn ios +``` + +If everything is set up _correctly_, you should see your new app running in your _Android Emulator_ or _iOS Simulator_ shortly provided you have set up your emulator/simulator correctly. + +This is one way to run your app — you can also run it directly from within Android Studio and Xcode respectively. + +## Step 3: Modifying your App + +Now that you have successfully run the app, let's modify it. + +1. Open `App.tsx` in your text editor of choice and edit some lines. +2. For **Android**: Press the R key twice or select **"Reload"** from the **Developer Menu** (Ctrl + M (on Window and Linux) or Cmd ⌘ + M (on macOS)) to see your changes! + + For **iOS**: Hit Cmd ⌘ + R in your iOS Simulator to reload the app and see your changes! + +## Congratulations! :tada: + +You've successfully run and modified your React Native App. :partying_face: + +### Now what? + +- If you want to add this new React Native code to an existing application, check out the [Integration guide](https://reactnative.dev/docs/integration-with-existing-apps). +- If you're curious to learn more about React Native, check out the [Introduction to React Native](https://reactnative.dev/docs/getting-started). + +# Troubleshooting + +If you can't get this to work, see the [Troubleshooting](https://reactnative.dev/docs/troubleshooting) page. + +# Learn More + +To learn more about React Native, take a look at the following resources: + +- [React Native Website](https://reactnative.dev) - learn more about React Native. +- [Getting Started](https://reactnative.dev/docs/environment-setup) - an **overview** of React Native and how setup your environment. +- [Learn the Basics](https://reactnative.dev/docs/getting-started) - a **guided tour** of the React Native **basics**. +- [Blog](https://reactnative.dev/blog) - read the latest official React Native **Blog** posts. +- [`@facebook/react-native`](https://github.com/facebook/react-native) - the Open Source; GitHub **repository** for React Native. diff --git a/agent-mobile/artimobile/Task b/agent-mobile/artimobile/Task new file mode 100644 index 0000000..e69de29 diff --git a/agent-mobile/artimobile/VoiceHandler.js b/agent-mobile/artimobile/VoiceHandler.js new file mode 100644 index 0000000..82fe92d --- /dev/null +++ b/agent-mobile/artimobile/VoiceHandler.js @@ -0,0 +1,212 @@ +// VoiceHandler.js +import React, { Component } from 'react'; +import { View, Text, Button } from 'react-native'; +import Voice from '@react-native-voice/voice'; + +// import Config from 'react-native-config'; +// process.env.TTS_BACKEND_URL = Config.TTS_BACKEND_URL; +// process.env.TTS_BACKEND_URL = "http://192.168.0.10:9008/asr" +process.env.TTS_BACKEND_URL = "https://tts.d-popov.com/asr" + +process.env.LLM_ENDPOINT = "https://ws.ai.d-popov.com/api/chat"; +// const LLM_ENDPOINT = "http://192.168.0.11:11434/api/chat"; + +process.env.LOG_ENDPOINT = "http://192.168.0.11:3005/log"; + +class VoiceHandler extends Component { + constructor(props) { + super(props); + this.state = { + status: '', + recognized: '', + started: '', + results: [], + isRecording: false, + isProcessing: false, + }; + + Voice.onSpeechStart = this.onSpeechStart.bind(this); + Voice.onSpeechRecognized = this.onSpeechRecognized.bind(this); + Voice.onSpeechResults = this.onSpeechResults.bind(this); + } + + + + componentWillUnmount() { + Voice.destroy().then(Voice.removeAllListeners); + } + + onSpeechStart(e) { + this.setState({ + started: '√', + status: "listening..." + }); + } + + onSpeechRecognized(e) { + this.setState({ + status: "Recognized" + }); + console.log("onSpeechRecognized()"); + } + + onSpeechResults(e) { + this.setState({ + recognized: [...recognized, e.value], + status: this.state.status+ "\nonSpeechResults():" + e.value + }); + + this.logRemote("onSpeechResults():" + e.value); + } + + + async logRemote(message){ + try{ + fetch(process.env.LOG_ENDPOINT, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({"message":message}), + }); + } + catch(e){ + console.log("error logging:" + e); + } + console.log( message ); + } + + async _startRecognizing(e) { + this.setState({ + recognized: '', + started: '', + results: [], + status: "Starting...", + isRecording: true, + }); + + try { + await Voice.start('en-US'); // Start the voice recognition + } catch (error) { + console.error('There was an error starting voice recognition:', error); + this.setState({ + isRecording: false, + }); + } + } + async _stopRecognizing() { + try { + await Voice.stop(); + this.setState({ + isRecording: false, + isProcessing:true, + status: this.state.status+ "\nstopRecognizing()" + this.state.recognized + }); + // Assuming you have the audio data, send it to your backend + this._sendTranscribedTextToLLM("who is the president of thr USA"); + //this._sendTranscribedTextToLLM(this.state.recognized); + } catch (e) { + console.error(e); + } + } + async _sendTranscribedTextToLLM(transcribedText) { + + const model = "openhermes:latest"; + const prompt = "I have a question. Answer briefly and precise as an expert: \n" + transcribedText ; + const data = { + model: model, + messages: [{ role: "user", content: `${prompt}`}], + stream: false, + }; + this.setState({ + status: this.state.status + "\nsending to LLM:\n" + prompt + }) + + try { + this.logRemote('sending text to LLM at ' + process.env.LLM_ENDPOINT + ": '" + transcribedText + "'"); + const response = await fetch(process.env.LLM_ENDPOINT, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify(data), + }); + + if (response.ok) { + const responseJson = await response.json(); + if (responseJson.error) { + console.error("LLM Error:", responseJson.error); + // Handle error appropriately in your app + } else { + // Handle successful response + console.log('LLM Response:', responseJson.message); + // Update your app state or UI based on LLM response + this.setState(prevState => ({ + status: "LLM responded", + results: [...prevState.results, responseJson.message.content], // Append the response to the existing results + })); + } + } else { + // Handle HTTP errors + console.error("HTTP Error:", response.status); + } + } catch (error) { + console.error('Request failed:', error); + // Handle request error + } + + finally{ + this.setState({ + isProcessing:false + }); + } + + } + + _sendAudioToBackend(results) { + // Placeholder: Convert `results` or actual audio data to a format acceptable by your backend + const formData = new FormData(); + //formData.append('audio', {uri: 'path_to_audio_file', type: 'audio/x-m4a', name: 'audio.m4a'}); + + fetch(process.env.TTS_BACKEND_URL, { + method: 'POST', + body: formData, + headers: { + 'Content-Type': 'multipart/form-data', + }, + }) + .then(response => response.text()) + .then(body => { + console.log('Audio sent to backend, response:', body); + this.setState(prevState => ({ + results: [...prevState.results, body], // Append the response to the existing results + })); + }) + .catch(error => { + console.error('Failed to send audio:', error); + }); + } + render() { + return ( + + Press the button and start speaking. +