// VoiceHandler.js import React, { Component } from 'react'; import { View, Text, Button } from 'react-native'; import Voice from '@react-native-voice/voice'; // import Config from 'react-native-config'; // process.env.TTS_BACKEND_URL = Config.TTS_BACKEND_URL; // process.env.TTS_BACKEND_URL = "http://192.168.0.10:9008/asr" process.env.TTS_BACKEND_URL = "https://tts.d-popov.com/asr" const LLM_ENDPOINT = "https://ws.ai.d-popov.com/api/chat"; // const LLM_ENDPOINT = "http://192.168.0.11:11434/api/chat"; const LOG_ENDPOINT = "ws://192.168.0.11:9999"; class VoiceHandler extends Component { constructor(props) { super(props); this.state = { status: '', recognized: '', started: '', results: [], isRecording: false, isProcessing: false, }; Voice.onSpeechStart = this.onSpeechStart.bind(this); Voice.onSpeechRecognized = this.onSpeechRecognized.bind(this); Voice.onSpeechResults = this.onSpeechResults.bind(this); } componentWillUnmount() { Voice.destroy().then(Voice.removeAllListeners); } onSpeechStart(e) { this.setState({ started: '√', status: "listening..." }); } onSpeechRecognized(e) { this.setState({ status: "Recognized" }); console.log("onSpeechRecognized()"); } onSpeechResults(e) { this.setState({ recognized: [...recognized, e.value], status: this.state.status+ "\nonSpeechResults():" + e.value }); console.log("onSpeechResults():" + e.value); } async _startRecognizing(e) { this.setState({ recognized: '', started: '', results: [], status: "Starting...", isRecording: true, }); try { await Voice.start('en-US'); // Start the voice recognition } catch (error) { console.error('There was an error starting voice recognition:', error); this.setState({ isRecording: false, }); } } async _stopRecognizing() { try { await Voice.stop(); this.setState({ isRecording: false, isProcessing:true, status: this.state.status+ "\nstopRecognizing()" + this.state.recognized }); // Assuming you have the audio data, send it to your backend this._sendTranscribedTextToLLM("who is the president of thr USA"); //this._sendTranscribedTextToLLM(this.state.recognized); } catch (e) { console.error(e); } } async _sendTranscribedTextToLLM(transcribedText) { const model = "openhermes:latest"; const prompt = "I have a question. Answer briefly and precise as an expert: \n" + transcribedText ; const data = { model: model, messages: [{ role: "user", content: `${prompt}`}], stream: false, }; this.setState({ status: this.state.status + "\nsending to LLM:\n" + prompt }) try { console.log('sending text to LLM at ', LLM_ENDPOINT, ": '", transcribedText, "'"); const response = await fetch(LLM_ENDPOINT, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify(data), }); if (response.ok) { const responseJson = await response.json(); if (responseJson.error) { console.error("LLM Error:", responseJson.error); // Handle error appropriately in your app } else { // Handle successful response console.log('LLM Response:', responseJson.message); // Update your app state or UI based on LLM response this.setState(prevState => ({ status: "LLM responded", results: [...prevState.results, responseJson.message.content], // Append the response to the existing results })); } } else { // Handle HTTP errors console.error("HTTP Error:", response.status); } } catch (error) { console.error('Request failed:', error); // Handle request error } finally{ this.setState({ isProcessing:false }); } } _sendAudioToBackend(results) { // Placeholder: Convert `results` or actual audio data to a format acceptable by your backend const formData = new FormData(); //formData.append('audio', {uri: 'path_to_audio_file', type: 'audio/x-m4a', name: 'audio.m4a'}); fetch(process.env.TTS_BACKEND_URL, { method: 'POST', body: formData, headers: { 'Content-Type': 'multipart/form-data', }, }) .then(response => response.text()) .then(body => { console.log('Audio sent to backend, response:', body); this.setState(prevState => ({ results: [...prevState.results, body], // Append the response to the existing results })); }) .catch(error => { console.error('Failed to send audio:', error); }); } render() { return ( Press the button and start speaking.