gogo2/agent-mobile/artimobile/VoiceHandler.js

// VoiceHandler.js
import React, { Component } from 'react';
import { View, Text, Button } from 'react-native';
import Voice from '@react-native-voice/voice';

// import Config from 'react-native-config';
// process.env.TTS_BACKEND_URL = Config.TTS_BACKEND_URL;
// process.env.TTS_BACKEND_URL = "http://192.168.0.10:9008/asr"
process.env.TTS_BACKEND_URL = "https://tts.d-popov.com/asr"

process.env.LLM_ENDPOINT = "https://ws.ai.d-popov.com/api/chat";
// const LLM_ENDPOINT = "http://192.168.0.11:11434/api/chat";

process.env.LOG_ENDPOINT = "https://dev.d-popov.com/log";

class VoiceHandler extends Component {
  constructor(props) {
    super(props);
    this.state = {
      status: '',
      recognized: '',
      started: '',
      results: [],
      response:[],
      isRecording: false,
      isProcessing: false,
    };

    Voice.onSpeechStart = this.onSpeechStart.bind(this);
    Voice.onSpeechRecognized = this.onSpeechRecognized.bind(this);
    Voice.onSpeechResults = this.onSpeechResults.bind(this);
  }

  

  componentWillUnmount() {
    Voice.destroy().then(Voice.removeAllListeners);
  }

  onSpeechStart(e) {
    this.setState({
      started: '√',
      status: "listening..."
    });
  }

  onSpeechRecognized(e) {
    this.setState({
      status: "Recognized"
    });
    this.logRemote("onSpeechRecognized()");
  }

  onSpeechResults(e) {
    this.setState({
      results: e.value,
      recognized: recognized + " " + e.value,
      status: this.state.status+ "\nonSpeechResults():" + e.value 
    });
    
    this.logRemote("onSpeechResults():" + e.value);
  }


  async logRemote(message){
    try{
       fetch(process.env.LOG_ENDPOINT, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({"message":message}),
      });
    }
    catch(e){
      console.log("error logging:" + e);
    }
    console.log( message );
  }

  async _startRecognizing(e) {
    this.setState({
      recognized: '',
      started: '',
      results: [],
      status: "Starting...",
      isRecording: true,
    });

    try {
      await Voice.start('en-US'); // Start the voice recognition
    } catch (error) {
      console.error('There was an error starting voice recognition:', error);
      this.setState({
        isRecording: false,
      });
    }
  }
  async _stopRecognizing() {
    try {
      await Voice.stop();
      this.setState({
        isRecording: false,
        isProcessing:true,
        recognized: this.state.results.join(' ')
      });
      // Assuming you have the audio data, send it to your backend
      // this._sendTranscribedTextToLLM("who is the president of thr USA");
      this._sendTranscribedTextToLLM(this.state.results.join(' '));
    } catch (e) {
      console.error(e);
    }
  }
  async _sendTranscribedTextToLLM(transcribedText) {

    const model = "openhermes:latest";
    const prompt = "I have a question. Answer briefly and precise as an expert: \n" + transcribedText ;
    const data = {
      model: model,
      messages: [{ role: "user", content: `${prompt}`}],
      stream: false,
    };
    this.setState({
      status: this.state.status +  "\nsending to LLM:\n" + prompt
    })
    
    try {
      this.logRemote('sending text to LLM at ' + process.env.LLM_ENDPOINT +  ": '" +  transcribedText +  "'");
      const response = await fetch(process.env.LLM_ENDPOINT, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify(data),
      });

      if (response.ok) {
        const responseJson = await response.json();
        if (responseJson.error) {
          console.error("LLM Error:", responseJson.error);
          // Handle error appropriately in your app
        } else {
          // Handle successful response
          this.logRemote('LLM Response:', responseJson.message);
          // Update your app state or UI based on LLM response
          this.setState(prevState => ({
            status: "LLM responded",
            results: [...prevState.results, responseJson.message.content], // Append the response to the existing results
          }));
        }
      } else {
        // Handle HTTP errors
        console.error("HTTP Error:", response.status);
      }
    } catch (error) {
      console.error('Request failed:', error);
      // Handle request error
    }

    finally{
      this.setState({
        isProcessing:false
      });
    }

  }

  _sendAudioToBackend(results) {
    // Placeholder: Convert `results` or actual audio data to a format acceptable by your backend
    const formData = new FormData();
    //formData.append('audio', {uri: 'path_to_audio_file', type: 'audio/x-m4a', name: 'audio.m4a'});

    fetch(process.env.TTS_BACKEND_URL, {
      method: 'POST',
      body: formData,
      headers: {
        'Content-Type': 'multipart/form-data',
      },
    })
      .then(response => response.text())
      .then(body => {
        this.logRemote('Audio sent to backend, response:', body);
        this.setState(prevState => ({
          results: [...prevState.results, body], // Append the response to the existing results
        }));
      })
      .catch(error => {
        console.error('Failed to send audio:', error);
      });
  }
  render() {
    return (
      <View>
        <Text>Press the button and start speaking.</Text>
        <Button
          onPress={() => this.state.isRecording ? this._stopRecognizing() : this._startRecognizing()}
          title={this.state.isRecording ? "Stop Recognizing" : "Start Recognizing"}
          color={this.state.isRecording ? "red" : this.state.isProcessing ? "orange" : "blue"} // Change color based on state
        />
        <Text>Status: {this.state.status}</Text>
        <Text>Recognized: {this.state.recognized}</Text>
        <Text>Started: {this.state.started}</Text>
        <Text>Results: </Text>
        <View>
          {this.state.results.map((r, index) => (
            <Text key={index}>{r}</Text>
          ))}
        </View>
      </View>
    );
  }
}

export default VoiceHandler;