gogo2/agent-mobile/artimobile/VoiceHandler.js

// VoiceHandler.js
import React, { Component } from 'react';
import { View, Text, Button } from 'react-native';
import Voice from '@react-native-voice/voice';

// import Config from 'react-native-config';
// process.env.TTS_BACKEND_URL = Config.TTS_BACKEND_URL;
process.env.TTS_BACKEND_URL = "http://192.168.0.10:9008/asr"

class VoiceHandler extends Component {
  constructor(props) {
    super(props);
    this.state = {
      status: '',
      recognized: '',
      started: '',
      results: [],
      isRecording: false,
      isProcessing: false,
    };

    Voice.onSpeechStart = this.onSpeechStart.bind(this);
    Voice.onSpeechRecognized = this.onSpeechRecognized.bind(this);
    Voice.onSpeechResults = this.onSpeechResults.bind(this);
  }

  componentWillUnmount() {
    Voice.destroy().then(Voice.removeAllListeners);
  }

  onSpeechStart(e) {
    this.setState({
      started: '√',
    });
  }

  onSpeechRecognized(e) {
    this.setState({
      recognized: '√',
    });
  }

  onSpeechResults(e) {
    this.setState({
      results: e.value,
    });
  }

  async _startRecognizing(e) {
    this.setState({
      recognized: '',
      started: '',
      results: [],
      isRecording: true,
    });

    try {
      await Voice.start('en-US'); // Start the voice recognition
    } catch (error) {
      console.error('There was an error starting voice recognition:', error);
      this.setState({
        isRecording: false,
      });
    }
  }
  async _stopRecognizing() {
    try {
      await Voice.stop();
      this.setState({
        isRecording: false,
        isProcessing:true,
        recognized: this.state.results.join(' ')
      });
      // Assuming you have the audio data, send it to your backend
      this._sendTranscribedTextToLLM(this.state.results.join(' '));
    } catch (e) {
      console.error(e);
    }
  }
  async _sendTranscribedTextToLLM(transcribedText) {

    const LLM_ENDPOINT = "http://192.168.0.11:11434/api/chat";
    const model = "openhermes:latest";
    const data = {
      model: model,
      messages: [{ role: "user", content: "I have a request: " + transcribedText }],
      stream: false,
    };
    this.setState({
      status: ["sending to LLM:" + transcribedText]
    })
    try {

      console.log('sending text to LLM at ', LLM_ENDPOINT, ": '", transcribedText, "'");
      const response = await fetch(LLM_ENDPOINT, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify(data),
      });

      if (response.ok) {
        const responseJson = await response.json();
        if (responseJson.error) {
          console.error("LLM Error:", responseJson.error);
          // Handle error appropriately in your app
        } else {
          // Handle successful response
          console.log('LLM Response:', responseJson.message);
          // Update your app state or UI based on LLM response
          this.setState(prevState => ({
            status: "LLM responded",
            results: [...prevState.results, responseJson.message.content], // Append the response to the existing results
          }));
        }
      } else {
        // Handle HTTP errors
        console.error("HTTP Error:", response.status);
      }
    } catch (error) {
      console.error('Request failed:', error);
      // Handle request error
    }

    finally{
      this.setState({
        isProcessing:false
      });
    }

  }
  _sendAudioToBackend(results) {
    // Placeholder: Convert `results` or actual audio data to a format acceptable by your backend
    const formData = new FormData();
    //formData.append('audio', {uri: 'path_to_audio_file', type: 'audio/x-m4a', name: 'audio.m4a'});

    fetch(process.env.TTS_BACKEND_URL, {
      method: 'POST',
      body: formData,
      headers: {
        'Content-Type': 'multipart/form-data',
      },
    })
      .then(response => response.text())
      .then(body => {
        console.log('Audio sent to backend, response:', body);
        this.setState(prevState => ({
          results: [...prevState.results, body], // Append the response to the existing results
        }));
      })
      .catch(error => {
        console.error('Failed to send audio:', error);
      });
  }
  render() {
    return (
      <View>
        <Text>Press the button and start speaking.</Text>
        <Button
          onPress={() => this.state.isRecording ? this._stopRecognizing() : this._startRecognizing()}
          title={this.state.isRecording ? "Stop Recognizing" : "Start Recognizing"}
          color={this.state.isRecording ? "red" : this.state.isProcessing ? "orange" : "blue"} // Change color based on state
        />
        <Text>Status: {this.state.status}</Text>
        <Text>Recognized: {this.state.recognized}</Text>
        <Text>Started: {this.state.started}</Text>
        <Text>Results: {this.state.results.join(' ')}</Text>
      </View>
    );
  }
}

export default VoiceHandler;