x

Search in
Sort by:

Question Status:

Search help

  • Simple searches use one or more words. Separate the words with spaces (cat dog) to search cat,dog or both. Separate the words with plus signs (cat +dog) to search for items that may contain cat but must contain dog.
  • You can further refine your search on the search results page, where you can search by keywords, author, topic. These can be combined with each other. Examples
    • cat dog --matches anything with cat,dog or both
    • cat +dog --searches for cat +dog where dog is a mandatory term
    • cat -dog -- searches for cat excluding any result containing dog
    • [cats] —will restrict your search to results with topic named "cats"
    • [cats] [dogs] —will restrict your search to results with both topics, "cats", and "dogs"

Native Voice Recognition for Android

I've been working for days on getting the built in Google speech recognition to work in Android but hit a wall where the "Speechrecognizer methods from service has no main thread" (see: https://stackoverflow.com/questions/...hread-or-activ - and no, the answer there hasn't helped me much) I am working, of course in concert with C++ and Java (jni) to create this plugin. By no means am I an expert in either C++ or Android Java jni, but I did get to this failure, where it would have worked if Speechrecognizer just worked without this special regulation. I should mention, that I did get the recognition to work with a microphone popup, but that isn't something I want (nor should most who use Unreal and want to use their own UI, I'd think). I hoped to get a silently working background recognition going. If anyone is interested in some of the code I've written so far to "jump start" I'll share it, let me know. I am aware of the Sphinx port, and it is nice and good for a prototype, having the native recognition for unlimited words/phrases is what I am looking for.

Here is a snippet of one of my three tries with VoiceRecog_APL.xml if this gives anyone else a head start. Note I give credit/attribution in the code where most of this came from.

     <!-- optional additions to the GameActivity imports in GameActivity.java -->
     <gameActivityImportAdditions>
         <insert>
       import android.util.Log;
       import android.app.Activity;
       import android.content.res.Configuration;
       import android.os.Bundle;
       import android.app.Service;
       import android.content.Context;
       import android.content.Intent;
       import android.content.ServiceConnection;
       import android.content.ComponentName;
       import android.os.Handler;
       import android.os.IBinder;
       import android.os.Message;
       import android.os.Messenger;
       import android.os.PowerManager;
       import android.os.CountDownTimer;
       import android.os.RemoteException;
       import android.widget.Toast;
       import android.content.ClipboardManager;
       import android.hardware.Camera;
       import android.hardware.Camera.CameraInfo;
       import android.hardware.Camera.Parameters;
       import android.hardware.Camera.PreviewCallback;
       import android.graphics.SurfaceTexture;
       import android.graphics.ImageFormat;
       import android.graphics.PixelFormat;
       import java.util.ArrayList;
       import java.util.List;
       import java.lang.Object;
       import java.lang.ref.WeakReference;
       import java.io.IOException;
       import android.speech.RecognitionListener;
       import android.speech.RecognizerIntent;
       import android.speech.SpeechRecognizer;
       import android.media.AudioManager;
       import java.util.Timer;
     </insert>
     </gameActivityImportAdditions>
 
   <gameActivityClassAdditions>
         <insert>
 
 
       /* Bulk of this comes from the thread: https://stackoverflow.com/questions/18650072/android-speech-speech-recognition-repeated-calling-of-speechrecognizer-startlis */
 
 public Context m_ctx;
 public class VoiceRecogService extends Service
 {
     protected AudioManager mAudioManager; 
     protected SpeechRecognizer mSpeechRecognizer;
     protected Intent mSpeechRecognizerIntent;
     protected RecognitionListener mSpeechRecognizerListner;
     //protected final Messenger mServerMessenger = new Messenger(new IncomingHandler(this));
 
     protected volatile boolean mIsListening;
     protected volatile boolean mIsCountDownOn;
 
     static final int MSG_RECOGNIZER_START_LISTENING = 1;
     static final int MSG_RECOGNIZER_CANCEL = 2;
 
     private int mBindFlag;
     private Messenger mServiceMessenger;
 
     
 
     private Handler mHandler = new Handler();
     //private boolean m_bReadyForSpeechReceived = false;
 
     //@Override
     public void onStartCommand()  //public void onCreate()
     {
         //super.onCreate();
         m_ctx = this;
 
         mAudioManager = (AudioManager) getSystemService(Context.AUDIO_SERVICE); 
 
         //do not mute beep when speech listening first kicks off
         Log.debug("TESTING: SPEECH SERVICE: CALL START"+ "onCreate()"); 
         startListening(false);
     }
     private void startListening(boolean bMuteSound){
         Log.debug("TESTING: SPEECH SERVICE: startListening()"); 
         if (bMuteSound==true <![CDATA[&&]]>Build.VERSION.SDK_INT >= 16)//Build.VERSION_CODES.JELLY_BEAN)
         {
             // turn off beep sound  
             mAudioManager.setStreamMute(AudioManager.STREAM_SYSTEM, true);
         }
         if (!mIsListening)
         {
              //mSpeechRecognizer.startListening(mSpeechRecognizerIntent);
              recognizeSpeechDirectly ();
              mIsListening = true;
 
         }
     }
 
     /////////////////////////////////////////////////////////////////////////
     /**
      * lazy initialize the speech recognizer
      */
     private SpeechRecognizer getSpeechRecognizer()
     {
         if (mSpeechRecognizer == null)
         {
             mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(m_ctx);
         }
         return mSpeechRecognizer;
     }
     private RecognitionListener getSpeechRecognizerListner()
     {
         if (mSpeechRecognizerListner == null)
         {
             mSpeechRecognizerListner = new SpeechRecognitionListener();
         }
         return mSpeechRecognizerListner;
     }
 
     private void recognizeSpeechDirectly()
     {
         Intent recognizerIntent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
         recognizerIntent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
         // accept partial results if they come
         recognizerIntent.putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true);
 
         recognizeSpeechDirectly(m_ctx,recognizerIntent, getSpeechRecognizerListner(), getSpeechRecognizer());
     }
     public void recognizeSpeechDirectly(Context context, 
                                                Intent recognizerIntent, 
                                                RecognitionListener listener,
                                                SpeechRecognizer recognizer)
     {
         //need to have a calling package for it to work
         if (!recognizerIntent.hasExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE))
         {
             recognizerIntent.putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, "com.dummy");
         }
 
         recognizer.setRecognitionListener(listener);
         recognizer.startListening(recognizerIntent);
     }
     ////////////////////////////////////////////////////////////////////////////
 
     public void stop()
     {
         if (getSpeechRecognizer() != null)
         {
             getSpeechRecognizer().stopListening();
             getSpeechRecognizer().cancel();
             getSpeechRecognizer().destroy();
 
             mIsListening = false;
             if (Build.VERSION.SDK_INT >= 16);//Build.VERSION_CODES.JELLY_BEAN)
                 mAudioManager.setStreamMute(AudioManager.STREAM_SYSTEM, false);
         }
     }
 
     // Count down timer for Jelly Bean work around
     protected CountDownTimer mNoSpeechCountDown = new CountDownTimer(5000, 5000)
     {
         @Override
         public void onTick(long millisUntilFinished)
         {
             // TODO Auto-generated method stub
         }
         @Override
         public void onFinish()
         {
             mIsCountDownOn = false;
             Log.debug("TESTING: SPEECH SERVICE: CALL START" + "onFinish()"); 
             startListening(true);
         }
     };
 
     @Override
     public void onDestroy()
     {
         super.onDestroy();
 
         if (mIsCountDownOn)
         {
             mNoSpeechCountDown.cancel();
         }
         if (mSpeechRecognizer != null)
         {
             mSpeechRecognizer.destroy();
         }
     }
 
     protected class SpeechRecognitionListener implements RecognitionListener
     {
         @Override
         public void onReadyForSpeech(Bundle params)
         {
             if (Build.VERSION.SDK_INT >= 16)//Build.VERSION_CODES.JELLY_BEAN)
             {
                 mIsCountDownOn = true;
                 mNoSpeechCountDown.start();
             }
             Log.debug("TESTING: SPEECH SERVICE"+ "onReadyForSpeech"); 
         }
         @Override
         public void onBeginningOfSpeech()
         {
             // speech input will be processed, so there is no need for count down anymore
             if (mIsCountDownOn)
             {
                 mIsCountDownOn = false;
                 mNoSpeechCountDown.cancel();
             }               
         }
         @Override
         public void onEndOfSpeech()
         {
             Log.debug("TESTING: SPEECH SERVICE" + "onEndOfSpeech"); 
         }
 
         @Override
         public void onBufferReceived(byte[] buffer)
         {
             //Log.debug("TESTING: SPEECH SERVICE"+ buffer + new String(new byte[] {0x63})); 
         }
 
         @Override
         public void onError(int error)
         {
             if ((error == SpeechRecognizer.ERROR_NO_MATCH)
                     || (error == SpeechRecognizer.ERROR_SPEECH_TIMEOUT)){
                 if (mIsCountDownOn)
                 {
                     mIsCountDownOn = false;
                     mNoSpeechCountDown.cancel();
                 }
                  mIsListening = false;
                  Log.debug("TESTING: SPEECH SERVICE: CALL START" + "onError()"); 
                  startListening(true);
             }
         }
 
         @Override
         public void onEvent(int eventType, Bundle params)
         {
 
         }
 
         @Override
         public void onPartialResults(Bundle partialResults)
         {
 
         }
 
         @Override
         public void onResults(Bundle results)
         {
              //String str = new String();
              //Log.d(TAG, "onResults " + results);
              ArrayList data = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
 
              //if(data.size() >=1){
              //  //check for save it:
              //}
 
              for (int i = 0; i <![CDATA[<]]> data.size(); i++)
              {
                  Log.debug("TESTING: SPEECH SERVICE "+(String)data.get(i));
              }
 
              //if no "save it" somewhere in there, then continue:
              if (mIsCountDownOn)
              {
                  mIsCountDownOn = false;
              }
              mIsListening = false;
              Log.debug("TESTING: SPEECH SERVICE: CALL START"+ "onResults()"); 
 
              startListening(true);
         }
         @Override
         public void onRmsChanged(float rmsdB)
         {
 
         }
     }
     @Override
     public IBinder onBind(Intent arg0) {
         // TODO Auto-generated method stub
         return null;
     }
 }
 
       public VoiceRecogService sVoiceRecognitionTest = new VoiceRecogService();
      
       //public listenUp heylistenUp = new listenUp();
       
       public Context activityContext;
 
       public void AndroidThunkJava_VoiceRecog()
       {
       //m_ctx = this;
       //sVoiceRecognitionTest.startListening(false);
             try
       {
     _activity.runOnUiThread(new Runnable()
     {
       public void run()
       {
       sVoiceRecognitionTest.onStartCommand();
       }
      });
       } // end of try
       catch (Exception e)
       {
       Log.debug("Recognition failed with exception " + e.getMessage());
       }
 

Product Version: UE 4.20
Tags:
more ▼

asked Nov 13 '18 at 12:12 AM in Everything Else

avatar image

RobMilliken
22 4 7

avatar image Naitguolf Nov 24 '18 at 01:16 PM

Hey! This looks really cool and fun to use? I hope you dont mind to share the plugin! :) Love to test it if needed!

avatar image RobMilliken Nov 25 '18 at 11:56 PM

Currently finessing. Successfully removed the beeping noises only the other day for start/end speech. Also communicating from .jni to blueprint requires unique thinking (eg: kludgy/not private and probably can be done better) though I've done it successfully. That and two projects I need to complete and a ton of cleanup. But if someone doesn't beat me to it, I'll upload a plugin as time permits. Thanks for interest!

avatar image Naitguolf Nov 26 '18 at 09:52 AM

You are very kind. Many thanks! :)

(comments are locked)
10|2000 characters needed characters left
Viewable by all users

1 answer: sort voted first

Okay. After days on working on this, I post the question here and within hours I believe I have figured it out. Note to myself. Ask questions here on here more often. Not only did I figure it out, I got it to request for voice recognition repeatedly after a phrase is done:

Beep Talk talk talk talk, I pause and Beep Beep Talk more... Beep Beep

... you get the idea. The odd thing is that I've tried _activity.runOnUiThread before and it didn't work because of context mix ups and nulls being passed (at least those were the errors). Anyhoo, all this works while still running a camera plugin showing a live view of what my camera sees in the background of a 3D scene (doesn't hurt that I am using a Pixel 2 in testing). Nice. I am so excited, I am going to go to bed before I write the getting results strings and parsing them, though I am certain there shouldn't be a problem there. Long term, I'll try to post a free plugin if someone doesn't beat me to it after reading this thread. At this hour for those interested in looking at my code snippets, feel free. The imports are the same as in the question.

 /* This code is based on this stackoverflow answer: https://stackoverflow.com/questions/52366828/speech-recognizer-not-working-when-called-normally-other-than-button-event-like */
 public SpeechRecognizer mSpeechRecognizer;
 public Intent mSpeechRecognizerIntent;
 
     RecognitionListener recognitionListener = (new RecognitionListener() {
         @Override
         public void onReadyForSpeech(Bundle params) {
 
         }
 
         @Override
         public void onBeginningOfSpeech() {
 
         }
 
         @Override
         public void onRmsChanged(float rmsdB) {
 
         }
 
         @Override
         public void onBufferReceived(byte[] buffer) {
 
         }
 
         @Override
         public void onEndOfSpeech() {
 
         }
 
         @Override
         public void onError(int error) {
             String message;
             switch (error) {
                 case SpeechRecognizer.ERROR_AUDIO:
                     message = "Audio recording error";
                     break;
                 case SpeechRecognizer.ERROR_CLIENT:
                     message = "Client side error";
                     break;
                 case SpeechRecognizer.ERROR_INSUFFICIENT_PERMISSIONS:
                     message = "Insufficient permissions";
                     break;
                 case SpeechRecognizer.ERROR_NETWORK:
                     message = "Network error";
                     break;
                 case SpeechRecognizer.ERROR_NETWORK_TIMEOUT:
                     message = "Network timeout";
                     break;
                 case SpeechRecognizer.ERROR_NO_MATCH:
                     message = "No match";
                     break;
                 case SpeechRecognizer.ERROR_RECOGNIZER_BUSY:
                     message = "RecognitionService busy";
                     break;
                 case SpeechRecognizer.ERROR_SERVER:
                     message = "error from server";
                     break;
                 case SpeechRecognizer.ERROR_SPEECH_TIMEOUT:
                     message = "No speech input";
                     break;
                 default:
                     message = "Didn't understand, please try again.";
                     break;
             }
             //textView.setText(message);
         }
 
         @Override
         public void onResults(Bundle results) {
             //ArrayList String matches = results.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION);
             //textView.setText(matches.get(0));
 
             //displaying the first match
             //if (matches != null)
                 //processResult(matches.get(0));
                  try
                   {
                 _activity.runOnUiThread(new Runnable()
                 {
                   public void run()
                   {
                   mSpeechRecognizer.startListening(mSpeechRecognizerIntent);
                   }
                  });
                   } // end of try
           catch (Exception e)
           {
           Log.debug("Recognition failed with exception " + e.getMessage());
           }
                     Log.debug("!!!!!!!!!!!!!!***************ON RESULTS CALLED*********************!!!!!!!!!!!!!!!!!");
         }
         
 
         @Override
         public void onPartialResults(Bundle partialResults) {
 
         }
 
         @Override
         public void onEvent(int eventType, Bundle params) {
 
         }
 });


Then toward the end of the XML add this to the oncreate additions for the first time:

 <!-- optional additions to GameActivity onCreate in GameActivity.java -->
     <gameActivityOnCreateAdditions>
         <insert>   
     mSpeechRecognizer = SpeechRecognizer.createSpeechRecognizer(this);
     mSpeechRecognizer.setRecognitionListener(recognitionListener);
 
     mSpeechRecognizerIntent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH);
     mSpeechRecognizerIntent.putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL,
             RecognizerIntent.LANGUAGE_MODEL_FREE_FORM);
     mSpeechRecognizerIntent.putExtra(RecognizerIntent.EXTRA_LANGUAGE,
             Locale.getDefault());
       mSpeechRecognizer.startListening(mSpeechRecognizerIntent);
  
     Log.debug("!!!!!!!!!!!!!!***************CREATE CALLED*********************!!!!!!!!!!!!!!!!!");
     
         </insert>
     </gameActivityOnCreateAdditions>



more ▼

answered Nov 13 '18 at 08:00 AM

avatar image

RobMilliken
22 4 7

(comments are locked)
10|2000 characters needed characters left
Viewable by all users
Your answer
toggle preview:

Up to 5 attachments (including images) can be used with a maximum of 5.2 MB each and 5.2 MB total.

Follow this question

Once you sign in you will be able to subscribe for any updates here

Answers to this question