protected void InitSpeechEngine(bool def) { try { WSRConfig cfg = WSRConfig.GetInstance(); WSRSpeechManager manager = WSRSpeechManager.GetInstance(); // File manager.InitEngines(); // Default if (def) { manager.AddDefaultEngine("Default", cfg.language, cfg.confidence); } // RTP if (rtpClient == null) { return; } var format = new SpeechAudioFormatInfo(16000, AudioBitsPerSample.Sixteen, AudioChannel.Stereo); manager.AddEngine("RTP", cfg.language, cfg.confidence, rtpClient.AudioStream, format); } catch (Exception ex) { WSRConfig.GetInstance().logError("ENGINE", "InitEngines: " + ex.Message); } }
protected void recognizer_SpeechRecognized(object sender, SpeechRecognizedEventArgs e) { RecognitionResult rr = e.Result; if (!WSRSpeechManager.GetInstance().Listening) { cfg.logInfo("ENGINE - " + Name, "REJECTED not listening"); return; } if (IsWorking) { cfg.logInfo("ENGINE - " + Name, "REJECTED Speech while working: " + rr.Confidence + " Text: " + rr.Text); return; } var start = DateTime.Now; IsWorking = true; try { SpeechRecognized(rr); } catch (Exception ex) { cfg.logError("ENGINE - " + Name, ex); } IsWorking = false; var stop = DateTime.Now; cfg.logInfo("ENGINE - " + Name, "SpeechRecognized: " + (stop - start).TotalMilliseconds + "ms Text: " + rr.Text); }
protected void HandleListen(XPathNavigator xnav) { XPathNavigator listen = xnav.SelectSingleNode("/SML/action/@listen"); if (listen != null) { WSRSpeechManager.GetInstance().Listening = bool.Parse(listen.Value); } }
protected String ProcessAudioStream(RecognitionResult rr, String language) { cfg.logInfo("GOOGLE", "ProcessAudioStream: " + language + " " + rr.Audio.Duration); using (MemoryStream audioStream = new MemoryStream()) { rr.Audio.WriteToWaveStream(audioStream); // rr.GetAudioForWordRange(rr.Words[word], rr.Words[word]).WriteToWaveStream(audioStream); audioStream.Position = 0; // WSRSpeechManager.GetInstance().DumpAudio(rr); return(WSRSpeechManager.GetInstance().ProcessAudioStream(audioStream, language)); } }
protected void HandleContext(XPathNavigator xnav) { XPathNavigator ctxt = xnav.SelectSingleNode("/SML/action/@context"); if (ctxt != null) { WSRSpeechManager.GetInstance().SetContext(new List <string>(ctxt.Value.Split(','))); WSRSpeechManager.GetInstance().SetContextTimeout(); WSRSpeechManager.GetInstance().ForwardContext(); } }
// ========================================== // RECOGNIZE // ========================================== protected void SpeechRecognized(RecognitionResult rr) { // 1. Prevent while speaking if (WSRSpeakerManager.GetInstance().Speaking) { cfg.logWarning("ENGINE - " + Name, "REJECTED Speech while speaking : " + rr.Confidence + " Text: " + rr.Text); return; } // 2. Prevent while working XPathNavigator xnav = HandleSpeech(rr); if (xnav == null) { return; } // 3. Reset context timeout if (rr.Grammar.Name != "Dyn") { WSRSpeechManager.GetInstance().ResetContextTimeout(); } else { cfg.logInfo("ENGINE - " + Name, "DYN reset to default context"); WSRSpeechManager.GetInstance().SetContext("default"); WSRSpeechManager.GetInstance().ForwardContext(); } // 4. Track Audio Pitch TrackPitch(rr); // 5. Reset Speech Buffer WSRSpeakerManager.GetInstance().SpeechBuffer = ""; // 6. Hook String path = cfg.WSR.HandleCustomAttributes(xnav); // 7. Parse Result's URL String url = GetURL(xnav, rr.Confidence); // 8. Parse Result's Dication url = HandleWildcard(rr, url); if (url == null) { return; } // 9. Send the request HandleRequest(url, path); }
// Perform cleanup on application exit public virtual void Dispose() { // Stop RTP Client DisposeRTPClient(); // Stop HttpServer WSRHttpManager.GetInstance().Dispose(); // Stop Speech Manager WSRSpeechManager.GetInstance().Dispose(); // Stop Speaker WSRSpeakerManager.GetInstance().Dispose(); }
public void SendRequest(string url) { if (url == null) { return; } if (working) { return; } working = true; // Clean URL url = CleanURL(url); // Append ClientId url = AppendURL(url, "client", WSRConfig.GetInstance().Id); // Append UserId var profile = WSRProfileManager.GetInstance().Current; if (null != profile) { url = AppendURL(url, "profile", profile.Name); } // Append Directory Path url = AppendURL(url, "directory", WSRSpeechManager.GetInstance().GetGrammarPath()); WSRConfig.GetInstance().logInfo("HTTP", "Build HttpRequest: " + url); HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url); req.Method = "GET"; WSRConfig.GetInstance().logInfo("HTTP", "Send HttpRequest: " + req.Address); try { HttpWebResponse res = (HttpWebResponse)req.GetResponse(); using (StreamReader sr = new StreamReader(res.GetResponseStream(), Encoding.UTF8)) { WSRSpeakerManager.GetInstance().Speak(sr.ReadToEnd(), false); } } catch (WebException ex) { WSRConfig.GetInstance().logError("HTTP", ex); } working = false; }
// ========================================== // CONSTRUCTOR // ========================================== public virtual void Init() { // Start RTP Client StartRTPClient(); // Start HttpServer WSRHttpManager.GetInstance().StartHttpServer(); // Start Speech Manager WSRSpeechManager.GetInstance().Init(); InitSpeechEngine(); // Start Speaker Manager WSRSpeakerManager.GetInstance(); // Start Timeout RestartTimeout(); }
protected XPathNavigator HandleSpeech(RecognitionResult rr) { XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator(); double confidence = GetConfidence(xnav); // Normal check if (rr.Grammar.Name != "Dyn") { if (rr.Confidence < confidence) { cfg.logWarning("ENGINE - " + Name, "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text); return(null); } if (rr.Words[0].Confidence < cfg.trigger) { cfg.logWarning("ENGINE - " + Name, "REJECTED Trigger: " + rr.Words[0].Confidence + " Text: " + rr.Words[0].Text); return(null); } } // Dynamic check else { confidence = confidence - 0.2; cfg.logInfo("ENGINE - " + Name + " - DYN", "Dynamic Grammar"); if (rr.Confidence < (confidence)) { cfg.logWarning("ENGINE - " + Name + " - DYN", "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text); return(null); } } cfg.logWarning("ENGINE - " + Name, "RECOGNIZED Speech: " + rr.Confidence + " (" + confidence + ") / " + rr.Words[0].Confidence + " (" + rr.Words[0].Text + ")" + " Device: " + " Text: " + rr.Text); cfg.logDebug("ENGINE - " + Name, xnav.OuterXml); if (cfg.DEBUG) { WSRSpeechManager.GetInstance().DumpAudio(rr); } return(xnav); }
public void LoadGrammar() { if (this.Name == "RTP") { return; } var cache = WSRSpeechManager.GetInstance().Cache; foreach (WSRSpeecGrammar g in cache.Values) { if (g.LastModified < loading) { continue; } cfg.logInfo("GRAMMAR", Name + ": Load Grammar to Engine"); g.LoadGrammar(this); } loading = DateTime.Now; }
// ========================================== // HANDLE SPEECH RECOGNITION // ========================================== public override void InitSpeechEngine() { base.InitSpeechEngine(false); try { WSRConfig cfg = WSRConfig.GetInstance(); WSRSpeechManager manager = WSRSpeechManager.GetInstance(); SpeechAudioFormatInfo format = new SpeechAudioFormatInfo(EncodingFormat.Pcm, 16000, 16, 1, 32000, 2, null); for (int i = 0; i < Sensors.Count; i++) { KinectAudioSource source = Sensors[i].Sensor.AudioSource; source.EchoCancellationMode = EchoCancellationMode.CancellationAndSuppression; source.NoiseSuppression = true; source.BeamAngleMode = BeamAngleMode.Adaptive; //set the beam to adapt to the surrounding source.AutomaticGainControlEnabled = false; if (WSRConfig.GetInstance().Echo >= 0) { source.EchoCancellationSpeakerIndex = WSRConfig.GetInstance().Echo; } String prefix = "KINECT_" + i; cfg.logInfo(prefix, "AutomaticGainControlEnabled : " + source.AutomaticGainControlEnabled); cfg.logInfo(prefix, "BeamAngle : " + source.BeamAngle); cfg.logInfo(prefix, "EchoCancellationMode : " + source.EchoCancellationMode); cfg.logInfo(prefix, "EchoCancellationSpeakerIndex : " + source.EchoCancellationSpeakerIndex); cfg.logInfo(prefix, "NoiseSuppression : " + source.NoiseSuppression); cfg.logInfo(prefix, "SoundSourceAngle : " + source.SoundSourceAngle); cfg.logInfo(prefix, "SoundSourceAngleConfidence : " + source.SoundSourceAngleConfidence); var stream = source.Start(); // streamer = new SpeechStreamer(stream); // FIXME manager.AddEngine(prefix, cfg.language, cfg.confidence, stream, format); } } catch (Exception ex) { WSRConfig.GetInstance().logError("ENGINE", "Init Kinect Engines: " + ex.Message); } }
protected void recognizer_SpeechRecognitionRejected(object sender, SpeechRecognitionRejectedEventArgs e) { var url = WSRSpeechManager.GetInstance().ProcessRejected(); if (url == null) { return; } // 1. Prevent while speaking if (WSRSpeakerManager.GetInstance().Speaking) { cfg.logWarning("ENGINE - " + Name, "REJECTED Speech while speaking (in SpeechRecognitionRejected)"); return; } // 2. Process Audio cfg.logInfo("ENGINE - " + Name, "DYN process speech2text: ..."); String speech2text = ProcessAudioStream(e.Result, WSRConfig.GetInstance().language); if (String.IsNullOrEmpty(speech2text)) { cfg.logInfo("ENGINE - " + Name, "DYN wrong speech2text: " + speech2text); return; } // 3. Reset Dynamic cfg.logInfo("ENGINE - " + Name, "DYN reset to default context (in SpeechRecognitionRejected)"); WSRSpeechManager.GetInstance().SetContext("default"); WSRSpeechManager.GetInstance().ForwardContext(); // 4. Send back request cfg.logInfo("ENGINE - " + Name, "DYN callback: " + speech2text); url += "?dictation=" + HttpUtility.UrlEncode(speech2text); HandleRequest(url, null); }
public virtual void StopSpeechEngine() { WSRSpeechManager.GetInstance().Dispose(); }
// ========================================== // HANDLE HTTPSERVER // ========================================== public virtual bool HandleCustomRequest(NHttp.HttpRequestEventArgs e, StreamWriter writer) { // Status String status = e.Request.Params.Get("status"); if (status != null) { if (WSRSpeakerManager.GetInstance().Speaking) { writer.Write("speaking"); } } // Askme var values = System.Web.HttpUtility.ParseQueryString(e.Request.Url.Query); var mgr = WSRSpeechManager.GetInstance(); String[] grammar = values.GetValues("grammar"); String[] tags = values.GetValues("tags"); WSRSpeechManager.GetInstance().DynamicGrammar(grammar, tags); // Stop Music String pause = e.Request.Params.Get("pause"); if (pause != null) { WSRSpeakerManager.GetInstance().Stop(pause); } // Play Music String mp3 = e.Request.Params.Get("play"); if (mp3 != null) { WSRSpeakerManager.GetInstance().Play(mp3, e.Request.Params.Get("sync") == null); } // Recognize String audio = e.Request.Params.Get("recognize"); if (audio != null) { WSRSpeechManager.GetInstance().RecognizeFile(audio); } // Listening String listen = e.Request.Params.Get("listen"); if (listen != null) { WSRSpeechManager.GetInstance().Listening = bool.Parse(listen); } // Recognize File NHttp.HttpPostedFile file = e.Request.Files.Get("recognize"); if (file != null) { byte[] data = null; using (var reader = new BinaryReader(file.InputStream)) { data = reader.ReadBytes(file.ContentLength); } var path = WSRConfig.GetInstance().audio + "/" + file.FileName; if (File.Exists(path)) { File.Delete(path); } File.WriteAllBytes(path, data); } // Text To Speech String tts = e.Request.Params.Get("tts"); if (tts != null) { tts = e.Server.HtmlDecode(tts); WSRSpeakerManager.GetInstance().Speak(tts, e.Request.Params.Get("sync") == null); } // Text To Speech - Stop String notts = e.Request.Params.Get("notts"); if (notts != null) { WSRSpeakerManager.GetInstance().ShutUp(); } // Text To Speech - Stop String restart = e.Request.Params.Get("restart"); if (restart != null) { Restart(); } // Set Context String ctxt = e.Request.Params.Get("context"); if (ctxt != null) { WSRSpeechManager.GetInstance().SetContext(new List <string>(ctxt.Split(','))); WSRSpeechManager.GetInstance().SetContextTimeout(); WSRSpeechManager.GetInstance().ForwardContext(); } // Process String activate = e.Request.Params.Get("activate"); if (activate != null) { WSRKeyboard.GetInstance().ActivateApp(activate); } String run = e.Request.Params.Get("run"); String param = e.Request.Params.Get("runp"); if (run != null) { WSRKeyboard.GetInstance().RunApp(run, param); } // Keyboard String keyMod = e.Request.Params.Get("keyMod"); String key = e.Request.Params.Get("keyPress"); if (key != null) { WSRKeyboard.GetInstance().SimulateKey(key, 0, keyMod); } key = e.Request.Params.Get("keyDown"); if (key != null) { WSRKeyboard.GetInstance().SimulateKey(key, 1, keyMod); } key = e.Request.Params.Get("keyUp"); if (key != null) { WSRKeyboard.GetInstance().SimulateKey(key, 2, keyMod); } String keyText = e.Request.Params.Get("keyText"); if (keyText != null) { WSRKeyboard.GetInstance().SimulateTextEntry(keyText); } // Recognize String String emulate = e.Request.Params.Get("emulate"); if (emulate != null) { WSRSpeechManager.GetInstance().RecognizeString(emulate); writer.Write(WSRSpeakerManager.GetInstance().SpeechBuffer); } return(false); }