protected void recognizer_SpeechRecognized(object sender, SpeechRecognizedEventArgs e)
{
RecognitionResult rr = e.Result;
if (!WSRSpeechManager.GetInstance().Listening)
{
cfg.logInfo("ENGINE - " + Name, "REJECTED not listening");
return;
}
if (IsWorking)
{
cfg.logInfo("ENGINE - " + Name, "REJECTED Speech while working: " + rr.Confidence + " Text: " + rr.Text);
return;
}
var start = DateTime.Now;
IsWorking = true;
try {
SpeechRecognized(rr);
}
catch (Exception ex) {
cfg.logError("ENGINE - " + Name, ex);
}
IsWorking = false;
var stop = DateTime.Now;
cfg.logInfo("ENGINE - " + Name, "SpeechRecognized: " + (stop - start).TotalMilliseconds + "ms Text: " + rr.Text);
}
protected void InitSpeechEngine(bool def)
{
try {
WSRConfig cfg = WSRConfig.GetInstance();
WSRSpeechManager manager = WSRSpeechManager.GetInstance();
// File
manager.InitEngines();
// Default
if (def)
{
manager.AddDefaultEngine("Default", cfg.language, cfg.confidence);
}
// RTP
if (rtpClient == null)
{
return;
}
var format = new SpeechAudioFormatInfo(16000, AudioBitsPerSample.Sixteen, AudioChannel.Stereo);
manager.AddEngine("RTP", cfg.language, cfg.confidence, rtpClient.AudioStream, format);
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("ENGINE", "InitEngines: " + ex.Message);
}
}
public static WSRSpeechManager GetInstance()
{
if (manager == null)
{
manager = new WSRSpeechManager();
}
return(manager);
}
protected void HandleListen(XPathNavigator xnav)
{
XPathNavigator listen = xnav.SelectSingleNode("/SML/action/@listen");
if (listen != null)
{
WSRSpeechManager.GetInstance().Listening = bool.Parse(listen.Value);
}
}
protected void HandleContext(XPathNavigator xnav)
{
XPathNavigator ctxt = xnav.SelectSingleNode("/SML/action/@context");
if (ctxt != null)
{
WSRSpeechManager.GetInstance().SetContext(new List <string>(ctxt.Value.Split(',')));
WSRSpeechManager.GetInstance().SetContextTimeout();
WSRSpeechManager.GetInstance().ForwardContext();
}
}
protected String ProcessAudioStream(RecognitionResult rr, String language)
{
cfg.logInfo("GOOGLE", "ProcessAudioStream: " + language + " " + rr.Audio.Duration);
using (MemoryStream audioStream = new MemoryStream()) {
rr.Audio.WriteToWaveStream(audioStream);
// rr.GetAudioForWordRange(rr.Words[word], rr.Words[word]).WriteToWaveStream(audioStream);
audioStream.Position = 0;
// WSRSpeechManager.GetInstance().DumpAudio(rr);
return(WSRSpeechManager.GetInstance().ProcessAudioStream(audioStream, language));
}
}
// ==========================================
// RECOGNIZE
// ==========================================
protected void SpeechRecognized(RecognitionResult rr)
{
// 1. Prevent while speaking
if (WSRSpeakerManager.GetInstance().Speaking)
{
cfg.logWarning("ENGINE - " + Name, "REJECTED Speech while speaking : " + rr.Confidence + " Text: " + rr.Text);
return;
}
// 2. Prevent while working
XPathNavigator xnav = HandleSpeech(rr);
if (xnav == null)
{
return;
}
// 3. Reset context timeout
if (rr.Grammar.Name != "Dyn")
{
WSRSpeechManager.GetInstance().ResetContextTimeout();
}
else
{
cfg.logInfo("ENGINE - " + Name, "DYN reset to default context");
WSRSpeechManager.GetInstance().SetContext("default");
WSRSpeechManager.GetInstance().ForwardContext();
}
// 4. Track Audio Pitch
TrackPitch(rr);
// 5. Reset Speech Buffer
WSRSpeakerManager.GetInstance().SpeechBuffer = "";
// 6. Hook
String path = cfg.WSR.HandleCustomAttributes(xnav);
// 7. Parse Result's URL
String url = GetURL(xnav, rr.Confidence);
// 8. Parse Result's Dication
url = HandleWildcard(rr, url);
if (url == null)
{
return;
}
// 9. Send the request
HandleRequest(url, path);
}
// Perform cleanup on application exit
public virtual void Dispose()
{
// Stop RTP Client
DisposeRTPClient();
// Stop HttpServer
WSRHttpManager.GetInstance().Dispose();
// Stop Speech Manager
WSRSpeechManager.GetInstance().Dispose();
// Stop Speaker
WSRSpeakerManager.GetInstance().Dispose();
}
public void SendRequest(string url)
{
if (url == null)
{
return;
}
if (working)
{
return;
}
working = true;
// Clean URL
url = CleanURL(url);
// Append ClientId
url = AppendURL(url, "client", WSRConfig.GetInstance().Id);
// Append UserId
var profile = WSRProfileManager.GetInstance().Current;
if (null != profile)
{
url = AppendURL(url, "profile", profile.Name);
}
// Append Directory Path
url = AppendURL(url, "directory", WSRSpeechManager.GetInstance().GetGrammarPath());
WSRConfig.GetInstance().logInfo("HTTP", "Build HttpRequest: " + url);
HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url);
req.Method = "GET";
WSRConfig.GetInstance().logInfo("HTTP", "Send HttpRequest: " + req.Address);
try {
HttpWebResponse res = (HttpWebResponse)req.GetResponse();
using (StreamReader sr = new StreamReader(res.GetResponseStream(), Encoding.UTF8)) {
WSRSpeakerManager.GetInstance().Speak(sr.ReadToEnd(), false);
}
}
catch (WebException ex) {
WSRConfig.GetInstance().logError("HTTP", ex);
}
working = false;
}
// ==========================================
// CONSTRUCTOR
// ==========================================
public virtual void Init()
{
// Start RTP Client
StartRTPClient();
// Start HttpServer
WSRHttpManager.GetInstance().StartHttpServer();
// Start Speech Manager
WSRSpeechManager.GetInstance().Init();
InitSpeechEngine();
// Start Speaker Manager
WSRSpeakerManager.GetInstance();
// Start Timeout
RestartTimeout();
}
protected XPathNavigator HandleSpeech(RecognitionResult rr)
{
XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator();
double confidence = GetConfidence(xnav);
// Normal check
if (rr.Grammar.Name != "Dyn")
{
if (rr.Confidence < confidence)
{
cfg.logWarning("ENGINE - " + Name, "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text);
return(null);
}
if (rr.Words[0].Confidence < cfg.trigger)
{
cfg.logWarning("ENGINE - " + Name, "REJECTED Trigger: " + rr.Words[0].Confidence + " Text: " + rr.Words[0].Text);
return(null);
}
}
// Dynamic check
else
{
confidence = confidence - 0.2;
cfg.logInfo("ENGINE - " + Name + " - DYN", "Dynamic Grammar");
if (rr.Confidence < (confidence))
{
cfg.logWarning("ENGINE - " + Name + " - DYN", "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text);
return(null);
}
}
cfg.logWarning("ENGINE - " + Name, "RECOGNIZED Speech: " + rr.Confidence + " (" + confidence + ") / " + rr.Words[0].Confidence + " (" + rr.Words[0].Text + ")" + " Device: " + " Text: " + rr.Text);
cfg.logDebug("ENGINE - " + Name, xnav.OuterXml);
if (cfg.DEBUG)
{
WSRSpeechManager.GetInstance().DumpAudio(rr);
}
return(xnav);
}
public void LoadGrammar()
{
if (this.Name == "RTP")
{
return;
}
var cache = WSRSpeechManager.GetInstance().Cache;
foreach (WSRSpeecGrammar g in cache.Values)
{
if (g.LastModified < loading)
{
continue;
}
cfg.logInfo("GRAMMAR", Name + ": Load Grammar to Engine");
g.LoadGrammar(this);
}
loading = DateTime.Now;
}
// ==========================================
// HANDLE SPEECH RECOGNITION
// ==========================================
public override void InitSpeechEngine()
{
base.InitSpeechEngine(false);
try {
WSRConfig cfg = WSRConfig.GetInstance();
WSRSpeechManager manager = WSRSpeechManager.GetInstance();
SpeechAudioFormatInfo format = new SpeechAudioFormatInfo(EncodingFormat.Pcm, 16000, 16, 1, 32000, 2, null);
for (int i = 0; i < Sensors.Count; i++)
{
KinectAudioSource source = Sensors[i].Sensor.AudioSource;
source.EchoCancellationMode = EchoCancellationMode.CancellationAndSuppression;
source.NoiseSuppression = true;
source.BeamAngleMode = BeamAngleMode.Adaptive; //set the beam to adapt to the surrounding
source.AutomaticGainControlEnabled = false;
if (WSRConfig.GetInstance().Echo >= 0)
{
source.EchoCancellationSpeakerIndex = WSRConfig.GetInstance().Echo;
}
String prefix = "KINECT_" + i;
cfg.logInfo(prefix, "AutomaticGainControlEnabled : " + source.AutomaticGainControlEnabled);
cfg.logInfo(prefix, "BeamAngle : " + source.BeamAngle);
cfg.logInfo(prefix, "EchoCancellationMode : " + source.EchoCancellationMode);
cfg.logInfo(prefix, "EchoCancellationSpeakerIndex : " + source.EchoCancellationSpeakerIndex);
cfg.logInfo(prefix, "NoiseSuppression : " + source.NoiseSuppression);
cfg.logInfo(prefix, "SoundSourceAngle : " + source.SoundSourceAngle);
cfg.logInfo(prefix, "SoundSourceAngleConfidence : " + source.SoundSourceAngleConfidence);
var stream = source.Start();
// streamer = new SpeechStreamer(stream); // FIXME
manager.AddEngine(prefix, cfg.language, cfg.confidence, stream, format);
}
}
catch (Exception ex) {
WSRConfig.GetInstance().logError("ENGINE", "Init Kinect Engines: " + ex.Message);
}
}
protected void recognizer_SpeechRecognitionRejected(object sender, SpeechRecognitionRejectedEventArgs e)
{
var url = WSRSpeechManager.GetInstance().ProcessRejected();
if (url == null)
{
return;
}
// 1. Prevent while speaking
if (WSRSpeakerManager.GetInstance().Speaking)
{
cfg.logWarning("ENGINE - " + Name, "REJECTED Speech while speaking (in SpeechRecognitionRejected)");
return;
}
// 2. Process Audio
cfg.logInfo("ENGINE - " + Name, "DYN process speech2text: ...");
String speech2text = ProcessAudioStream(e.Result, WSRConfig.GetInstance().language);
if (String.IsNullOrEmpty(speech2text))
{
cfg.logInfo("ENGINE - " + Name, "DYN wrong speech2text: " + speech2text);
return;
}
// 3. Reset Dynamic
cfg.logInfo("ENGINE - " + Name, "DYN reset to default context (in SpeechRecognitionRejected)");
WSRSpeechManager.GetInstance().SetContext("default");
WSRSpeechManager.GetInstance().ForwardContext();
// 4. Send back request
cfg.logInfo("ENGINE - " + Name, "DYN callback: " + speech2text);
url += "?dictation=" + HttpUtility.UrlEncode(speech2text);
HandleRequest(url, null);
}
// ==========================================
// HANDLE HTTPSERVER
// ==========================================
public virtual bool HandleCustomRequest(NHttp.HttpRequestEventArgs e, StreamWriter writer)
{
// Status
String status = e.Request.Params.Get("status");
if (status != null)
{
if (WSRSpeakerManager.GetInstance().Speaking)
{
writer.Write("speaking");
}
}
// Askme
var values = System.Web.HttpUtility.ParseQueryString(e.Request.Url.Query);
var mgr = WSRSpeechManager.GetInstance();
String[] grammar = values.GetValues("grammar");
String[] tags = values.GetValues("tags");
WSRSpeechManager.GetInstance().DynamicGrammar(grammar, tags);
// Stop Music
String pause = e.Request.Params.Get("pause");
if (pause != null)
{
WSRSpeakerManager.GetInstance().Stop(pause);
}
// Play Music
String mp3 = e.Request.Params.Get("play");
if (mp3 != null)
{
WSRSpeakerManager.GetInstance().Play(mp3, e.Request.Params.Get("sync") == null);
}
// Recognize
String audio = e.Request.Params.Get("recognize");
if (audio != null)
{
WSRSpeechManager.GetInstance().RecognizeFile(audio);
}
// Listening
String listen = e.Request.Params.Get("listen");
if (listen != null)
{
WSRSpeechManager.GetInstance().Listening = bool.Parse(listen);
}
// Recognize File
NHttp.HttpPostedFile file = e.Request.Files.Get("recognize");
if (file != null)
{
byte[] data = null;
using (var reader = new BinaryReader(file.InputStream)) {
data = reader.ReadBytes(file.ContentLength);
}
var path = WSRConfig.GetInstance().audio + "/" + file.FileName;
if (File.Exists(path))
{
File.Delete(path);
}
File.WriteAllBytes(path, data);
}
// Text To Speech
String tts = e.Request.Params.Get("tts");
if (tts != null)
{
tts = e.Server.HtmlDecode(tts);
WSRSpeakerManager.GetInstance().Speak(tts, e.Request.Params.Get("sync") == null);
}
// Text To Speech - Stop
String notts = e.Request.Params.Get("notts");
if (notts != null)
{
WSRSpeakerManager.GetInstance().ShutUp();
}
// Text To Speech - Stop
String restart = e.Request.Params.Get("restart");
if (restart != null)
{
Restart();
}
// Set Context
String ctxt = e.Request.Params.Get("context");
if (ctxt != null)
{
WSRSpeechManager.GetInstance().SetContext(new List <string>(ctxt.Split(',')));
WSRSpeechManager.GetInstance().SetContextTimeout();
WSRSpeechManager.GetInstance().ForwardContext();
}
// Process
String activate = e.Request.Params.Get("activate");
if (activate != null)
{
WSRKeyboard.GetInstance().ActivateApp(activate);
}
String run = e.Request.Params.Get("run");
String param = e.Request.Params.Get("runp");
if (run != null)
{
WSRKeyboard.GetInstance().RunApp(run, param);
}
// Keyboard
String keyMod = e.Request.Params.Get("keyMod");
String key = e.Request.Params.Get("keyPress");
if (key != null)
{
WSRKeyboard.GetInstance().SimulateKey(key, 0, keyMod);
}
key = e.Request.Params.Get("keyDown");
if (key != null)
{
WSRKeyboard.GetInstance().SimulateKey(key, 1, keyMod);
}
key = e.Request.Params.Get("keyUp");
if (key != null)
{
WSRKeyboard.GetInstance().SimulateKey(key, 2, keyMod);
}
String keyText = e.Request.Params.Get("keyText");
if (keyText != null)
{
WSRKeyboard.GetInstance().SimulateTextEntry(keyText);
}
// Recognize String
String emulate = e.Request.Params.Get("emulate");
if (emulate != null)
{
WSRSpeechManager.GetInstance().RecognizeString(emulate);
writer.Write(WSRSpeakerManager.GetInstance().SpeechBuffer);
}
return(false);
}
public static WSRSpeechManager GetInstance() {
if (manager == null) {
manager = new WSRSpeechManager();
}
return manager;
}
public virtual void StopSpeechEngine()
{
WSRSpeechManager.GetInstance().Dispose();
}
