public RecognizerData(string _data)
{
string[] tokens = _data.Split(_separator.ToCharArray());
if (tokens.Length != 4)
throw new Exception("RecognizerData: Cannot instantiate object. Error in input string");
try
{
if (tokens[0].Equals("recognized", StringComparison.CurrentCultureIgnoreCase))
_spRec = RecognitionResult.RECOGNIZED;
else
_spRec = RecognitionResult.UNRECOGNIZED;
_confidence = Convert.ToDouble(tokens[1]);
if (tokens[2].Equals("null"))
_propName = null;
else
_propName = tokens[2];
if (tokens[3].Equals("null"))
_textEqt = null;
else
_textEqt = tokens[3];
}
catch(Exception e)
{
throw new Exception("RecognizerData: Cannot instantiate object. Error in input string");
}
}
private string _textEqt; // If speech was recognized, then contains corresponding text, else null string
#endregion Fields
#region Constructors
public RecognizerData()
{
_spRec = RecognitionResult.UNRECOGNIZED;
_confidence = 0;
_propName = null;
_textEqt = null;
}
public RecognizerData(double _conf, string _pName, string _txtEqt)
{
_spRec = RecognitionResult.RECOGNIZED;
_confidence = _conf;
_propName = _pName;
_textEqt = _txtEqt;
}
public void AddItemToList(RecognitionResult speech, ListBox list, Color color)
{
ListBoxItem item = CreateDefaultListBoxItem();
item.Content = "S:" + speech.Text + " C:" + speech.Confidence;
item.Foreground = new SolidColorBrush(color);
AddToList(list, item);
}
protected override void RecognizedSpeech(RecognitionResult result)
{
if (result.Confidence < CONFIDENCE_THRESHOLD)
{
this.RejectSpeech(result);
return;
}
string message = "";
switch (result.Text.ToUpperInvariant())
{
case "HELLO":
message += "Hello! I am your computer.";
break;
case "COMPUTER":
message += "Computer here. What would you like me to do?";
break;
case "ACTION":
message += "What action shall I perform?";
break;
default:
message += "I recognized your speech but I'm not sure what you mean.";
break;
}
string status = "Recognized: " + result.Text + " " + result.Confidence + "\nResponse: " + message;
this.ReportSpeechStatus(status);
this.gestureDetected = true;
}
public IRecog Process(RecognitionResult r)
{
var semanticsValue = r.Semantics.Value.ToString();
if (string.IsNullOrEmpty(semanticsValue))
return null;
string[] args;
switch (semanticsValue)
{
case "CANCEL":
return Listen.ResetRecog;
case "GO":
args = _lastKnownGood.Semantics.Value.ToString().Split('|');
Vlc.Play(args[2]);
_listenCtx.BalloonTip(args[1], r);
return Listen.ResetRecog;
default:
args = r.Semantics.Value.ToString().Split('|');
if (args.Length == 3)
{
_lastKnownGood = r;
_listenCtx.BalloonTip(args[1], r);
}
else
_listenCtx.BalloonTip("Play", r);
return null;
}
}
public IRecog Process(RecognitionResult r)
{
switch (r.Semantics.Value.ToString())
{
case "CANCEL":
SystemSounds.Exclamation.Play();
return Listen.ResetRecog;
case "EXIT":
if (r.Confidence >= 0.6)
{
SystemSounds.Exclamation.Play();
_listenCtx.Exit();
return null;
}
break;
case "PLAY":
if (r.Confidence >= 0.5)
{
_listenCtx.BalloonTip(2, "Play", "Name, Series, Episode, Go", ListenIcon.Info);
return _recogPlay;
}
break;
case "PAUSE":
Vlc.Pause();
return Listen.ResetRecog;
}
return null;
}
public static RecognitionResult Parse(String toParse)
{
// cap
Regex regexCommonInfo = new Regex(@"""status"":(?<status>\d),""id"":""(?<id>[\w-]+)""");
RecognitionResult result = new RecognitionResult();
var match = regexCommonInfo.Match(toParse);
result.id = match.Groups["id"].Value;
result.status = match.Groups["status"].Value;
// hypotheses
Regex regexUtter = new Regex(@"""utterance"":""(?<utter>[а-яА-Я\s\w.,]+)"",""confidence"":(?<conf>[\d.]+)");
float confidence;
var matches = regexUtter.Matches(toParse);
List<RecognizedItem> hypos = new List<RecognizedItem>();
foreach (Match m in matches)
{
var g = m.Groups;
confidence = float.Parse(g["conf"].Value.Replace(".", ","));
hypos.Add(new RecognizedItem { confidence = confidence, utterance = g["utter"].Value });
}
result.hypotheses = hypos;
return result;
}
/// <summary>
/// Handles the SpeechRecognized event from the Recognition Engine
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
public virtual void SpeechRecognized_Handler(object sender, SpeechRecognizedEventArgs e)
{
string text = e.Result.Text;
SemanticValue semantics = e.Result.Semantics;
NewInput = true;
LastResult = e.Result;
}
public static Int16[] GetAudioData(RecognitionResult Result)
{
MemoryStream Stream = new MemoryStream();
Result.Audio.WriteToAudioStream(Stream);
Byte[] ByteData = Stream.ToArray();
Int16[] AudioData = new Int16[ByteData.Length/2];
Buffer.BlockCopy(ByteData, 0, AudioData, 0, ByteData.Length);
return AudioData;
}
public void Start(ListenContext listenCtx, SpeechRecognitionEngine sre)
{
_listenCtx = listenCtx;
using (var memoryStream = new MemoryStream(Encoding.ASCII.GetBytes(Resources.RecogPlay)))
sre.LoadGrammar(new Grammar(memoryStream));
var gb = new GrammarBuilder { Culture = new CultureInfo("en-US") };
gb.Append(_choices);
sre.LoadGrammar(new Grammar(gb));
_lastKnownGood = null;
}
public IRecog Process(RecognitionResult r)
{
if (!string.IsNullOrEmpty(r.Semantics.Value.ToString()))
{
SystemSounds.Exclamation.Play();
_listenCtx.BalloonTip(2, "KinectHaus\u2122", "Say: Play, Pause, Cancel", ListenIcon.Info);
return _recogCommand;
}
return null;
}
public string Interpret(string persName, RecognitionResult rec)
{
if (persName == null)
throw new ArgumentNullException("persName");
if (rec == null)
throw new ArgumentNullException("sem");
string parsedUri = String.Format("/{0}/{1}", persName.ToLower(), RecInterpret(rec.Semantics));
logger.Debug("Parsed URI : {0}\n", parsedUri);
return parsedUri;
}
public static void Actions(RecognitionResult recognitionResult)
{
switch (recognitionResult.Text)
{
case "wordfiller move":
MouseImpersonator.Grab(KinectSettings.movement.HandX, KinectSettings.movement.HandY);
break;
case "wordfiller drop":
MouseImpersonator.Release(KinectSettings.movement.HandX, KinectSettings.movement.HandY);
break;
case "wordfiller mark":
foreach (var w in ManageWindows.Forms)
{
if (w is FrmMain)
{
(w as FrmMain).UserRequestCurrentMark();
}
}
break;
case "wordfiller click":
MouseImpersonator.Grab(KinectSettings.movement.HandX, KinectSettings.movement.HandY);
MouseImpersonator.Release(KinectSettings.movement.HandX, KinectSettings.movement.HandY);
break;
case "wordfiller close menu":
foreach (var w in ManageWindows.Forms)
{
if (w is FrmMain)
{
(w as FrmMain).CloseMenu();
}
}
break;
case "wordfiller open menu":
foreach (var w in ManageWindows.Forms)
{
if (w is FrmMain)
{
(w as FrmMain).OpenMenu();
}
}
break;
}
}
public void DoSpellEffect(RecognitionResult result)
{
if (result.score.score > 0.95f)
{
enemyTarget.SetIsBlinded(3);
}
else if (result.score.score > 0.85f)
{
enemyTarget.SetIsBlinded(2);
}
else
{
enemyTarget.SetIsBlinded(1);
}
enemyTarget.DecreaseHealthBySpellDamage(damage, spellType);
Debug.Log("Void hits and blinds the target.");
}
public void DoSpellEffect(RecognitionResult result)
{
if (result.score.score > 0.95f)
{
enemyTarget.SetIsConfused(3);
}
else if (result.score.score > 0.85f)
{
enemyTarget.SetIsConfused(2);
}
else
{
enemyTarget.SetIsConfused(1);
}
player.Attack(damage, spellType);
Debug.Log("Mind Blow hits and confuses the target.");
}
public override void OnTrigger(RecognitionResult rResult, string phrase)
{
var turnOn = !_apOffStrings.Any(phrase.Contains);
var apChar = phrase.Contains("auto pilot b") ? "b" : "a";
var apStatus = (int)XPlaneInterface.GetDataRef <double>($"laminar/B738/autopilot/cmd_{apChar}_pos").Value;
if (turnOn && apStatus == 0)
{
PressButton(apChar);
SpeechSynthesizer.SpeakAsync($"auto pilot {apChar} engaged");
}
else if (!turnOn && apStatus == 1)
{
PressButton(apChar);
SpeechSynthesizer.SpeakAsync($"autopilot {apChar} disengaged");
}
}
public override void OnTrigger(RecognitionResult rResult, string phrase)
{
var stringHeading = Constants.StringNumbersToDigits(phrase);
try
{
var startingIndexNumber = Constants.NumbersInDigits.First(stringHeading.Contains);
var startingIndex = stringHeading.IndexOf(startingIndexNumber, StringComparison.Ordinal);
stringHeading = stringHeading.Substring(startingIndex, stringHeading.Length - startingIndex);
}
catch
{
// ignored
}
var splittedString = stringHeading.Split(' ');
float verticalSpeedToSet = 0;
for (var i = 0; i < splittedString.Length; i += 2)
{
var nextIndex = i + 1;
if (nextIndex > splittedString.Length - 1)
{
break;
}
if (splittedString[i + 1] == "thousand")
{
verticalSpeedToSet += int.Parse(splittedString[i]) * 1000;
}
if (splittedString[i + 1] == "hundred")
{
verticalSpeedToSet += int.Parse(splittedString[i]) * 100;
}
}
if (phrase.Contains("fifty") && verticalSpeedToSet < 1000 && verticalSpeedToSet > -1000)
{
verticalSpeedToSet += 50;
}
if (phrase.Contains("negative"))
{
verticalSpeedToSet *= -1;
}
XPlaneInterface.SetDataRef("sim/cockpit/autopilot/vertical_velocity", verticalSpeedToSet);
}
void SpeechRecognitionEngine_SpeechRecognized(object sender, SpeechRecognizedEventArgs e)
{
RecognitionResult result = e.Result;
if (result != null)
{
Console.WriteLine("Speech recognized: " + result.Text);
imMessageContent[index++] = result.Text;
if (result.Text.Equals("send"))
{
SendIM(); //Call the method to send IM messages.
_waitForXXXCompleted.Set();
}
}
}
protected Dictionary <String, String> ConvertResultToDictionary(RecognitionResult result)
{
Dictionary <String, String> vals = new Dictionary <string, string>();
foreach (KeyValuePair <String, SemanticValue> val in result.Semantics)
{
if (val.Value.Value == null)
{
vals[val.Key] = "";
}
else
{
vals[val.Key] = val.Value.Value.ToString();
}
}
return(vals);
}
public override void OnTrigger(RecognitionResult rResult, string phrase)
{
var dataref = XPlaneInterface.GetDataRef <float[]>("sim/cockpit2/switches/landing_lights_switch");
var landingLights = new float[4];
Array.Copy(dataref.Value, 0, landingLights, 0, 4);
if (phrase.Contains("on") && !landingLights.SequenceEqual(_landingLightsAllOn))
{
XPlaneInterface.SetExecutingCommand("sim/lights/landing_lights_on");
SpeechSynthesizer.SpeakAsync("Landing lights on");
}
else if (phrase.Contains("off") && !landingLights.SequenceEqual(_landingLightsAllOff))
{
XPlaneInterface.SetExecutingCommand("sim/lights/landing_lights_off");
SpeechSynthesizer.SpeakAsync("Landing lights off");
}
}
/// <summary>
/// Identify a stream of audio
/// </summary>
/// <param name="stream">Audio buffer to be recognized</param>
/// <param name="serviceClient">Client used in identifying the streamed audio wave</param>
/// <param name="clientId">Client ID</param>
/// <param name="requestId">Request ID</param>
public async Task IdentifyStreamAsync(Stream stream, SpeakerIdentificationServiceClient serviceClient, Guid clientId, int requestId)
{
try
{
OperationLocation processPollingLocation;
processPollingLocation = await serviceClient.IdentifyAsync(stream, this.speakerIds, forceShortAudio : true).ConfigureAwait(false);
IdentificationOperation identificationResponse = null;
int numOfRetries = int.Parse(ConfigurationManager.AppSettings["NumberOfPollingRetries"]);
TimeSpan timeBetweenRetries = TimeSpan.FromSeconds(int.Parse(ConfigurationManager.AppSettings["TimeSpanBetweenPollingRetries"]));
while (numOfRetries > 0)
{
await Task.Delay(timeBetweenRetries);
identificationResponse = await serviceClient.CheckIdentificationStatusAsync(processPollingLocation);
if (identificationResponse.Status == Status.Succeeded)
{
var result = new RecognitionResult(identificationResponse.ProcessingResult, clientId, requestId);
this.resultCallback(result);
break;
}
else if (identificationResponse.Status == Status.Failed)
{
var failureResult = new RecognitionResult(false, identificationResponse.Message, requestId);
this.resultCallback(failureResult);
return;
}
numOfRetries--;
}
if (numOfRetries <= 0)
{
var failureResult = new RecognitionResult(false, "Request timeout.", requestId);
this.resultCallback(failureResult);
return;
}
}
catch (Exception ex)
{
var result = new RecognitionResult(false, ex.Message, requestId);
this.resultCallback(result);
}
}
private async void Camera_FinishedPickingMedia(object sender, UIImagePickerMediaPickedEventArgs mediaPicker)
{
ToggleLoading(true);
_takePhotoButton.Hidden = true;
_galleryPhotoButton.Hidden = true;
UIImage originalImage = mediaPicker.Info[UIImagePickerController.OriginalImage] as UIImage;
nfloat height = originalImage.PreferredPresentationSizeForItemProvider.Height;
nfloat width = originalImage.PreferredPresentationSizeForItemProvider.Width;
if (!_fromGallery)
{
originalImage = MaxResizeImage(originalImage, (float)width / 8, (float)height / 8);
}
NSData imgData = originalImage.AsJPEG(0.1f);
byte[] dataBytesArray = imgData.ToArray();
this.DismissModalViewController(true);
try
{
RecognitionResult response = await new ComputerVisionService().RecognizeTextService(dataBytesArray);
if (response != null)
{
this.NavigationController.PushViewController(new TextDisplayController(response, height, width, 0), true);
}
else
{
UIAlertView alert = new UIAlertView()
{
Title = "Error",
Message = "API Fail"
};
alert.AddButton("Cancel");
alert.Show();
}
}
finally
{
_takePhotoButton.Hidden = false;
_galleryPhotoButton.Hidden = false;
ToggleLoading(false);
}
}
public void Talk()
{
syn.Speak(parser.ParsePrompt());
if (parser.GetGrammarPath().Equals(""))
{
if (parser.GetFieldName() == "Repertuar")
{
ObsluzRepertuar();
}
ObsluzZakonczenie();
}
else
{
System.Diagnostics.Debug.WriteLine("Oczekiwanie na odpowiedź użytkownika... ");
if (parser.GetFieldName() == "CzasRezerwacji")
{
ObsluzCzasRezerwacji();
}
else
{
recognizer.LoadGrammar(new Grammar(@parser.GetGrammarPath()));
RecognitionResult wynik = recognizer.Recognize();
if (wynik == null || wynik.Confidence < 0.5)
{
syn.Speak(parser.ParseNomatch());
}
else
{
ObsluzPrzejscie(wynik);
}
}
}
if (parser.GetFieldName() == "WynikRezerwacji")
{
WyswietlTekst?.Invoke(this, new ArgumentyRozpoznania() { Text = "WynikRezerwacji", Confidence = 0.9F });
ObsluzPrzypadki(parser.GetFieldName(), "");
syn.SayGoodbye();
pRunnig = false;
}
}
/// <summary>
/// Recognizes the strokes of the sketch using the Ink.
/// </summary>
/// <param name="args">An unlabeled Sketch and a UserTriggered flag</param>
/// <returns>A Labeled Sketch and </returns>
public override RecognitionResult Recognize(RecognitionArgs args)
{
RecognitionResult result = new RecognitionResult();
result.UserTriggered = args.UserTriggered;
// Only recognize when necessary
if (!args.UserTriggered)
{
return(result);
}
// Run recognition and fill result
Sketcher sketcher = new Sketcher();
iTool = sketcher.InkPanel.InkTool;
iTool.Clusterer.ImageRecognizer.RecognitionCompleted += new RecognitionCompletedEventHandler(ImageRecognizer_RecognitionCompleted);
settingsFilename = sketcher.SettingsFilename;
dir = sketcher.BaseDirectory;
try
{
iTool.Clusterer.ImageRecognizer.m_RecognitionComplete = false;
// Try to use InkTool
bool success = iTool.ClassifySketch(args.Sketch, settingsFilename, dir);
StrokeClassifier.StrokeClassifierResult clResult = iTool.Clusterer.Classifier.Classify();
StrokeGrouper.StrokeGrouperResult grResult = iTool.Clusterer.Grouper.Group(clResult.AllClassifications);
List <Cluster> initialClusters = new List <Cluster>();//iTool.Clusterer.CreateClusters(grResult);
ImageRecognizerWrapper.ImageRecognizerResults imgResult = iTool.Clusterer.GetImageResults(initialClusters);
//while (!iTool.Clusterer.InitialClustersDone);
//iTool.Clusterer.ImageRecognizer.RecognizeST(iTool.Clusterer.
rSketch = iTool.MakeSketch(imgResult.ScoredClusters);
result.Sketch = rSketch;
}
catch (Exception e)
{
// Catch all other exceptions
System.Windows.MessageBox.Show("General Exception from sketch recognizer component: \n" + e.Message);
// Return unrecognized sketch
result.Sketch = args.Sketch;
}
return(result);
}
private void btnSpeak_Click(object sender, EventArgs e)
{
SpeechRecognitionEngine sr = new SpeechRecognitionEngine();
Grammar gramer = new DictationGrammar();
sr.LoadGrammar(gramer);
try
{
btnSpeak.Text = "Please, Speak!";
sr.SetInputToDefaultAudioDevice();
RecognitionResult result = sr.Recognize();
richTextBox1.Text = result.Text;
}
catch (Exception)
{
btnSpeak.Text = "Error";
}
}
public string recognize()
{
string response = "";
try
{
this.recognizer.SetInputToDefaultAudioDevice();
RecognitionResult result = this.recognizer.Recognize();
response = result.Text;
}
catch (Exception ex)
{
response = "Invalid command";
//response = "Error " + ex.ToString();
// Console.WriteLine(String.Format("Could not recognize input from default aduio device. Is a microphone or sound card available?\r\n{0} - {1}.", exception.Source, exception.Message));
}
return(response);
}
private RecognitionResult RecognizeEmotions(FaceCropResult faces)
{
RecognitionResult result = new RecognitionResult();
result.CaptureTime = faces.CaptureTime;
result.Tag = faces.Tag;
try
{
result.Faces = _emotionApi.DetectEmotions(faces.Faces).GetAwaiter().GetResult();
}
catch (Exception ex)
{
Debug.WriteLine(ex);
}
return(result);
}
public async Task <ActionResult <RecognitionResultDto> > GetByIdAsync(int id)
{
if (id <= 0)
{
return(BadRequest());
}
RecognitionResult recognitionResult = await _service.GetByIdAsync(id);
if (recognitionResult == null)
{
return(NotFound());
}
RecognitionResultDto dto = _mapper.MapToDto(recognitionResult);
return(Ok(dto));
}
private void button1_Click(object sender, EventArgs e)
{
SpeechRecognitionEngine sr = new SpeechRecognitionEngine();
Grammar gr = new DictationGrammar();
sr.LoadGrammar(gr);
try
{
button1.Text = "SPEAK";
sr.SetInputToDefaultAudioDevice();
RecognitionResult rec = sr.Recognize();
button1.Text = rec.Text;
}
catch (Exception)
{
button1.Text = "ERROR";
}
}
private bool WhetherConfirmed(SpeechSynthesizer synthEngine)
{
if (recognizedResult != null)
{
// recognizer.RecognizeAsyncCancel();
synthEngine.SpeakAsync("Do you confirm " + recognizedResult.Text);
RecognitionResult confirmationCommand = recognizer.Recognize(TimeSpan.FromSeconds(5));
if (confirmationCommand != null && confirmationCommand.Text == "yes please confirm command")
{
return(true);
}
else if (confirmationCommand != null && confirmationCommand.Text == "no do not confirm")
{
return(false);
}
}
return(false);
}
// Speech recognition with backup subscription region.
public static async Task RecognitionOnceWithFileAsyncSwitchSecondaryRegion()
{
// Create a speech resource with primary subscription key and service region.
// Also create a speech resource with secondary subscription key and service region
RecognitionResult recognitionResult = await RecognizeOnceAsyncInternal("PrimarySubscriptionKey", "PrimarySubscriptionRegion");
if (recognitionResult.Reason == ResultReason.Canceled)
{
CancellationDetails details = CancellationDetails.FromResult(recognitionResult);
if (details.ErrorCode == CancellationErrorCode.ConnectionFailure ||
details.ErrorCode == CancellationErrorCode.ServiceUnavailable ||
details.ErrorCode == CancellationErrorCode.ServiceTimeout)
{
recognitionResult = await RecognizeOnceAsyncInternal("SecondarySubscriptionRegion", "SecondarySubscriptionRegion");
}
}
Console.WriteLine("Recognized {0}", recognitionResult.Text);
}
public static bool CorrectlyInterrupted(RecognitionResult recognition)
{
if (recognition == null)
{
return(false);
}
else
{
if (recognition.Confidence < 0.90)
{
return(false);
}
else
{
return(true);
}
}
}
private void Run()
{
try {
SpeechRecognitionEngine engine = new SpeechRecognitionEngine(new System.Globalization.CultureInfo("en-US"));
engine.SetInputToWaveFile(_fileName);
engine.LoadGrammar(new DictationGrammar());
engine.BabbleTimeout = TimeSpan.FromSeconds(10.0);
engine.EndSilenceTimeout = TimeSpan.FromSeconds(10.0);
engine.EndSilenceTimeoutAmbiguous = TimeSpan.FromSeconds(10.0);
engine.InitialSilenceTimeout = TimeSpan.FromSeconds(10.0);
_result = engine.Recognize();
}
finally {
_operation.PostOperationCompleted(delegate {
RaiseCompleted();
}, null);
}
}
public static bool ProcessCommand(RecognitionResult rr)
{
Trace.WriteLine("---------------------", "DEBUG");
foreach (var alt in rr.Alternates)
{
Trace.WriteLine(string.Format("{0:00.00}%, {1}", alt.Confidence * 100f, alt.Text), "DEBUG");
}
if (rr.Confidence < 0.9)
{
Trace.WriteLine("Ignoring poor match", "INFO");
Trace.WriteLine("---------------------", "DEBUG");
return(false);
}
Trace.WriteLine("---------------------", "DEBUG");
return(true);
}
public CommandResult Listen()
{
RecognitionResult result = recognizer.Recognize();
CommandResult commandResult;
if (result == null)
{
commandResult = new CommandResult(false, null);
}
else
{
// Extract command
commandResult = new CommandResult(true, result);
}
return(commandResult);
}
public override void OnTrigger(RecognitionResult rResult, string phrase)
{
var radioToSwap = _navRadios.First(phrase.Contains);
var stringFreq =
Constants.StringNumbersToDigits(phrase).Split(new[] { $"{radioToSwap} to " }, StringSplitOptions.None)[1]
.Replace(" ", "");
stringFreq += new string('0', 5 - stringFreq.Length);
var freq = int.Parse(stringFreq);
if (!Constants.IsValidNavFreq(freq))
{
return;
}
var dataRef = $"sim/cockpit2/radios/actuators/{radioToSwap}_standby_frequency_hz";
XPlaneInterface.SetDataRef(dataRef, freq);
}
private string[] recognize()
{
List <string> results = new List <string>(10);
this.recognizerContext.Strokes = this.ink.Strokes;
RecognitionStatus status;
RecognitionResult result = this.recognizerContext.Recognize(out status);
if (status == RecognitionStatus.NoError)
{
RecognitionAlternates alts = result.GetAlternatesFromSelection(0, -1, 10);
foreach (var alt in alts)
{
results.Add(alt.ToString());
}
}
return(results.ToArray());
}
protected XPathNavigator HandleSpeech(RecognitionResult rr)
{
XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator();
double confidence = GetConfidence(xnav);
// Normal check
if (rr.Grammar.Name != "Dyn")
{
if (rr.Confidence < confidence)
{
cfg.logWarning("ENGINE - " + Name, "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text);
return(null);
}
if (rr.Words[0].Confidence < cfg.trigger)
{
cfg.logWarning("ENGINE - " + Name, "REJECTED Trigger: " + rr.Words[0].Confidence + " Text: " + rr.Words[0].Text);
return(null);
}
}
// Dynamic check
else
{
confidence = confidence - 0.2;
cfg.logInfo("ENGINE - " + Name + " - DYN", "Dynamic Grammar");
if (rr.Confidence < (confidence))
{
cfg.logWarning("ENGINE - " + Name + " - DYN", "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text);
return(null);
}
}
cfg.logWarning("ENGINE - " + Name, "RECOGNIZED Speech: " + rr.Confidence + " (" + confidence + ") / " + rr.Words[0].Confidence + " (" + rr.Words[0].Text + ")" + " Device: " + " Text: " + rr.Text);
cfg.logDebug("ENGINE - " + Name, xnav.OuterXml);
if (cfg.DEBUG)
{
WSRSpeechManager.GetInstance().DumpAudio(rr);
}
return(xnav);
}
private void recognizeButton_Click(object sender, EventArgs e)
{
WAVSound copiedTestSound = soundVisualizer.Sound.Copy();
if (setMaximumNonclippingVolumeToolStripMenuItem.Checked)
{
copiedTestSound.SetMaximumNonClippingVolume();
soundVisualizer.SetSound(copiedTestSound);
}
recognitionResult = recognizer.RecognizeSingle(copiedTestSound);
// 20170114
if (recognitionResult == null)
{
recognitionResultTextBox.Text = "No sound data!";
return;
}
// end 20170114
deviationListBox.Items.Clear();
for (int ii = 0; ii < recognitionResult.DeviationList.Count; ii++)
{
deviationListBox.Items.Add(recognitionResult.DeviationList[ii].Item1.PadRight(16) + " " +
recognitionResult.DeviationList[ii].Item2.ToString("0.0000"));
}
if (recognitionResult.DeviationList.Count > 0)
{
if (recognitionResult.DeviationList[0].Item2 < recognizer.RecognitionThreshold)
{
recognitionResultTextBox.Text = recognitionResult.DeviationList[0].Item1;
}
else
{
recognitionResultTextBox.Text = "UNKNOWN";
}
featureComparisonComboBox.Items.Clear();
if (recognizer.AverageSoundFeatureSetList.Count > 0)
{
for (int ii = 0; ii < recognizer.AverageSoundFeatureSetList[0].FeatureList.Count; ii++)
{
featureComparisonComboBox.Items.Add(recognizer.AverageSoundFeatureSetList[0].FeatureList[ii].Name);
}
featureComparisonComboBox.SelectedIndex = 0;
}
}
}
public string Nasluch()
{
RecognitionResult result = sre.Recognize(new TimeSpan(0, 0, 20)); //rozpoznawaj przez 20 sekund
if (result == null)
{
return(null);
}
if (result.Semantics.Value != null)
{
return(result.Semantics.Value.ToString());
}
else if (result.Semantics.Count > 0)
{
return(string.Join(";", result.Semantics.Select(x => x.Key + "=" + x.Value.Value).ToArray()));
}
return(null);
}
public override void doCommand(RecognitionResult speechResult)
{
base.doCommand(speechResult);
if (speechResult.Semantics["item"].Value.ToString().Contains("NPC"))
{
Creature newCreature = Catalog.current.GetData <CreatureData>(speechResult.Semantics["item"].Value.ToString().Replace(" NPC", "")).Spawn(Player.local.head.transform.position + 0.5f * Player.local.body.transform.right, Quaternion.identity, false);
return;
}
Debug.Log("Spawning you a " + speechResult.Semantics["item"].Value);
Item item = Catalog.current.GetData <ItemData>(speechResult.Semantics["item"].Value.ToString(), true).Spawn(true, null);
PlayerHand hand = speechResult.Semantics["optional"].Value.ToString().Contains("left") ? Player.local.handLeft : Player.local.handRight;
item.transform.position = hand.transform.position;
if (Creature.player.body.GetHand(hand.side).interactor.grabbedHandle)
{
Creature.player.body.GetHand(hand.side).interactor.UnGrab(true);
}
Creature.player.body.GetHand(hand.side).interactor.Grab(item.GetMainHandle(hand.side));
}
private void HandleMode1(RecognitionResult result)
{
// Si no hay un resultado claro al reconocer el caracter solo se limpia el canvas
if (result == null)
{
ResetCanvasAndTimer();
return;
}
var resultString = result.TopString;
// Si el caracter reconocido no es valido solo se limpia el canvas
if (!IsAcceptedChar(resultString))
{
ResetCanvasAndTimer();
return;
}
// Cuando se reconoce ">" se comprueba la expresión introducida hasta el momento y
// si es valida se muestra su resultado
if (resultString.Equals(">"))
{
if (!IsAcceptedExpression(resultBox.Text))
{
MessageBox.Show("Expresión no valida", "Error", MessageBoxButton.OK, MessageBoxImage.Error);
expressionDone = true;
ResetCanvasAndTimer();
return;
}
DataTable dt = new DataTable();
var value = dt.Compute(resultBox.Text, "");
resultString = "=" + value;
expressionDone = true;
}
resultString = Regex.Replace(resultString, @"[xX]", "*");
resultBox.Text += resultString;
ResetCanvasAndTimer();
}
/// <summary>
/// Builds a SpeechRecognitionResult object from a RecognitionResult returned by the recognizer.
/// </summary>
/// <param name="result">The RecognitionResult object.</param>
/// <returns>A SpeechRecognitionResult object containing the recognition results.</returns>
private StreamingSpeechRecognitionResult BuildSpeechRecognitionResult(RecognitionResult result)
{
// Allocate a buffer large enough to hold the raw audio bytes. Round up to account for precision errors.
byte[] audioBytes = new byte[(int)Math.Ceiling(result.Audio.Duration.TotalSeconds * result.Audio.Format.AverageBytesPerSecond)];
// Write recognized audio in the RecognitionResult to a MemoryStream and create an AudioBuffer of the entire utterance
using (MemoryStream ms = new MemoryStream(audioBytes))
{
result.Audio.WriteToAudioStream(ms);
}
return(new StreamingSpeechRecognitionResult(
true,
result.Text,
result.Confidence,
result.Alternates.Select(p => new SpeechRecognitionAlternate(p.Text, p.Confidence)),
new AudioBuffer(audioBytes, this.Configuration.InputFormat),
result.Audio?.Duration));
}
// byte protocol (full):
// 0. first block length in bytes (including this one)
// 1. message format (0 == minimal)
// 2. message confidence (0-100)
// 3. opcode
// 4. arg1 (optional)
// 5. arg2 (optional)
// 6. checksum (of this block)
// 7. culture code (0 = en-US, 1 = fr-CA)
// 8. N - number of words
// repeated N times:
// 9. confidence (0-100)
// 10. word (array of 11 chars, always null-terminated)
/// <summary>
/// recognition result to byte array - full format
/// </summary>
/// <param name="result"></param>
/// <param name="cultureinfo"></param>
/// <param name="full"></param>
/// <returns></returns>
public byte[] ParseResult(RecognitionResult result, byte cultureinfo, bool full)
{
Byte[] res = ParseResult(result);
if (!full)
return res;
res[1] = 1; // full format;
List<byte> block2 = new List<byte>();
block2.Add(cultureinfo);
block2.Add((byte)(result.Words.Count)); // number of words
foreach (var v in result.Words)
{
block2.AddRange(ParseWord(v));
}
List<byte> lst = res.ToList();
lst.AddRange(block2);
return lst.ToArray();
}
public void ProcessResult(RecognitionResult result)
{
Console.WriteLine(result.Text);
if (result.Semantics.ContainsKey(expecting.ToString()))
{
if (result.Semantics.ContainsKey(Slots.Major.ToString()))
{
if (result.Semantics[Slots.Major.ToString()].Value.ToString().Equals("CSC"))
{
// ignore major
promptForGradYear();
}
else
{
repromptForMajor();
}
}
else if (result.Semantics.ContainsKey(Slots.GradYear.ToString()))
{
String year = result.Semantics[Slots.GradYear.ToString()].Value.ToString();
int.TryParse(year, out GradYear);
confirmGradYear();
}
else if (result.Semantics.ContainsKey(Slots.YesNo.ToString()))
{
if (result.Semantics[Slots.YesNo.ToString()].Value.ToString().Equals("yes"))
{
gradYearConfirmed();
}
else
{
repromptForGradYear();
}
}
}
}
// byte protocol (minimal):
// 0. message length in bytes (including this one)
// 1. message format (0 == minimal)
// 2. message confidence (0-100)
// 3. opcode
// 4. arg1 (optional)
// 5. arg2 (optional)
// 6. checksum
/// <summary>
/// recognition result to byte array - minimal format
/// </summary>
/// <param name="result"></param>
/// <returns></returns>
public byte[] ParseResult(RecognitionResult result)
{
String opcode;
List<int> args;
byte[] res = null;
int opval;
if (GetOpcodeArgs(result, out opcode, out args))
{
int len = args.Count + 5; // header, format, confidence, opcode, checksum
int k = 0;
res = new byte[len];
res[k++] = (byte)len;
res[k++] = 0; // minimal format
res[k++] = (byte)(0.5 + 100 * result.Semantics.Confidence);
res[k++] = Cfg.Instance.Opcodes.TryGetValue(opcode, out opval) ? (byte)opval : (byte)(0);
foreach (int i in args)
res[k++] = (byte)i;
res[k] = (byte)((256 - res.Checksum()) % 256);
return res;
}
return new byte[0];
}
// ==========================================
// RECOGNIZE
// ==========================================
protected void SpeechRecognized(RecognitionResult rr)
{
// 1. Prevent while speaking
if (WSRSpeakerManager.GetInstance().Speaking) {
cfg.logWarning("ENGINE - " + Name, "REJECTED Speech while speaking : " + rr.Confidence + " Text: " + rr.Text);
return;
}
// 2. Prevent while working
XPathNavigator xnav = HandleSpeech(rr);
if (xnav == null) {
return;
}
// 3. Reset context timeout
if (rr.Grammar.Name != "Dyn") {
WSRSpeechManager.GetInstance().ResetContextTimeout();
}
else {
cfg.logInfo("ENGINE - " + Name, "DYN reset to default context");
WSRSpeechManager.GetInstance().SetContext("default");
WSRSpeechManager.GetInstance().ForwardContext();
}
// 4. Track Audio Pitch
TrackPitch(rr);
// 5. Reset Speech Buffer
WSRSpeakerManager.GetInstance().SpeechBuffer = "";
// 6. Hook
String path = cfg.WSR.HandleCustomAttributes(xnav);
// 7. Parse Result's URL
String url = GetURL(xnav, rr.Confidence);
// 8. Parse Result's Dication
url = HandleWildcard(rr, url);
if (url == null) { return; }
// 9. Send the request
HandleRequest(url, path);
}
/// <summary>
/// wordTypeResult returns the alternative pronunciation deemed best by some method
/// </summary>
/// <param name="method">Method to decide the best pronunciation</param>
/// <param name="result">Recognition result</param>
/// <returns>best result</returns>
private String[] wordTypeResult(int method, RecognitionResult result)
{
String[] output = null;
switch (method)
{
case(0):
{
output = result.Alternates[0].Words[0].Text.Split('_');
break;
}
case (1):
{
Double count = 1;
Double alpha = 2;
Dictionary<String, Double> scoreTable = new Dictionary<String, Double>();
// Updating the results using the rank formula: 1/(r^alpha)
foreach (RecognizedPhrase altPhrase in result.Alternates)
{
String altPronun = altPhrase.Words[0].Text;
Double score = 1.0 / (Math.Pow(count, alpha));
Double tempScore;
if (scoreTable.ContainsKey(altPronun))
{
tempScore = scoreTable[altPronun] + score;
scoreTable.Remove(altPronun);
}
else
{
tempScore = score;
}
scoreTable.Add(altPronun, tempScore);
}
// Sorting and returning the best result
String[] values = new String[scoreTable.Count];
Double[] keys = new Double[scoreTable.Count];
scoreTable.Values.CopyTo(keys, 0);
scoreTable.Keys.CopyTo(values, 0);
Array.Sort(keys, values);
output = values[scoreTable.Count - 1].Split('_');
break;
}
case (2):
{
Dictionary<String, Double> scoreTable = new Dictionary<String, Double>();
// Updating the results using the rank formula: 1/(r^alpha)
foreach (RecognizedPhrase altPhrase in result.Alternates)
{
String altPronun = altPhrase.Words[0].Text;
Double score = altPhrase.Words[0].Confidence;
Double tempScore;
if (scoreTable.ContainsKey(altPronun))
{
tempScore = scoreTable[altPronun] + score;
scoreTable.Remove(altPronun);
}
else
{
tempScore = score;
}
scoreTable.Add(altPronun, tempScore);
}
// Sorting and returning the best result
String[] values = new String[scoreTable.Count];
Double[] keys = new Double[scoreTable.Count];
scoreTable.Values.CopyTo(keys, 0);
scoreTable.Keys.CopyTo(values, 0);
Array.Sort(keys, values);
output = values[scoreTable.Count - 1].Split('_');
break;
}
default:
throw new Exception("Tester.wordTypeResult: wrong case\n");
}
return output;
}
/// \brief Translates recognized command to plugin method, then Executes.
private void doCommand(RecognitionResult speechResult)
{
string[] metaData = speechResult.Grammar.Name.Split(':');
// plugin to look in
string className = metaData[0];
// unique method - for overloading support
string methodUniqueName = metaData[1];
// method name - for invoking
string methodName = methodUniqueName.Split('-')[0];
// method words - for filtering
int methodParts = methodName.Split('_').Count();
PluginBase plugin = pluginsDict[className];
ParameterInfo[] parameterInfos = plugin.MethodParameterInfo[methodUniqueName];
int paramIndex = 0;
List<object> parameterList = new List<object>();
List<Type> parameterTypes = new List<Type>();
// skip activation word + method words
foreach (RecognizedWordUnit word in speechResult.Words.Where(word => word.Text != "...").Skip(1 + methodParts))
{
// add type for method init
parameterTypes.Add(parameterInfos[paramIndex].ParameterType);
ParameterInfo parameter = parameterInfos[paramIndex++];
// if Enum
if (parameter.ParameterType.IsEnum)
{
parameterList.Add(Enum.Parse(parameter.ParameterType, word.Text));
}
else
{
// get param name
string paramName = parameter.Name;
// get corrosponding Dictionary
IDictionary tempDict = (IDictionary)plugin.GetType().GetField(paramName).GetValue(plugin);
Dictionary<string, object> paramDict = CastDict(tempDict).ToDictionary(entry => (string)entry.Key,
entry => entry.Value);
dynamic realParam = paramDict[word.Text];
parameterList.Add(realParam);
}
}
object[] parameters;
if (parameterList.Count == 0)
parameters = null;
else
parameters = parameterList.ToArray();
if (parameterInfos == null)
plugin.GetType().GetMethod(methodName).Invoke(plugin, parameters);
else
plugin.GetType().GetMethod(methodName, parameterTypes.ToArray()).Invoke(plugin, parameters);
}
private void RejectSpeech(RecognitionResult result)
{
string status = "Rejected: " + (result == null ? string.Empty : result.Text + " " + result.Confidence);
this.ReportSpeechStatus(status);
// Dispatcher.BeginInvoke(new Action(() => { tbColor.Background = blackBrush; }), DispatcherPriority.Normal);
}
//if speech is rejected
private void RejectSpeech(RecognitionResult result)
{
//TextBlock1.Text = "Pardon Moi?";
}
/// <summary>
/// Try to recognize the image and return its label
/// </summary>
/// <param name="image">The image to be recognized</param>
/// <returns>
/// Recognition result. Null if the eigen distance of the result is greater than the preset threshold.
/// </returns>
public RecognitionResult Recognize(Image<Gray, Byte> image)
{
float[] dist = GetEigenDistances(image);
int index = 0;
float eigenDistance = dist[0];
for (int i = 1; i < dist.Length; i++)
{
if (dist[i] < eigenDistance)
{
index = i;
eigenDistance = dist[i];
}
}
RecognitionResult result = new RecognitionResult();
result.Distance = dist[index];
result.Index = index;
result.Label = Labels[index];
return result.Distance <= _eigenDistanceThreshold ? result : null;
}
// -------------------------------------------
// SPEECH
// -------------------------------------------
protected bool Confidence(String device, RecognitionResult rr, XPathNavigator xnav, double threashold) {
double confidence = rr.Confidence;
// Override Engine threashold by grammar
XPathNavigator level = xnav.SelectSingleNode("/SML/action/@threashold");
if (level != null) {
Log("Override confidence: " + level.Value);
threashold = level.ValueAsDouble;
}
// Search for bot name
double match = 0;
double bot = ConfigManager.GetInstance().Find("bot.confidence", 0.0);
string name = ConfigManager.GetInstance().Find("bot.name", "SARAH").ToUpper();
foreach (var word in rr.Words) {
if (!name.Equals(word.Text.ToUpper())) { continue; }
if (word.Confidence < threashold + bot) { // Check the bot name with threashold confidence
Log("REJECTED " + name + ": " + word.Confidence + " < " + (threashold + bot) + " (" + rr.Text + ")");
return false;
}
match = word.Confidence;
break;
}
// Must have bot name in sentence
var grammar = GrammarManager.GetInstance().FindGrammar(rr.Grammar.Name);
if (match == 0 && grammar.HasName && !AddOnManager.GetInstance().IsEngaged(device)) { return false; }
// Check full sentence
if (confidence < threashold) {
Log("REJECTED: " + confidence + " (" + match + ") " + " < " + threashold + " (" + rr.Text + ")");
return false;
}
// Speech recognized
Log("RECOGNIZED: " + confidence + " (" + match + ") " + " > " + threashold + " (" + rr.Text + ")");
return true;
}
private void SpeechRecognized(RecognitionResult speechRecognized)
{
SemanticValue semanticCommandValue = speechRecognized.Semantics["command"];
float value = 1;
if(speechRecognized.Semantics.ContainsKey("multiplier"))
{
SemanticValue semanticMultiplierValue = speechRecognized.Semantics["multiplier"];
value = float.Parse((string)semanticMultiplierValue.Value);
}
if (!skeletonRepository.FirstUser.IsFacingForward)
{
Output.Debug("KinectControlWindow", "User not facing forward, speech ignored.");
this.listBox1.Items.Clear();
listHelper.AddItemToList(speechRecognized, listBox1, Colors.Red);
return;
}
this.listBox1.Items.Clear();
listHelper.AddItemToList(speechRecognized, listBox1, Colors.DarkBlue);
string speech = (string)semanticCommandValue.Value;
if (PowerPointGrammar.NEXT_SLIDE.Equals(speech))
{
commands.ProcessNextSlide();
}
else if (PowerPointGrammar.PREVIOUS_SLIDE.Equals(speech))
{
commands.ProcessPreviousSlide();
}
else if (PowerPointGrammar.CLOSE_PRESENTATION.Equals(speech))
{
commands.ProcessClosePresentation();
}
else if (ImagePresentationGrammar.MOVE_RIGHT.Equals(speech))
{
//move port view to right
commands.ProcessMoveLeft(value);
}
else if (ImagePresentationGrammar.MOVE_LEFT.Equals(speech))
{
//move port view to left
commands.ProcessMoveRight(value);
}
else if (ImagePresentationGrammar.MOVE_UP.Equals(speech))
{
//move port view up
commands.ProcessMoveDown(value);
}
else if (ImagePresentationGrammar.MOVE_DOWN.Equals(speech))
{
//move port view down
commands.ProcessMoveUp(value);
}
else if (ImagePresentationGrammar.ROTATE_RIGHT.Equals(speech))
{
commands.ProcessRotateRight();
}
else if (ImagePresentationGrammar.ROTATE_LEFT.Equals(speech))
{
commands.ProcessRotateLeft();
}
else if (ImagePresentationGrammar.ZOOM_IN.Equals(speech))
{
commands.ProcessZoomIn(value);
}
else if (ImagePresentationGrammar.ZOOM_OUT.Equals(speech))
{
commands.ProcessZoomOut(value);
}
else if (ImagePresentationGrammar.CLOSE_IMAGE.Equals(speech))
{
commands.ProcessCloseImage();
}
}
private void SpeechHypothesized(RecognitionResult speechHypothesized)
{
listHelper.AddItemToList(speechHypothesized, listBox1, Colors.DarkRed);
}
/// <summary>
/// Initializes a new instance of the <see cref="GestureEventArgs" /> class.
/// </summary>
/// <param name="result">The result.</param>
public GestureEventArgs(RecognitionResult result, GestureType type)
{
this.Result = result;
this.GestureType = type;
}
protected XPathNavigator HandleSpeech(RecognitionResult rr)
{
XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator();
double confidence = GetConfidence(xnav);
if (rr.Confidence < confidence) {
cfg.logWarning("ENGINE - "+Name, "REJECTED Speech: " + rr.Confidence + " < " + confidence + " Device: " + " Text: " + rr.Text);
return null;
}
if (rr.Words[0].Confidence < cfg.trigger) {
cfg.logWarning("ENGINE - "+Name, "REJECTED Trigger: " + rr.Words[0].Confidence + " Text: " + rr.Words[0].Text);
return null;
}
cfg.logWarning("ENGINE - "+Name, "RECOGNIZED Speech: " + rr.Confidence + " / " + rr.Words[0].Confidence + " (" + rr.Words[0].Text + ")" + " Device: " + " Text: " + rr.Text);
cfg.logDebug("ENGINE - "+Name, xnav.OuterXml);
if (cfg.DEBUG) { WSRSpeechManager.GetInstance().DumpAudio(rr); }
return xnav;
}
protected String HandleWildcard(RecognitionResult rr, String url)
{
if (rr.Audio == null) { return url; }
XPathNavigator xnav = rr.ConstructSmlFromSemantics().CreateNavigator();
XPathNavigator wildcard = xnav.SelectSingleNode("/SML/action/@dictation");
if (wildcard == null) { return url; }
// Retrieve language
String language = cfg.language;
if (wildcard.Value != "true") {
language = wildcard.Value;
}
// Google
using (MemoryStream audioStream = new MemoryStream()) {
rr.Audio.WriteToWaveStream(audioStream);
// rr.GetAudioForWordRange(rr.Words[word], rr.Words[word]).WriteToWaveStream(audioStream);
audioStream.Position = 0;
var speech2text = WSRSpeechManager.GetInstance().ProcessAudioStream(audioStream, language);
if (url != null) {
url += "dictation=" + HttpUtility.UrlEncode(speech2text);
}
}
return url;
}
protected void TrackPitch(RecognitionResult rr)
{
if (rr.Audio == null) { return; }
using (MemoryStream audioStream = new MemoryStream()) {
rr.Audio.WriteToWaveStream(audioStream);
audioStream.Position = 0;
byte[] audioBytes = audioStream.ToArray();
float[] audioBuffer = new float[audioBytes.Length / 2];
for (int i = 0, j = 0; i < audioBytes.Length / 2; i += 2, j++) {
// convert two bytes to one short
short s = BitConverter.ToInt16(audioBytes, i);
// convert to range from -1 to (just below) 1
audioBuffer[j] = s / 32768.0f;
}
// Reset
tracker.Reset();
pitch.Clear();
// Process
tracker.ProcessBuffer(audioBuffer);
// Notify
WSRProfileManager.GetInstance().UpdatePitch(pitch.Mean());
}
}
