/// <summary>
/// Default Constructor
/// </summary>
public CreateGesture()
{
InitializeComponent();
//this.Gesture = new tempuri.org.GestureDefinition.xsd.Gesture();
this.Gesture = new IavaGesture();
// If we are in design mode, do nothing...
if (DesignerProperties.GetIsInDesignMode(this)) { return; }
// Subscribe to the Camera events we are interested in...
try {
IavaCamera.ImageFrameReady += OnCameraImageFrameReady;
IavaCamera.SkeletonFrameReady += OnCameraSkeletonFrameReady;
}
catch (Exception e) {
}
// Set up the Audio Recognizer...
AudioRecognizer = new AudioRecognizer();
AudioRecognizer.SyncCommand = "IAVA";
AudioRecognizer.Subscribe("Capture", CaptureCallback);
AudioRecognizer.Subscribe("Snapshot", CaptureCallback);
AudioRecognizer.Start();
}
public MainWindow()
{
InitializeComponent();
AudioRecognizer = new Iava.Audio.AudioRecognizer();
AudioRecognizer.Subscribe("Create Gesture", CreateAudioCallback);
AudioRecognizer.Subscribe("Test Gesture", TestAudioCallback);
AudioRecognizer.Subscribe("Exit", ExitAudioCallback);
AudioRecognizer.Start();
CreateGesture.Visibility = System.Windows.Visibility.Hidden;
TestGesture.Visibility = System.Windows.Visibility.Hidden;
}
/// <summary>
/// Default Constructor
/// </summary>
public MainWindow()
{
InitializeComponent();
const string gesturesPath = @"..\..\..\Gestures\";
m_pGestureRecognizer = new Gesture.GestureRecognizer(gesturesPath);
m_pAudioRecognizer = new AudioRecognizer();
m_pAudioRecognizer.AudioConfidenceThreshold = 0.8f;
// Initialize the CityLocations class by calling a method on it
CityLocations.GetCityLocation("test");
// Events
m_pAudioRecognizer.StatusChanged += OnAudioRecognizerStatusChanged;
m_pGestureRecognizer.StatusChanged += OnGestureRecognizerStatusChanged;
m_pAudioRecognizer.Synced += OnAudioRecognizerSynced;
m_pGestureRecognizer.Synced += OnGestureRecognizerSynced;
m_pAudioRecognizer.Unsynced += OnAudioRecognizerUnsynced;
m_pGestureRecognizer.Unsynced += OnGestureRecognizerUnsynced;
// Gesture Callbacks
string[] mappedGestureNames = new[]
{
"Zoom In",
"Zoom Out",
"Swipe Left",
"Swipe Right",
"Swipe Up",
"Swipe Down"
};
m_pGestureRecognizer.Subscribe(mappedGestureNames[0], GestureZoomInCallback);
m_pGestureRecognizer.Subscribe(mappedGestureNames[1], GestureZoomOutCallback);
m_pGestureRecognizer.Subscribe(mappedGestureNames[2], GestureSwipeLeftCallback);
m_pGestureRecognizer.Subscribe(mappedGestureNames[3], GestureSwipeRightCallback);
m_pGestureRecognizer.Subscribe(mappedGestureNames[4], GestureSwipeUpCallback);
m_pGestureRecognizer.Subscribe(mappedGestureNames[5], GestureSwipeDownCallback);
// Add unmapped gestures so that they will be recognized
foreach (IavaGesture gesture in GestureFolderReader.Read(gesturesPath))
{
if (!Equals("Sync", gesture.Name) && !mappedGestureNames.Contains<string>(gesture.Name))
{
m_pGestureRecognizer.Subscribe(gesture.Name, GestureUnmappedCallback);
}
}
// Audio Callbacks
m_pAudioRecognizer.Subscribe("Zoom In", ZoomInCallback);
m_pAudioRecognizer.Subscribe("Zoom Out", ZoomOutCallback);
m_pAudioRecognizer.Subscribe("Move North", MoveNorthCallback);
m_pAudioRecognizer.Subscribe("Move South", MoveSouthCallback);
m_pAudioRecognizer.Subscribe("Move East", MoveEastCallback);
m_pAudioRecognizer.Subscribe("Move West", MoveWestCallback);
m_pAudioRecognizer.Subscribe("Exit Application", BlowUp);
m_pAudioRecognizer.Subscribe("Locate *", LocateCityCallback);
m_pAudioRecognizer.Subscribe("Get Recognizer Statuses", GetRecognizerStatusesCallback);
IavaCamera.ImageFrameReady += OnCameraImageFrameReady;
IavaCamera.SkeletonFrameReady += OnCameraSkeletonFrameReady;
m_pAudioSyncTimer = new System.Timers.Timer(1000);
m_pAudioSyncTimer.Elapsed += OnAudioSyncTimerElapsed;
m_pGestureSyncTimer = new System.Timers.Timer(1000);
m_pGestureSyncTimer.Elapsed += OnGestureSyncTimerElapsed;
m_pConsoleTxtBox = new TextBoxStreamWriter(this.txtConsole);
Console.SetOut(m_pConsoleTxtBox);
// UI theme stuff...
lblAudioStatus.Background = lblGestureStatus.Background = unsyncedBrush;
lblAudioSyncTime.Background = lblGestureSyncTime.Background = backgroundBrush;
lblAudioTTL.Background = lblGestureTTL.Background = backgroundBrush;
}
public void AudioWildcardTest()
{
// Create a mock speech engine and set it up
var mockEngine = SetupMockSpeechRecognitionEngine();
recognizer = new AudioRecognizer(mockEngine.Object);
const string commandString = "Test Callback *";
string eventArgsCommand = null;
List<string> commands = null;
recognizer.Subscribe(commandString, (eventArgs) =>
{
eventArgsCommand = eventArgs.Command;
commands = eventArgs.CommandWildcards;
resetEvent.Set();
});
recognizer.Start();
// Sync the recognizer
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(recognizer.SyncCommand, recognizer.AudioConfidenceThreshold + 0.01f));
Thread.Sleep(50);
// Raise the recognized command
const string wildcardPhrase = "one two three";
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString.Replace('*', ' ') + wildcardPhrase, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
Assert.AreEqual(commandString, eventArgsCommand, "Command string returned did not match expected value.");
Assert.IsNotNull(commands);
Assert.AreEqual(3, commands.Count);
Assert.AreEqual("one", commands[0]);
Assert.AreEqual("two", commands[1]);
Assert.AreEqual("three", commands[2]);
}
public void AudioUnsubscribeTest()
{
const string commandString = "Test Callback";
// Create a mock speech engine and set it up
var mockEngine = SetupMockSpeechRecognitionEngine();
recognizer = new AudioRecognizer(mockEngine.Object);
// Test the audio confidence level
try
{ recognizer.AudioConfidenceThreshold = -0.5f; }
catch (Exception e)
{
Assert.IsInstanceOfType(e, typeof(ArgumentOutOfRangeException));
}
try
{ recognizer.AudioConfidenceThreshold = 1.01f; }
catch (Exception e)
{
Assert.IsInstanceOfType(e, typeof(ArgumentOutOfRangeException));
}
recognizer.AudioConfidenceThreshold = 0.6f;
// Raise the speech recognized event, unsubscribe the event, and try the command again
string eventArgsCommand = null;
recognizer.Subscribe(commandString, (eventArgs) =>
{
eventArgsCommand = eventArgs.Command;
resetEvent.Set();
});
recognizer.Start();
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(recognizer.SyncCommand, recognizer.AudioConfidenceThreshold + 0.01f));
Thread.Sleep(50);
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
Assert.AreEqual(commandString, eventArgsCommand, "Command string returned did not match expected value.");
resetEvent.Reset();
// Unsubscribe the callback and ensure it is not called when the audio command was recognized
recognizer.Unsubscribe(commandString);
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsFalse(resetEvent.WaitOne(TimeoutValue));
// For code coverage
try
{ recognizer.Unsubscribe(null); }
catch (Exception e)
{
Assert.IsInstanceOfType(e, typeof(ArgumentException));
}
try
{ recognizer.Unsubscribe("Non Existant Key"); }
catch (Exception e)
{
Assert.IsInstanceOfType(e, typeof(ArgumentException));
}
}
public void AudioSyncUnsyncTest()
{
const string commandString = "Test Callback";
// Create a mock speech engine and set it up
var mockEngine = SetupMockSpeechRecognitionEngine();
recognizer = new AudioRecognizer(mockEngine.Object);
recognizer.SyncTimeoutValue = 500;
recognizer.Synced +=
(sender, args) =>
{
resetEvent.Set();
};
recognizer.Unsynced +=
(sender, args) =>
{
resetEvent.Set();
};
// Sync the recognizer, call a command, wait until un-sync and re-call the same command.
// Ensure the callback is not called twice.
string eventArgsCommand = null;
recognizer.Subscribe(commandString, (eventArgs) =>
{
eventArgsCommand = eventArgs.Command;
resetEvent.Set();
});
recognizer.Start();
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(recognizer.SyncCommand, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
resetEvent.Reset();
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
Assert.AreEqual(commandString, eventArgsCommand, "Command string returned did not match expected value.");
resetEvent.Reset();
Assert.IsTrue(resetEvent.WaitOne(recognizer.SyncTimeoutValue + 100));
resetEvent.Reset();
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsFalse(resetEvent.WaitOne(TimeoutValue));
}
public void AudioSyncCommandTest()
{
// Create a mock speech engine and set it up
var mockEngine = SetupMockSpeechRecognitionEngine();
recognizer = new AudioRecognizer(mockEngine.Object);
// Test changing the sync command
try { recognizer.SyncCommand = null; }
catch (Exception exception)
{
Assert.IsInstanceOfType(exception, typeof(ArgumentException));
}
try { recognizer.SyncCommand = string.Empty; }
catch (Exception exception)
{
Assert.IsInstanceOfType(exception, typeof(ArgumentException));
}
try { recognizer.SyncCommand = " "; }
catch (Exception exception)
{
Assert.IsInstanceOfType(exception, typeof(ArgumentException));
}
recognizer.SyncCommand = "Test Sync Command";
const string commandString = "Test Callback";
string eventArgsCommand = null;
recognizer.Subscribe(commandString, (eventArgs) =>
{
eventArgsCommand = eventArgs.Command;
resetEvent.Set();
});
recognizer.Start();
// Sync the callback first then raise spoken event
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(recognizer.SyncCommand, recognizer.AudioConfidenceThreshold + 0.01f));
Thread.Sleep(50);
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
Assert.AreEqual(commandString, eventArgsCommand, "Command string returned did not match expected value.");
resetEvent.Reset();
// Change the sync command, re-sync and recognize the command again
recognizer.SyncCommand = "Open the pod bay doors HAL...";
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(recognizer.SyncCommand, recognizer.AudioConfidenceThreshold + 0.01f));
Thread.Sleep(50);
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
}
public void AudioSubscribeTest()
{
const string commandString = "Test Callback";
// Create a mock speech engine and set it up
var mockEngine = SetupMockSpeechRecognitionEngine();
recognizer = new AudioRecognizer(mockEngine.Object);
string eventArgsCommand = null;
recognizer.Subscribe(commandString, (eventArgs) =>
{
eventArgsCommand = eventArgs.Command;
resetEvent.Set();
});
// Test for exception throwing on invalid parameters entered
try
{ recognizer.Subscribe(string.Empty, (eventArgs) => { }); }
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentException));
}
try
{ recognizer.Subscribe(commandString, null); }
catch (Exception ex)
{
Assert.IsInstanceOfType(ex, typeof(ArgumentException));
}
recognizer.Start();
const string commandString2 = "Command String 2";
string eventArgsCommand2 = null;
recognizer.Subscribe(commandString2, (eventArgs) =>
{
eventArgsCommand2 = eventArgs.Command;
resetEvent.Set();
});
// Sync the callback first then raise spoken event
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs("Blah Blah blah", recognizer.AudioConfidenceThreshold + 0.01f));
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(recognizer.SyncCommand, recognizer.AudioConfidenceThreshold + 0.01f));
Thread.Sleep(200);
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold + 0.01f));
Thread.Sleep(200);
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs("Blah Blah blah", recognizer.AudioConfidenceThreshold + 0.01f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
Assert.AreEqual(commandString, eventArgsCommand, "Command string returned did not match expected value.");
resetEvent.Reset();
// Callback with the same command but with confidence below the threshold level to ensure the audio callback method is not called
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString, recognizer.AudioConfidenceThreshold - 0.01f));
Thread.Sleep(200);
Assert.IsFalse(resetEvent.WaitOne(TimeoutValue));
// Call second command and wait for callback to be invoked
mockEngine.Raise(m => m.SpeechRecognized += null, new IavaSpeechRecognizedEventArgs(commandString2, 0.95f));
Assert.IsTrue(resetEvent.WaitOne(TimeoutValue));
Assert.AreEqual(commandString2, eventArgsCommand2, "Command string returned did not match expected value.");
resetEvent.Reset();
// For code coverage
try
{ recognizer.Subscribe(commandString2, (eventArgs) => { }); }
catch (Exception e)
{
Assert.IsInstanceOfType(e, typeof(ArgumentException));
}
}
