private void ThisAddIn_Startup(object sender, System.EventArgs e)
{
// only one sensor is currently supported
kinectSensor = KinectSensor.GetDefault();
// set IsAvailableChanged event notifier
kinectSensor.IsAvailableChanged += Sensor_IsAvailableChanged;
// open the sensor
kinectSensor.Open();
// open the reader for the body frames
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
// initialize the gesture detection objects for our gestures
gestureDetectorList = new List <GestureDetector>();
// create a gesture detector for each body (6 bodies => 6 detectors) and create content controls to display results in the UI
int maxBodies = 6;
for (int i = 0; i < maxBodies; ++i)
{
GestureResultView result = new GestureResultView(i, false, false, 0.0f);
GestureDetector detector = new GestureDetector(kinectSensor, result);
gestureDetectorList.Add(detector);
}
// set the BodyFramedArrived event notifier
bodyFrameReader.FrameArrived += Reader_BodyFrameArrived;
}
// this class is loaded when we want to open kinect after using the app manually
void OnOpenSensor(object sender, RoutedEventArgs e) // called when the button is clicked to open sensor
{
// just to tell kinect started
speaker = new SpeechSynthesizer(); // Used for getting output voice from computer
speaker.Speak("App is started");
System.Windows.Forms.MessageBox.Show("Kinect started - Click on Map Now");
// FOR THE VGB ONE
// open the sensor
this.kinectSensor.Open();
// open the reader for the body frames
this.bodyFrameReader = this.kinectSensor.BodyFrameSource.OpenReader();
// set the BodyFramedArrived event notifier
this.bodyFrameReader.FrameArrived += this.Reader_BodyFrameArrived;
// initialize the BodyViewer object for displaying tracked bodies in the UI
this.kinectBodyView = new KinectBodyView(this.kinectSensor);
// initialize the gesture detection objects for our gestures
this.gestureDetectorList = new List<GestureDetector>();
// set our data context objects for display in UI
// It is usd to show the bone stucture in UI
this.DataContext = this;
this.kinectBodyViewbox.DataContext = this.kinectBodyView;
// As Kinect V2 can detect upto 6 bdies and so can detect gesture from all six bodies so we can used that by assigning gesture detector for each body
// create a gesture detector for each body (6 bodies => 6 detectors) and create content controls to display results in the UI
int maxBodies = this.kinectSensor.BodyFrameSource.BodyCount; // if body is six then it goes to check for each.
for (int i = 0; i < maxBodies; ++i)
{
GestureResultView result = new GestureResultView(i, false, false, 0.0f, webView1,gestureState.Text);
// i is body number,webview1 is the html control which have to pass to access it in this method. // others defiend later
GestureDetector detector = new GestureDetector(this.kinectSensor, result);
this.gestureDetectorList.Add(detector);
}
// FOR THE JOINT ONE
if (_sensor != null)
{
_sensor.Open();
_reader = _sensor.OpenMultiSourceFrameReader(FrameSourceTypes.Color | FrameSourceTypes.Depth | FrameSourceTypes.Infrared | FrameSourceTypes.Body);
_reader.MultiSourceFrameArrived += Reader_MultiSourceFrameArrived;
}
// FOR THE SOUND
this.sensor = KinectSensor.GetDefault();
// open the sensor
this.sensor.Open();
// grab the audio stream
IReadOnlyList<AudioBeam> audioBeamList = this.sensor.AudioSource.AudioBeams;
System.IO.Stream audioStream = audioBeamList[0].OpenInputStream();
// create the convert stream
this.convertStream = new KinectAudioStream(audioStream);
RecognizerInfo ri = TryGetKinectRecognizer();
// these are engine where we put what to analyse in speech recogniition it can not be done if we can use bing speech recognizer but it is not possible to use now in germany as in beta phase
this.speechEngine = new SpeechRecognitionEngine(ri.Id); // speech recognition engine
var directions = new Choices(); // direction is the vaariable used to direct where we have to go
directions.Add(new SemanticResultValue("ZoomIn", "ZOOMIN")); // first one is the text which system see in audio and if match then it save the second name in it which we can access to run what we want
directions.Add(new SemanticResultValue("ZoomIn", "ZOOMIN"));
directions.Add(new SemanticResultValue("In", "ZOOMIN"));
directions.Add(new SemanticResultValue("ZoomOut", "ZOOMOUT"));
directions.Add(new SemanticResultValue("Out", "ZOOMOUT"));
directions.Add(new SemanticResultValue("ZoomOut", "ZOOMOUT"));
directions.Add(new SemanticResultValue("Left", "LEFT"));
directions.Add(new SemanticResultValue("Right", "RIGHT"));
directions.Add(new SemanticResultValue("Up", "UP"));
directions.Add(new SemanticResultValue("Down", "DOWN"));
// places
directions.Add(new SemanticResultValue("Go India", "INDIA"));
directions.Add(new SemanticResultValue("india", "INDIA"));
directions.Add(new SemanticResultValue("Go AMERICA", "AMERICA"));
directions.Add(new SemanticResultValue("america", "AMERICA"));
directions.Add(new SemanticResultValue("Go SanDiego", "SANDIEGO"));
directions.Add(new SemanticResultValue("SanDiego", "SANDIEGO"));
directions.Add(new SemanticResultValue("MY PLACE", "MYPLACE")); // this will not work through window app as cant activate gps through eo web browser
directions.Add(new SemanticResultValue("San Francisco Bay", "SANFRANCISCO"));
directions.Add(new SemanticResultValue("Mount Everest", "MOUNTEVEREST"));
directions.Add(new SemanticResultValue("Grand Canyon", "GRANDCANYON"));
directions.Add(new SemanticResultValue("hannover", "HANOVER"));
directions.Add(new SemanticResultValue("newyork", "NEWYORK"));
directions.Add(new SemanticResultValue("Delhi", "DELHI"));
directions.Add(new SemanticResultValue("Goa", "GOA"));
directions.Add(new SemanticResultValue("Mumbai", "MUMBAI"));
directions.Add(new SemanticResultValue("Banglore", "BANGLORE"));
directions.Add(new SemanticResultValue("Europe", "EUROPE"));
directions.Add(new SemanticResultValue("Germany", "GERMANY"));
directions.Add(new SemanticResultValue("Switzerland", "SWITZERLAND"));
directions.Add(new SemanticResultValue("Amsterdam", "AMSTERDAM"));
directions.Add(new SemanticResultValue("Belgium", "BELGIUM"));
directions.Add(new SemanticResultValue("Hildesheim", "HILDESHEIM"));
directions.Add(new SemanticResultValue("Hamburg", "HAMBURG"));
directions.Add(new SemanticResultValue("Berlin", "BERLIN"));
directions.Add(new SemanticResultValue("Prague", "PRAGUE"));
directions.Add(new SemanticResultValue("Sylt", "SYLT"));
directions.Add(new SemanticResultValue("Paris", "PARIS"));
directions.Add(new SemanticResultValue("Great Pyramid", "GREAT"));
directions.Add(new SemanticResultValue("Effiel Tower", "Tower"));
directions.Add(new SemanticResultValue("Taj Mahal", "TAj"));
directions.Add(new SemanticResultValue("Pisa", "PISA"));
directions.Add(new SemanticResultValue("Venice", "VENICE"));
//tools
directions.Add(new SemanticResultValue("Fly", "FLY"));
directions.Add(new SemanticResultValue("walk", "WALK"));
directions.Add(new SemanticResultValue("valk", "WALK"));
directions.Add(new SemanticResultValue("Back", "BACK"));
directions.Add(new SemanticResultValue("photo", "PHOTO"));
directions.Add(new SemanticResultValue("PHOTO", "PHOTO"));
var gb = new GrammarBuilder { Culture = ri.Culture }; // to run recognizer
gb.Append(directions);
gb.AppendWildcard(); // this is used so that second word of saying wil not be count to detect the acccuracy of word
var grr = new Grammar(gb);
this.speechEngine.LoadGrammar(grr); // to load the grammer
this.speechEngine.SpeechRecognized += this.SpeechRecognized; // called if speech recognized
this.speechEngine.SpeechRecognitionRejected += this.SpeechRejected; // called if speech rejected
// let the convertStream know speech is going active
this.convertStream.SpeechActive = true;
this.speechEngine.SetInputToAudioStream(
this.convertStream, new SpeechAudioFormatInfo(EncodingFormat.Pcm, 16000, 16, 1, 32000, 2, null));
this.speechEngine.RecognizeAsync(RecognizeMode.Multiple);
}
/// <summary>
/// Initializes a new instance of the MainWindow class
/// </summary>
public MainWindow()
{
try
{
InitializeComponent();
this.kinectSensor = KinectSensor.GetDefault();
// set IsAvailableChanged event notifier
//this.kinectSensor.IsAvailableChanged += this.Sensor_IsAvailableChanged;
// open the sensor
this.kinectSensor.Open();
// open the reader for the body frames
this.bodyFrameReader = this.kinectSensor.BodyFrameSource.OpenReader();
// set the BodyFramedArrived event notifier
this.bodyFrameReader.FrameArrived += this.Reader_BodyFrameArrived;
// initialize the BodyViewer object for displaying tracked bodies in the UI
this.kinectBodyView = new KinectBodyView(this.kinectSensor);
// initialize the gesture detection objects for our gestures
//this.gestureDetectorList = new GestureDetector(this.kinectSensor,this.gestureResultView);
// initialize the MainWindow
this.InitializeComponent();
// set our data context objects for display in UI
this.DataContext = this;
//this.kinectBodyViewbox.DataContext = this.kinectBodyView;
// GestureResultView result = new GestureResultView(0, false, false, 0.0f);
//GestureDetector detector = new GestureDetector(this.kinectSensor, result);
this.gestureDetectorList = new List <GestureDetector>();
// set our data context objects for display in UI
this.DataContext = this;
this.kinectBodyViewbox.DataContext = this.kinectBodyView;
// create a gesture detector for each body (6 bodies => 6 detectors) and create content controls to display results in the UI
//int col0Row = 0;
//int col1Row = 0;
int maxBodies = this.kinectSensor.BodyFrameSource.BodyCount;
//for (int i = 0; i < maxBodies; ++i)
//{
GestureResultView result = new GestureResultView(0, false, false, 0.0f);
GestureDetector detector = new GestureDetector(this.kinectSensor, result);
this.gestureDetectorList.Add(detector);
// split gesture results across the first two columns of the content grid
/*ContentControl contentControl = new ContentControl();
* contentControl.Content = this.gestureDetectorList[i].GestureResultView;
*
* if (i % 2 == 0)
* {
* // Gesture results for bodies: 0, 2, 4
* Grid.SetColumn(contentControl, 0);
* Grid.SetRow(contentControl, col0Row);
++col0Row;
* }
* else
* {
* // Gesture results for bodies: 1, 3, 5
* Grid.SetColumn(contentControl, 1);
* Grid.SetRow(contentControl, col1Row);
++col1Row;
* }
*
* this.contentGrid.Children.Add(contentControl);
* }*/
// split gesture results across the first two columns of the content grid
// ContentControl contentControl = new ContentControl();
//contentControl.Content = this.gestureDetectorList.GestureResultView;
}
catch (Exception e)
{
System.Windows.MessageBox.Show(e.StackTrace);
//throw;
}
}
/// <summary>
/// Initializes a new instance of the GestureDetector class along with the gesture frame source and reader
/// </summary>
/// <param name="kinectSensor">Active sensor to initialize the VisualGestureBuilderFrameSource object with</param>
/// <param name="gestureResultView">GestureResultView object to store gesture results of a single body to</param>
public GestureDetector(KinectSensor kinectSensor, GestureResultView gestureResultView)
{
if (kinectSensor == null)
{
throw new ArgumentNullException("kinectSensor");
}
if (gestureResultView == null)
{
throw new ArgumentNullException("gestureResultView");
}
this.GestureResultView = gestureResultView;
// create the vgb source. The associated body tracking ID will be set when a valid body frame arrives from the sensor.
this.vgbFrameSource = new VisualGestureBuilderFrameSource(kinectSensor, 0);
this.vgbFrameSource.TrackingIdLost += this.Source_TrackingIdLost;
// open the reader for the vgb frames
this.vgbFrameReader = this.vgbFrameSource.OpenReader();
if (this.vgbFrameReader != null)
{
this.vgbFrameReader.IsPaused = true;
this.vgbFrameReader.FrameArrived += this.Reader_GestureFrameArrived;
}
// load the gestures from the gesture database
using (VisualGestureBuilderDatabase database = new VisualGestureBuilderDatabase(this.gestureDatabase))
{
vgbFrameSource.AddGestures(database.AvailableGestures);
}
}
