/// <summary>
/// Initializes a new instance of the GestureDetector class along with the gesture frame source and reader
/// </summary>
/// <param name="kinectSensor">Active sensor to initialize the VisualGestureBuilderFrameSource object with</param>
/// <param name="gestureResultView">GestureResultView object to store gesture results of a single body to</param>
public GestureDetector(KinectSensor kinectSensor, GestureResultView gestureResultView)
{
if (kinectSensor == null)
{
throw new ArgumentNullException("kinectSensor");
}
if (gestureResultView == null)
{
throw new ArgumentNullException("gestureResultView");
}
GestureResultView = gestureResultView;
// create the vgb source. The associated body tracking ID will be set when a valid body frame arrives from the sensor.
_vgbFrameSource = new VisualGestureBuilderFrameSource(kinectSensor, 0);
_vgbFrameSource.TrackingIdLost += Source_TrackingIdLost;
// open the reader for the vgb frames
_vgbFrameReader = _vgbFrameSource.OpenReader();
if (_vgbFrameReader != null)
{
_vgbFrameReader.IsPaused = true;
_vgbFrameReader.FrameArrived += Reader_GestureFrameArrived;
}
// load the gestures from the gesture database
using (VisualGestureBuilderDatabase database = new VisualGestureBuilderDatabase(_gestureDatabase))
{
_vgbFrameSource.AddGestures(database.AvailableGestures);
}
}
/// <summary>
/// Handles gesture detection results arriving from the sensor for the associated body tracking Id
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void Reader_GestureFrameArrived(object sender, VisualGestureBuilderFrameArrivedEventArgs e)
{
_gestureName = null;
using (VisualGestureBuilderFrame frame = _vgbFrameReader.CalculateAndAcquireLatestFrame())
{
if (frame != null)
{
// get the discrete gesture results which arrived with the latest frame
IReadOnlyDictionary <Gesture, DiscreteGestureResult> discreteResults = frame.DiscreteGestureResults;
if (discreteResults != null)
{
foreach (Gesture gesture in _vgbFrameSource.Gestures)
{
DiscreteGestureResult result;
discreteResults.TryGetValue(gesture, out result);
if (result != null)
{
if (gesture.Name.Equals(_pickUp) && gesture.GestureType == GestureType.Discrete && result.Confidence > 0.05)
{
// update the GestureResultView object with new gesture result values
GestureResultView.UpdateGestureResult(true, result.Detected, result.Confidence, gesture.Name);
_gestureName = "PickUp";
}
else if (gesture.Name.Equals(_putDown) && gesture.GestureType == GestureType.Discrete && result.Confidence > 0.05)
{
// update the GestureResultView object with new gesture result values
GestureResultView.UpdateGestureResult(true, result.Detected, result.Confidence, gesture.Name);
_gestureName = "PutDown";
}
else if (gesture.Name.Equals(_openDoor) && gesture.GestureType == GestureType.Discrete && result.Confidence > 0.05)
{
// update the GestureResultView object with new gesture result values
GestureResultView.UpdateGestureResult(true, result.Detected, result.Confidence, gesture.Name);
_gestureName = "OpenDoor";
}
else if (gesture.Name.Equals(_handToMouth) && gesture.GestureType == GestureType.Discrete && result.Confidence > 0.05)
{
// update the GestureResultView object with new gesture result values
GestureResultView.UpdateGestureResult(true, result.Detected, result.Confidence, gesture.Name);
_gestureName = "HandToMouth";
}
else if (gesture.Name.Equals(_pour) && gesture.GestureType == GestureType.Discrete && result.Confidence > 0.05)
{
// update the GestureResultView object with new gesture result values
GestureResultView.UpdateGestureResult(true, result.Detected, result.Confidence, gesture.Name);
_gestureName = "Pour";
}
else
{
_gestureName = "No Gesture";
}
}
}
}
}
}
}
public MainPage()
{
// one sensor is currently supported
_kinectSensor = KinectSensor.GetDefault();
_coordinateMapper = _kinectSensor.CoordinateMapper;
var multiSourceFrameReader = _kinectSensor.OpenMultiSourceFrameReader(FrameSourceTypes.Infrared | FrameSourceTypes.Color | FrameSourceTypes.Depth | FrameSourceTypes.BodyIndex | FrameSourceTypes.Body);
multiSourceFrameReader.MultiSourceFrameArrived += Reader_MultiSourceFrameArrived;
// set IsAvailableChanged event notifier
_kinectSensor.IsAvailableChanged += Sensor_IsAvailableChanged;
// use the window object as the view model in this simple example
DataContext = this;
// open the sensor
_kinectSensor.Open();
InitializeComponent();
// new
Loaded += MainPage_Loaded;
// Initialize the gesture detection objects for our gestures
_gestureDetectorList = new List <GestureDetector>();
// Create a gesture detector for each body (6 bodies => 6 detectors)
int maxBodies = _kinectSensor.BodyFrameSource.BodyCount;
for (int i = 0; i < maxBodies; ++i)
{
GestureResultView result = new GestureResultView(i, false, false, 0.0f, null);
GestureDetector detector = new GestureDetector(_kinectSensor, result);
result.PropertyChanged += GestureResult_PropertyChanged;
_gestureDetectorList.Add(detector);
}
}
/// <summary>
/// Handles the TrackingIdLost event for the VisualGestureBuilderSource object
/// </summary>
/// <param name="sender">object sending the event</param>
/// <param name="e">event arguments</param>
private void Source_TrackingIdLost(object sender, TrackingIdLostEventArgs e)
{
// update the GestureResultView object to show the 'Not Tracked' image in the UI
GestureResultView.UpdateGestureResult(false, false, 0.0f, null);
}
/// <summary>
///
/// </summary>
private void GestureResult_PropertyChanged(object sender, PropertyChangedEventArgs e)
{
GestureResultView result = sender as GestureResultView;
if (result == null)
{
return;
}
var detectedGestureName = result.GestureName;
GestureVisual.Opacity = result.Confidence;
BodyTracked.Text = result.BodyIndex.ToString();
GestureName.FontSize = 30;
GestureName.Foreground = new SolidColorBrush(Colors.White);
// Using the array bodies[], the specific body recording the gesture is found.
// The location of the left and right fingertips, as well as the midpoint of the spine are found and stored as [x,y,z] coordinates.
// These coordinates correspond to the location of the camera
Body body = _bodies[result.BodyIndex];
if (body != null)
{
// FloorClipPlane results, used to normalise the coordinate system of the detector to the floor of the room
float floorX = _floorPlane.X;
float floorY = _floorPlane.Y;
float floorZ = _floorPlane.Z;
float floorW = _floorPlane.W;
float cameraAngleRadians = (float)Math.Atan(floorZ / floorY);
float cosCameraAngle = (float)Math.Cos(cameraAngleRadians);
float sinCameraAngle = (float)Math.Sin(cameraAngleRadians);
// Output the FloorClipPlane coordinates to the GUI for debug purposes
FloorPosition.Text = "X: " + floorX + ", Y: " + floorY + ", Z: " + floorZ + ", W: " + floorW;
// Collect the coordinates of the left and right finger tips and the spine midpoint
Joint fingerTipLeft = body.Joints[JointType.HandTipLeft];
Joint fingerTipRight = body.Joints[JointType.HandTipRight];
Joint spineMid = body.Joints[JointType.SpineMid];
// Location of the left and right finger tips and the spine midpoint, after trigonometric correction for the FloorClipPlane.
// Gives the coordinate locations with the origin on the floor at the base of the camera, with y-axis normal to the floor.
float[] fingerTipLeftXyz = { fingerTipLeft.Position.X *floorX, (fingerTipLeft.Position.Y * floorY) + floorW, fingerTipLeft.Position.Z *floorZ };
float[] fingerTipRightXyz = { fingerTipRight.Position.X, (fingerTipRight.Position.Y * cosCameraAngle) + (fingerTipRight.Position.Z * sinCameraAngle) + floorW, (fingerTipRight.Position.Z * cosCameraAngle) + (fingerTipRight.Position.Y * sinCameraAngle) };
float[] spineMidXyz = { spineMid.Position.X, (spineMid.Position.Y * cosCameraAngle) + (spineMid.Position.Z * sinCameraAngle) + floorW, (spineMid.Position.Z * cosCameraAngle) + (spineMid.Position.Y * sinCameraAngle) };
// Output the coordinates of the spine and the left fingertip to the GUI for debug purposes
BodyPosition.Text = "X: " + spineMidXyz[0].ToString("n4") + ", Y: " + spineMidXyz[1].ToString("n4") + ", Z: " + spineMidXyz[2].ToString("n4");
LHandPosition.Text = "X: " + fingerTipLeftXyz[0].ToString("n4") + ", Y: " + fingerTipLeftXyz[1].ToString("n4") + ", Z: " + fingerTipLeftXyz[2].ToString("n4");
// Check whether the location where the gesture is performed matches the key locations
string gestureLocation = GestureLocationChecker(fingerTipLeftXyz, fingerTipRightXyz, spineMidXyz, detectedGestureName);
GestureLocation.Text = gestureLocation;
if (detectedGestureName == null || !(result.Confidence > 0.1f) || GestureLocation.Text == null)
{
return;
}
GestureConfidence.Text = result.Confidence < 0.3f ? "Low" : (result.Confidence * 100f).ToString();
// Output gesture name to GUI
GestureName.Text = detectedGestureName;
if (result.Confidence < 0.7f)
{
return;
}
// if confidence over 0.4 change gesture name in GUI to green
GestureName.Foreground = new SolidColorBrush(Colors.Green);
switch (detectedGestureName)
{
case "PickUp":
switch (GestureLocation.Text)
{
case "Medication":
_pickUpMedication.Push(DateTime.Now);
break;
case "Pantry":
_pickUpPantry.Push(DateTime.Now);
break;
case "Fridge":
_pickUpFridge.Push(DateTime.Now);
break;
case "BowlCupboard":
_pickUpBowlCupboard.Push(DateTime.Now);
break;
case "FoodPrep":
_pickUpFoodPrep.Push(DateTime.Now);
break;
default:
break;
}
break;
case "PutDown":
if (GestureLocation.Text == "FoodPrep")
{
_putDownFoodPrep.Push(DateTime.Now);
}
break;
case "OpenDoor":
switch (GestureLocation.Text)
{
case "Pantry":
_openDoorPantry.Push(DateTime.Now);
break;
case "Fridge":
_openDoorFridge.Push(DateTime.Now);
break;
case "BowlCupboard":
_openDoorBowlCupboard.Push(DateTime.Now);
break;
default:
break;
}
break;
case "HandToMouth":
if (GestureLocation.Text == "Dining")
{
_handToMouthDining.Push(DateTime.Now);
}
break;
case "Pour":
if (GestureLocation.Text == "FoodPrep")
{
_pourFoodPrep.Push(DateTime.Now);
}
break;
case "No Gesture":
GestureName.Foreground = new SolidColorBrush(Colors.Red);
GestureConfidence.Text = "";
break;
default:
break;
}
//Check for an event, such as taking medication or eatin meal
EventChecker();
}
else
{
BodyPosition.Text = "body = null";
}
}
