MainWindow.xaml.cs

﻿//------------------------------------------------------------------------------
// <copyright file="MainWindow.xaml.cs" company="Microsoft">
//     Copyright (c) Microsoft Corporation.  All rights reserved.
// </copyright>
//------------------------------------------------------------------------------

namespace Microsoft.Samples.Kinect.SkeletonBasics
{
    using System;
    using System.IO;
    using System.Windows;
    using System.Windows.Media;
    using Microsoft.Kinect;
    using System.Collections.Generic;
    using Newtonsoft.Json;
    using System.Diagnostics;
    
    /// <summary>
    /// Interaction logic for MainWindow.xaml
    /// </summary>
    public partial class MainWindow : Window
    {
        /// <summary>
        /// Width of output drawing
        /// </summary>
        private const float RenderWidth = 640.0f;

        /// <summary>
        /// Height of our output drawing
        /// </summary>
        private const float RenderHeight = 480.0f;

        /// <summary>
        /// Thickness of drawn joint lines
        /// </summary>
        private const double JointThickness = 3;

        /// <summary>
        /// Thickness of body center ellipse
        /// </summary>
        private const double BodyCenterThickness = 10;

        /// <summary>
        /// Thickness of clip edge rectangles
        /// </summary>
        private const double ClipBoundsThickness = 10;

        /// <summary>
        /// Brush used to draw skeleton center point
        /// </summary>
        private readonly Brush centerPointBrush = Brushes.Blue;

        /// <summary>
        /// Brush used for drawing joints that are currently tracked
        /// </summary>
        private readonly Brush trackedJointBrush = new SolidColorBrush(Color.FromArgb(255, 68, 192, 68));

        /// <summary>
        /// Brush used for drawing joints that are currently inferred
        /// </summary>        
        private readonly Brush inferredJointBrush = Brushes.Yellow;

        /// <summary>
        /// Pen used for drawing bones that are currently tracked
        /// </summary>
        private readonly Pen trackedBonePen = new Pen(Brushes.Green, 6);

        /// <summary>
        /// Pen used for drawing bones that are currently inferred
        /// </summary>        
        private readonly Pen inferredBonePen = new Pen(Brushes.Gray, 1);

        /// <summary>
        /// Active Kinect sensor
        /// </summary>
        private KinectSensor sensor;

        /// <summary>
        /// Drawing group for skeleton rendering output
        /// </summary>
        private DrawingGroup drawingGroup;

        /// <summary>
        /// Drawing image that we will display
        /// </summary>
        private DrawingImage imageSource;

        /// <summary>
        /// Initializes a new instance of the MainWindow class.
        /// </summary>
        /// 


        private class quaternionPoint
        {
            public long time { get; set; }
            public Vector4 value { get; set; }

            public quaternionPoint(long t, Vector4 v)
            {
                time = t;
                value = v;
            }
        }

        private class rotationData
        {
            public List<quaternionPoint> HipCenter;
            public List<quaternionPoint> Spine;
            public List<quaternionPoint> ShoulderCenter;
            public List<quaternionPoint> Head;
            public List<quaternionPoint> ShoulderLeft;
            public List<quaternionPoint> ElbowLeft;
            public List<quaternionPoint> WristLeft;
            public List<quaternionPoint> HandLeft;
            public List<quaternionPoint> ShoulderRight;
            public List<quaternionPoint> ElbowRight;
            public List<quaternionPoint> WristRight;
            public List<quaternionPoint> HandRight;
            public List<quaternionPoint> HipLeft;
            public List<quaternionPoint> KneeLeft;
            public List<quaternionPoint> AnkleLeft;
            public List<quaternionPoint> FootLeft;
            public List<quaternionPoint> HipRight;
            public List<quaternionPoint> KneeRight;
            public List<quaternionPoint> AnkleRight;
            public List<quaternionPoint> FootRight;

            public rotationData()
            {
                HipCenter = new List<quaternionPoint>();
                Spine  = new List<quaternionPoint>();
                ShoulderCenter = new List<quaternionPoint>();
                Head = new List<quaternionPoint>();
                ShoulderLeft = new List<quaternionPoint>();
                ElbowLeft = new List<quaternionPoint>();
                WristLeft = new List<quaternionPoint>();
                HandLeft = new List<quaternionPoint>();
                ShoulderRight = new List<quaternionPoint>();
                ElbowRight = new List<quaternionPoint>();
                WristRight = new List<quaternionPoint>();
                HandRight = new List<quaternionPoint>();
                HipLeft = new List<quaternionPoint>();
                KneeLeft = new List<quaternionPoint>();
                AnkleLeft = new List<quaternionPoint>();
                FootLeft = new List<quaternionPoint>();
                HipRight = new List<quaternionPoint>();
                KneeRight = new List<quaternionPoint>();
                AnkleRight = new List<quaternionPoint>();
                FootRight = new List<quaternionPoint>(); 
            }
        }

        // Variables to track and store data
        Stopwatch stopwatch = new Stopwatch();
        private bool isRecording = false;
        private rotationData skeletonData;

        // Given a skeleton, this function stores all the rotations and the current time
        //  to the skeletonData object, so we can export that later.
        private void storeSkeletonData(Skeleton skeleton)
        {
            // Get the current time, so all our actions are synchronized in the data
            long time = stopwatch.ElapsedMilliseconds;
            // Get the bone orientations
            BoneOrientationCollection skeletonRotations = skeleton.BoneOrientations;
            // From the joint orientations, get easy names for all of them
            //  so we can store these quaternion rotations easily and sensically
            Vector4 HipCenterRotation = skeletonRotations[JointType.HipCenter].HierarchicalRotation.Quaternion;
            Vector4 SpineRotation = skeletonRotations[JointType.Spine].HierarchicalRotation.Quaternion;
            Vector4 ShoulderCenterRotation = skeletonRotations[JointType.ShoulderCenter].HierarchicalRotation.Quaternion;
            Vector4 HeadRotation = skeletonRotations[JointType.Head].HierarchicalRotation.Quaternion;

            Vector4 ShoulderLeftRotation = skeletonRotations[JointType.ShoulderLeft].HierarchicalRotation.Quaternion;
            Vector4 ElbowLeftRotation = skeletonRotations[JointType.ElbowLeft].HierarchicalRotation.Quaternion;
            Vector4 WristLeftRotation = skeletonRotations[JointType.WristLeft].HierarchicalRotation.Quaternion;
            Vector4 HandLeftRotation = skeletonRotations[JointType.HandLeft].HierarchicalRotation.Quaternion;
            Vector4 ShoulderRightRotation = skeletonRotations[JointType.ShoulderRight].HierarchicalRotation.Quaternion;
            Vector4 ElbowRightRotation = skeletonRotations[JointType.ElbowRight].HierarchicalRotation.Quaternion;
            Vector4 WristRightRotation = skeletonRotations[JointType.WristRight].HierarchicalRotation.Quaternion;
            Vector4 HandRightRotation = skeletonRotations[JointType.HandRight].HierarchicalRotation.Quaternion;
            
            Vector4 HipLeftRotation = skeletonRotations[JointType.HipLeft].HierarchicalRotation.Quaternion;
            Vector4 KneeLeftRotation = skeletonRotations[JointType.KneeLeft].HierarchicalRotation.Quaternion;
            Vector4 AnkleLeftRotation = skeletonRotations[JointType.AnkleLeft].HierarchicalRotation.Quaternion;
            Vector4 FootLeftRotation = skeletonRotations[JointType.FootLeft].HierarchicalRotation.Quaternion;
            Vector4 HipRightRotation = skeletonRotations[JointType.HipRight].HierarchicalRotation.Quaternion;
            Vector4 KneeRightRotation = skeletonRotations[JointType.KneeRight].HierarchicalRotation.Quaternion;
            Vector4 AnkleRightRotation = skeletonRotations[JointType.AnkleRight].HierarchicalRotation.Quaternion;
            Vector4 FootRightRotation = skeletonRotations[JointType.FootRight].HierarchicalRotation.Quaternion;

            // Now add new points with the time and values to the storage object
            skeletonData.HipCenter.Add(new quaternionPoint(time, HipCenterRotation));
            skeletonData.Spine.Add(new quaternionPoint(time, SpineRotation));
            skeletonData.ShoulderCenter.Add(new quaternionPoint(time, ShoulderCenterRotation));
            skeletonData.Head.Add(new quaternionPoint(time, HeadRotation));

            skeletonData.ShoulderLeft.Add(new quaternionPoint(time, ShoulderLeftRotation));
            skeletonData.ElbowLeft.Add(new quaternionPoint(time, ElbowLeftRotation));
            skeletonData.WristLeft.Add(new quaternionPoint(time, WristLeftRotation));
            skeletonData.HandLeft.Add(new quaternionPoint(time, HandLeftRotation));
            skeletonData.ShoulderRight.Add(new quaternionPoint(time, ShoulderRightRotation));
            skeletonData.ElbowRight.Add(new quaternionPoint(time, ElbowRightRotation));
            skeletonData.WristRight.Add(new quaternionPoint(time, WristRightRotation));
            skeletonData.HandRight.Add(new quaternionPoint(time, HandRightRotation));

            skeletonData.HipLeft.Add(new quaternionPoint(time, HipLeftRotation));
            skeletonData.KneeLeft.Add(new quaternionPoint(time, KneeLeftRotation));
            skeletonData.AnkleLeft.Add(new quaternionPoint(time, AnkleLeftRotation));
            skeletonData.FootLeft.Add(new quaternionPoint(time, FootLeftRotation));
            skeletonData.HipRight.Add(new quaternionPoint(time, HipRightRotation));
            skeletonData.KneeRight.Add(new quaternionPoint(time, KneeRightRotation));
            skeletonData.AnkleRight.Add(new quaternionPoint(time, AnkleRightRotation));
            skeletonData.FootRight.Add(new quaternionPoint(time, FootRightRotation));
        }


        public MainWindow()
        {
            InitializeComponent();
        }

        /// <summary>
        /// Draws indicators to show which edges are clipping skeleton data
        /// </summary>
        /// <param name="skeleton">skeleton to draw clipping information for</param>
        /// <param name="drawingContext">drawing context to draw to</param>
        private static void RenderClippedEdges(Skeleton skeleton, DrawingContext drawingContext)
        {
            if (skeleton.ClippedEdges.HasFlag(FrameEdges.Bottom))
            {
                drawingContext.DrawRectangle(
                    Brushes.Red,
                    null,
                    new Rect(0, RenderHeight - ClipBoundsThickness, RenderWidth, ClipBoundsThickness));
            }

            if (skeleton.ClippedEdges.HasFlag(FrameEdges.Top))
            {
                drawingContext.DrawRectangle(
                    Brushes.Red,
                    null,
                    new Rect(0, 0, RenderWidth, ClipBoundsThickness));
            }

            if (skeleton.ClippedEdges.HasFlag(FrameEdges.Left))
            {
                drawingContext.DrawRectangle(
                    Brushes.Red,
                    null,
                    new Rect(0, 0, ClipBoundsThickness, RenderHeight));
            }

            if (skeleton.ClippedEdges.HasFlag(FrameEdges.Right))
            {
                drawingContext.DrawRectangle(
                    Brushes.Red,
                    null,
                    new Rect(RenderWidth - ClipBoundsThickness, 0, ClipBoundsThickness, RenderHeight));
            }
        }

        /// <summary>
        /// Execute startup tasks
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void WindowLoaded(object sender, RoutedEventArgs e)
        {
            // Create the drawing group we'll use for drawing
            this.drawingGroup = new DrawingGroup();

            // Create an image source that we can use in our image control
            this.imageSource = new DrawingImage(this.drawingGroup);

            // Display the drawing using our image control
            Image.Source = this.imageSource;

            // Look through all sensors and start the first connected one.
            // This requires that a Kinect is connected at the time of app startup.
            // To make your app robust against plug/unplug, 
            // it is recommended to use KinectSensorChooser provided in Microsoft.Kinect.Toolkit
            foreach (var potentialSensor in KinectSensor.KinectSensors)
            {
                if (potentialSensor.Status == KinectStatus.Connected)
                {
                    this.sensor = potentialSensor;
                    break;
                }
            }

            if (null != this.sensor)
            {
                // Turn on the skeleton stream to receive skeleton frames
                this.sensor.SkeletonStream.Enable();
                
                // Add an event handler to be called whenever there is new color frame data
                this.sensor.SkeletonFrameReady += this.SensorSkeletonFrameReady;

                // Start the sensor!
                try
                {
                    this.sensor.Start();
                }
                catch (IOException)
                {
                    this.sensor = null;
                }
            }

            if (null == this.sensor)
            {
                this.statusBarText.Text = Properties.Resources.NoKinectReady;
            }
        }

        /// <summary>
        /// Execute shutdown tasks
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void WindowClosing(object sender, System.ComponentModel.CancelEventArgs e)
        {
            if (null != this.sensor)
            {
                this.sensor.Stop();
            }
        }

        /// <summary>
        /// Event handler for Kinect sensor's SkeletonFrameReady event
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void SensorSkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
        {
            Skeleton[] skeletons = new Skeleton[0];

            using (SkeletonFrame skeletonFrame = e.OpenSkeletonFrame())
            {
                if (skeletonFrame != null)
                {
                    skeletons = new Skeleton[skeletonFrame.SkeletonArrayLength];
                    skeletonFrame.CopySkeletonDataTo(skeletons);
                }
            }

            using (DrawingContext dc = this.drawingGroup.Open())
            {
                // Draw a transparent background to set the render size
                dc.DrawRectangle(Brushes.Black, null, new Rect(0.0, 0.0, RenderWidth, RenderHeight));

                if (skeletons.Length != 0)
                {
                    // Let's store the data from skeleton 0 into our data object...
                    if (skeletons[0].TrackingState == SkeletonTrackingState.Tracked &&
                        skeletons[0].ClippedEdges.HasFlag(FrameEdges.None) &&
                        isRecording == true)
                    {
                        storeSkeletonData(skeletons[0]);
                    }
                    foreach (Skeleton skel in skeletons)
                    {
                        RenderClippedEdges(skel, dc);

                        if (skel.TrackingState == SkeletonTrackingState.Tracked)
                        {
                            this.DrawBonesAndJoints(skel, dc);
                        }
                        else if (skel.TrackingState == SkeletonTrackingState.PositionOnly)
                        {
                            dc.DrawEllipse(
                            this.centerPointBrush,
                            null,
                            this.SkeletonPointToScreen(skel.Position),
                            BodyCenterThickness,
                            BodyCenterThickness);
                        }
                    }
                }

                // prevent drawing outside of our render area
                this.drawingGroup.ClipGeometry = new RectangleGeometry(new Rect(0.0, 0.0, RenderWidth, RenderHeight));
            }
        }

        /// <summary>
        /// Draws a skeleton's bones and joints
        /// </summary>
        /// <param name="skeleton">skeleton to draw</param>
        /// <param name="drawingContext">drawing context to draw to</param>
        private void DrawBonesAndJoints(Skeleton skeleton, DrawingContext drawingContext)
        {
            // Render Torso
            this.DrawBone(skeleton, drawingContext, JointType.Head, JointType.ShoulderCenter);
            this.DrawBone(skeleton, drawingContext, JointType.ShoulderCenter, JointType.ShoulderLeft);
            this.DrawBone(skeleton, drawingContext, JointType.ShoulderCenter, JointType.ShoulderRight);
            this.DrawBone(skeleton, drawingContext, JointType.ShoulderCenter, JointType.Spine);
            this.DrawBone(skeleton, drawingContext, JointType.Spine, JointType.HipCenter);
            this.DrawBone(skeleton, drawingContext, JointType.HipCenter, JointType.HipLeft);
            this.DrawBone(skeleton, drawingContext, JointType.HipCenter, JointType.HipRight);

            // Left Arm
            this.DrawBone(skeleton, drawingContext, JointType.ShoulderLeft, JointType.ElbowLeft);
            this.DrawBone(skeleton, drawingContext, JointType.ElbowLeft, JointType.WristLeft);
            this.DrawBone(skeleton, drawingContext, JointType.WristLeft, JointType.HandLeft);

            // Right Arm
            this.DrawBone(skeleton, drawingContext, JointType.ShoulderRight, JointType.ElbowRight);
            this.DrawBone(skeleton, drawingContext, JointType.ElbowRight, JointType.WristRight);
            this.DrawBone(skeleton, drawingContext, JointType.WristRight, JointType.HandRight);

            // Left Leg
            this.DrawBone(skeleton, drawingContext, JointType.HipLeft, JointType.KneeLeft);
            this.DrawBone(skeleton, drawingContext, JointType.KneeLeft, JointType.AnkleLeft);
            this.DrawBone(skeleton, drawingContext, JointType.AnkleLeft, JointType.FootLeft);

            // Right Leg
            this.DrawBone(skeleton, drawingContext, JointType.HipRight, JointType.KneeRight);
            this.DrawBone(skeleton, drawingContext, JointType.KneeRight, JointType.AnkleRight);
            this.DrawBone(skeleton, drawingContext, JointType.AnkleRight, JointType.FootRight);
 
            // Render Joints
            foreach (Joint joint in skeleton.Joints)
            {
                Brush drawBrush = null;

                if (joint.TrackingState == JointTrackingState.Tracked)
                {
                    drawBrush = this.trackedJointBrush;                    
                }
                else if (joint.TrackingState == JointTrackingState.Inferred)
                {
                    drawBrush = this.inferredJointBrush;                    
                }

                if (drawBrush != null)
                {
                    drawingContext.DrawEllipse(drawBrush, null, this.SkeletonPointToScreen(joint.Position), JointThickness, JointThickness);
                }
            }
        }

        /// <summary>
        /// Maps a SkeletonPoint to lie within our render space and converts to Point
        /// </summary>
        /// <param name="skelpoint">point to map</param>
        /// <returns>mapped point</returns>
        private Point SkeletonPointToScreen(SkeletonPoint skelpoint)
        {
            // Convert point to depth space.  
            // We are not using depth directly, but we do want the points in our 640x480 output resolution.
            DepthImagePoint depthPoint = this.sensor.CoordinateMapper.MapSkeletonPointToDepthPoint(skelpoint, DepthImageFormat.Resolution640x480Fps30);
            return new Point(depthPoint.X, depthPoint.Y);
        }

        /// <summary>
        /// Draws a bone line between two joints
        /// </summary>
        /// <param name="skeleton">skeleton to draw bones from</param>
        /// <param name="drawingContext">drawing context to draw to</param>
        /// <param name="jointType0">joint to start drawing from</param>
        /// <param name="jointType1">joint to end drawing at</param>
        private void DrawBone(Skeleton skeleton, DrawingContext drawingContext, JointType jointType0, JointType jointType1)
        {
            Joint joint0 = skeleton.Joints[jointType0];
            Joint joint1 = skeleton.Joints[jointType1];

            // If we can't find either of these joints, exit
            if (joint0.TrackingState == JointTrackingState.NotTracked ||
                joint1.TrackingState == JointTrackingState.NotTracked)
            {
                return;
            }

            // Don't draw if both points are inferred
            if (joint0.TrackingState == JointTrackingState.Inferred &&
                joint1.TrackingState == JointTrackingState.Inferred)
            {
                return;
            }

            // We assume all drawn bones are inferred unless BOTH joints are tracked
            Pen drawPen = this.inferredBonePen;
            if (joint0.TrackingState == JointTrackingState.Tracked && joint1.TrackingState == JointTrackingState.Tracked)
            {
                drawPen = this.trackedBonePen;
            }

            drawingContext.DrawLine(drawPen, this.SkeletonPointToScreen(joint0.Position), this.SkeletonPointToScreen(joint1.Position));
        }

        /// <summary>
        /// Handles the checking or unchecking of the seated mode combo box
        /// </summary>
        /// <param name="sender">object sending the event</param>
        /// <param name="e">event arguments</param>
        private void CheckBoxSeatedModeChanged(object sender, RoutedEventArgs e)
        {
            if (null != this.sensor)
            {
                if (this.checkBoxSeatedMode.IsChecked.GetValueOrDefault())
                {
                    this.sensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Seated;
                }
                else
                {
                    this.sensor.SkeletonStream.TrackingMode = SkeletonTrackingMode.Default;
                }
            }
        }


        // Tracks when record button is click and starts recording to JSON
        private void RecordButton_Click(object sender, RoutedEventArgs e)
        {
            // Toggles on record that will start recording AU coefficients to buffer
            if (!isRecording)
            {
                RecordButton.Content = "Stop";
                RecordButton.ToolTip = "Click to stop recording";
                skeletonData = new rotationData();
                isRecording = true;
                // Start stopwatch
                stopwatch.Start();
            }
            // When record is disabled write stored AUs to JSON object
            else
            {
                RecordButton.Content = "Record";
                RecordButton.ToolTip = "Click to start recording";
                isRecording = false;
                stopwatch.Stop();
                JsonSerializer serializer = new JsonSerializer();

                // File path settings -- currently json.txt in My Documents folder
                string myDocuments = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
                string path = System.IO.Path.Combine(myDocuments, "bodyData.json");

                // Writes AUCoefficients
                using (StreamWriter sw = new StreamWriter(path))
                using (JsonWriter jw = new JsonTextWriter(sw))
                {
                    serializer.Serialize(jw, skeletonData);
                }
                Console.WriteLine("Test");
                if (skeletonData.ShoulderLeft.Count > 0)
                {
                    Console.WriteLine(skeletonData.ShoulderLeft[0].time);
                    Console.WriteLine(skeletonData.ShoulderLeft[0].value);
                }
                else
                {
                    Console.WriteLine("No data in ShoulderLeft");
                }
            }
        }
    }
}