private void Timer_Tick(object sender, EventArgs e)
{
// check if the PoseRecognized is properly initialized
if (!initialized)
{
return;
}
// a boolean flag to see if the next frame from the Kinect was successfully processed
bool nextFrame = false;
try
{
// try to process the next frame from the Kinect
// this will do everyintg: detect people, extract skeletons/joints,
// recognize the current pose, etc.
nextFrame = poseRecognizer.processNextFrame();
}
catch
{
MessageBox.Show("next frame exception");
timer.Stop();
return;
}
if (nextFrame)
{
Bitmap frame = null;
try
{
// this is how you can extract the current frame (image) from the Kinect
switch (toggleType)
{
case ImageType.Skeleton:
// extract the "index" image with the user's outline and overlaid skeleton
frame = poseRecognizer.getSkeletonImage();
break;
case ImageType.Rgb:
// extract the normal 1080p RGB frame
frame = poseRecognizer.getRgbFrame();
break;
case ImageType.Depth:
// extract the depth map
frame = poseRecognizer.getDepthMap();
break;
default:
break;
}
}
catch
{
MessageBox.Show("get frame exception");
}
bool userPresent = false;
try
{
// check if there is at least one user present in the Kinect's point of view
userPresent = poseRecognizer.userPresent();
}
catch
{
MessageBox.Show("user present exception");
}
label2.Text = "User present: " + userPresent.ToString();
if (userPresent)
{
/*
* populate the list of currently seen users:
* this list will only contain the 64-bit (8-byte) IDs of the users
* these IDs are generated by the Kinect API and are unique for every
* currently tracked user
* You can use these IDs to check how many users are present and
* to access the properties (pose, hands, etc.) for specific users by IDs
*/
UInt64[] list = poseRecognizer.getUserList();
try
{
// this will recognize the pose (out of all available ones) for THE FIRST SEEN user
label5AllPoses.Text = poseRecognizer.getPoseName();
// You can also ask to recognize the pose of a specific user by his ID:
// string poseName = poseRecognizer.getPoseName(list[0]);
// recognize one specified pose and get the similarity score in range [0..1]:
if (comboBox1.Text != "")
{
string poseName = comboBox1.Text;
// "expectPose(String poseName)" will return the similarity score for the pose in range [0..1]
// this is also done for THE FIRST AVAILABLE user (i.e. the user that was seen first by the Kinect)
label3.Text = poseName.ElementAt <char>(0) + ": " + poseRecognizer.expectPose(poseName);
// again, we can do that for a specific user:
// double score = poseRecognizer.expectPose(list[0], "Flute");
/*
* For every user there's also a whole bunch of properties available, such as:
* - hand state (unknown, untracked, open, closed, lasso)
* - is the hand moving? (boolean)
* - hand movement speed (I think it's in m/sec, but I'm not sure)
* - hand movement direction (up, down, left, right, forwards, backwards)
* - did the hand just change movement direction?
*/
Direction rightHandMovementDirection = poseRecognizer.getHandDirection(list[0], Hand.RIGHT);
if (rightHandMovementDirection == Direction.RIGHT)
{
// Hand is moving right
}
double rightHandMovementSpeed = poseRecognizer.getHandMovementSpeed(list[0], Hand.RIGHT);
if (rightHandMovementSpeed > 0.85)
{
// the hand is moving very fast!
}
bool isRightHandMoving = poseRecognizer.isHandMoving(list[0], Hand.RIGHT);
if (!isRightHandMoving)
{
// right hand stopped moving
}
bool rightHandChangedMovementDirection = poseRecognizer.handChangedDirection(list[0], Hand.RIGHT);
if (rightHandChangedMovementDirection)
{
// hand changed direction! Do something! Play a different sound, show some effect!
}
HandState rightHandState = poseRecognizer.getHandState(list[0], Hand.RIGHT);
if (rightHandState == HandState.Closed)
{
// right hand is clenched into a fist!
}
}
// see how the joints for the user are extracted:
if (toggleType == ImageType.Skeleton)
{
drawSkeleton(ref frame);
}
}
catch (NullReferenceException ex)
{
MessageBox.Show("get pose exception");
}
}
pictureBox1.Image = frame as Image;
}
timer.Start();
}
private void playMusic(UInt64 user)
{
if (!userList.ContainsKey(user))
{
return;
}
string instrument = poseRecognizer.getPoseName(user).ToLower();
var LeftHand = new {
state = poseRecognizer.getHandState(user, Hand.LEFT),
speed = poseRecognizer.getHandMovementSpeed(user, Hand.LEFT),
direction = poseRecognizer.getHandDirection(user, Hand.LEFT),
changedDirection = poseRecognizer.handChangedDirection(user, Hand.LEFT),
isMoving = poseRecognizer.isHandMoving(user, Hand.LEFT)
};
var RightHand = new
{
state = poseRecognizer.getHandState(user, Hand.RIGHT),
speed = poseRecognizer.getHandMovementSpeed(user, Hand.RIGHT),
direction = poseRecognizer.getHandDirection(user, Hand.RIGHT),
changedDirection = poseRecognizer.handChangedDirection(user, Hand.RIGHT),
isMoving = poseRecognizer.isHandMoving(user, Hand.RIGHT)
};
//if (userList[user].changedRight(RightHand.direction))
// Console.WriteLine(Enum.GetName(typeof(Direction), RightHand.direction));
string soundName = String.Copy(instrument);
// go instrument by instrument:
if (instrument == "flute" || instrument == "clarinet")
{
// both hands closed
if (userList[user].isStateChanged(LeftHand.state, RightHand.state))
{
if ((int)LeftHand.state >= (int)HandState.Closed && (int)RightHand.state >= (int)HandState.Closed)
{
soundName += "_A";
}
else if ((int)LeftHand.state <= (int)HandState.Open && (int)RightHand.state >= (int)HandState.Closed)
{
soundName += "_B";
}
else if ((int)LeftHand.state >= (int)HandState.Closed && (int)RightHand.state <= (int)HandState.Open)
{
soundName += "_C";
}
else if ((int)LeftHand.state <= (int)HandState.Open && (int)RightHand.state <= (int)HandState.Open)
{
soundName += "_D";
}
userList[user].playSound(soundName);
}
}
else if (instrument == "violin" || instrument == "cello")
{
if (userList[user].changedRight(RightHand.direction))
{
if (RightHand.direction == Direction.RIGHT || RightHand.direction == Direction.BACKWARDS)
{
soundName += "_right";
}
else if (RightHand.direction == Direction.LEFT || RightHand.direction == Direction.FORWARDS)
{
soundName += "_left";
}
double slow = 0.45, medium = 0.85;
if (RightHand.speed <= slow)
{
soundName += "_slow";
}
else if (RightHand.speed > slow && RightHand.speed <= medium)
{
soundName += "_medium";
}
else if (RightHand.speed > medium)
{
soundName += "_fast";
}
else
{
soundName += "_slow";
}
userList[user].playSound(soundName);
}
}
else if (instrument == "drums")
{
if (userList[user].changedRight(RightHand.direction))
{
if (RightHand.direction == Direction.DOWN)
{
userList[user].playSound(soundName + "_fast");
}
}
if (userList[user].changedLeft(LeftHand.direction))
{
if (LeftHand.direction == Direction.DOWN)
{
userList[user].playSound(soundName + "_medium");
}
}
}
else if (instrument == "guitar")
{
if (userList[user].changedRight(RightHand.direction))
{
//Console.WriteLine(Enum.GetName(typeof(Direction), RightHand.direction));
if (RightHand.direction == Direction.UP)
{
userList[user].playSound("guitar_up");
}
else if (RightHand.direction == Direction.DOWN)
{
userList[user].playSound("guitar_down");
}
}
}
else if (instrument == "conductor")
{
if (RightHand.changedDirection && !RightHand.isMoving)
{
userList[user].playSound("conductor");
}
}
}