private static void UpdateFacePoint(TargetData faceBone, KinectDevice.FaceBone faceBoneId, Vector referencePoint)
{
Vector facePointPosition = faceBone.startPosition + KinectDevice.GetFacePoint(faceBoneId) - referencePoint;
faceBone.position = Filter(faceBone.position, facePointPosition);
faceBone.confidence.position = 0.7F;
}
private void UpdateEyes()
{
leftEye.closed = KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.EyeClosedLeft);
leftEye.closed = Clamp01((leftEye.closed - 0.5F) * 2);
rightEye.closed = KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.EyeClosedRight);
rightEye.closed = Clamp01((rightEye.closed - 0.5F) * 2);
}
private void UpdateFace()
{
UpdateEyeBrows();
UpdateEyes();
UpdateCheeks();
UpdateNose();
UpdateMouth();
UpdateJaw();
smile = KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.LipCornerPullerLeft);
}
private void UpdateJaw()
{
float jawOpen = KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.JawOpen);
float jawAngleX = minAngles.x + jawOpen * (maxAngles.x - minAngles.x);
Rotation newRotation = Rotation.AngleAxis(jawAngleX, Vector.right);
jaw.rotation = Rotation.Slerp(newRotation, jaw.rotation, beta);
jaw.confidence.rotation = 0.7F;
}
private void UpdateCheeks()
{
float confidence = 0.7F;
Vector referencePoint = KinectDevice.GetFacePoint((int)HighDetailFacePoints.NoseBottom);
Vector cheekPosition;
cheekPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.LeftCheek) - referencePoint;
leftCheek.position = Filter(leftCheek.position, leftCheek.startPosition + cheekPosition);
leftCheek.confidence.position = confidence;
cheekPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.RightCheek) - referencePoint;
rightCheek.position = Filter(rightCheek.position, rightCheek.startPosition + cheekPosition);
rightCheek.confidence.position = confidence;
}
private static void CalibrateEyeBrows()
{
Vector leftEyePositon = KinectDevice.GetFacePoint((int)HighDetailFacePoints.LefteyeMidbottom);
for (int i = (int)FaceBone.LeftOuterBrow; i <= (int)FaceBone.LeftInnerBrow; i++)
{
faceBias[i] = leftEyePositon - GetFacePointRaw((FaceBone)i);
}
Vector rightEyePosition = KinectDevice.GetFacePoint((int)HighDetailFacePoints.RighteyeMidbottom);
for (int i = (int)FaceBone.RightInnerBrow; i <= (int)FaceBone.RightOuterBrow; i++)
{
faceBias[i] = rightEyePosition - GetFacePointRaw((FaceBone)i);
}
}
public override Status Update()
{
if (!KinectDevice.JointIsTracked(handJoint))
{
status = Status.Present;
return(status);
}
status = Status.Tracking;
UpdateArm();
if (handTracking)
{
UpdateFingers();
}
return(status);
}
private void UpdateMouth()
{
float confidence = 0.7F;
Vector referencePoint = KinectDevice.GetFacePoint((int)HighDetailFacePoints.NoseBottom);
Vector lipPosition;
lipPosition = mouth.upperLipLeft.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.UpperLipLeft) - referencePoint;
mouth.upperLipLeft.position = Filter(mouth.upperLipLeft.position, lipPosition);
mouth.upperLipLeft.confidence.position = confidence;
lipPosition = mouth.upperLip.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.UpperLip) - referencePoint;
mouth.upperLip.position = Filter(mouth.upperLip.position, lipPosition);
mouth.upperLip.confidence.position = confidence;
lipPosition = mouth.upperLipRight.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.UpperLipRight) - referencePoint;
mouth.upperLipRight.position = Filter(mouth.upperLipRight.position, lipPosition);
mouth.upperLipRight.confidence.position = confidence;
lipPosition = mouth.lipLeft.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.LipLeft) - referencePoint;
lipPosition += (KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.LipPucker)) * 0.01F * Vector.right;
mouth.lipLeft.position = Filter(mouth.lipLeft.position, lipPosition);
mouth.lipLeft.confidence.position = confidence;
lipPosition = mouth.lipRight.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.LipRight) - referencePoint;
lipPosition += (KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.LipPucker)) * -0.01F * Vector.right;
mouth.lipRight.position = Filter(mouth.lipRight.position, lipPosition);
mouth.lipRight.confidence.position = confidence;
lipPosition = mouth.lowerLipLeft.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.LowerLipLeft) - referencePoint;
mouth.lowerLipLeft.position = Filter(mouth.lowerLipLeft.position, lipPosition);
mouth.lowerLipLeft.confidence.position = confidence;
lipPosition = mouth.lowerLip.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.LowerLip) - referencePoint;
mouth.lowerLip.position = Filter(mouth.lowerLip.position, lipPosition);
mouth.lowerLip.confidence.position = confidence;
lipPosition = mouth.lowerLipRight.startPosition + KinectDevice.GetFacePoint(KinectDevice.FaceBone.LowerLipRight) - referencePoint;
mouth.lowerLipRight.position = Filter(mouth.lowerLipRight.position, lipPosition);
mouth.lowerLipRight.confidence.position = confidence;
}
private void UpdateFingers()
{
bool handLasso, handClosed;
if (isLeft)
{
handLasso = KinectDevice.GetLeftHandState() == Windows.Kinect.HandState.Lasso;
handClosed = KinectDevice.GetLeftHandState() == Windows.Kinect.HandState.Closed;
}
else
{
handLasso = KinectDevice.GetRightHandState() == Windows.Kinect.HandState.Lasso;
handClosed = KinectDevice.GetRightHandState() == Windows.Kinect.HandState.Closed;
}
thumb.curl = MaxChange(thumb.curl, (handLasso || handClosed) ? 1 : 0, 0.1F);
indexFinger.curl = MaxChange(indexFinger.curl, handClosed ? 1 : 0, 0.1F);
middleFinger.curl = MaxChange(middleFinger.curl, handClosed ? 1 : 0, 0.1F);
ringFinger.curl = MaxChange(ringFinger.curl, (handLasso || handClosed) ? 1 : 0, 0.1F);
littleFinger.curl = MaxChange(littleFinger.curl, (handLasso || handClosed) ? 1 : 0, 0.1F);
}
private void UpdateEyeBrows()
{
float confidence = 0.7F;
Vector browPosition;
Vector leftEyePositon = KinectDevice.GetFacePoint((int)HighDetailFacePoints.LefteyeMidbottom);
Vector browExpressionPosition = Vector.zero; // (KinectDevice.GetFaceExpression(KinectDevice.FaceExpression.BrowLowererLeft) - 0.75F) * 0.01F * Vector.up;
// This is very unstable
browPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.LeftOuterBrow) - leftEyePositon;
browPosition -= browExpressionPosition;
leftBrow.outer.position = Filter(leftBrow.outer.position, leftBrow.outer.startPosition + browPosition);
leftBrow.outer.confidence.position = confidence;
browPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.LeftBrow) - leftEyePositon;
browPosition -= browExpressionPosition;
leftBrow.center.position = Filter(leftBrow.center.position, leftBrow.center.startPosition + browPosition);
leftBrow.center.confidence.position = confidence;
browPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.LeftInnerBrow) - leftEyePositon;
browPosition -= browExpressionPosition;
leftBrow.inner.position = Filter(leftBrow.inner.position, leftBrow.inner.startPosition + browPosition);
leftBrow.inner.confidence.position = confidence;
Vector rightEyePosition = KinectDevice.GetFacePoint((int)HighDetailFacePoints.RighteyeMidbottom);
browPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.RightInnerBrow) - rightEyePosition;
rightBrow.inner.position = Filter(rightBrow.inner.position, rightBrow.inner.startPosition + browPosition);
rightBrow.inner.confidence.position = confidence;
browPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.RightBrow) - rightEyePosition;
rightBrow.center.position = Filter(rightBrow.center.position, rightBrow.center.startPosition + browPosition);
rightBrow.center.confidence.position = confidence;
browPosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.RightOuterBrow) - rightEyePosition;
rightBrow.outer.position = Filter(rightBrow.outer.position, rightBrow.outer.startPosition + browPosition);
rightBrow.outer.confidence.position = confidence;
}
public Status Update(Rotation hipsOrientation)
{
if (!KinectDevice.JointIsTracked(footJoint))
{
status = Status.Present;
return(status);
}
status = Status.Tracking;
Vector hipsPosition = kinectDevice.GetTargetPosition(hipJoint);
Vector kneePosition = kinectDevice.GetTargetPosition(kneeJoint);
Vector anklePosition = kinectDevice.GetTargetPosition(ankleJoint);
//Vector footPosition = kinectDevice.GetTargetPosition(footJoint);
// Position
upperLeg.confidence.position = KinectDevice.TrackingConfidence(upperLeg.position, hipsPosition);
upperLeg.position = hipsPosition;
lowerLeg.confidence.position = KinectDevice.TrackingConfidence(lowerLeg.position, kneePosition);
lowerLeg.position = kneePosition;
foot.confidence.position = KinectDevice.TrackingConfidence(foot.position, anklePosition);
foot.position = anklePosition;
// Rotation
upperLeg.rotation = device.ToWorldOrientation(CalculateLegOrientation(hipsPosition, kneePosition, hipsOrientation));
upperLeg.confidence.rotation = Math.Min(upperLeg.confidence.position, lowerLeg.confidence.position);
lowerLeg.rotation = device.ToWorldOrientation(CalculateLegOrientation(kneePosition, anklePosition, hipsOrientation));
lowerLeg.confidence.rotation = Math.Min(lowerLeg.confidence.position, foot.confidence.position);
foot.rotation = device.ToWorldOrientation(Rotation.AngleAxis(180, Vector.up));
foot.confidence.rotation = foot.confidence.position;
return(status);
}
private void UpdateNose()
{
float confidence = 0.7F;
// We could also use NoseTip as reference point...
Vector referencePoint = KinectDevice.GetFacePoint((int)HighDetailFacePoints.NoseBottom);
Vector nosePosition;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseTop) - referencePoint;
nose.top.position = Filter(nose.top.position, nose.top.startPosition + nosePosition);
nose.top.confidence.position = confidence;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseTopLeft) - referencePoint;
nose.topLeft.position = Filter(nose.topLeft.position, nose.topLeft.startPosition + nosePosition);
nose.topLeft.confidence.position = confidence;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseTopRight) - referencePoint;
nose.topRight.position = Filter(nose.topRight.position, nose.topRight.startPosition + nosePosition);
nose.topRight.confidence.position = confidence;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseTip) - referencePoint;
nose.tip.position = Filter(nose.tip.position, nose.tip.startPosition + nosePosition);
nose.tip.confidence.position = confidence;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseBottomLeft) - referencePoint;
nose.bottomLeft.position = Filter(nose.bottomLeft.position, nose.bottomLeft.startPosition + nosePosition);
nose.bottomLeft.confidence.position = confidence;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseBottom) - referencePoint;
nose.bottom.position = Filter(nose.bottom.position, nose.bottom.startPosition + nosePosition);
nose.bottom.confidence.position = confidence;
nosePosition = KinectDevice.GetFacePoint(KinectDevice.FaceBone.NoseBottomRight) - referencePoint;
nose.bottomRight.position = Filter(nose.bottomRight.position, nose.bottomRight.startPosition + nosePosition);
nose.bottomRight.confidence.position = confidence;
}
public Vector GetTargetPosition(KinectDevice.JointID boneType)
{
return(ToWorldPosition(KinectDevice.GetTargetLocalPosition(boneType)));
}
private void UpdateArm()
{
Vector shoulderPosition = kinectDevice.GetTargetPosition(shoulderJoint);
Vector elbowPosition = kinectDevice.GetTargetPosition(elbowJoint);
Vector wristPosition = kinectDevice.GetTargetPosition(wristJoint);
Vector upperArmDirection = elbowPosition - shoulderPosition;
Vector forearmDirection = wristPosition - elbowPosition;
float upperArmLength = Vector.Magnitude(upperArmDirection);
float forearmLength = Vector.Magnitude(forearmDirection);
Vector elbowUp = Vector.Cross(upperArmDirection, forearmDirection);
if (!isLeft)
{
elbowUp = -elbowUp;
}
upperArm.confidence.position = KinectDevice.TrackingConfidence(upperArm.position, shoulderPosition);
upperArm.position = shoulderPosition;
upperArm.rotation = CalculateUpperArmOrientation(shoulderPosition, upperArmLength, elbowUp, forearmLength, wristPosition, isLeft);
forearm.confidence.position = KinectDevice.TrackingConfidence(forearm.position, elbowPosition);
forearm.position = elbowPosition;
upperArm.confidence.rotation = Math.Min(upperArm.confidence.position, forearm.confidence.position);
//float handPositionConfidence = KinectDevice.TrackingConfidence(hand.position, wristPosition);
//if (handPositionConfidence >= hand.confidence.position) { // Is this tracking info better than we already have?
hand.confidence.position = 0.9F;
Vector thumbPosition = kinectDevice.GetTargetPosition(thumbJoint);
Vector handPosition = kinectDevice.GetTargetPosition(handTipJoint);
Vector handDirection = handPosition - wristPosition;
Vector thumbDirection = wristPosition - thumbPosition;
Vector handUp = Vector.Cross(handDirection, thumbDirection);
if (isLeft)
{
handUp = -handUp;
}
forearm.rotation = CalculateArmOrientation(elbowPosition, elbowUp, wristPosition, isLeft);
forearm.confidence.rotation = Math.Min(forearm.confidence.position, hand.confidence.position);
hand.position = wristPosition;
// hand rotational tracking is quite unreliable
hand.rotation = CalculateArmOrientation(wristPosition, handUp, handPosition, isLeft);
hand.confidence.rotation = 0.6F;
//}
//else {
// float weight = 0.01F;
// hand.confidence.position = handPositionConfidence * weight + hand.confidence.position * (1 - weight);
// hand.position = wristPosition * weight + hand.position * (1 - weight);
// // do not use wristRotation here, quality is low, we expect any other tracker is better
// forearm.rotation = CalculateArmOrientation(elbowPosition, elbowUp, hand.position, isLeft);
// forearm.confidence.rotation = Math.Min(forearm.confidence.position, hand.confidence.position);
//}
}