private void GetHandStateAndConf(uint handState, InteractionHandEventType handEvent,
ref KinectInterop.HandState refHandState, ref KinectInterop.TrackingConfidence refHandConf)
{
bool bHandPrimary = (handState & (uint)NuiHandpointerState.PrimaryForUser) != 0;
refHandConf = bHandPrimary ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low;
if (bHandPrimary)
{
switch (handEvent)
{
case InteractionHandEventType.Grip:
refHandState = KinectInterop.HandState.Closed;
break;
case InteractionHandEventType.Release:
//case InteractionHandEventType.None:
refHandState = KinectInterop.HandState.Open;
break;
}
}
else
{
refHandState = KinectInterop.HandState.NotTracked;
}
}
private Visibility TrackingToVisibility(Joint hand, InteractionHandEventType leftHandEvent)
{
if (hand.JointType == JointType.HandLeft)
{
if (hand.TrackingState == JointTrackingState.Tracked)
{
return(Visibility.Visible);
}
else
{
return(Visibility.Hidden);
}
}
else
{
if (hand.TrackingState == JointTrackingState.Tracked && leftHandEvent == InteractionHandEventType.Grip)
{
return(Visibility.Visible);
}
else
{
return(Visibility.Hidden);
}
}
}
private SolidColorBrush StateToColor(InteractionHandEventType state)
{
if (state == InteractionHandEventType.Grip)
{
return(new SolidColorBrush(Colors.Green));
}
else
{
return(new SolidColorBrush(Colors.Red));
}
}
/// <summary>
/// Evènement : si on a les informations de profondeur et du squelette -> définit l'état actuel de la main
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void EventInteractionFrameReady(object sender, InteractionFrameReadyEventArgs e)
{
UserInfo[] userInfos = new UserInfo[InteractionFrame.UserInfoArrayLength];
using (InteractionFrame interactionFrame = e.OpenInteractionFrame())
{
if (interactionFrame == null)
{
return;
}
interactionFrame.CopyInteractionDataTo(userInfos);
}
// On parcourt les informations trouvées
bool handFound = false;
foreach (UserInfo userInfo in userInfos)
{
// Si mal reconnu ou on déjà trouvé une main qui est affichée
if (userInfo.SkeletonTrackingId == 0 || handFound)
{
continue;
}
var hands = userInfo.HandPointers;
if (hands.Count == 0) // Si aucune main reconnue
{
continue;
}
foreach (var hand in hands)
{
// La main doit être trackée, active et la main 'primaire' (= première levée des deux)
if (!hand.IsActive || !hand.IsTracked)
{
continue;
}
handFound = true;
actualHandType = hand.HandType;
// Si la main devient grip et c'était pas son ancien état
if (hand.HandEventType == InteractionHandEventType.Grip && hand.HandEventType != actualHandState)
{
actualHandState = InteractionHandEventType.Grip;
}
// Si la main lâche le grip
else if (hand.HandEventType == InteractionHandEventType.GripRelease && hand.HandEventType != actualHandState)
{
actualHandState = InteractionHandEventType.GripRelease;
}
}
}
}
public void UpdateUI(KinectSensor kinectSensor, Joint handJoint, InteractionHandEventType handEvent, InteractionHandEventType handColor)
{
//calculate values
var leftImagePoint = kinectSensor.CoordinateMapper.MapSkeletonPointToDepthPoint(handJoint.Position,
DepthImageFormat.Resolution640x480Fps30);
//update UI
CanvasLeft = leftImagePoint.X;
CanvasTop = leftImagePoint.Y;
Visibility = TrackingToVisibility(handJoint, handEvent);
Color = StateToColor(handColor);
}
/// <summary>
/// Convert an InteractionHandEventType value to corresponding HandEventType value.
/// </summary>
/// <param name="interactionHandEventType">
/// InteractionHandEventType value to convert.
/// </param>
/// <returns>
/// Corresponding HandEventType value.
/// </returns>
internal static HandEventType ConvertHandEventType(InteractionHandEventType interactionHandEventType)
{
switch (interactionHandEventType)
{
case InteractionHandEventType.Grip:
return(HandEventType.Grip);
case InteractionHandEventType.GripRelease:
return(HandEventType.GripRelease);
default:
Debug.Assert(interactionHandEventType == InteractionHandEventType.None, "HandEventType and InteractionHandEventType are out of sync.");
return(HandEventType.None);
}
}
/// <summary>
/// Convert an InteractionHandEventType value to corresponding HandEventType value.
/// </summary>
/// <param name="interactionHandEventType">
/// InteractionHandEventType value to convert.
/// </param>
/// <returns>
/// Corresponding HandEventType value.
/// </returns>
internal static HandEventType ConvertHandEventType(InteractionHandEventType interactionHandEventType)
{
switch (interactionHandEventType)
{
case InteractionHandEventType.Grip:
return HandEventType.Grip;
case InteractionHandEventType.GripRelease:
return HandEventType.GripRelease;
default:
Debug.Assert(interactionHandEventType == InteractionHandEventType.None, "HandEventType and InteractionHandEventType are out of sync.");
return HandEventType.None;
}
}
public override bool isInteractionActive(ReadOnlyCollection <InteractionHandPointer> hands)
{
// if primary hand is event is grip -> return true
foreach (var hand in hands)
{
if (hand.HandType != InteractionHandType.None)
{
if (hand.IsPrimaryForUser)
{
primaryHand = hand;
break;
}
else
{
continue;
}
}
}
if (primaryHand == null)
{
return(false);
}
if (primaryHand.HandEventType != InteractionHandEventType.None)
{
lastHandEvent = primaryHand.HandEventType;
}
if (primaryHand == null || !primaryHand.IsActive)
{
return(false);
}
if (lastHandEvent == InteractionHandEventType.Grip)
{
return(true);
}
else
{
return(false);
}
}
public override bool isInteractionActive(ReadOnlyCollection<InteractionHandPointer> hands)
{
// if primary hand is event is grip -> return true
foreach (var hand in hands)
{
if (hand.HandType != InteractionHandType.None)
{
if (hand.IsPrimaryForUser)
{
primaryHand = hand;
break;
}
else
{
continue;
}
}
}
if (primaryHand == null)
return false;
if(primaryHand.HandEventType != InteractionHandEventType.None)
lastHandEvent = primaryHand.HandEventType;
if (primaryHand == null || !primaryHand.IsActive)
return false;
if (lastHandEvent == InteractionHandEventType.Grip)
{
return true;
}
else
{
return false;
}
}
private void interStream_InteractionFrameReady(object sender, InteractionFrameReadyEventArgs e)
{
using (InteractionFrame inf = e.OpenInteractionFrame())
{
if (inf == null)
{
return;
}
inf.CopyInteractionDataTo(userInfos);
}
foreach (UserInfo ui in userInfos)
{
var hands = ui.HandPointers;
if (ui.SkeletonTrackingId == 0)
{
continue;
}
foreach (InteractionHandPointer hand in hands)
{
last = hand.HandEventType == InteractionHandEventType.None ? last : hand.HandEventType;
if (last == InteractionHandEventType.Grip)
{
checkDraw.IsChecked = true;
}
else
{
if (checkDraw.IsChecked == true)
{
checkDraw.IsChecked = false;
}
}
}
}
}
private void interStream_InteractionFrameReady(object sender, InteractionFrameReadyEventArgs e)
{
using (InteractionFrame inf = e.OpenInteractionFrame())
{
if (inf == null)
{
return;
}
inf.CopyInteractionDataTo(userInfos);
}
foreach (UserInfo ui in userInfos)
{
var hands = ui.HandPointers;
if (ui.SkeletonTrackingId == 0)
continue;
foreach (InteractionHandPointer hand in hands)
{
last = hand.HandEventType == InteractionHandEventType.None ? last : hand.HandEventType;
if (last == InteractionHandEventType.Grip)
{
checkDraw.IsChecked = true;
}
else
{
if(checkDraw.IsChecked == true)
checkDraw.IsChecked = false;
}
}
}
}
private void GetHandStateAndConf(uint skeletonId, bool isRightHand, uint handState, InteractionHandEventType handEvent,
ref Dictionary<uint, InteractionHandEventType> lastHandEvent,
ref KinectInterop.HandState refHandState,
ref KinectInterop.TrackingConfidence refHandConf)
{
bool bHandPrimary = (handState & (uint)NuiHandpointerState.Active) != 0;
refHandConf = bHandPrimary ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low;
if(!lastHandEvent.ContainsKey(skeletonId))
{
lastHandEvent[skeletonId] = InteractionHandEventType.Release;
}
if(handEvent == InteractionHandEventType.Grip ||
handEvent == InteractionHandEventType.Release)
{
if(lastHandEvent[skeletonId] != handEvent)
{
//Debug.Log(string.Format("{0} - {1}, event: {2}", skeletonId, !isRightHand ? "left" : "right", handEvent));
lastHandEvent[skeletonId] = handEvent;
}
}
if(bHandPrimary)
{
switch(lastHandEvent[skeletonId])
{
case InteractionHandEventType.Grip:
refHandState = KinectInterop.HandState.Closed;
break;
case InteractionHandEventType.Release:
refHandState = KinectInterop.HandState.Open;
break;
}
}
else
{
// if(lastHandEvent[skeletonId] != InteractionHandEventType.Release)
// {
// Debug.Log(string.Format("{0} - old: {1}, NONE: {2}", skeletonId, lastHandEvent[skeletonId], InteractionHandEventType.Release));
// lastHandEvent[skeletonId] = InteractionHandEventType.Release;
// }
refHandState = KinectInterop.HandState.NotTracked;
}
}
private void GetHandStateAndConf(uint handState, InteractionHandEventType handEvent,
ref KinectInterop.HandState refHandState, ref KinectInterop.TrackingConfidence refHandConf)
{
bool bHandPrimary = (handState & (uint)NuiHandpointerState.PrimaryForUser) != 0;
refHandConf = bHandPrimary ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low;
if(bHandPrimary)
{
switch(handEvent)
{
case InteractionHandEventType.Grip:
refHandState = KinectInterop.HandState.Closed;
break;
case InteractionHandEventType.Release:
//case InteractionHandEventType.None:
refHandState = KinectInterop.HandState.Open;
break;
}
}
else
{
refHandState = KinectInterop.HandState.NotTracked;
}
}
public bool PollBodyFrame(KinectInterop.SensorData sensorData, ref KinectInterop.BodyFrameData bodyFrame, ref Matrix4x4 kinectToWorld)
{
bool newSkeleton = false;
int hr = NuiSkeletonGetNextFrame(0, ref skeletonFrame);
if (hr == 0)
{
newSkeleton = true;
}
if (newSkeleton)
{
hr = NuiTransformSmooth(ref skeletonFrame, ref smoothParameters);
if (hr != 0)
{
Debug.LogError("Skeleton Data Smoothing failed");
}
for (uint i = 0; i < sensorData.bodyCount; i++)
{
NuiSkeletonData body = skeletonFrame.SkeletonData[i];
bodyFrame.bodyData[i].bIsTracked = (short)(body.eTrackingState == NuiSkeletonTrackingState.SkeletonTracked ? 1 : 0);
if (body.eTrackingState == NuiSkeletonTrackingState.SkeletonTracked)
{
// transfer body and joints data
bodyFrame.bodyData[i].liTrackingID = (long)body.dwTrackingID;
for (int j = 0; j < sensorData.jointCount; j++)
{
KinectInterop.JointData jointData = bodyFrame.bodyData[i].joint[j];
jointData.jointType = GetJointAtIndex(j);
jointData.trackingState = (KinectInterop.TrackingState)body.eSkeletonPositionTrackingState[j];
if (jointData.trackingState != KinectInterop.TrackingState.NotTracked)
{
jointData.kinectPos = body.SkeletonPositions[j];
jointData.position = kinectToWorld.MultiplyPoint3x4(jointData.kinectPos);
}
jointData.orientation = Quaternion.identity;
// Windows.Kinect.Vector4 vQ = body.JointOrientations[jointData.jointType].Orientation;
// jointData.orientation = new Quaternion(vQ.X, vQ.Y, vQ.Z, vQ.W);
if (j == 0)
{
bodyFrame.bodyData[i].position = jointData.position;
bodyFrame.bodyData[i].orientation = jointData.orientation;
}
bodyFrame.bodyData[i].joint[j] = jointData;
}
// tranfer hand states
uint intCount = GetInteractorsCount();
for (uint intIndex = 0; intIndex < intCount; intIndex++)
{
uint skeletonId = GetSkeletonTrackingID(intIndex);
if (skeletonId == body.dwTrackingID)
{
uint leftHandState = GetLeftHandState(intIndex);
InteractionHandEventType leftHandEvent = GetLeftHandEvent(intIndex);
uint rightHandState = GetRightHandState(intIndex);
InteractionHandEventType rightHandEvent = GetRightHandEvent(intIndex);
GetHandStateAndConf(leftHandState, leftHandEvent,
ref bodyFrame.bodyData[i].leftHandState,
ref bodyFrame.bodyData[i].leftHandConfidence);
GetHandStateAndConf(rightHandState, rightHandEvent,
ref bodyFrame.bodyData[i].rightHandState,
ref bodyFrame.bodyData[i].rightHandConfidence);
}
}
}
}
}
return(newSkeleton);
}
/// <summary>
/// Update the hand datas
/// </summary>
/// <param name="rawPosition">Hand's raw position</param>
/// <param name="handEventType">Event on the hand</param>
/// <param name="isActive">Hand is active or not</param>
/// <param name="isPrimaryHand">Hand is primary or not</param>
public void UpdateHandData(PointF rawPosition, InteractionHandEventType handEventType, bool isActive, bool isPrimaryHand)
{
// Get the hand's raw position in the kinect hand landmark
HandRawPosition = rawPosition;
// Transform the hand's raw position in the kinec hand landmark to the MGRE landmark
// This transformation take account the parameters 'SpaceBetweenHands' and 'PointingHandsAmplitude'
if (HandType == InteractionHandType.Left)
{
// I consider the hand left landmark corresponding to the left half of the screen (/2)
m_HandRawPosition.X = ((m_HandRawPosition.X + PropertiesPluginKinect.Instance.PointingSpaceBetweenHands + PropertiesPluginKinect.Instance.PointingHandsAmplitude)
/
( (1-PropertiesPluginKinect.Instance.PointingSpaceBetweenHands) +
(PropertiesPluginKinect.Instance.PointingHandsAmplitude + PropertiesPluginKinect.Instance.PointingSpaceBetweenHands))
) / 2;
}
else
{
// I concider the hand right landmark corresponding to the right half of the screen (/2 + 0.5f)
m_HandRawPosition.X = ((m_HandRawPosition.X - PropertiesPluginKinect.Instance.PointingSpaceBetweenHands) / ((1 - PropertiesPluginKinect.Instance.PointingSpaceBetweenHands) +
(PropertiesPluginKinect.Instance.PointingHandsAmplitude + PropertiesPluginKinect.Instance.PointingSpaceBetweenHands))) / 2 + 0.5f;
}
PointF handScreen = new PointF();
// Transform the hand's raw position to the hand's screen position
handScreen.X = m_HandRawPosition.X * PropertiesPluginKinect.Instance.ExperienceIntuiFaceWidth;
handScreen.Y = m_HandRawPosition.Y * PropertiesPluginKinect.Instance.ExperienceIntuifaceHeight;
HandScreenPosition = handScreen;
// Get the hand's event type
if (handEventType == InteractionHandEventType.Grip)
{
IsGrip = true;
}
else if (handEventType == InteractionHandEventType.GripRelease)
{
IsGrip = false;
}
// Update the primary hand and active hand
IsPrimaryHand = isPrimaryHand;
IsActive = isActive;
}
