private void HandleKeyPoseChanged(ManagedHand hand, MLHandTracking.HandKeyPose pose)
{
//both hands holding thumbs?
if (HandInput.Left.Hand.KeyPose == MLHandTracking.HandKeyPose.Thumb && HandInput.Right.Hand.KeyPose == MLHandTracking.HandKeyPose.Thumb)
{
StartCoroutine("ThumbsUpReset");
}
else
{
StopCoroutine("ThumbsUpReset");
}
}
private bool GetGesture(MLHandTracking.Hand hand, MLHandTracking.HandKeyPose type)
{
if (hand != null)
{
if (hand.KeyPose == type)
{
if (hand.HandKeyPoseConfidence > 0.9f)
{
return(true);
}
}
}
return(false);
}
private void HandleGestureChanged(ManagedHand hand, MLHandTracking.HandKeyPose pose)
{
//snap transform changes to avoid force pops:
switch (pose)
{
case MLHandTracking.HandKeyPose.Fist:
_handCollider.transform.position = _managedHand.Skeleton.Position;
_handCollider.transform.localScale = Vector3.one * _managedHand.Gesture.Grasp.radius;
break;
case MLHandTracking.HandKeyPose.L:
case MLHandTracking.HandKeyPose.Finger:
_handCollider.transform.localScale = Vector3.one * _pointColliderSize;
_handCollider.transform.position = _managedHand.Skeleton.Index.Tip.positionFiltered;
break;
case MLHandTracking.HandKeyPose.OpenHand:
_handCollider.transform.localScale = new Vector3(_managedHand.Gesture.Grasp.radius, _openHandColliderWidth, _managedHand.Gesture.Grasp.radius);
_handCollider.transform.position = _managedHand.Skeleton.Position;
_handCollider.transform.rotation = _managedHand.Skeleton.Rotation;
break;
}
}
/// <summary>
/// Sets the values of a specific key pose.
/// </summary>
/// <param name="config">The native config object of key poses to use.</param>
/// <param name="handKeyPose">The key pose to set.</param>
/// <param name="enable">The value to set the key pose with.</param>
private static void SetKeyPoseConfig(NativeBindings.ConfigurationNative config, MLHandTracking.HandKeyPose handKeyPose, bool enable)
{
if (handKeyPose > MLHandTracking.HandKeyPose.NoPose)
{
MLPluginLog.WarningFormat("KeyPoseManager.SetKeyPoseConfig trying to set {0}. Ignoring.", handKeyPose);
return;
}
config.KeyposeConfig[(uint)handKeyPose] = Convert.ToByte(enable);
}
//Event Handlers:
private void HandleKeyposeChanged(MLHandTracking.HandKeyPose keyPose)
{
OnKeyPoseChanged?.Invoke(_managedHand, keyPose);
}
//Public Methods:
public void Update()
{
if (!_managedHand.Visible)
{
return;
}
//pinch rotation offset mirror:
Vector3 rotationOffset = _pinchAbsoluteRotationOffset;
if (_managedHand.Hand.Type == MLHandTracking.HandType.Left)
{
rotationOffset.y *= -1;
}
//holders:
Vector3 pinchPosition = Vector3.zero;
Quaternion pinchRotation = Quaternion.identity;
//pinch interaction point radius:
if (_managedHand.Skeleton.Thumb.Tip.Visible && _managedHand.Skeleton.Index.Tip.Visible)
{
Pinch.radius = Vector3.Distance(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered);
}
if (_managedHand.Skeleton.Thumb.Tip.Visible) //absolute placement:
{
//are we swapping modes?
if (_pinchIsRelative)
{
_pinchIsRelative = false;
_pinchTransitioning = true;
_pinchTransitionStartTime = Time.realtimeSinceStartup;
}
pinchPosition = _managedHand.Skeleton.Thumb.Tip.positionFiltered;
pinchRotation = TransformUtilities.RotateQuaternion(_managedHand.Skeleton.Rotation, rotationOffset);
//gather offset distance:
if (_managedHand.Skeleton.Index.Knuckle.Visible && _managedHand.Skeleton.Thumb.Knuckle.Visible)
{
Vector3 mcpMidpoint = Vector3.Lerp(_managedHand.Skeleton.Index.Knuckle.positionFiltered, _managedHand.Skeleton.Thumb.Knuckle.positionFiltered, .5f);
_pinchRelativePositionDistance = Vector3.Distance(mcpMidpoint, pinchPosition);
}
}
else //relative placement:
{
//are we swapping modes?
if (!_pinchIsRelative)
{
_pinchIsRelative = true;
_pinchTransitioning = true;
_pinchTransitionStartTime = Time.realtimeSinceStartup;
}
//place between available mcps:
if (_managedHand.Skeleton.Index.Knuckle.Visible && _managedHand.Skeleton.Thumb.Knuckle.Visible)
{
pinchPosition = Vector3.Lerp(_managedHand.Skeleton.Index.Knuckle.positionFiltered, _managedHand.Skeleton.Thumb.Knuckle.positionFiltered, .5f);
//rotate:
pinchRotation = TransformUtilities.RotateQuaternion(_managedHand.Skeleton.Rotation, _pinchRelativeRotationOffset);
//move out along rotation forward:
pinchPosition += pinchRotation * Vector3.forward * _pinchRelativePositionDistance;
}
else
{
//just use previous:
pinchPosition = Pinch.position;
pinchRotation = Pinch.rotation;
}
}
//sticky release reduction:
if (_collapsed)
{
if (_managedHand.Skeleton.Thumb.Tip.Visible && _managedHand.Skeleton.Index.Tip.Visible)
{
//if starting to release, start using a point between the thumb and index tips:
if (Vector3.Distance(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered) > _dynamicReleaseDistance)
{
pinchPosition = Vector3.Lerp(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered, .3f);
}
}
}
//apply pinch pose - to avoid jumps when relative placement is used we smooth until close enough:
if (_pinchTransitioning)
{
//position:
Pinch.position = Vector3.SmoothDamp(Pinch.position, pinchPosition, ref _pinchArrivalPositionVelocity, _pinchTransitionTime);
float positionDelta = Vector3.Distance(Pinch.position, pinchPosition);
//rotation:
Pinch.rotation = MotionUtilities.SmoothDamp(Pinch.rotation, pinchRotation, ref _pinchArrivalRotationVelocity, _pinchTransitionTime);
float rotationDelta = Quaternion.Angle(Pinch.rotation, pinchRotation);
//close enough to hand off?
if (positionDelta < .001f && rotationDelta < 5)
{
_pinchTransitioning = false;
}
//taking too long?
if (Time.realtimeSinceStartup - _pinchTransitionStartTime > _pinchTransitionMaxDuration)
{
_pinchTransitioning = false;
}
}
else
{
Pinch.position = pinchPosition;
Pinch.rotation = pinchRotation;
}
//grasp interaction point:
Bounds graspBounds = CalculateGraspBounds
(
_managedHand.Skeleton.Thumb.Knuckle,
_managedHand.Skeleton.Thumb.Joint,
_managedHand.Skeleton.Thumb.Tip,
_managedHand.Skeleton.Index.Knuckle,
_managedHand.Skeleton.Index.Joint,
_managedHand.Skeleton.Index.Tip,
_managedHand.Skeleton.Middle.Knuckle,
_managedHand.Skeleton.Middle.Joint,
_managedHand.Skeleton.Middle.Tip
);
Grasp.position = _managedHand.Skeleton.Position;
//when points are being initially found they can be wildly off and this could cause a massively large volume:
Grasp.radius = Mathf.Min(graspBounds.size.magnitude, _maxGraspRadius);
Grasp.rotation = _managedHand.Skeleton.Rotation;
//intent updated:
if (_currentInteractionState != null)
{
_currentInteractionState.FireUpdate();
}
//keypose change proposed:
if (_managedHand.Hand.KeyPose != VerifiedGesture && _managedHand.Hand.KeyPose != _proposedKeyPose)
{
//queue a new proposed change to keypose:
_proposedKeyPose = _managedHand.Hand.KeyPose;
_keyPoseChangedTime = Time.realtimeSinceStartup;
}
//keypose change acceptance:
if (_managedHand.Hand.KeyPose != VerifiedGesture && Time.realtimeSinceStartup - _keyPoseChangedTime > _keyPoseStabailityDuration)
{
//reset:
Point.active = false;
Pinch.active = false;
Grasp.active = false;
if (_collapsed)
{
//intent end:
if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.C || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.OpenHand || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.L || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Finger)
{
if (_managedHand.Skeleton.Thumb.Tip.Visible && _managedHand.Skeleton.Index.Tip.Visible)
{
//dynamic release:
if (Vector3.Distance(_managedHand.Skeleton.Thumb.Tip.positionFiltered, _managedHand.Skeleton.Index.Tip.positionFiltered) > _dynamicReleaseDistance)
{
//end intent:
_collapsed = false;
_currentInteractionState.FireEnd();
_currentInteractionState = null;
//accept keypose change:
VerifiedGesture = _managedHand.Hand.KeyPose;
_proposedKeyPose = _managedHand.Hand.KeyPose;
OnVerifiedGestureChanged?.Invoke(_managedHand, VerifiedGesture);
if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Finger || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.L)
{
Intent = IntentPose.Pointing;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
else if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.C || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.OpenHand || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Thumb)
{
Intent = IntentPose.Relaxed;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
}
}
}
}
else
{
//intent begin:
if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Pinch || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Ok || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Fist)
{
_collapsed = true;
if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Pinch || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Ok)
{
Intent = IntentPose.Pinching;
Pinch.active = true;
_currentInteractionState = Pinch.Touch;
_currentInteractionState.FireBegin();
OnIntentChanged?.Invoke(_managedHand, Intent);
}
else if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Fist)
{
Intent = IntentPose.Grasping;
Grasp.active = true;
_currentInteractionState = Grasp.Touch;
_currentInteractionState.FireBegin();
OnIntentChanged?.Invoke(_managedHand, Intent);
}
}
if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Finger || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.L)
{
Intent = IntentPose.Pointing;
Point.active = true;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
else if (_managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.C || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.OpenHand || _managedHand.Hand.KeyPose == MLHandTracking.HandKeyPose.Thumb)
{
Intent = IntentPose.Relaxed;
OnIntentChanged?.Invoke(_managedHand, Intent);
}
//accept keypose change:
VerifiedGesture = _managedHand.Hand.KeyPose;
_proposedKeyPose = _managedHand.Hand.KeyPose;
OnVerifiedGestureChanged?.Invoke(_managedHand, VerifiedGesture);
}
}
UpdateCurrentInteractionPoint();
UpdatePalmEvents();
}
private bool IsHandInGesture(MLHandTracking.HandKeyPose keyPose, float confidence = 0.8f)
{
return(_managedHand.Hand.KeyPose == keyPose &&
_managedHand.Hand.HandKeyPoseConfidence >= confidence);
}
/// <summary>
/// Method to call when a pose is no longer detected.
/// </summary>
/// <param name="pose">The pose that has no longer been detected.</param>
public void EndKeyPose(MLHandTracking.HandKeyPose pose)
{
this.KeyPose = pose;
this.OnHandKeyPoseEnd?.Invoke(pose);
}
/// <summary>
/// Method to call when a pose is detected for the first time.
/// </summary>
/// <param name="pose">The pose that has no longer been detected.</param>
public void BeginKeyPose(MLHandTracking.HandKeyPose pose)
{
this.KeyPose = pose;
this.OnHandKeyPoseBegin?.Invoke(pose);
}
