public static bool IsGrabbing(Handedness trackedHand)
{
return(!IsPinching(trackedHand) &&
HandPoseUtils.MiddleFingerCurl(trackedHand) > GrabThreshold &&
HandPoseUtils.RingFingerCurl(trackedHand) > GrabThreshold &&
HandPoseUtils.PinkyFingerCurl(trackedHand) > GrabThreshold &&
HandPoseUtils.ThumbFingerCurl(trackedHand) > GrabThreshold);
}
/// <inheritdoc/>
void IMixedRealityHandJointHandler.OnHandJointsUpdated(InputEventData <IDictionary <TrackedHandJoint, MixedRealityPose> > eventData)
{
if (eventData.InputSource.SourceId != Controller.InputSource.SourceId)
{
return;
}
Debug.Assert(eventData.Handedness == Controller.ControllerHandedness);
IMixedRealityInputSystem inputSystem = CoreServices.InputSystem;
// Only runs if render skeleton joints is true
MixedRealityHandTrackingProfile handTrackingProfile = inputSystem?.InputSystemProfile.HandTrackingProfile;
if (handTrackingProfile != null && handTrackingProfile.EnableHandJointVisualization)
{
// This starts at 1 to skip over TrackedHandJoint.None.
for (int i = 1; i < ArticulatedHandPose.JointCount; i++)
{
TrackedHandJoint handJoint = (TrackedHandJoint)i;
MixedRealityPose handJointPose = eventData.InputData[handJoint];
if (skeletonJoints.TryGetValue(handJoint, out Transform skeletonJointTransform))
{
skeletonJointTransform.SetPositionAndRotation(handJointPose.Position, handJointPose.Rotation);
}
else
{
GameObject prefab;
if (handJoint == TrackedHandJoint.Palm)
{
prefab = inputSystem.InputSystemProfile.HandTrackingProfile.PalmJointPrefab;
}
else if (handJoint == TrackedHandJoint.IndexTip)
{
prefab = inputSystem.InputSystemProfile.HandTrackingProfile.FingerTipPrefab;
}
else
{
prefab = inputSystem.InputSystemProfile.HandTrackingProfile.JointPrefab;
}
GameObject jointObject;
if (prefab != null)
{
jointObject = Instantiate(prefab);
}
else
{
jointObject = new GameObject();
}
jointObject.name = handJoint.ToString() + " Proxy Transform";
jointObject.transform.SetPositionAndRotation(handJointPose.Position, handJointPose.Rotation);
jointObject.transform.parent = transform;
skeletonJoints.Add(handJoint, jointObject.transform);
}
}
}
else
{
// clear existing joint GameObjects / meshes
foreach (var joint in skeletonJoints)
{
Destroy(joint.Value.gameObject);
}
skeletonJoints.Clear();
}
// Only runs if render hand mesh is true
bool renderHandmesh = handTrackingProfile != null && handTrackingProfile.EnableHandMeshVisualization;
HandRenderer.enabled = renderHandmesh;
if (renderHandmesh)
{
// Render the rigged hand mesh itself
// Apply updated TrackedHandJoint pose data to the assigned transforms
// This starts at 1 to skip over TrackedHandJoint.None.
for (int i = 1; i < ArticulatedHandPose.JointCount; i++)
{
TrackedHandJoint handJoint = (TrackedHandJoint)i;
MixedRealityPose handJointPose = eventData.InputData[handJoint];
if (joints.TryGetValue(handJoint, out Transform jointTransform) && jointTransform != null)
{
if (handJoint == TrackedHandJoint.Palm)
{
if (ModelPalmAtLeapWrist)
{
Palm.position = eventData.InputData[TrackedHandJoint.Wrist].Position;
}
else
{
Palm.position = handJointPose.Position;
}
Palm.rotation = handJointPose.Rotation * UserBoneRotation;
}
else if (handJoint == TrackedHandJoint.Wrist)
{
if (!ModelPalmAtLeapWrist)
{
Wrist.position = handJointPose.Position;
}
}
else
{
// Finger joints
jointTransform.rotation = handJointPose.Rotation * Reorientation();
if (DeformPosition)
{
jointTransform.position = handJointPose.Position;
}
if (ScaleLastFingerBone &&
(handJoint == TrackedHandJoint.ThumbDistalJoint ||
handJoint == TrackedHandJoint.IndexDistalJoint ||
handJoint == TrackedHandJoint.MiddleDistalJoint ||
handJoint == TrackedHandJoint.RingDistalJoint ||
handJoint == TrackedHandJoint.PinkyDistalJoint))
{
ScaleFingerTip(eventData, jointTransform, handJoint + 1, jointTransform.position);
}
}
}
}
// Update the hand material
float pinchStrength = HandPoseUtils.CalculateIndexPinch(Controller.ControllerHandedness);
// Hand Curl Properties:
float indexFingerCurl = HandPoseUtils.IndexFingerCurl(Controller.ControllerHandedness);
float middleFingerCurl = HandPoseUtils.MiddleFingerCurl(Controller.ControllerHandedness);
float ringFingerCurl = HandPoseUtils.RingFingerCurl(Controller.ControllerHandedness);
float pinkyFingerCurl = HandPoseUtils.PinkyFingerCurl(Controller.ControllerHandedness);
if (handMaterial != null && handRendererInitialized)
{
float gripStrength = indexFingerCurl + middleFingerCurl + ringFingerCurl + pinkyFingerCurl;
gripStrength /= 4.0f;
gripStrength = gripStrength > 0.8f ? 1.0f : gripStrength;
pinchStrength = Mathf.Pow(Mathf.Max(pinchStrength, gripStrength), 2.0f);
if (handRenderer.sharedMaterial.HasProperty(pinchStrengthMaterialProperty))
{
handRenderer.sharedMaterial.SetFloat(pinchStrengthMaterialProperty, pinchStrength);
}
// Only show this warning once
else if (!displayedMaterialPropertyWarning)
{
Debug.LogWarning(String.Format("The property {0} for reacting to pinch strength was not found. A material with this property is required to visualize pinch strength.", pinchStrengthMaterialProperty));
displayedMaterialPropertyWarning = true;
}
}
}
}
/// <inheritdoc/>
public override void OnHandJointsUpdated(InputEventData <IDictionary <TrackedHandJoint, MixedRealityPose> > eventData)
{
using (OnHandJointsUpdatedPerfMarker.Auto())
{
// The base class takes care of updating all of the joint data
base.OnHandJointsUpdated(eventData);
// exit early and disable the rigged hand model if we've gotten a hand mesh from the underlying platform
if (receivingPlatformHandMesh)
{
HandRenderer.enabled = false;
return;
}
// Ensures that the hand only renders when the event data matches the controller this visualizer represents
if (eventData.InputSource.SourceId != Controller.InputSource.SourceId)
{
return;
}
Debug.Assert(eventData.Handedness == Controller.ControllerHandedness);
IMixedRealityInputSystem inputSystem = CoreServices.InputSystem;
MixedRealityHandTrackingProfile handTrackingProfile = inputSystem?.InputSystemProfile.HandTrackingProfile;
// Only runs if render hand mesh is true
bool renderHandmesh = handTrackingProfile != null && handTrackingProfile.EnableHandMeshVisualization;
HandRenderer.enabled = renderHandmesh;
if (renderHandmesh)
{
// Render the rigged hand mesh itself
// Apply updated TrackedHandJoint pose data to the assigned transforms
// This starts at 1 to skip over TrackedHandJoint.None.
for (int i = 1; i < ArticulatedHandPose.JointCount; i++)
{
TrackedHandJoint handJoint = (TrackedHandJoint)i;
// Skip this hand joint if the event data doesn't have an entry for it
if (!eventData.InputData.ContainsKey(handJoint))
{
continue;
}
MixedRealityPose handJointPose = eventData.InputData[handJoint];
Transform jointTransform = riggedVisualJointsArray[i];
if (jointTransform != null)
{
if (handJoint == TrackedHandJoint.Palm)
{
if (ModelPalmAtLeapWrist)
{
Palm.position = eventData.InputData[TrackedHandJoint.Wrist].Position;
}
else
{
Palm.position = handJointPose.Position;
}
Palm.rotation = handJointPose.Rotation * UserBoneRotation;
}
else if (handJoint == TrackedHandJoint.Wrist)
{
if (!ModelPalmAtLeapWrist)
{
Wrist.position = handJointPose.Position;
}
}
else
{
// Finger riggedVisualJointsArray
jointTransform.rotation = handJointPose.Rotation * Reorientation();
if (DeformPosition)
{
jointTransform.position = handJointPose.Position;
}
if (ScaleLastFingerBone &&
(handJoint == TrackedHandJoint.ThumbDistalJoint ||
handJoint == TrackedHandJoint.IndexDistalJoint ||
handJoint == TrackedHandJoint.MiddleDistalJoint ||
handJoint == TrackedHandJoint.RingDistalJoint ||
handJoint == TrackedHandJoint.PinkyDistalJoint))
{
ScaleFingerTip(eventData, jointTransform, handJoint + 1, jointTransform.position);
}
}
}
}
// Update the hand material
float pinchStrength = HandPoseUtils.CalculateIndexPinch(Controller.ControllerHandedness);
// Hand Curl Properties:
float indexFingerCurl = HandPoseUtils.IndexFingerCurl(Controller.ControllerHandedness);
float middleFingerCurl = HandPoseUtils.MiddleFingerCurl(Controller.ControllerHandedness);
float ringFingerCurl = HandPoseUtils.RingFingerCurl(Controller.ControllerHandedness);
float pinkyFingerCurl = HandPoseUtils.PinkyFingerCurl(Controller.ControllerHandedness);
if (handMaterial != null && handRendererInitialized)
{
float gripStrength = indexFingerCurl + middleFingerCurl + ringFingerCurl + pinkyFingerCurl;
gripStrength /= 4.0f;
gripStrength = gripStrength > 0.8f ? 1.0f : gripStrength;
pinchStrength = Mathf.Pow(Mathf.Max(pinchStrength, gripStrength), 2.0f);
if (handRenderer.sharedMaterial.HasProperty(pinchStrengthMaterialProperty))
{
handRenderer.sharedMaterial.SetFloat(pinchStrengthMaterialProperty, pinchStrength);
}
// Only show this warning once
else if (!displayedMaterialPropertyWarning)
{
Debug.LogWarning(String.Format("The property {0} for reacting to pinch strength was not found. A material with this property is required to visualize pinch strength.", pinchStrengthMaterialProperty));
displayedMaterialPropertyWarning = true;
}
}
}
}
}
protected bool UpdateHandData(OVRHand ovrHand, OVRSkeleton ovrSkeleton)
{
bool isTracked = ovrHand.IsTracked;
if (ovrHand.HandConfidence == OVRHand.TrackingConfidence.High)
{
_lastHighConfidenceTime = Time.unscaledTime;
}
if (ovrHand.HandConfidence == OVRHand.TrackingConfidence.Low)
{
if (settingsProfile.MinimumHandConfidence == OVRHand.TrackingConfidence.High)
{
isTracked = false;
}
else
{
float lowConfidenceTime = Time.time - _lastHighConfidenceTime;
if (settingsProfile.LowConfidenceTimeThreshold > 0 &&
settingsProfile.LowConfidenceTimeThreshold < lowConfidenceTime)
{
isTracked = false;
}
}
}
if (ControllerHandedness == Handedness.Left)
{
settingsProfile.CurrentLeftHandTrackingConfidence = ovrHand.HandConfidence;
}
else
{
settingsProfile.CurrentRightHandTrackingConfidence = ovrHand.HandConfidence;
}
if (ovrSkeleton != null)
{
var bones = ovrSkeleton.Bones;
foreach (var bone in bones)
{
UpdateBone(bone);
}
UpdatePalm();
}
HandDefinition?.UpdateHandJoints(jointPoses);
// Note: After some testing, it seems when moving your hand fast, Oculus's pinch estimation data gets frozen, which leads to stuck pinches.
// To counter this, we perform a distance check between thumb and index to determine if we should force the pinch to a false state.
float pinchStrength = HandPoseUtils.CalculateIndexPinch(ControllerHandedness);
if (pinchStrength == 0.0f)
{
IsPinching = false;
}
else
{
if (IsPinching)
{
// If we are already pinching, we make the pinch a bit sticky
IsPinching = pinchStrength > 0.5f;
}
else
{
// If not yet pinching, only consider pinching if finger confidence is high
IsPinching = pinchStrength > 0.85f && ovrHand.GetFingerConfidence(OVRHand.HandFinger.Index) == OVRHand.TrackingConfidence.High;
}
}
isIndexGrabbing = HandPoseUtils.IsIndexGrabbing(ControllerHandedness);
isMiddleGrabbing = HandPoseUtils.IsMiddleGrabbing(ControllerHandedness);
// Pinch was also used as grab, we want to allow hand-curl grab not just pinch.
// Determine pinch and grab separately
if (isTracked)
{
IsGrabbing = isIndexGrabbing && isMiddleGrabbing;
}
return(isTracked);
}
protected bool UpdateHandData(OVRHand ovrHand, OVRSkeleton ovrSkeleton)
{
bool isTracked = ovrHand.IsTracked;
if (ovrHand.HandConfidence == OVRHand.TrackingConfidence.High)
{
_lastHighConfidenceTime = Time.unscaledTime;
}
if (ovrHand.HandConfidence == OVRHand.TrackingConfidence.Low)
{
if (MRTKOculusConfig.Instance.MinimumHandConfidence == OVRHand.TrackingConfidence.High)
{
isTracked = false;
}
else
{
float lowConfidenceTime = Time.time - _lastHighConfidenceTime;
if (MRTKOculusConfig.Instance.LowConfidenceTimeThreshold > 0 &&
MRTKOculusConfig.Instance.LowConfidenceTimeThreshold < lowConfidenceTime)
{
isTracked = false;
}
}
}
if (ControllerHandedness == Handedness.Left)
{
MRTKOculusConfig.Instance.CurrentLeftHandTrackingConfidence = ovrHand.HandConfidence;
}
else
{
MRTKOculusConfig.Instance.CurrentRightHandTrackingConfidence = ovrHand.HandConfidence;
}
// Disable hand if not tracked
if (handRenderer != null)
{
handRenderer.enabled = isTracked;
}
if (ovrSkeleton != null)
{
var bones = ovrSkeleton.Bones;
foreach (var bone in bones)
{
UpdateBone(bone);
}
UpdatePalm();
}
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, jointPoses);
// Note: After some testing, it seems when moving your hand fast, Oculus's pinch estimation data gets frozen, which leads to stuck pinches.
// To counter this, we perform a distance check between thumb and index to determine if we should force the pinch to a false state.
float pinchStrength;
if (AreIndexAndThumbFarApart())
{
pinchStrength = 0f;
IsPinching = false;
}
else
{
pinchStrength = ovrHand.GetFingerPinchStrength(OVRHand.HandFinger.Index);
if (IsPinching)
{
// If we are already pinching, we make the pinch a bit sticky
IsPinching = ovrHand.GetFingerPinchStrength(OVRHand.HandFinger.Index) > 0.85f;
}
else
{
// If not yet pinching, only consider pinching if finger confidence is high
IsPinching = ovrHand.GetFingerIsPinching(OVRHand.HandFinger.Index) &&
ovrHand.GetFingerConfidence(OVRHand.HandFinger.Index) == OVRHand.TrackingConfidence.High;
}
}
isIndexGrabbing = HandPoseUtils.IsIndexGrabbing(ControllerHandedness);
isMiddleGrabbing = HandPoseUtils.IsMiddleGrabbing(ControllerHandedness);
isThumbGrabbing = HandPoseUtils.IsThumbGrabbing(ControllerHandedness);
// Hand Curl Properties:
float indexFingerCurl = HandPoseUtils.IndexFingerCurl(ControllerHandedness);
float middleFingerCurl = HandPoseUtils.MiddleFingerCurl(ControllerHandedness);
float ringFingerCurl = HandPoseUtils.RingFingerCurl(ControllerHandedness);
float pinkyFingerCurl = HandPoseUtils.PinkyFingerCurl(ControllerHandedness);
// Pinch was also used as grab, we want to allow hand-curl grab not just pinch.
// Determine pinch and grab separately
if (isTracked)
{
IsGrabbing = isIndexGrabbing && isMiddleGrabbing;
}
if (MRTKOculusConfig.Instance.UpdateMaterialPinchStrengthValue && handMaterial != null)
{
float gripStrength = indexFingerCurl + middleFingerCurl + ringFingerCurl + pinkyFingerCurl;
gripStrength /= 4.0f;
gripStrength = gripStrength > 0.8f ? 1.0f : gripStrength;
pinchStrength = Mathf.Max(pinchStrength, gripStrength);
handMaterial.SetFloat(pinchStrengthProp, pinchStrength);
}
return(isTracked);
}
public static bool IsPinching(Handedness trackedHand)
{
return(HandPoseUtils.CalculateIndexPinch(trackedHand) > PinchThreshold);
}
