public void UpdateHandMesh(SpatialInteractionSourceState sourceState) { using (UpdateHandMeshPerfMarker.Auto()) { MixedRealityHandTrackingProfile handTrackingProfile = null; MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile; if (inputSystemProfile != null) { handTrackingProfile = inputSystemProfile.HandTrackingProfile; } if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization) { // If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created if (handMeshObserver != null) { // Notify that hand mesh has been updated (cleared) HandMeshInfo handMeshInfo = new HandMeshInfo(); CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo); hasRequestedHandMeshObserver = false; handMeshObserver = null; } return; } HandPose handPose = sourceState.TryGetHandPose(); // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it. if (handMeshObserver == null && !hasRequestedHandMeshObserver) { SetHandMeshObserver(sourceState); hasRequestedHandMeshObserver = true; } if (handMeshObserver != null && handPose != null) { if (handMeshTriangleIndices == null) { handMeshTriangleIndices = new ushort[handMeshObserver.TriangleIndexCount]; handMeshTriangleIndicesUnity = new int[handMeshObserver.TriangleIndexCount]; handMeshObserver.GetTriangleIndices(handMeshTriangleIndices); Array.Copy(handMeshTriangleIndices, handMeshTriangleIndicesUnity, (int)handMeshObserver.TriangleIndexCount); // Compute neutral pose Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount]; HandPose neutralPose = handMeshObserver.NeutralPose; var neutralVertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose); handMeshVertexState.GetVertices(neutralVertexAndNormals); Parallel.For(0, handMeshObserver.VertexCount, i => { neutralVertexAndNormals[i].Position.ConvertToUnityVector3(ref neutralPoseVertices[i]); }); // Compute UV mapping InitializeUVs(neutralPoseVertices); } if (vertexAndNormals == null) { vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; handMeshVerticesUnity = new Vector3[handMeshObserver.VertexCount]; handMeshNormalsUnity = new Vector3[handMeshObserver.VertexCount]; } if (vertexAndNormals != null && handMeshTriangleIndices != null) { var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose); handMeshVertexState.GetVertices(vertexAndNormals); var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem); if (meshTransform.HasValue) { System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out System.Numerics.Vector3 scale, out System.Numerics.Quaternion rotation, out System.Numerics.Vector3 translation); Parallel.For(0, handMeshObserver.VertexCount, i => { vertexAndNormals[i].Position.ConvertToUnityVector3(ref handMeshVerticesUnity[i]); vertexAndNormals[i].Normal.ConvertToUnityVector3(ref handMeshNormalsUnity[i]); }); /// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform. Vector3 positionUnity = MixedRealityPlayspace.TransformPoint(translation.ToUnityVector3()); Quaternion rotationUnity = MixedRealityPlayspace.Rotation * rotation.ToUnityQuaternion(); HandMeshInfo handMeshInfo = new HandMeshInfo { vertices = handMeshVerticesUnity, normals = handMeshNormalsUnity, triangles = handMeshTriangleIndicesUnity, uvs = handMeshUVsUnity, position = positionUnity, rotation = rotationUnity }; CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo); } } } } }
/// <summary> /// Updates the current hand mesh based on the passed in state of the hand. /// </summary> /// <param name="sourceState">The current hand state.</param> public void UpdateHandMesh(SpatialInteractionSourceState sourceState) { MixedRealityHandTrackingProfile handTrackingProfile = null; MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile; if (inputSystemProfile != null) { handTrackingProfile = inputSystemProfile.HandTrackingProfile; } if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization) { // If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created if (handMeshObserver != null) { // Notify that hand mesh has been updated (cleared) HandMeshInfo handMeshInfo = new HandMeshInfo(); CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo); hasRequestedHandMeshObserver = false; handMeshObserver = null; } return; } HandPose handPose = sourceState.TryGetHandPose(); // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it. if (handMeshObserver == null && !hasRequestedHandMeshObserver) { SetHandMeshObserver(sourceState); hasRequestedHandMeshObserver = true; } if (handMeshObserver != null && handMeshTriangleIndices == null) { uint indexCount = handMeshObserver.TriangleIndexCount; ushort[] indices = new ushort[indexCount]; handMeshObserver.GetTriangleIndices(indices); handMeshTriangleIndices = new int[indexCount]; Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount); // Compute neutral pose Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount]; HandPose neutralPose = handMeshObserver.NeutralPose; var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose); handMeshVertexState.GetVertices(vertexAndNormals); for (int i = 0; i < handMeshObserver.VertexCount; i++) { neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnityVector3(); } // Compute UV mapping InitializeUVs(neutralPoseVertices); } if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null) { var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose); handMeshVertexState.GetVertices(vertexAndNormals); var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem); if (meshTransform.HasValue) { System.Numerics.Vector3 scale; System.Numerics.Quaternion rotation; System.Numerics.Vector3 translation; System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation); var handMeshVertices = new Vector3[handMeshObserver.VertexCount]; var handMeshNormals = new Vector3[handMeshObserver.VertexCount]; for (int i = 0; i < handMeshObserver.VertexCount; i++) { handMeshVertices[i] = vertexAndNormals[i].Position.ToUnityVector3(); handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnityVector3(); } HandMeshInfo handMeshInfo = new HandMeshInfo { vertices = handMeshVertices, normals = handMeshNormals, triangles = handMeshTriangleIndices, uvs = handMeshUVs, position = translation.ToUnityVector3(), rotation = rotation.ToUnityQuaternion() }; CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo); } } }
/// <summary> /// Update the hand data from the device. /// </summary> /// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param> private void UpdateHandData(InteractionSourceState interactionSourceState) { #if WINDOWS_UWP || DOTNETWINRT_PRESENT // Articulated hand support is only present in the 18362 version and beyond Windows // SDK (which contains the V8 drop of the Universal API Contract). In particular, // the HandPose related APIs are only present on this version and above. if (!articulatedHandApiAvailable) { return; } PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now); IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp); foreach (SpatialInteractionSourceState sourceState in sources) { if (sourceState.Source.Id.Equals(interactionSourceState.source.id)) { HandPose handPose = sourceState.TryGetHandPose(); #if WINDOWS_UWP if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization) { // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it. if (handMeshObserver == null && !hasRequestedHandMeshObserver) { SetHandMeshObserver(sourceState); hasRequestedHandMeshObserver = true; } if (handMeshObserver != null && handMeshTriangleIndices == null) { uint indexCount = handMeshObserver.TriangleIndexCount; ushort[] indices = new ushort[indexCount]; handMeshObserver.GetTriangleIndices(indices); handMeshTriangleIndices = new int[indexCount]; Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount); // Compute neutral pose Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount]; HandPose neutralPose = handMeshObserver.NeutralPose; var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose); handMeshVertexState.GetVertices(vertexAndNormals); for (int i = 0; i < handMeshObserver.VertexCount; i++) { neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position); } // Compute UV mapping InitializeUVs(neutralPoseVertices); } if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null) { var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose); handMeshVertexState.GetVertices(vertexAndNormals); var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem); if (meshTransform.HasValue) { System.Numerics.Vector3 scale; System.Numerics.Quaternion rotation; System.Numerics.Vector3 translation; System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation); var handMeshVertices = new Vector3[handMeshObserver.VertexCount]; var handMeshNormals = new Vector3[handMeshObserver.VertexCount]; for (int i = 0; i < handMeshObserver.VertexCount; i++) { handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position); handMeshNormals[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal); } /// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform. Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation); unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition); Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation); unityRotation = MixedRealityPlayspace.Rotation * unityRotation; HandMeshInfo handMeshInfo = new HandMeshInfo { vertices = handMeshVertices, normals = handMeshNormals, triangles = handMeshTriangleIndices, uvs = handMeshUVs, position = unityPosition, rotation = unityRotation }; CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo); } } } else { // if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created if (handMeshObserver != null) { // notify that hand mesh has been updated (cleared) HandMeshInfo handMeshInfo = new HandMeshInfo(); CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo); hasRequestedHandMeshObserver = false; handMeshObserver = null; } } #endif // WINDOWS_UWP if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses)) { for (int i = 0; i < jointPoses.Length; i++) { unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation); unityJointPositions[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position); // We want the controller to follow the Playspace, so fold in the playspace transform here to // put the controller pose into world space. unityJointPositions[i] = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]); unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i]; if (jointIndices[i] == HandJointKind.IndexTip) { lastIndexTipRadius = jointPoses[i].Radius; } TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]); if (!unityJointPoses.ContainsKey(handJoint)) { unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i])); } else { unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]); } } CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses); } } } #endif // WINDOWS_UWP || DOTNETWINRT_PRESENT }
/// <summary> /// Attempts to get updated hand mesh data. /// </summary> /// <param name="spatialInteractionSourceState">Platform provided current input source state for the hand.</param> /// <param name="handPose">Hand pose information retrieved for joint conversion.</param> /// <param name="data">Mesh information retrieved in case of success.</param> /// <returns>True, if mesh data could be loaded.</returns> private bool TryGetUpdatedHandMeshData(SpatialInteractionSourceState spatialInteractionSourceState, HandPose handPose, out HandMeshData data) { if (!handMeshObservers.ContainsKey(spatialInteractionSourceState.Source.Handedness) && !HasRequestedHandMeshObserver(spatialInteractionSourceState.Source.Handedness)) { SetHandMeshObserver(spatialInteractionSourceState); } if (handMeshObservers.TryGetValue(spatialInteractionSourceState.Source.Handedness, out var handMeshObserver) && handMeshTriangleIndices == null) { var indexCount = handMeshObserver.TriangleIndexCount; var indices = new ushort[indexCount]; handMeshObserver.GetTriangleIndices(indices); handMeshTriangleIndices = new int[indexCount]; Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount); // Compute neutral pose var neutralPoseVertices = new Vector3[handMeshObserver.VertexCount]; var neutralPose = handMeshObserver.NeutralPose; var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; var handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose); handMeshVertexState.GetVertices(vertexAndNormals); for (int i = 0; i < handMeshObserver.VertexCount; i++) { neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnity(); } // Compute UV mapping InitializeHandMeshUVs(neutralPoseVertices); } if (handMeshObserver != null && handMeshTriangleIndices != null) { var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount]; var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose); handMeshVertexState.GetVertices(vertexAndNormals); var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(spatialCoordinateSystem); if (meshTransform.HasValue) { System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out var scale, out var rotation, out var translation); var handMeshVertices = new Vector3[handMeshObserver.VertexCount]; var handMeshNormals = new Vector3[handMeshObserver.VertexCount]; for (int i = 0; i < handMeshObserver.VertexCount; i++) { handMeshVertices[i] = vertexAndNormals[i].Position.ToUnity(); handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnity(); } data = new HandMeshData( handMeshVertices, handMeshTriangleIndices, handMeshNormals, handMeshUVs); return(true); } } return(false); }