public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
        {
            using (UpdateHandMeshPerfMarker.Auto())
            {
                MixedRealityHandTrackingProfile handTrackingProfile = null;
                MixedRealityInputSystemProfile  inputSystemProfile  = CoreServices.InputSystem?.InputSystemProfile;
                if (inputSystemProfile != null)
                {
                    handTrackingProfile = inputSystemProfile.HandTrackingProfile;
                }

                if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
                {
                    // If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
                    if (handMeshObserver != null)
                    {
                        // Notify that hand mesh has been updated (cleared)
                        HandMeshInfo handMeshInfo = new HandMeshInfo();
                        CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo);
                        hasRequestedHandMeshObserver = false;
                        handMeshObserver             = null;
                    }
                    return;
                }

                HandPose handPose = sourceState.TryGetHandPose();

                // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
                if (handMeshObserver == null && !hasRequestedHandMeshObserver)
                {
                    SetHandMeshObserver(sourceState);
                    hasRequestedHandMeshObserver = true;
                }

                if (handMeshObserver != null && handPose != null)
                {
                    if (handMeshTriangleIndices == null)
                    {
                        handMeshTriangleIndices      = new ushort[handMeshObserver.TriangleIndexCount];
                        handMeshTriangleIndicesUnity = new int[handMeshObserver.TriangleIndexCount];
                        handMeshObserver.GetTriangleIndices(handMeshTriangleIndices);

                        Array.Copy(handMeshTriangleIndices, handMeshTriangleIndicesUnity, (int)handMeshObserver.TriangleIndexCount);

                        // Compute neutral pose
                        Vector3[]           neutralPoseVertices     = new Vector3[handMeshObserver.VertexCount];
                        HandPose            neutralPose             = handMeshObserver.NeutralPose;
                        var                 neutralVertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
                        HandMeshVertexState handMeshVertexState     = handMeshObserver.GetVertexStateForPose(neutralPose);
                        handMeshVertexState.GetVertices(neutralVertexAndNormals);

                        Parallel.For(0, handMeshObserver.VertexCount, i =>
                        {
                            neutralVertexAndNormals[i].Position.ConvertToUnityVector3(ref neutralPoseVertices[i]);
                        });

                        // Compute UV mapping
                        InitializeUVs(neutralPoseVertices);
                    }

                    if (vertexAndNormals == null)
                    {
                        vertexAndNormals      = new HandMeshVertex[handMeshObserver.VertexCount];
                        handMeshVerticesUnity = new Vector3[handMeshObserver.VertexCount];
                        handMeshNormalsUnity  = new Vector3[handMeshObserver.VertexCount];
                    }

                    if (vertexAndNormals != null && handMeshTriangleIndices != null)
                    {
                        var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
                        handMeshVertexState.GetVertices(vertexAndNormals);

                        var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
                        if (meshTransform.HasValue)
                        {
                            System.Numerics.Matrix4x4.Decompose(meshTransform.Value,
                                                                out System.Numerics.Vector3 scale,
                                                                out System.Numerics.Quaternion rotation,
                                                                out System.Numerics.Vector3 translation);

                            Parallel.For(0, handMeshObserver.VertexCount, i =>
                            {
                                vertexAndNormals[i].Position.ConvertToUnityVector3(ref handMeshVerticesUnity[i]);
                                vertexAndNormals[i].Normal.ConvertToUnityVector3(ref handMeshNormalsUnity[i]);
                            });

                            /// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
                            Vector3    positionUnity = MixedRealityPlayspace.TransformPoint(translation.ToUnityVector3());
                            Quaternion rotationUnity = MixedRealityPlayspace.Rotation * rotation.ToUnityQuaternion();

                            HandMeshInfo handMeshInfo = new HandMeshInfo
                            {
                                vertices  = handMeshVerticesUnity,
                                normals   = handMeshNormalsUnity,
                                triangles = handMeshTriangleIndicesUnity,
                                uvs       = handMeshUVsUnity,
                                position  = positionUnity,
                                rotation  = rotationUnity
                            };

                            CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo);
                        }
                    }
                }
            }
        }
        /// <summary>
        /// Updates the current hand mesh based on the passed in state of the hand.
        /// </summary>
        /// <param name="sourceState">The current hand state.</param>
        public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
        {
            MixedRealityHandTrackingProfile handTrackingProfile = null;
            MixedRealityInputSystemProfile  inputSystemProfile  = CoreServices.InputSystem?.InputSystemProfile;

            if (inputSystemProfile != null)
            {
                handTrackingProfile = inputSystemProfile.HandTrackingProfile;
            }

            if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
            {
                // If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
                if (handMeshObserver != null)
                {
                    // Notify that hand mesh has been updated (cleared)
                    HandMeshInfo handMeshInfo = new HandMeshInfo();
                    CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
                    hasRequestedHandMeshObserver = false;
                    handMeshObserver             = null;
                }
                return;
            }

            HandPose handPose = sourceState.TryGetHandPose();

            // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
            if (handMeshObserver == null && !hasRequestedHandMeshObserver)
            {
                SetHandMeshObserver(sourceState);
                hasRequestedHandMeshObserver = true;
            }

            if (handMeshObserver != null && handMeshTriangleIndices == null)
            {
                uint     indexCount = handMeshObserver.TriangleIndexCount;
                ushort[] indices    = new ushort[indexCount];
                handMeshObserver.GetTriangleIndices(indices);
                handMeshTriangleIndices = new int[indexCount];
                Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);

                // Compute neutral pose
                Vector3[]           neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
                HandPose            neutralPose         = handMeshObserver.NeutralPose;
                var                 vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
                handMeshVertexState.GetVertices(vertexAndNormals);

                for (int i = 0; i < handMeshObserver.VertexCount; i++)
                {
                    neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnityVector3();
                }

                // Compute UV mapping
                InitializeUVs(neutralPoseVertices);
            }

            if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
            {
                var vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
                handMeshVertexState.GetVertices(vertexAndNormals);

                var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
                if (meshTransform.HasValue)
                {
                    System.Numerics.Vector3    scale;
                    System.Numerics.Quaternion rotation;
                    System.Numerics.Vector3    translation;
                    System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);

                    var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
                    var handMeshNormals  = new Vector3[handMeshObserver.VertexCount];

                    for (int i = 0; i < handMeshObserver.VertexCount; i++)
                    {
                        handMeshVertices[i] = vertexAndNormals[i].Position.ToUnityVector3();
                        handMeshNormals[i]  = vertexAndNormals[i].Normal.ToUnityVector3();
                    }

                    HandMeshInfo handMeshInfo = new HandMeshInfo
                    {
                        vertices  = handMeshVertices,
                        normals   = handMeshNormals,
                        triangles = handMeshTriangleIndices,
                        uvs       = handMeshUVs,
                        position  = translation.ToUnityVector3(),
                        rotation  = rotation.ToUnityQuaternion()
                    };

                    CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
                }
            }
        }
Пример #3
0
        /// <summary>
        /// Update the hand data from the device.
        /// </summary>
        /// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
        private void UpdateHandData(InteractionSourceState interactionSourceState)
        {
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
            // Articulated hand support is only present in the 18362 version and beyond Windows
            // SDK (which contains the V8 drop of the Universal API Contract). In particular,
            // the HandPose related APIs are only present on this version and above.
            if (!articulatedHandApiAvailable)
            {
                return;
            }

            PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
            IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
            foreach (SpatialInteractionSourceState sourceState in sources)
            {
                if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
                {
                    HandPose handPose = sourceState.TryGetHandPose();

#if WINDOWS_UWP
                    if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization)
                    {
                        // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
                        if (handMeshObserver == null && !hasRequestedHandMeshObserver)
                        {
                            SetHandMeshObserver(sourceState);
                            hasRequestedHandMeshObserver = true;
                        }

                        if (handMeshObserver != null && handMeshTriangleIndices == null)
                        {
                            uint     indexCount = handMeshObserver.TriangleIndexCount;
                            ushort[] indices    = new ushort[indexCount];
                            handMeshObserver.GetTriangleIndices(indices);
                            handMeshTriangleIndices = new int[indexCount];
                            Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);

                            // Compute neutral pose
                            Vector3[]           neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
                            HandPose            neutralPose         = handMeshObserver.NeutralPose;
                            var                 vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                            HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
                            handMeshVertexState.GetVertices(vertexAndNormals);

                            for (int i = 0; i < handMeshObserver.VertexCount; i++)
                            {
                                neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
                            }

                            // Compute UV mapping
                            InitializeUVs(neutralPoseVertices);
                        }

                        if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
                        {
                            var vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                            var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
                            handMeshVertexState.GetVertices(vertexAndNormals);

                            var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
                            if (meshTransform.HasValue)
                            {
                                System.Numerics.Vector3    scale;
                                System.Numerics.Quaternion rotation;
                                System.Numerics.Vector3    translation;
                                System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);

                                var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
                                var handMeshNormals  = new Vector3[handMeshObserver.VertexCount];

                                for (int i = 0; i < handMeshObserver.VertexCount; i++)
                                {
                                    handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
                                    handMeshNormals[i]  = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal);
                                }

                                /// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
                                Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation);
                                unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition);
                                Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation);
                                unityRotation = MixedRealityPlayspace.Rotation * unityRotation;

                                HandMeshInfo handMeshInfo = new HandMeshInfo
                                {
                                    vertices  = handMeshVertices,
                                    normals   = handMeshNormals,
                                    triangles = handMeshTriangleIndices,
                                    uvs       = handMeshUVs,
                                    position  = unityPosition,
                                    rotation  = unityRotation
                                };

                                CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
                            }
                        }
                    }
                    else
                    {
                        // if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
                        if (handMeshObserver != null)
                        {
                            // notify that hand mesh has been updated (cleared)
                            HandMeshInfo handMeshInfo = new HandMeshInfo();
                            CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
                            hasRequestedHandMeshObserver = false;
                            handMeshObserver             = null;
                        }
                    }
#endif // WINDOWS_UWP

                    if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
                    {
                        for (int i = 0; i < jointPoses.Length; i++)
                        {
                            unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation);
                            unityJointPositions[i]    = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position);

                            // We want the controller to follow the Playspace, so fold in the playspace transform here to
                            // put the controller pose into world space.
                            unityJointPositions[i]    = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]);
                            unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i];

                            if (jointIndices[i] == HandJointKind.IndexTip)
                            {
                                lastIndexTipRadius = jointPoses[i].Radius;
                            }

                            TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);

                            if (!unityJointPoses.ContainsKey(handJoint))
                            {
                                unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]));
                            }
                            else
                            {
                                unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]);
                            }
                        }
                        CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses);
                    }
                }
            }
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
        }
Пример #4
0
        /// <summary>
        /// Attempts to get updated hand mesh data.
        /// </summary>
        /// <param name="spatialInteractionSourceState">Platform provided current input source state for the hand.</param>
        /// <param name="handPose">Hand pose information retrieved for joint conversion.</param>
        /// <param name="data">Mesh information retrieved in case of success.</param>
        /// <returns>True, if mesh data could be loaded.</returns>
        private bool TryGetUpdatedHandMeshData(SpatialInteractionSourceState spatialInteractionSourceState, HandPose handPose, out HandMeshData data)
        {
            if (!handMeshObservers.ContainsKey(spatialInteractionSourceState.Source.Handedness) &&
                !HasRequestedHandMeshObserver(spatialInteractionSourceState.Source.Handedness))
            {
                SetHandMeshObserver(spatialInteractionSourceState);
            }

            if (handMeshObservers.TryGetValue(spatialInteractionSourceState.Source.Handedness, out var handMeshObserver) && handMeshTriangleIndices == null)
            {
                var indexCount = handMeshObserver.TriangleIndexCount;
                var indices    = new ushort[indexCount];
                handMeshObserver.GetTriangleIndices(indices);
                handMeshTriangleIndices = new int[indexCount];
                Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);

                // Compute neutral pose
                var neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
                var neutralPose         = handMeshObserver.NeutralPose;
                var vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                var handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
                handMeshVertexState.GetVertices(vertexAndNormals);

                for (int i = 0; i < handMeshObserver.VertexCount; i++)
                {
                    neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnity();
                }

                // Compute UV mapping
                InitializeHandMeshUVs(neutralPoseVertices);
            }

            if (handMeshObserver != null && handMeshTriangleIndices != null)
            {
                var vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
                handMeshVertexState.GetVertices(vertexAndNormals);

                var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(spatialCoordinateSystem);
                if (meshTransform.HasValue)
                {
                    System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out var scale, out var rotation, out var translation);

                    var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
                    var handMeshNormals  = new Vector3[handMeshObserver.VertexCount];

                    for (int i = 0; i < handMeshObserver.VertexCount; i++)
                    {
                        handMeshVertices[i] = vertexAndNormals[i].Position.ToUnity();
                        handMeshNormals[i]  = vertexAndNormals[i].Normal.ToUnity();
                    }

                    data = new HandMeshData(
                        handMeshVertices,
                        handMeshTriangleIndices,
                        handMeshNormals,
                        handMeshUVs);

                    return(true);
                }
            }

            return(false);
        }