public override void Update()
        {
#if WINDOWS_UWP
            if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || !WindowsApiChecker.UniversalApiContractV8_IsAvailable)
            {
                return;
            }

            SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
            if (pointerPose != null)
            {
                var eyes = pointerPose.Eyes;
                if ((eyes != null) && (eyes.Gaze.HasValue))
                {
                    Ray newGaze = new Ray(WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Origin), WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Direction));

                    if (SmoothEyeTracking)
                    {
                        newGaze = SmoothGaze(newGaze);
                    }

                    InputSystem?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
                }
            }
#endif // WINDOWS_UWP
        }
示例#2
0
        public override void Update()
        {
#if UNITY_WSA
            if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || typeof(SpatialPointerPose).GetProperty("Eyes") == null)
            {
                return;
            }

            SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
            if (pointerPose != null)
            {
                var eyes = pointerPose.Eyes;
                if (eyes != null)
                {
                    InputSystem?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, eyes.IsCalibrationValid);

                    if (eyes.Gaze.HasValue)
                    {
                        Ray newGaze = new Ray(WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Origin), WindowsMixedRealityUtilities.SystemVector3ToUnity(eyes.Gaze.Value.Direction));

                        if (SmoothEyeTracking)
                        {
                            newGaze = SmoothGaze(newGaze);
                        }

                        InputSystem?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
                    }
                }
            }
#endif // UNITY_WSA
        }
示例#3
0
        /// <summary>
        /// Update the hand data from the device.
        /// </summary>
        /// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
        private void UpdateHandData(InteractionSourceState interactionSourceState)
        {
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
            // Articulated hand support is only present in the 18362 version and beyond Windows
            // SDK (which contains the V8 drop of the Universal API Contract). In particular,
            // the HandPose related APIs are only present on this version and above.
            if (!articulatedHandApiAvailable)
            {
                return;
            }

            PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
            IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
            foreach (SpatialInteractionSourceState sourceState in sources)
            {
                if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
                {
                    HandPose handPose = sourceState.TryGetHandPose();

#if WINDOWS_UWP
                    if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization)
                    {
                        // Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
                        if (handMeshObserver == null && !hasRequestedHandMeshObserver)
                        {
                            SetHandMeshObserver(sourceState);
                            hasRequestedHandMeshObserver = true;
                        }

                        if (handMeshObserver != null && handMeshTriangleIndices == null)
                        {
                            uint     indexCount = handMeshObserver.TriangleIndexCount;
                            ushort[] indices    = new ushort[indexCount];
                            handMeshObserver.GetTriangleIndices(indices);
                            handMeshTriangleIndices = new int[indexCount];
                            Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);

                            // Compute neutral pose
                            Vector3[]           neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
                            HandPose            neutralPose         = handMeshObserver.NeutralPose;
                            var                 vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                            HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
                            handMeshVertexState.GetVertices(vertexAndNormals);

                            for (int i = 0; i < handMeshObserver.VertexCount; i++)
                            {
                                neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
                            }

                            // Compute UV mapping
                            InitializeUVs(neutralPoseVertices);
                        }

                        if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
                        {
                            var vertexAndNormals    = new HandMeshVertex[handMeshObserver.VertexCount];
                            var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
                            handMeshVertexState.GetVertices(vertexAndNormals);

                            var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
                            if (meshTransform.HasValue)
                            {
                                System.Numerics.Vector3    scale;
                                System.Numerics.Quaternion rotation;
                                System.Numerics.Vector3    translation;
                                System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);

                                var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
                                var handMeshNormals  = new Vector3[handMeshObserver.VertexCount];

                                for (int i = 0; i < handMeshObserver.VertexCount; i++)
                                {
                                    handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
                                    handMeshNormals[i]  = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal);
                                }

                                /// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
                                Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation);
                                unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition);
                                Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation);
                                unityRotation = MixedRealityPlayspace.Rotation * unityRotation;

                                HandMeshInfo handMeshInfo = new HandMeshInfo
                                {
                                    vertices  = handMeshVertices,
                                    normals   = handMeshNormals,
                                    triangles = handMeshTriangleIndices,
                                    uvs       = handMeshUVs,
                                    position  = unityPosition,
                                    rotation  = unityRotation
                                };

                                CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
                            }
                        }
                    }
                    else
                    {
                        // if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
                        if (handMeshObserver != null)
                        {
                            // notify that hand mesh has been updated (cleared)
                            HandMeshInfo handMeshInfo = new HandMeshInfo();
                            CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
                            hasRequestedHandMeshObserver = false;
                            handMeshObserver             = null;
                        }
                    }
#endif // WINDOWS_UWP

                    if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
                    {
                        for (int i = 0; i < jointPoses.Length; i++)
                        {
                            unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation);
                            unityJointPositions[i]    = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position);

                            // We want the controller to follow the Playspace, so fold in the playspace transform here to
                            // put the controller pose into world space.
                            unityJointPositions[i]    = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]);
                            unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i];

                            if (jointIndices[i] == HandJointKind.IndexTip)
                            {
                                lastIndexTipRadius = jointPoses[i].Radius;
                            }

                            TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);

                            if (!unityJointPoses.ContainsKey(handJoint))
                            {
                                unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]));
                            }
                            else
                            {
                                unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]);
                            }
                        }
                        CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses);
                    }
                }
            }
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
        }