/// <inheritdoc />
public override void Update()
{
using (UpdatePerfMarker.Auto())
{
if (WindowsMixedRealityUtilities.SpatialCoordinateSystem == null || !eyesApiAvailable)
{
return;
}
SpatialPointerPose pointerPose = SpatialPointerPose.TryGetAtTimestamp(WindowsMixedRealityUtilities.SpatialCoordinateSystem, PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
if (pointerPose != null)
{
var eyes = pointerPose.Eyes;
if (eyes != null)
{
Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, eyes.IsCalibrationValid);
if (eyes.Gaze.HasValue)
{
Vector3 origin = MixedRealityPlayspace.TransformPoint(eyes.Gaze.Value.Origin.ToUnityVector3());
Vector3 direction = MixedRealityPlayspace.TransformDirection(eyes.Gaze.Value.Direction.ToUnityVector3());
Ray newGaze = new Ray(origin, direction);
if (SmoothEyeTracking)
{
newGaze = SmoothGaze(newGaze);
}
Service?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, eyes.UpdateTimestamp.TargetTime.UtcDateTime);
}
}
}
}
}
/// <summary>
/// Update the source input from the device.
/// </summary>
/// <param name="inputDevice">The InputDevice retrieved from the platform.</param>
public void UpdateSourceData(InputDevice inputDevice)
{
using (UpdateSourceDataPerfMarker.Auto())
{
var lastState = TrackingState;
LastControllerPose = CurrentControllerPose;
// Check for position and rotation.
IsPositionAvailable = inputDevice.TryGetFeatureValue(CommonUsages.devicePosition, out CurrentControllerPosition);
IsPositionApproximate = false;
IsRotationAvailable = inputDevice.TryGetFeatureValue(CommonUsages.deviceRotation, out CurrentControllerRotation);
// Devices are considered tracked if we receive position OR rotation data from the sensors.
TrackingState = (IsPositionAvailable || IsRotationAvailable) ? TrackingState.Tracked : TrackingState.NotTracked;
CurrentControllerPosition = MixedRealityPlayspace.TransformPoint(CurrentControllerPosition);
CurrentControllerRotation = MixedRealityPlayspace.Rotation * CurrentControllerRotation;
CurrentControllerPose.Position = CurrentControllerPosition;
CurrentControllerPose.Rotation = CurrentControllerRotation;
// Raise input system events if it is enabled.
if (lastState != TrackingState)
{
CoreServices.InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
}
}
/// <summary>
/// Update spatial pointer and spatial grip data.
/// </summary>
protected override void UpdatePoseData(MixedRealityInteractionMapping interactionMapping, InputDevice inputDevice)
{
Debug.Assert(interactionMapping.AxisType == AxisType.SixDof);
base.UpdatePoseData(interactionMapping, inputDevice);
// Update the interaction data source
switch (interactionMapping.InputType)
{
case DeviceInputType.SpatialPointer:
if (inputDevice.TryGetFeatureValue(WindowsMRUsages.PointerPosition, out currentPointerPosition))
{
currentPointerPose.Position = MixedRealityPlayspace.TransformPoint(currentPointerPosition);
}
if (inputDevice.TryGetFeatureValue(WindowsMRUsages.PointerRotation, out currentPointerRotation))
{
currentPointerPose.Rotation = MixedRealityPlayspace.Rotation * currentPointerRotation;
}
interactionMapping.PoseData = currentPointerPose;
// If our value changed raise it.
if (interactionMapping.Changed)
{
// Raise input system event if it's enabled
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction, interactionMapping.PoseData);
}
break;
default:
return;
}
}
private MixedRealityPose GetJointMixedRealityPose(WebXRJoint joint)
{
var position = MixedRealityPlayspace.TransformPoint(joint.Position);
var rotation = MixedRealityPlayspace.Rotation * joint.Rotation;
return(new MixedRealityPose(position, rotation));
}
/// <summary>
/// Update the spatial pointer input from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdatePointerData(InteractionSourceState interactionSourceState, MixedRealityInteractionMapping interactionMapping)
{
using (UpdatePointerDataPerfMarker.Auto())
{
if (interactionSourceState.source.supportsPointing)
{
interactionSourceState.sourcePose.TryGetPosition(out currentPointerPosition, InteractionSourceNode.Pointer);
interactionSourceState.sourcePose.TryGetRotation(out currentPointerRotation, InteractionSourceNode.Pointer);
// We want the source to follow the Playspace, so fold in the playspace transform here to
// put the source pose into world space.
currentPointerPose.Position = MixedRealityPlayspace.TransformPoint(currentPointerPosition);
currentPointerPose.Rotation = MixedRealityPlayspace.Rotation * currentPointerRotation;
}
// Update the interaction data source
interactionMapping.PoseData = currentPointerPose;
// If our value changed raise it.
if (interactionMapping.Changed)
{
// Raise input system event if it's enabled
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction, currentPointerPose);
}
}
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InputDevice inputDevice)
{
using (UpdateHandDataPerfMarker.Auto())
{
#if WINDOWS_UWP && WMR_ENABLED
XRSubsystemHelpers.InputSubsystem?.GetCurrentSourceStates(states);
foreach (SpatialInteractionSourceState sourceState in states)
{
if (sourceState.Source.Handedness.ToMRTKHandedness() == ControllerHandedness)
{
handMeshProvider?.UpdateHandMesh(sourceState);
break;
}
}
#endif // WINDOWS_UWP && WMR_ENABLED
Hand hand;
if (inputDevice.TryGetFeatureValue(CommonUsages.handData, out hand))
{
foreach (HandFinger finger in handFingers)
{
if (hand.TryGetFingerBones(finger, fingerBones))
{
for (int i = 0; i < fingerBones.Count; i++)
{
TrackedHandJoint trackedHandJoint = ConvertToTrackedHandJoint(finger, i);
Bone bone = fingerBones[i];
Vector3 position = Vector3.zero;
Quaternion rotation = Quaternion.identity;
bool positionAvailable = bone.TryGetPosition(out position);
bool rotationAvailable = bone.TryGetRotation(out rotation);
// If either position or rotation is available, use both pieces of data given.
// This might result in using a zeroed out position or rotation. Most likely,
// either both are available or both are unavailable.
if (positionAvailable || rotationAvailable)
{
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the controller pose into world space.
position = MixedRealityPlayspace.TransformPoint(position);
rotation = MixedRealityPlayspace.Rotation * rotation;
unityJointPoses[trackedHandJoint] = new MixedRealityPose(position, rotation);
}
}
// Unity doesn't provide a palm joint, so we synthesize one here
MixedRealityPose palmPose = CurrentControllerPose;
palmPose.Rotation *= (ControllerHandedness == Handedness.Left ? leftPalmOffset : rightPalmOffset);
unityJointPoses[TrackedHandJoint.Palm] = palmPose;
}
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
}
}
/// <summary>
/// Update the spatial grip input from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
/// <param name="interactionMapping"></param>
private void UpdateGripData(InteractionSourceState interactionSourceState, MixedRealityInteractionMapping interactionMapping)
{
switch (interactionMapping.AxisType)
{
case AxisType.SixDof:
{
interactionSourceState.sourcePose.TryGetPosition(out currentGripPosition, InteractionSourceNode.Grip);
interactionSourceState.sourcePose.TryGetRotation(out currentGripRotation, InteractionSourceNode.Grip);
currentGripPose.Position = MixedRealityPlayspace.TransformPoint(currentGripPosition);
currentGripPose.Rotation = Quaternion.Euler(MixedRealityPlayspace.TransformDirection(currentGripRotation.eulerAngles));
// Update the interaction data source
interactionMapping.PoseData = currentGripPose;
// If our value changed raise it.
if (interactionMapping.Changed)
{
// Raise input system Event if it enabled
InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction, currentGripPose);
}
}
break;
}
}
/// <inheritdoc />
protected override void UpdatePoseData(MixedRealityInteractionMapping interactionMapping, InputDevice inputDevice)
{
using (UpdatePoseDataPerfMarker.Auto())
{
switch (interactionMapping.InputType)
{
case DeviceInputType.SpatialPointer:
if (inputDevice.TryGetFeatureValue(CustomUsages.PointerPosition, out currentPointerPosition))
{
currentPointerPose.Position = MixedRealityPlayspace.TransformPoint(currentPointerPosition);
}
if (inputDevice.TryGetFeatureValue(CustomUsages.PointerRotation, out currentPointerRotation))
{
currentPointerPose.Rotation = MixedRealityPlayspace.Rotation * currentPointerRotation;
}
interactionMapping.PoseData = currentPointerPose;
// If our value changed raise it.
if (interactionMapping.Changed)
{
// Raise input system event if it's enabled
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction, interactionMapping.PoseData);
}
break;
default:
base.UpdatePoseData(interactionMapping, inputDevice);
break;
}
}
}
/// <summary>
/// Update the "Controller" input from the device
/// </summary>
protected void UpdateControllerData(XRNodeState state)
{
Profiler.BeginSample("[MRTK] GenericOpenVRController.UpdateControllerData");
var lastState = TrackingState;
LastControllerPose = CurrentControllerPose;
if (nodeType == XRNode.LeftHand || nodeType == XRNode.RightHand)
{
// The source is either a hand or a controller that supports pointing.
// We can now check for position and rotation.
IsPositionAvailable = state.TryGetPosition(out CurrentControllerPosition);
IsPositionApproximate = false;
IsRotationAvailable = state.TryGetRotation(out CurrentControllerRotation);
// Devices are considered tracked if we receive position OR rotation data from the sensors.
TrackingState = (IsPositionAvailable || IsRotationAvailable) ? TrackingState.Tracked : TrackingState.NotTracked;
CurrentControllerPosition = MixedRealityPlayspace.TransformPoint(CurrentControllerPosition);
CurrentControllerRotation = MixedRealityPlayspace.Rotation * CurrentControllerRotation;
}
else
{
// The input source does not support tracking.
TrackingState = TrackingState.NotApplicable;
}
CurrentControllerPose.Position = CurrentControllerPosition;
CurrentControllerPose.Rotation = CurrentControllerRotation;
// Raise input system events if it is enabled.
if (lastState != TrackingState)
{
CoreServices.InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked && LastControllerPose != CurrentControllerPose)
{
if (IsPositionAvailable && IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, CurrentControllerPose);
}
else if (IsPositionAvailable && !IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourcePositionChanged(InputSource, this, CurrentControllerPosition);
}
else if (!IsPositionAvailable && IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourceRotationChanged(InputSource, this, CurrentControllerRotation);
}
}
Profiler.EndSample(); // UpdateControllerData
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
Profiler.BeginSample("[MRTK] WindowsMixedRealityArticulatedHand.UpdateHandData");
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
#if WINDOWS_UWP
handDefinition?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 jointPosition = jointPoses[i].Position.ToUnityVector3();
Quaternion jointOrientation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
jointPosition = MixedRealityPlayspace.TransformPoint(jointPosition);
jointOrientation = MixedRealityPlayspace.Rotation * jointOrientation;
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (handJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[handJoint] = new MixedRealityPose(jointPosition, jointOrientation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
break;
}
}
Profiler.EndSample(); // UpdateHandData
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InputDevice inputDevice)
{
Profiler.BeginSample("[MRTK] WindowsMixedRealityXRSDKArticulatdHand.UpdateHandData");
#if WINDOWS_UWP && WMR_ENABLED
XRSDKSubsystemHelpers.InputSubsystem?.GetCurrentSourceStates(states);
foreach (SpatialInteractionSourceState sourceState in states)
{
if (sourceState.Source.Handedness.ToMRTKHandedness() == ControllerHandedness)
{
handDefinition?.UpdateHandMesh(sourceState);
break;
}
}
#endif // WINDOWS_UWP && WMR_ENABLED
Hand hand;
if (inputDevice.TryGetFeatureValue(CommonUsages.handData, out hand))
{
foreach (HandFinger finger in handFingers)
{
if (hand.TryGetFingerBones(finger, fingerBones))
{
for (int i = 0; i < fingerBones.Count; i++)
{
TrackedHandJoint trackedHandJoint = ConvertToTrackedHandJoint(finger, i);
Bone bone = fingerBones[i];
Vector3 position = Vector3.zero;
Quaternion rotation = Quaternion.identity;
if (bone.TryGetPosition(out position) || bone.TryGetRotation(out rotation))
{
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the controller pose into world space.
position = MixedRealityPlayspace.TransformPoint(position);
rotation = MixedRealityPlayspace.Rotation * rotation;
unityJointPoses[trackedHandJoint] = new MixedRealityPose(position, rotation);
}
}
// Unity doesn't provide a palm joint, so we synthesize one here
MixedRealityPose palmPose = CurrentControllerPose;
palmPose.Rotation *= (ControllerHandedness == Handedness.Left ? leftPalmOffset : rightPalmOffset);
unityJointPoses[TrackedHandJoint.Palm] = palmPose;
}
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
Profiler.EndSample(); // UpdateHandData
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
using (UpdateHandDataPerfMarker.Auto())
{
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
SpatialInteractionSourceState sourceState = interactionSourceState.source.GetSpatialInteractionSourceState();
if (sourceState == null)
{
return;
}
#if WINDOWS_UWP
handMeshProvider?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 position = jointPoses[i].Position.ToUnityVector3();
Quaternion rotation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
position = MixedRealityPlayspace.TransformPoint(position);
rotation = MixedRealityPlayspace.Rotation * rotation;
TrackedHandJoint trackedHandJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (trackedHandJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[trackedHandJoint] = new MixedRealityPose(position, rotation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
protected virtual void UpdateSixDofData(InputDevice inputDevice)
{
var lastState = TrackingState;
LastControllerPose = CurrentControllerPose;
// Check for position and rotation.
IsPositionAvailable = inputDevice.TryGetFeatureValue(CommonUsages.devicePosition, out CurrentControllerPosition);
IsPositionApproximate = false;
IsRotationAvailable = inputDevice.TryGetFeatureValue(CommonUsages.deviceRotation, out CurrentControllerRotation);
// Devices are considered tracked if we receive position OR rotation data from the sensors.
TrackingState = (IsPositionAvailable || IsRotationAvailable) ? TrackingState.Tracked : TrackingState.NotTracked;
CurrentControllerPosition = MixedRealityPlayspace.TransformPoint(CurrentControllerPosition);
CurrentControllerRotation = MixedRealityPlayspace.Rotation * CurrentControllerRotation;
CurrentControllerPose.Position = CurrentControllerPosition;
CurrentControllerPose.Rotation = CurrentControllerRotation;
// Raise input system events if it is enabled.
if (lastState != TrackingState)
{
CoreServices.InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked && LastControllerPose != CurrentControllerPose)
{
if (IsPositionAvailable && IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, CurrentControllerPose);
}
else if (IsPositionAvailable && !IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourcePositionChanged(InputSource, this, CurrentControllerPosition);
}
else if (!IsPositionAvailable && IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourceRotationChanged(InputSource, this, CurrentControllerRotation);
}
}
for (int i = 0; i < Interactions?.Length; i++)
{
switch (Interactions[i].AxisType)
{
case AxisType.SixDof:
UpdatePoseData(Interactions[i], inputDevice);
break;
}
}
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InputDevice inputDevice)
{
using (UpdateHandDataPerfMarker.Auto())
{
handMeshProvider?.UpdateHandMesh();
#if MSFT_OPENXR && (UNITY_STANDALONE_WIN || UNITY_WSA)
if (handTracker != null && handTracker.TryLocateHandJoints(FrameTime.OnUpdate, locations))
{
foreach (HandJoint handJoint in HandJoints)
{
HandJointLocation handJointLocation = locations[(int)handJoint];
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the pose into world space.
#if MSFT_OPENXR_0_2_0_OR_NEWER
Vector3 position = MixedRealityPlayspace.TransformPoint(handJointLocation.Pose.position);
Quaternion rotation = MixedRealityPlayspace.Rotation * handJointLocation.Pose.rotation;
#else
Vector3 position = MixedRealityPlayspace.TransformPoint(handJointLocation.Position);
Quaternion rotation = MixedRealityPlayspace.Rotation * handJointLocation.Rotation;
#endif // MSFT_OPENXR_0_2_0_OR_NEWER
unityJointPoses[ConvertToTrackedHandJoint(handJoint)] = new MixedRealityPose(position, rotation);
}
#else
if (inputDevice.TryGetFeatureValue(CommonUsages.handData, out Hand hand))
{
foreach (HandFinger finger in handFingers)
{
if (hand.TryGetRootBone(out Bone rootBone))
{
ReadHandJoint(TrackedHandJoint.Wrist, rootBone);
}
if (hand.TryGetFingerBones(finger, fingerBones))
{
for (int i = 0; i < fingerBones.Count; i++)
{
ReadHandJoint(ConvertToTrackedHandJoint(finger, i), fingerBones[i]);
}
}
}
#endif // MSFT_OPENXR && (UNITY_STANDALONE_WIN || UNITY_WSA)
handDefinition?.UpdateHandJoints(unityJointPoses);
}
}
}
private void ReadHandJoint(TrackedHandJoint trackedHandJoint, Bone bone)
{
bool positionAvailable = bone.TryGetPosition(out Vector3 position);
bool rotationAvailable = bone.TryGetRotation(out Quaternion rotation);
if (positionAvailable && rotationAvailable)
{
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the pose into world space.
position = MixedRealityPlayspace.TransformPoint(position);
rotation = MixedRealityPlayspace.Rotation * rotation;
unityJointPoses[trackedHandJoint] = new MixedRealityPose(position, rotation);
}
}
/// <inheritdoc />
public override void Update()
{
using (UpdatePerfMarker.Auto())
{
if (!eyeTrackingDevice.isValid)
{
InputDevices.GetDevicesWithCharacteristics(InputDeviceCharacteristics.EyeTracking, InputDeviceList);
if (InputDeviceList.Count > 0)
{
eyeTrackingDevice = InputDeviceList[0];
}
if (!eyeTrackingDevice.isValid)
{
Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, false);
return;
}
}
#if UNITY_OPENXR
if (eyeTrackingDevice.TryGetFeatureValue(CommonUsages.isTracked, out bool gazeAvailable))
{
Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, gazeAvailable);
if (gazeAvailable &&
eyeTrackingDevice.TryGetFeatureValue(EyeTrackingUsages.gazePosition, out Vector3 eyeGazePosition) &&
eyeTrackingDevice.TryGetFeatureValue(EyeTrackingUsages.gazeRotation, out Quaternion eyeGazeRotation))
{
Vector3 worldPosition = MixedRealityPlayspace.TransformPoint(eyeGazePosition);
Vector3 worldRotation = MixedRealityPlayspace.TransformDirection(eyeGazeRotation * Vector3.forward);
Ray newGaze = new Ray(worldPosition, worldRotation);
if (SmoothEyeTracking)
{
newGaze = SmoothGaze(newGaze);
}
Service?.EyeGazeProvider?.UpdateEyeGaze(this, newGaze, DateTime.UtcNow);
}
}
else
{
Service?.EyeGazeProvider?.UpdateEyeTrackingStatus(this, false);
}
#endif // UNITY_OPENXR
}
}
public void UpdateHandJoints(Hand hand, ref MixedRealityPose[] jointPoses)
{
#if MSFT_OPENXR && (UNITY_STANDALONE_WIN || UNITY_WSA)
if (handTracker != null && handTracker.TryLocateHandJoints(FrameTime.OnUpdate, locations))
{
if (jointPoses == null)
{
jointPoses = new MixedRealityPose[ArticulatedHandPose.JointCount];
}
foreach (HandJoint handJoint in HandJoints)
{
HandJointLocation handJointLocation = locations[(int)handJoint];
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the pose into world space.
Vector3 position = MixedRealityPlayspace.TransformPoint(handJointLocation.Pose.position);
Quaternion rotation = MixedRealityPlayspace.Rotation * handJointLocation.Pose.rotation;
jointPoses[ConvertToArrayIndex(handJoint)] = new MixedRealityPose(position, rotation);
}
#else
if (jointPoses == null)
{
jointPoses = new MixedRealityPose[ArticulatedHandPose.JointCount];
}
foreach (HandFinger finger in HandFingers)
{
if (hand.TryGetRootBone(out Bone rootBone) && TryReadHandJoint(rootBone, out MixedRealityPose rootPose))
{
jointPoses[(int)TrackedHandJoint.Palm] = rootPose;
}
if (hand.TryGetFingerBones(finger, fingerBones))
{
for (int i = 0; i < fingerBones.Count; i++)
{
if (TryReadHandJoint(fingerBones[i], out MixedRealityPose pose))
{
jointPoses[ConvertToArrayIndex(finger, i)] = pose;
}
}
}
#endif // MSFT_OPENXR && (UNITY_STANDALONE_WIN || UNITY_WSA)
}
}
internal void UpdateController(MLInputController controller)
{
if (controller.Type == MLInputControllerType.Control)
{
CurrentControllerPosition = controller.Position;
CurrentControllerRotation = controller.Orientation;
// Update the interaction data source
CurrentControllerPosition = MixedRealityPlayspace.TransformPoint(CurrentControllerPosition);
CurrentControllerRotation = MixedRealityPlayspace.Rotation * CurrentControllerRotation;
CurrentControllerPose.Position = CurrentControllerPosition;
CurrentControllerPose.Rotation = CurrentControllerRotation;
InputSystem?.RaiseSourcePoseChanged(InputSource, this, CurrentControllerPose);
var interactionMapping = Interactions[0];
interactionMapping.PoseData = CurrentControllerPose;
if (interactionMapping.Changed)
{
// Raise input system Event if it enabled
InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction, interactionMapping.PoseData);
}
// Fire Select event
var trigger = controller.TriggerValue;
if (triggerDown)
{
if (trigger < triggerUpThreshold)
{
triggerDown = false;
RaiseOnSelectUp();
}
}
else
{
if (trigger > triggerDownThreshold)
{
triggerDown = true;
RaiseOnSelectDown();
}
}
}
}
private bool TryReadHandJoint(Bone bone, out MixedRealityPose pose)
{
bool positionAvailable = bone.TryGetPosition(out Vector3 position);
bool rotationAvailable = bone.TryGetRotation(out Quaternion rotation);
if (positionAvailable && rotationAvailable)
{
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the pose into world space.
position = MixedRealityPlayspace.TransformPoint(position);
rotation = MixedRealityPlayspace.Rotation * rotation;
pose = new MixedRealityPose(position, rotation);
return(true);
}
pose = MixedRealityPose.ZeroIdentity;
return(false);
}
/// <inheritdoc/>
public bool TryGetRectangularBoundsParams(out Vector2 center, out float angle, out float width, out float height)
{
if (rectangularBounds == null || !rectangularBounds.IsValid)
{
center = EdgeUtilities.InvalidPoint;
angle = 0f;
width = 0f;
height = 0f;
return(false);
}
// Handle the user teleporting (boundary moves with them).
Vector3 transformedCenter = MixedRealityPlayspace.TransformPoint(
new Vector3(rectangularBounds.Center.x, 0f, rectangularBounds.Center.y));
center = new Vector2(transformedCenter.x, transformedCenter.z);
angle = rectangularBounds.Angle;
width = rectangularBounds.Width;
height = rectangularBounds.Height;
return(true);
}
public void UpdateHandJoints(InputDevice inputDevice, Dictionary <TrackedHandJoint, MixedRealityPose> jointPoses)
{
#if MSFT_OPENXR && (UNITY_STANDALONE_WIN || UNITY_WSA)
if (handTracker != null && handTracker.TryLocateHandJoints(FrameTime.OnUpdate, locations))
{
foreach (HandJoint handJoint in HandJoints)
{
HandJointLocation handJointLocation = locations[(int)handJoint];
// We want input sources to follow the playspace, so fold in the playspace transform here to
// put the pose into world space.
Vector3 position = MixedRealityPlayspace.TransformPoint(handJointLocation.Pose.position);
Quaternion rotation = MixedRealityPlayspace.Rotation * handJointLocation.Pose.rotation;
jointPoses[ConvertToTrackedHandJoint(handJoint)] = new MixedRealityPose(position, rotation);
}
#else
if (inputDevice.TryGetFeatureValue(CommonUsages.handData, out Hand hand))
{
foreach (HandFinger finger in HandFingers)
{
if (hand.TryGetRootBone(out Bone rootBone) && TryReadHandJoint(rootBone, out MixedRealityPose rootPose))
{
jointPoses[TrackedHandJoint.Palm] = rootPose;
}
if (hand.TryGetFingerBones(finger, fingerBones))
{
for (int i = 0; i < fingerBones.Count; i++)
{
if (TryReadHandJoint(fingerBones[i], out MixedRealityPose pose))
{
jointPoses[ConvertToTrackedHandJoint(finger, i)] = pose;
}
}
}
}
#endif // MSFT_OPENXR && (UNITY_STANDALONE_WIN || UNITY_WSA)
}
}
public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
{
using (UpdateHandMeshPerfMarker.Auto())
{
MixedRealityHandTrackingProfile handTrackingProfile = null;
MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile;
if (inputSystemProfile != null)
{
handTrackingProfile = inputSystemProfile.HandTrackingProfile;
}
if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
{
// If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// Notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
return;
}
HandPose handPose = sourceState.TryGetHandPose();
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handPose != null)
{
if (handMeshTriangleIndices == null)
{
handMeshTriangleIndices = new ushort[handMeshObserver.TriangleIndexCount];
handMeshTriangleIndicesUnity = new int[handMeshObserver.TriangleIndexCount];
handMeshObserver.GetTriangleIndices(handMeshTriangleIndices);
Array.Copy(handMeshTriangleIndices, handMeshTriangleIndicesUnity, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var neutralVertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(neutralVertexAndNormals);
Parallel.For(0, handMeshObserver.VertexCount, i =>
{
neutralVertexAndNormals[i].Position.ConvertToUnityVector3(ref neutralPoseVertices[i]);
});
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (vertexAndNormals == null)
{
vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
handMeshVerticesUnity = new Vector3[handMeshObserver.VertexCount];
handMeshNormalsUnity = new Vector3[handMeshObserver.VertexCount];
}
if (vertexAndNormals != null && handMeshTriangleIndices != null)
{
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Matrix4x4.Decompose(meshTransform.Value,
out System.Numerics.Vector3 scale,
out System.Numerics.Quaternion rotation,
out System.Numerics.Vector3 translation);
Parallel.For(0, handMeshObserver.VertexCount, i =>
{
vertexAndNormals[i].Position.ConvertToUnityVector3(ref handMeshVerticesUnity[i]);
vertexAndNormals[i].Normal.ConvertToUnityVector3(ref handMeshNormalsUnity[i]);
});
/// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 positionUnity = MixedRealityPlayspace.TransformPoint(translation.ToUnityVector3());
Quaternion rotationUnity = MixedRealityPlayspace.Rotation * rotation.ToUnityQuaternion();
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVerticesUnity,
normals = handMeshNormalsUnity,
triangles = handMeshTriangleIndicesUnity,
uvs = handMeshUVsUnity,
position = positionUnity,
rotation = rotationUnity
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo);
}
}
}
}
}
public void UpdateController()
{
if (!Enabled)
{
return;
}
// hand pose
var lastState = TrackingState;
TrackingState = (hand.IsTracked) ? TrackingState.Tracked : TrackingState.NotTracked;
if (lastState != TrackingState)
{
CoreServices.InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked)
{
var pose = new MixedRealityPose();
pose.Position = MixedRealityPlayspace.TransformPoint(hand.transform.position);
pose.Rotation = MixedRealityPlayspace.Rotation * hand.transform.rotation;
CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, pose);
}
// hand interaction
if (Interactions == null)
{
Debug.LogError($"No interaction configuration for Oculus Quest Hand {ControllerHandedness} Source");
Enabled = false;
}
if (TrackingState == TrackingState.Tracked)
{
for (int i = 0; i < Interactions?.Length; i++)
{
var interaction = Interactions[i];
switch (interaction.InputType)
{
case DeviceInputType.None:
break;
case DeviceInputType.SpatialPointer:
// hand pointer
var pointer = new MixedRealityPose();
pointer.Position = MixedRealityPlayspace.TransformPoint(hand.PointerPose.position);
pointer.Rotation = MixedRealityPlayspace.Rotation * hand.PointerPose.rotation;
interaction.PoseData = pointer;
if (interaction.Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interaction.MixedRealityInputAction, pointer);
}
break;
case DeviceInputType.SpatialGrip:
if (interaction.AxisType == AxisType.SixDof)
{
var grip = new MixedRealityPose();
grip.Position = MixedRealityPlayspace.TransformPoint(hand.transform.position);
grip.Rotation = MixedRealityPlayspace.Rotation * hand.transform.rotation;
interaction.PoseData = grip;
if (interaction.Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interaction.MixedRealityInputAction, grip);
}
}
break;
case DeviceInputType.Select:
case DeviceInputType.TriggerPress:
interaction.BoolData = hand.GetFingerIsPinching(OVRHand.HandFinger.Index);
if (interaction.Changed)
{
if (interaction.BoolData)
{
CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, interaction.MixedRealityInputAction);
}
else
{
CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, interaction.MixedRealityInputAction);
}
}
break;
case DeviceInputType.IndexFinger:
if (jointPose.ContainsKey(TrackedHandJoint.IndexTip))
{
var indexFinger = jointPose[TrackedHandJoint.IndexTip];
interaction.PoseData = indexFinger;
if (interaction.Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, interaction.MixedRealityInputAction, indexFinger);
}
}
break;
}
}
}
// hand joint
if (TrackingState == TrackingState.Tracked)
{
for (int i = 0; i < skeleton.Bones.Count; i++)
{
var bones = skeleton.Bones[i];
var handJoint = convertBoneIdToTrackedHandJoint(bones.Id);
var position = MixedRealityPlayspace.TransformPoint(bones.Transform.position);
var rotation = MixedRealityPlayspace.Rotation * bones.Transform.rotation;
if (jointPose.ContainsKey(handJoint))
{
jointPose[handJoint] = new MixedRealityPose(position, rotation);
}
else
{
jointPose.Add(handJoint, new MixedRealityPose(position, rotation));
}
}
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, jointPose);
}
}
public void UpdateController(WebXRInputSource controller)
{
if (!Enabled)
{
return;
}
IsPositionAvailable = IsRotationAvailable = controller.Hand.Available;
jointPoses[TrackedHandJoint.Wrist] = GetJointMixedRealityPose(controller.Hand.Joints[WebXRHand.WRIST]);
for (int i = WebXRHand.THUMB_METACARPAL; i < WebXRHand.JOINT_COUNT; i++)
{
var joint = controller.Hand.Joints[i];
jointPoses[(TrackedHandJoint)(i + 2)] = GetJointMixedRealityPose(joint);
}
jointPoses[TrackedHandJoint.Palm] = new MixedRealityPose((jointPoses[TrackedHandJoint.MiddleMetacarpal].Position + jointPoses[TrackedHandJoint.MiddleMetacarpal].Position) / 2, jointPoses[TrackedHandJoint.MiddleMetacarpal].Rotation);
var indexPose = jointPoses[TrackedHandJoint.IndexTip];
bool isSelecting;
MixedRealityPose spatialPointerPose;
if (controller.IsPositionTracked)
{
isSelecting = controller.Selected;
spatialPointerPose = new MixedRealityPose(MixedRealityPlayspace.TransformPoint(controller.Position), MixedRealityPlayspace.Rotation * controller.Rotation);
}
else
{
// Is selecting if thumb tip and index tip are close
isSelecting = Vector3.Distance(controller.Hand.Joints[WebXRHand.THUMB_PHALANX_TIP].Position, controller.Hand.Joints[WebXRHand.INDEX_PHALANX_TIP].Position) < 0.04;
// The hand ray starts from the middle of thumb tip and index tip
HandRay.Update((controller.Hand.Joints[WebXRHand.THUMB_PHALANX_TIP].Position + controller.Hand.Joints[WebXRHand.INDEX_PHALANX_TIP].Position) / 2, new Vector3(0.3f, -0.4f, 0.9f), CameraCache.Main.transform, ControllerHandedness);
Ray ray = HandRay.Ray;
spatialPointerPose = new MixedRealityPose(ray.origin, Quaternion.LookRotation(ray.direction));
}
CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, spatialPointerPose);
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, jointPoses);
UpdateVelocity();
for (int i = 0; i < Interactions?.Length; i++)
{
switch (Interactions[i].InputType)
{
case DeviceInputType.SpatialPointer:
Interactions[i].PoseData = spatialPointerPose;
if (Interactions[i].Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, Interactions[i].PoseData);
}
break;
case DeviceInputType.SpatialGrip:
Interactions[i].PoseData = indexPose;
if (Interactions[i].Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, Interactions[i].PoseData);
}
break;
case DeviceInputType.Select:
Interactions[i].BoolData = isSelecting || controller.TargetRayMode == WebXRTargetRayModes.Screen;
if (Interactions[i].Changed)
{
if (Interactions[i].BoolData)
{
CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
else
{
CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
}
break;
case DeviceInputType.TriggerPress:
Interactions[i].BoolData = isSelecting;
if (Interactions[i].Changed)
{
if (Interactions[i].BoolData)
{
CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
else
{
CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
}
break;
case DeviceInputType.IndexFinger:
Interactions[i].PoseData = indexPose;
if (Interactions[i].Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, Interactions[i].PoseData);
}
break;
case DeviceInputType.ThumbStick:
HandDefinition?.UpdateCurrentTeleportPose(Interactions[i]);
break;
}
}
}
/// <summary>
/// Compute the world position corresponding to the input local position in playspace.
/// </summary>
/// <param name="localPosition">The local position.</param>
/// <returns>The world position.</returns>
public static Vector3 PositionRelativeToPlayspace(Vector3 localPosition)
{
return(MixedRealityPlayspace.TransformPoint(localPosition));
}
public void UpdateController(WebXRInputSource controller)
{
if (!Enabled)
{
return;
}
var position = MixedRealityPlayspace.TransformPoint(controller.Position);
var rotation = MixedRealityPlayspace.Rotation * controller.Rotation;
var pose = new MixedRealityPose(position, rotation);
for (int i = 0; i < Interactions?.Length; i++)
{
switch (Interactions[i].InputType)
{
case DeviceInputType.SpatialPointer:
Interactions[i].PoseData = pose;
if (Interactions[i].Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, pose);
}
break;
case DeviceInputType.SpatialGrip:
Interactions[i].PoseData = pose;
if (Interactions[i].Changed)
{
CoreServices.InputSystem?.RaisePoseInputChanged(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction, pose);
}
break;
case DeviceInputType.Select:
Interactions[i].BoolData = controller.Selected;
if (Interactions[i].Changed)
{
if (Interactions[i].BoolData)
{
CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
else
{
CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
}
break;
case DeviceInputType.TriggerPress:
Interactions[i].BoolData = controller.Squeezed;
if (Interactions[i].Changed)
{
if (Interactions[i].BoolData)
{
CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
else
{
CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, Interactions[i].MixedRealityInputAction);
}
}
break;
}
}
}
/// <summary>
/// Update the controller data from XR SDK.
/// </summary>
public virtual void UpdateController(InputDevice inputDevice)
{
using (UpdateControllerPerfMarker.Auto())
{
if (!Enabled)
{
return;
}
if (Interactions == null)
{
Debug.LogError($"No interaction configuration for {GetType().Name}");
Enabled = false;
}
var lastState = TrackingState;
LastControllerPose = CurrentControllerPose;
// Check for position and rotation.
IsPositionAvailable = inputDevice.TryGetFeatureValue(CommonUsages.devicePosition, out CurrentControllerPosition);
IsPositionApproximate = false;
IsRotationAvailable = inputDevice.TryGetFeatureValue(CommonUsages.deviceRotation, out CurrentControllerRotation);
// Devices are considered tracked if we receive position OR rotation data from the sensors.
TrackingState = (IsPositionAvailable || IsRotationAvailable) ? TrackingState.Tracked : TrackingState.NotTracked;
CurrentControllerPosition = MixedRealityPlayspace.TransformPoint(CurrentControllerPosition);
CurrentControllerRotation = MixedRealityPlayspace.Rotation * CurrentControllerRotation;
CurrentControllerPose.Position = CurrentControllerPosition;
CurrentControllerPose.Rotation = CurrentControllerRotation;
// Raise input system events if it is enabled.
if (lastState != TrackingState)
{
CoreServices.InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked && LastControllerPose != CurrentControllerPose)
{
if (IsPositionAvailable && IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourcePoseChanged(InputSource, this, CurrentControllerPose);
}
else if (IsPositionAvailable && !IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourcePositionChanged(InputSource, this, CurrentControllerPosition);
}
else if (!IsPositionAvailable && IsRotationAvailable)
{
CoreServices.InputSystem?.RaiseSourceRotationChanged(InputSource, this, CurrentControllerRotation);
}
}
for (int i = 0; i < Interactions?.Length; i++)
{
switch (Interactions[i].AxisType)
{
case AxisType.None:
break;
case AxisType.Digital:
UpdateButtonData(Interactions[i], inputDevice);
break;
case AxisType.SingleAxis:
UpdateSingleAxisData(Interactions[i], inputDevice);
break;
case AxisType.DualAxis:
UpdateDualAxisData(Interactions[i], inputDevice);
break;
case AxisType.SixDof:
UpdatePoseData(Interactions[i], inputDevice);
break;
}
}
}
}
/// <summary>
/// Update the source input from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateSourceData(InteractionSourceState interactionSourceState)
{
var lastState = TrackingState;
var sourceKind = interactionSourceState.source.kind;
lastSourcePose = currentSourcePose;
if (sourceKind == InteractionSourceKind.Hand ||
(sourceKind == InteractionSourceKind.Controller && interactionSourceState.source.supportsPointing))
{
// The source is either a hand or a controller that supports pointing.
// We can now check for position and rotation.
IsPositionAvailable = interactionSourceState.sourcePose.TryGetPosition(out currentSourcePosition);
if (IsPositionAvailable)
{
IsPositionApproximate = (interactionSourceState.sourcePose.positionAccuracy == InteractionSourcePositionAccuracy.Approximate);
}
else
{
IsPositionApproximate = false;
}
IsRotationAvailable = interactionSourceState.sourcePose.TryGetRotation(out currentSourceRotation);
// We want the source to follow the Playspace, so fold in the playspace transform here to
// put the source pose into world space.
currentSourcePosition = MixedRealityPlayspace.TransformPoint(currentSourcePosition);
currentSourceRotation = MixedRealityPlayspace.Rotation * currentSourceRotation;
// Devices are considered tracked if we receive position OR rotation data from the sensors.
TrackingState = (IsPositionAvailable || IsRotationAvailable) ? TrackingState.Tracked : TrackingState.NotTracked;
}
else
{
// The input source does not support tracking.
TrackingState = TrackingState.NotApplicable;
}
currentSourcePose.Position = currentSourcePosition;
currentSourcePose.Rotation = currentSourceRotation;
// Raise input system events if it is enabled.
if (lastState != TrackingState)
{
InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked && lastSourcePose != currentSourcePose)
{
if (IsPositionAvailable && IsRotationAvailable)
{
InputSystem?.RaiseSourcePoseChanged(InputSource, this, currentSourcePose);
}
else if (IsPositionAvailable && !IsRotationAvailable)
{
InputSystem?.RaiseSourcePositionChanged(InputSource, this, currentSourcePosition);
}
else if (!IsPositionAvailable && IsRotationAvailable)
{
InputSystem?.RaiseSourceRotationChanged(InputSource, this, currentSourceRotation);
}
}
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
HandPose handPose = sourceState.TryGetHandPose();
#if WINDOWS_UWP
if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization)
{
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handMeshTriangleIndices == null)
{
uint indexCount = handMeshObserver.TriangleIndexCount;
ushort[] indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
}
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Vector3 scale;
System.Numerics.Quaternion rotation;
System.Numerics.Vector3 translation;
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
handMeshNormals[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal);
}
/// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation);
unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition);
Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation);
unityRotation = MixedRealityPlayspace.Rotation * unityRotation;
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVertices,
normals = handMeshNormals,
triangles = handMeshTriangleIndices,
uvs = handMeshUVs,
position = unityPosition,
rotation = unityRotation
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
}
}
}
else
{
// if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
}
#endif // WINDOWS_UWP
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation);
unityJointPositions[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position);
// We want the controller to follow the Playspace, so fold in the playspace transform here to
// put the controller pose into world space.
unityJointPositions[i] = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]);
unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i];
if (jointIndices[i] == HandJointKind.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (!unityJointPoses.ContainsKey(handJoint))
{
unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]));
}
else
{
unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]);
}
}
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses);
}
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}