public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
{
using (UpdateHandMeshPerfMarker.Auto())
{
MixedRealityHandTrackingProfile handTrackingProfile = null;
MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile;
if (inputSystemProfile != null)
{
handTrackingProfile = inputSystemProfile.HandTrackingProfile;
}
if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
{
// If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// Notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
return;
}
HandPose handPose = sourceState.TryGetHandPose();
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handPose != null)
{
if (handMeshTriangleIndices == null)
{
handMeshTriangleIndices = new ushort[handMeshObserver.TriangleIndexCount];
handMeshTriangleIndicesUnity = new int[handMeshObserver.TriangleIndexCount];
handMeshObserver.GetTriangleIndices(handMeshTriangleIndices);
Array.Copy(handMeshTriangleIndices, handMeshTriangleIndicesUnity, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var neutralVertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(neutralVertexAndNormals);
Parallel.For(0, handMeshObserver.VertexCount, i =>
{
neutralVertexAndNormals[i].Position.ConvertToUnityVector3(ref neutralPoseVertices[i]);
});
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (vertexAndNormals == null)
{
vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
handMeshVerticesUnity = new Vector3[handMeshObserver.VertexCount];
handMeshNormalsUnity = new Vector3[handMeshObserver.VertexCount];
}
if (vertexAndNormals != null && handMeshTriangleIndices != null)
{
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Matrix4x4.Decompose(meshTransform.Value,
out System.Numerics.Vector3 scale,
out System.Numerics.Quaternion rotation,
out System.Numerics.Vector3 translation);
Parallel.For(0, handMeshObserver.VertexCount, i =>
{
vertexAndNormals[i].Position.ConvertToUnityVector3(ref handMeshVerticesUnity[i]);
vertexAndNormals[i].Normal.ConvertToUnityVector3(ref handMeshNormalsUnity[i]);
});
/// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 positionUnity = MixedRealityPlayspace.TransformPoint(translation.ToUnityVector3());
Quaternion rotationUnity = MixedRealityPlayspace.Rotation * rotation.ToUnityQuaternion();
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVerticesUnity,
normals = handMeshNormalsUnity,
triangles = handMeshTriangleIndicesUnity,
uvs = handMeshUVsUnity,
position = positionUnity,
rotation = rotationUnity
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, Handedness, handMeshInfo);
}
}
}
}
}
/// <summary>
/// Updates the current hand mesh based on the passed in state of the hand.
/// </summary>
/// <param name="sourceState">The current hand state.</param>
public void UpdateHandMesh(SpatialInteractionSourceState sourceState)
{
MixedRealityHandTrackingProfile handTrackingProfile = null;
MixedRealityInputSystemProfile inputSystemProfile = CoreServices.InputSystem?.InputSystemProfile;
if (inputSystemProfile != null)
{
handTrackingProfile = inputSystemProfile.HandTrackingProfile;
}
if (handTrackingProfile == null || !handTrackingProfile.EnableHandMeshVisualization)
{
// If hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// Notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
return;
}
HandPose handPose = sourceState.TryGetHandPose();
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handMeshTriangleIndices == null)
{
uint indexCount = handMeshObserver.TriangleIndexCount;
ushort[] indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnityVector3();
}
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Vector3 scale;
System.Numerics.Quaternion rotation;
System.Numerics.Vector3 translation;
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = vertexAndNormals[i].Position.ToUnityVector3();
handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnityVector3();
}
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVertices,
normals = handMeshNormals,
triangles = handMeshTriangleIndices,
uvs = handMeshUVs,
position = translation.ToUnityVector3(),
rotation = rotation.ToUnityQuaternion()
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(inputSource, handedness, handMeshInfo);
}
}
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
HandPose handPose = sourceState.TryGetHandPose();
#if WINDOWS_UWP
if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization)
{
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handMeshTriangleIndices == null)
{
uint indexCount = handMeshObserver.TriangleIndexCount;
ushort[] indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
}
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Vector3 scale;
System.Numerics.Quaternion rotation;
System.Numerics.Vector3 translation;
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
handMeshNormals[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal);
}
/// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation);
unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition);
Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation);
unityRotation = MixedRealityPlayspace.Rotation * unityRotation;
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVertices,
normals = handMeshNormals,
triangles = handMeshTriangleIndices,
uvs = handMeshUVs,
position = unityPosition,
rotation = unityRotation
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
}
}
}
else
{
// if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
}
#endif // WINDOWS_UWP
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation);
unityJointPositions[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position);
// We want the controller to follow the Playspace, so fold in the playspace transform here to
// put the controller pose into world space.
unityJointPositions[i] = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]);
unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i];
if (jointIndices[i] == HandJointKind.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (!unityJointPoses.ContainsKey(handJoint))
{
unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]));
}
else
{
unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]);
}
}
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses);
}
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
/// <summary>
/// Attempts to get updated hand mesh data.
/// </summary>
/// <param name="spatialInteractionSourceState">Platform provided current input source state for the hand.</param>
/// <param name="handPose">Hand pose information retrieved for joint conversion.</param>
/// <param name="data">Mesh information retrieved in case of success.</param>
/// <returns>True, if mesh data could be loaded.</returns>
private bool TryGetUpdatedHandMeshData(SpatialInteractionSourceState spatialInteractionSourceState, HandPose handPose, out HandMeshData data)
{
if (!handMeshObservers.ContainsKey(spatialInteractionSourceState.Source.Handedness) &&
!HasRequestedHandMeshObserver(spatialInteractionSourceState.Source.Handedness))
{
SetHandMeshObserver(spatialInteractionSourceState);
}
if (handMeshObservers.TryGetValue(spatialInteractionSourceState.Source.Handedness, out var handMeshObserver) && handMeshTriangleIndices == null)
{
var indexCount = handMeshObserver.TriangleIndexCount;
var indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
var neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
var neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnity();
}
// Compute UV mapping
InitializeHandMeshUVs(neutralPoseVertices);
}
if (handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(spatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out var scale, out var rotation, out var translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = vertexAndNormals[i].Position.ToUnity();
handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnity();
}
data = new HandMeshData(
handMeshVertices,
handMeshTriangleIndices,
handMeshNormals,
handMeshUVs);
return(true);
}
}
return(false);
}