/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
Profiler.BeginSample("[MRTK] WindowsMixedRealityArticulatedHand.UpdateHandData");
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
#if WINDOWS_UWP
handDefinition?.UpdateHandMesh(sourceState);
#endif // WINDOWS_UWP
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
Vector3 jointPosition = jointPoses[i].Position.ToUnityVector3();
Quaternion jointOrientation = jointPoses[i].Orientation.ToUnityQuaternion();
// We want the joints to follow the playspace, so fold in the playspace transform here to
// put the joint pose into world space.
jointPosition = MixedRealityPlayspace.TransformPoint(jointPosition);
jointOrientation = MixedRealityPlayspace.Rotation * jointOrientation;
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (handJoint == TrackedHandJoint.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
unityJointPoses[handJoint] = new MixedRealityPose(jointPosition, jointOrientation);
}
handDefinition?.UpdateHandJoints(unityJointPoses);
}
break;
}
}
Profiler.EndSample(); // UpdateHandData
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
private bool GetHistoricalPose(out Vector3 cameraPosition, out Quaternion cameraRotation)
{
#if !UNITY_EDITOR && UNITY_WSA
SpatialCoordinateSystem unityCoordinateSystem = Marshal.GetObjectForIUnknown(WorldManager.GetNativeISpatialCoordinateSystemPtr()) as SpatialCoordinateSystem;
if (unityCoordinateSystem == null)
{
Debug.LogError("Failed to get the native SpatialCoordinateSystem");
cameraPosition = default(Vector3);
cameraRotation = default(Quaternion);
return(false);
}
if (timeConversionCalendar == null)
{
timeConversionCalendar = new Calendar();
}
timeConversionCalendar.SetToNow();
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(timeConversionCalendar.GetDateTime());
if (perceptionTimestamp != null)
{
SpatialLocator locator = SpatialLocator.GetDefault();
if (locator != null)
{
SpatialLocation headPose = locator.TryLocateAtTimestamp(perceptionTimestamp, unityCoordinateSystem);
if (headPose != null)
{
var systemOrientation = headPose.Orientation;
var systemPostion = headPose.Position;
// Convert the orientation and position from Windows to Unity coordinate spaces
cameraRotation.x = -systemOrientation.X;
cameraRotation.y = -systemOrientation.Y;
cameraRotation.z = systemOrientation.Z;
cameraRotation.w = systemOrientation.W;
cameraPosition.x = systemPostion.X;
cameraPosition.y = systemPostion.Y;
cameraPosition.z = -systemPostion.Z;
return(true);
}
}
}
cameraPosition = default(Vector3);
cameraRotation = default(Quaternion);
return(false);
#else
cameraPosition = Camera.main.transform.position;
cameraRotation = Camera.main.transform.rotation;
return(true);
#endif
}
// TODO: which is the smoothest experience? FixedUpdate for physics, or LateUpdate/Prerender for more accurate onscreen representation
// (or if we move away from physics, then see if LateUpdate is just better.
private void FixedUpdate()
{
#if WINDOWS_UWP
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(
DateTimeOffset.Now + TimeSpan.FromSeconds(Time.fixedDeltaTime * 3.0f));
if (!UpdateAtTime(perceptionTimestamp))
{
// prediction failed, fall back to current timestamp
UpdateAtTime(PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now));
}
#endif
}
private bool UpdateAtTime(PerceptionTimestamp timestamp)
{
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(timestamp);
bool leftHandFound = false;
bool rightHandFound = false;
foreach (var source in sources)
{
if (source.Source.Kind == SpatialInteractionSourceKind.Hand)
{
var handPose = source.TryGetHandPose();
if (handPose != null)
{
int handIdx = HandednessToIndex(source.Source.Handedness);
if (source.Source.Handedness == SpatialInteractionSourceHandedness.Right)
{
rightHandFound = true;
}
else if (source.Source.Handedness == SpatialInteractionSourceHandedness.Left)
{
leftHandFound = true;
}
UpdateHand(handPose, source.Source.Handedness);
}
}
}
if (!leftHandFound)
{
foreach (var go in leftHandObjects)
{
go.SetActive(false);
}
}
if (!rightHandFound)
{
foreach (var go in rightHandObjects)
{
go.SetActive(false);
}
}
return(rightHandFound || leftHandFound);
}
Node AddNode(SpatialAnchor anchor, PerceptionTimestamp perceptionTimestamp)
{
var position = Vector3.Zero;
var forward = Vector3.Zero;
var anchorPose = SpatialPointerPose.TryGetAtTimestamp(anchor.CoordinateSystem, perceptionTimestamp);
if (anchorPose != null)
{
position = anchorPose.Head.Position;
forward = anchorPose.Head.ForwardDirection;
}
var node = new Node(anchor, position, forward);
nodes.Add(node);
return(node);
}
public void Update(PerceptionTimestamp timeStamp, SpatialCoordinateSystem coordinateSystem)
{
var states = interactionManager.GetDetectedSourcesAtTimestamp(timeStamp);
foreach (SpatialInteractionSourceState state in states)
{
if (state.Source.Handedness == hand)
{
SpatialInteractionSourceLocation location = state.Properties.TryGetLocation(coordinateSystem);
if (location != null)
{
SetSpatialInteractionSourceLocation(location);
}
previousState = currentState;
currentState = state;
internalState = previousState != null ? DeviceState.Valid : DeviceState.Invalid;
}
}
}
private void UpdateHands()
{
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
HandPose handPose = sourceState.TryGetHandPose();
if (handPose != null && handPose.TryGetJoints(SpatialCoordinateSystem, jointIndices, jointPoses))
{
SpatialInteractionSourceHandedness handIndex = sourceState.Source.Handedness;
if (handIndex == SpatialInteractionSourceHandedness.Left)
{
ApplyTransforms(leftHandProxy, jointPoses);
}
else
{
ApplyTransforms(rightHandProxy, jointPoses);
}
}
}
}
/// <summary>
/// Update the hand data from the device.
/// </summary>
/// <param name="interactionSourceState">The InteractionSourceState retrieved from the platform.</param>
private void UpdateHandData(InteractionSourceState interactionSourceState)
{
#if WINDOWS_UWP || DOTNETWINRT_PRESENT
// Articulated hand support is only present in the 18362 version and beyond Windows
// SDK (which contains the V8 drop of the Universal API Contract). In particular,
// the HandPose related APIs are only present on this version and above.
if (!articulatedHandApiAvailable)
{
return;
}
PerceptionTimestamp perceptionTimestamp = PerceptionTimestampHelper.FromHistoricalTargetTime(DateTimeOffset.Now);
IReadOnlyList <SpatialInteractionSourceState> sources = SpatialInteractionManager?.GetDetectedSourcesAtTimestamp(perceptionTimestamp);
foreach (SpatialInteractionSourceState sourceState in sources)
{
if (sourceState.Source.Id.Equals(interactionSourceState.source.id))
{
HandPose handPose = sourceState.TryGetHandPose();
#if WINDOWS_UWP
if (CoreServices.InputSystem.InputSystemProfile.HandTrackingProfile.EnableHandMeshVisualization)
{
// Accessing the hand mesh data involves copying quite a bit of data, so only do it if application requests it.
if (handMeshObserver == null && !hasRequestedHandMeshObserver)
{
SetHandMeshObserver(sourceState);
hasRequestedHandMeshObserver = true;
}
if (handMeshObserver != null && handMeshTriangleIndices == null)
{
uint indexCount = handMeshObserver.TriangleIndexCount;
ushort[] indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
Vector3[] neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
HandPose neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
HandMeshVertexState handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
}
// Compute UV mapping
InitializeUVs(neutralPoseVertices);
}
if (handPose != null && handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(WindowsMixedRealityUtilities.SpatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Vector3 scale;
System.Numerics.Quaternion rotation;
System.Numerics.Vector3 translation;
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out scale, out rotation, out translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Position);
handMeshNormals[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(vertexAndNormals[i].Normal);
}
/// Hands should follow the Playspace to accommodate teleporting, so fold in the Playspace transform.
Vector3 unityPosition = WindowsMixedRealityUtilities.SystemVector3ToUnity(translation);
unityPosition = MixedRealityPlayspace.TransformPoint(unityPosition);
Quaternion unityRotation = WindowsMixedRealityUtilities.SystemQuaternionToUnity(rotation);
unityRotation = MixedRealityPlayspace.Rotation * unityRotation;
HandMeshInfo handMeshInfo = new HandMeshInfo
{
vertices = handMeshVertices,
normals = handMeshNormals,
triangles = handMeshTriangleIndices,
uvs = handMeshUVs,
position = unityPosition,
rotation = unityRotation
};
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
}
}
}
else
{
// if hand mesh visualization is disabled make sure to destroy our hand mesh observer if it has already been created
if (handMeshObserver != null)
{
// notify that hand mesh has been updated (cleared)
HandMeshInfo handMeshInfo = new HandMeshInfo();
CoreServices.InputSystem?.RaiseHandMeshUpdated(InputSource, ControllerHandedness, handMeshInfo);
hasRequestedHandMeshObserver = false;
handMeshObserver = null;
}
}
#endif // WINDOWS_UWP
if (handPose != null && handPose.TryGetJoints(WindowsMixedRealityUtilities.SpatialCoordinateSystem, jointIndices, jointPoses))
{
for (int i = 0; i < jointPoses.Length; i++)
{
unityJointOrientations[i] = WindowsMixedRealityUtilities.SystemQuaternionToUnity(jointPoses[i].Orientation);
unityJointPositions[i] = WindowsMixedRealityUtilities.SystemVector3ToUnity(jointPoses[i].Position);
// We want the controller to follow the Playspace, so fold in the playspace transform here to
// put the controller pose into world space.
unityJointPositions[i] = MixedRealityPlayspace.TransformPoint(unityJointPositions[i]);
unityJointOrientations[i] = MixedRealityPlayspace.Rotation * unityJointOrientations[i];
if (jointIndices[i] == HandJointKind.IndexTip)
{
lastIndexTipRadius = jointPoses[i].Radius;
}
TrackedHandJoint handJoint = ConvertHandJointKindToTrackedHandJoint(jointIndices[i]);
if (!unityJointPoses.ContainsKey(handJoint))
{
unityJointPoses.Add(handJoint, new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]));
}
else
{
unityJointPoses[handJoint] = new MixedRealityPose(unityJointPositions[i], unityJointOrientations[i]);
}
}
CoreServices.InputSystem?.RaiseHandJointsUpdated(InputSource, ControllerHandedness, unityJointPoses);
}
}
}
#endif // WINDOWS_UWP || DOTNETWINRT_PRESENT
}
Node UpdateCurrentNode(SpatialCoordinateSystem referenceFrameCoordinateSystem, PerceptionTimestamp perceptionTimestamp, float nodeRadius = 1.0f)
{
SpatialPointerPose pose = SpatialPointerPose.TryGetAtTimestamp(referenceFrameCoordinateSystem, perceptionTimestamp);
if (pose == null)
{
return(currentNode);
}
if (currentNode == null)
{
// create current node
var nodeAnchor = SpatialAnchor.TryCreateRelativeTo(referenceFrameCoordinateSystem, pose.Head.ForwardDirection * 0.1f);
if (nodeAnchor == null)
{
Debug.WriteLine($"WARN: Failed to create Anchor");
return(null);
}
Debug.WriteLine($"Creating new node Head position {pose.Head.Position} and direction {pose.Head.ForwardDirection}");
AddNode(nodeAnchor, perceptionTimestamp);
return(nodes[nodes.Count - 1]);
}
else
{
// outside the current nodes threshold?
var distance = currentNode.TryGetDistance(referenceFrameCoordinateSystem, pose.Head.Position);
if (distance.HasValue && distance.Value > nodeRadius)
{
// search for node
var closestNodes = GetClosestNodes(referenceFrameCoordinateSystem, pose, nodeRadius);
if (closestNodes != null && closestNodes.Count > 0)
{
foreach (var node in closestNodes)
{
if (node == currentNode)
{
continue;
}
return(node);
}
}
// no node exist... try to create one
// position of current node in respect to the reference frame
var currentNodesPosition = currentNode.TryGetTransformedPosition(referenceFrameCoordinateSystem);
if (currentNodesPosition.HasValue)
{
var direction = Vector3.Normalize(
new Vector3(pose.Head.Position.X, 0f, pose.Head.Position.Z) -
new Vector3(currentNodesPosition.Value.X, 0f, currentNodesPosition.Value.Z)
);
var targetPosition = currentNodesPosition.Value + direction * nodeRadius;
var distanceFromPose = (targetPosition - new Vector3(pose.Head.Position.X, 0f, pose.Head.Position.Z)).Length();
var nodeAnchor = SpatialAnchor.TryCreateRelativeTo(referenceFrameCoordinateSystem, (direction * distanceFromPose));
if (nodeAnchor != null)
{
var newNode = AddNode(nodeAnchor, perceptionTimestamp);
// create a new edge connecting the current node and this node
edges.Add(new Edge
{
NodeA = currentNode,
NodeB = newNode
});
Debug.WriteLine($"Creating new node ({newNode.Name}) Head position {pose.Head.Position} and direction {pose.Head.ForwardDirection}, direction from current node {direction}.. Edge created {currentNode.Name}");
return(nodes[nodes.Count - 1]);
}
else
{
Debug.WriteLine($"WARN: Failed to create Anchor");
}
}
}
}
return(currentNode);
}
int RebuildTrailToTarget(SpatialCoordinateSystem referenceFrameCoordinateSystem, PerceptionTimestamp perceptionTimestamp, Node startNode, Node endNode,
int lookAhead = 100)
{
Debug.WriteLine($"RebuildTrailToTarget {startNode.Name} -> {endNode.Name}");
entities.Clear();
Stack <Node> trail = new Stack <Node>();
BuildPath(endNode, startNode, trail);
if (trail.Count == 0)
{
Debug.WriteLine($"Unable to find Path for startNode {startNode.Name} and {endNode.Name}");
return(-1);
}
var rootEntity = GetRootEntity(referenceFrameCoordinateSystem);
int i = 0;
while (i < lookAhead && trail.Count > 0)
{
var node = trail.Pop();
var entity = new Entity($"node_{i}");
entity.Node = node;
entity.Renderer = entity.Node == targetNode ? targetNodeRenderer : nodeRenderer;
entity.UpdateTransform(referenceFrameCoordinateSystem);
var targetPosition = rootEntity.Transform.Translation;
entity.Position = new Vector3(0, (targetPosition - entity.Transform.Translation).Y, 0f);
entities.Add(entity);
i += 1;
}
return(1);
}
int RebuildTrailToTarget(SpatialCoordinateSystem referenceFrameCoordinateSystem, PerceptionTimestamp perceptionTimestamp, Node startNode, Node endNode, int lookAhead = 3)
{
Debug.WriteLine($"RebuildTrailToTarget {startNode.Name} -> {endNode.Name}");
entities.Clear();
var trail = new List <Node>();
if (startNode == endNode)
{
trail.Add(startNode);
}
else
{
BuildPath(endNode, startNode, trail, new List <Node>());
}
if (trail.Count == 0)
{
Debug.WriteLine($"Unable to find Path for startNode {startNode.Name} and {endNode.Name}");
return(-1);
}
Debug.WriteLine($"Creating trials {trail.ToArray()}");
var baseEntity = GetReferenceEntitySpatialCoordinateSystem(referenceFrameCoordinateSystem);
for (var i = 0; i < Math.Min(trail.Count, lookAhead); i++)
{
var node = trail[i];
var entity = new Entity($"node_{i}");
entity.Node = node;
entity.Renderer = nodeRenderer;
entity.UpdateTransform(referenceFrameCoordinateSystem);
// offset from baseEntity (to keep the y positions uniform and consistent)
var targetPosition = baseEntity.Transform.Translation;
entity.Position = new Vector3(0, (targetPosition - entity.Transform.Translation).Y, 0f);
entities.Add(entity);
}
return(1);
}
