/// <inheritdoc />
public override void Update()
{
base.Update();
if (!Application.isPlaying)
{
return;
}
if (handTrackingHandle.IsValid)
{
if (MlHandTracking.MLHandTrackingGetDataEx(handTrackingHandle, ref handTrackingDataEx).IsOk)
{
leftHandData.TrackingState = handTrackingDataEx.left_hand_state.keypose < MlHandTracking.MLHandTrackingKeyPose.NoHand && !handTrackingDataEx.left_hand_state.is_holding_control ? TrackingState.Tracked : TrackingState.NotTracked;
rightHandData.TrackingState = handTrackingDataEx.right_hand_state.keypose < MlHandTracking.MLHandTrackingKeyPose.NoHand && !handTrackingDataEx.right_hand_state.is_holding_control ? TrackingState.Tracked : TrackingState.NotTracked;
}
else
{
Debug.LogError($"{nameof(MlHandTracking.MLHandTrackingGetDataEx)} Failed!");
}
if (leftHandData.TrackingState == TrackingState.Tracked || rightHandData.TrackingState == TrackingState.Tracked)
{
if (MlHandTracking.MLHandTrackingGetStaticData(handTrackingHandle, ref staticHandTrackingData).IsOk)
{
var now = DateTimeOffset.UtcNow.Ticks;
if (!MlPerception.MLPerceptionGetSnapshot(out var snapshot).IsOk)
{
Debug.LogError($"{nameof(MlPerception.MLPerceptionGetSnapshot)} Failed!");
return;
}
if (leftHandData.TrackingState == TrackingState.Tracked)
{
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Hand_Center] = GetPoseData(ref staticHandTrackingData.left.hand_center, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Wrist_Center] = GetPoseData(ref staticHandTrackingData.left.wrist.center, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Wrist_Radial] = GetPoseData(ref staticHandTrackingData.left.wrist.radial, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Wrist_Ulnar] = GetPoseData(ref staticHandTrackingData.left.wrist.ulnar, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_Tip] = GetPoseData(ref staticHandTrackingData.left.index.tip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_DIP] = GetPoseData(ref staticHandTrackingData.left.index.dip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_PIP] = GetPoseData(ref staticHandTrackingData.left.index.pip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_MCP] = GetPoseData(ref staticHandTrackingData.left.index.mcp, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_Tip] = GetPoseData(ref staticHandTrackingData.left.middle.tip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_DIP] = GetPoseData(ref staticHandTrackingData.left.middle.dip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_PIP] = GetPoseData(ref staticHandTrackingData.left.middle.pip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_MCP] = GetPoseData(ref staticHandTrackingData.left.middle.mcp, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_Tip] = GetPoseData(ref staticHandTrackingData.left.ring.tip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_DIP] = GetPoseData(ref staticHandTrackingData.left.ring.dip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_PIP] = GetPoseData(ref staticHandTrackingData.left.ring.pip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_MCP] = GetPoseData(ref staticHandTrackingData.left.ring.mcp, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_Tip] = GetPoseData(ref staticHandTrackingData.left.pinky.tip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_DIP] = GetPoseData(ref staticHandTrackingData.left.pinky.dip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_PIP] = GetPoseData(ref staticHandTrackingData.left.pinky.pip, ref snapshot);
tempLeftKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_MCP] = GetPoseData(ref staticHandTrackingData.left.pinky.mcp, ref snapshot);
leftHandData = SyncHandPoseData(leftHandData, Handedness.Left);
leftHandData.UpdatedAt = now;
}
if (rightHandData.TrackingState == TrackingState.Tracked)
{
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Hand_Center] = GetPoseData(ref staticHandTrackingData.right.hand_center, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Wrist_Center] = GetPoseData(ref staticHandTrackingData.right.wrist.center, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Wrist_Radial] = GetPoseData(ref staticHandTrackingData.right.wrist.radial, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Wrist_Ulnar] = GetPoseData(ref staticHandTrackingData.right.wrist.ulnar, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_Tip] = GetPoseData(ref staticHandTrackingData.right.index.tip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_DIP] = GetPoseData(ref staticHandTrackingData.right.index.dip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_PIP] = GetPoseData(ref staticHandTrackingData.right.index.pip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Index_MCP] = GetPoseData(ref staticHandTrackingData.right.index.mcp, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_Tip] = GetPoseData(ref staticHandTrackingData.right.middle.tip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_DIP] = GetPoseData(ref staticHandTrackingData.right.middle.dip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_PIP] = GetPoseData(ref staticHandTrackingData.right.middle.pip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Middle_MCP] = GetPoseData(ref staticHandTrackingData.right.middle.mcp, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_Tip] = GetPoseData(ref staticHandTrackingData.right.ring.tip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_DIP] = GetPoseData(ref staticHandTrackingData.right.ring.dip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_PIP] = GetPoseData(ref staticHandTrackingData.right.ring.pip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Ring_MCP] = GetPoseData(ref staticHandTrackingData.right.ring.mcp, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_Tip] = GetPoseData(ref staticHandTrackingData.right.pinky.tip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_DIP] = GetPoseData(ref staticHandTrackingData.right.pinky.dip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_PIP] = GetPoseData(ref staticHandTrackingData.right.pinky.pip, ref snapshot);
tempRightKeyPoses[(int)MlHandTracking.MLHandTrackingKeyPoint.Pinky_MCP] = GetPoseData(ref staticHandTrackingData.right.pinky.mcp, ref snapshot);
rightHandData = SyncHandPoseData(rightHandData, Handedness.Right);
rightHandData.UpdatedAt = now;
}
if (!MlPerception.MLPerceptionReleaseSnapshot(snapshot).IsOk)
{
Debug.LogError($"{nameof(MlPerception.MLPerceptionReleaseSnapshot)} Failed!");
}
}
else
{
Debug.LogError($"{nameof(MlHandTracking.MLHandTrackingGetStaticData)} Failed!");
}
}
GetOrAddController(Handedness.Left).UpdateController(leftHandData);
GetOrAddController(Handedness.Right).UpdateController(rightHandData);
}
}
private void UpdateControllerData(MlInput.MLInputControllerState inputState, MlController.MLControllerState controllerState)
{
var lastState = TrackingState;
lastControllerPose = currentControllerPose;
var bestStream = controllerState.stream[0];
for (int i = 0; i < controllerState.stream.Length; i++)
{
if (!controllerState.stream[i].is_active)
{
continue;
}
if (bestStream.mode <= controllerState.stream[i].mode)
{
bestStream = controllerState.stream[i];
}
}
if (!MlPerception.MLPerceptionGetSnapshot(out var snapshot).IsOk)
{
Debug.LogError("Failed to get perception snapshot!");
}
if (bestStream.is_active)
{
if (!MlSnapshot.MLSnapshotGetTransform(snapshot, bestStream.coord_frame_controller, ref controllerTransform).IsOk)
{
Debug.LogError($"Failed to get snapshot transform for controller {controllerState.controller_id}:{bestStream}");
}
}
if (!MlPerception.MLPerceptionReleaseSnapshot(snapshot).IsOk)
{
Debug.LogError("Failed to release perception snapshot!");
}
if (inputState.type == MlInput.MLInputControllerType.Device)
{
// The source is either a hand or a controller that supports pointing.
// We can now check for position and rotation.
IsPositionAvailable = bestStream.mode > MlController.MLControllerMode.Imu3Dof;
if (IsPositionAvailable)
{
IsPositionApproximate = controllerState.accuracy <= MlController.MLControllerCalibAccuracy.Medium;
}
else
{
IsPositionApproximate = false;
}
IsRotationAvailable = bestStream.is_active;
// Devices are considered tracked if we receive position OR rotation data from the sensors.
TrackingState = (IsPositionAvailable || IsRotationAvailable) ? TrackingState.Tracked : TrackingState.NotTracked;
}
else
{
// The input source does not support tracking.
TrackingState = TrackingState.NotApplicable;
}
currentControllerPose.Position = controllerTransform.position;
currentControllerPose.Rotation = controllerTransform.rotation;
// Raise input system events if it is enabled.
if (lastState != TrackingState)
{
InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked && lastControllerPose != currentControllerPose)
{
if (IsPositionAvailable && IsRotationAvailable)
{
InputSystem?.RaiseSourcePoseChanged(InputSource, this, currentControllerPose);
}
else if (IsPositionAvailable && !IsRotationAvailable)
{
InputSystem?.RaiseSourcePositionChanged(InputSource, this, currentControllerPose.Position);
}
else if (!IsPositionAvailable && IsRotationAvailable)
{
InputSystem?.RaiseSourceRotationChanged(InputSource, this, currentControllerPose.Rotation);
}
}
}
