/// <inheritdoc />
public bool TryGetHandMeshData(out HandMeshData handMeshData)
{
if (!lastHandMeshData.IsEmpty)
{
handMeshData = lastHandMeshData;
return(true);
}
handMeshData = HandMeshData.Empty;
return(false);
}
/// <summary>
/// Updates the hand controller with new hand data input.
/// </summary>
/// <param name="handData">Updated hand data.</param>
public void UpdateController(HandData handData)
{
if (!Enabled)
{
return;
}
var lastTrackingState = TrackingState;
TrackingState = handData.TrackingState;
if (lastTrackingState != TrackingState)
{
MixedRealityToolkit.InputSystem?.RaiseSourceTrackingStateChanged(InputSource, this, TrackingState);
}
if (TrackingState == TrackingState.Tracked)
{
IsPositionAvailable = true;
IsPositionApproximate = false;
IsRotationAvailable = true;
lastHandRootPose = handData.RootPose;
lastHandMeshData = handData.Mesh;
LastIsPinching = IsPinching;
LastIsGripping = IsGripping;
LastIsPointing = IsPointing;
UpdateJoints(handData);
UpdateIsPinching(handData);
UpdateIsIsPointing(handData);
UpdateIsIsGripping(handData);
UpdateIndexFingerTipPose();
UpdateGripPose();
UpdateFingerCurlStrength(handData);
UpdateBounds();
UpdateVelocity();
TrackedPoseId = handData.TrackedPoseId;
PinchStrength = handData.PinchStrength;
SpatialPointerPose = handData.PointerPose;
MixedRealityToolkit.InputSystem?.RaiseSourcePoseChanged(InputSource, this, handData.RootPose);
}
UpdateInteractionMappings();
}
/// <summary>
/// Updates the mesh visuailzation using latest hand mesh data.
/// </summary>
/// <param name="handMeshData">New hand mesh data.</param>
public void UpdateVisualization(HandMeshData handMeshData)
{
if (handMeshData.IsEmpty)
{
return;
}
if (meshFilter != null)
{
Mesh mesh = meshFilter.mesh;
if (lastHandMeshVertices == null)
{
lastHandMeshVertices = mesh.vertices;
}
bool meshChanged = false;
// On some platforms, mesh length counts may change as the hand mesh is updated.
// In order to update the vertices when the array sizes change, the mesh
// must be cleared per instructions here:
// https://docs.unity3d.com/ScriptReference/Mesh.html
if (lastHandMeshVertices != null &&
lastHandMeshVertices.Length != handMeshData.Vertices.Length)
{
meshChanged = true;
mesh.Clear();
}
mesh.vertices = handMeshData.Vertices;
mesh.normals = handMeshData.Normals;
lastHandMeshVertices = mesh.vertices;
if (meshChanged)
{
mesh.triangles = handMeshData.Triangles;
if (handMeshData.Uvs != null && handMeshData.Uvs.Length > 0)
{
mesh.uv = handMeshData.Uvs;
}
mesh.RecalculateBounds();
}
//meshFilter.transform.position = handMeshData.Position;
//meshFilter.transform.rotation = handData.RootPose.Rotation;
}
}
/// <summary>
/// Attempts to get updated hand mesh data.
/// </summary>
/// <param name="handedness">The handedness of the hand to get mesh data for.</param>
/// <param name="data">Mesh information retrieved in case of success.</param>
/// <returns>True, if mesh data could be loaded.</returns>
private bool TryGetUpdatedHandMeshData(Handedness handedness, out HandMeshData data)
{
if (OculusApi.GetMesh(handedness.ToMeshType(), out handMesh))
{
var vertices = new Vector3[handMesh.NumVertices];
for (int i = 0; i < handMesh.NumVertices; ++i)
{
vertices[i] = handMesh.VertexPositions[i];
}
var uvs = new Vector2[handMesh.NumVertices];
for (int i = 0; i < handMesh.NumVertices; ++i)
{
uvs[i] = new Vector2(handMesh.VertexUV0[i].x, -handMesh.VertexUV0[i].y);
}
var triangles = new int[handMesh.NumIndices];
for (int i = 0; i < handMesh.NumIndices; ++i)
{
triangles[i] = handMesh.Indices[handMesh.NumIndices - i - 1];
}
var normals = new Vector3[handMesh.NumVertices];
for (int i = 0; i < handMesh.NumVertices; ++i)
{
normals[i] = handMesh.VertexNormals[i];
}
data = new HandMeshData(vertices, triangles, normals, uvs);
return(true);
}
data = default;
return(false);
}
/// <summary>
/// Attempts to get updated hand mesh data.
/// </summary>
/// <param name="spatialInteractionSourceState">Platform provided current input source state for the hand.</param>
/// <param name="handPose">Hand pose information retrieved for joint conversion.</param>
/// <param name="data">Mesh information retrieved in case of success.</param>
/// <returns>True, if mesh data could be loaded.</returns>
private bool TryGetUpdatedHandMeshData(SpatialInteractionSourceState spatialInteractionSourceState, HandPose handPose, out HandMeshData data)
{
if (!handMeshObservers.ContainsKey(spatialInteractionSourceState.Source.Handedness) &&
!HasRequestedHandMeshObserver(spatialInteractionSourceState.Source.Handedness))
{
SetHandMeshObserver(spatialInteractionSourceState);
}
if (handMeshObservers.TryGetValue(spatialInteractionSourceState.Source.Handedness, out var handMeshObserver) && handMeshTriangleIndices == null)
{
var indexCount = handMeshObserver.TriangleIndexCount;
var indices = new ushort[indexCount];
handMeshObserver.GetTriangleIndices(indices);
handMeshTriangleIndices = new int[indexCount];
Array.Copy(indices, handMeshTriangleIndices, (int)handMeshObserver.TriangleIndexCount);
// Compute neutral pose
var neutralPoseVertices = new Vector3[handMeshObserver.VertexCount];
var neutralPose = handMeshObserver.NeutralPose;
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(neutralPose);
handMeshVertexState.GetVertices(vertexAndNormals);
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
neutralPoseVertices[i] = vertexAndNormals[i].Position.ToUnity();
}
// Compute UV mapping
InitializeHandMeshUVs(neutralPoseVertices);
}
if (handMeshObserver != null && handMeshTriangleIndices != null)
{
var vertexAndNormals = new HandMeshVertex[handMeshObserver.VertexCount];
var handMeshVertexState = handMeshObserver.GetVertexStateForPose(handPose);
handMeshVertexState.GetVertices(vertexAndNormals);
var meshTransform = handMeshVertexState.CoordinateSystem.TryGetTransformTo(spatialCoordinateSystem);
if (meshTransform.HasValue)
{
System.Numerics.Matrix4x4.Decompose(meshTransform.Value, out var scale, out var rotation, out var translation);
var handMeshVertices = new Vector3[handMeshObserver.VertexCount];
var handMeshNormals = new Vector3[handMeshObserver.VertexCount];
for (int i = 0; i < handMeshObserver.VertexCount; i++)
{
handMeshVertices[i] = vertexAndNormals[i].Position.ToUnity();
handMeshNormals[i] = vertexAndNormals[i].Normal.ToUnity();
}
data = new HandMeshData(
handMeshVertices,
handMeshTriangleIndices,
handMeshNormals,
handMeshUVs);
return(true);
}
}
return(false);
}
