static void MapToAuthorLandmarks(Vector3[] vertexPositions, ARFoundationFace face)
{
var rotation = Quaternion.identity;
var chin = new Pose(vertexPositions[(int)ARKitFaceMeshLandmark.ChinMiddle], rotation);
var noseBridge = new Pose(vertexPositions[(int)ARKitFaceMeshLandmark.NoseBridge], rotation);
var noseTip = new Pose(vertexPositions[(int)ARKitFaceMeshLandmark.NoseTip], rotation);
var rightEyeOuter = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeOuter];
var rightEyeUpper = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeTop];
var rightEyeInner = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeInner];
var rightEyeLower = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeBottom];
var rightEyeAverage = (rightEyeOuter + rightEyeUpper + rightEyeInner + rightEyeLower) * 0.25f;
var rightEye = new Pose(rightEyeAverage, rotation);
var leftEyeOuter = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeOuter];
var leftEyeUpper = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeTop];
var leftEyeInner = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeInner];
var leftEyeLower = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeBottom];
var leftEyeAverage = (leftEyeOuter + leftEyeUpper + leftEyeInner + leftEyeLower) * 0.25f;
var leftEye = new Pose(leftEyeAverage, rotation);
var rightEyebrowInner = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyebrowInner];
var rightEyebrowOuter = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyebrowOuter];
var rightEyebrowAverage = (rightEyebrowInner + rightEyebrowOuter) * 0.5f;
var rightEyebrow = new Pose(rightEyebrowAverage, rotation);
var leftEyebrowInner = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyebrowInner];
var leftEyebrowOuter = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyebrowOuter];
var leftEyebrowAverage = (leftEyebrowInner + leftEyebrowOuter) * 0.5f;
var leftEyebrow = new Pose(leftEyebrowAverage, rotation);
var upperLipLeft = vertexPositions[(int)ARKitFaceMeshLandmark.UpperLipLeft];
var upperLipRight = vertexPositions[(int)ARKitFaceMeshLandmark.UpperLipRight];
var upperLipAverage = (upperLipLeft + upperLipRight) * 0.5f;
var upperLip = new Pose(upperLipAverage, rotation);
var lowerLipLeft = vertexPositions[(int)ARKitFaceMeshLandmark.LowerLipLeft];
var lowerLipRight = vertexPositions[(int)ARKitFaceMeshLandmark.LowerLipRight];
var lowerLipAverage = (lowerLipLeft + lowerLipRight) * 0.5f;
var lowerLip = new Pose(lowerLipAverage, rotation);
var leftLipCorner = vertexPositions[(int)ARKitFaceMeshLandmark.LeftLipCorner];
var rightLipCorner = vertexPositions[(int)ARKitFaceMeshLandmark.RightLipCorner];
var mouthAverage = (upperLipAverage + lowerLipAverage + leftLipCorner + rightLipCorner) * 0.25f;
var mouth = new Pose(mouthAverage, rotation);
var facePose = face.pose;
face.LandmarkPoses[MRFaceLandmark.Chin] = facePose.ApplyOffsetTo(chin);
face.LandmarkPoses[MRFaceLandmark.NoseBridge] = facePose.ApplyOffsetTo(noseBridge);
face.LandmarkPoses[MRFaceLandmark.NoseTip] = facePose.ApplyOffsetTo(noseTip);
face.LandmarkPoses[MRFaceLandmark.Mouth] = facePose.ApplyOffsetTo(mouth);
face.LandmarkPoses[MRFaceLandmark.UpperLip] = facePose.ApplyOffsetTo(upperLip);
face.LandmarkPoses[MRFaceLandmark.LowerLip] = facePose.ApplyOffsetTo(lowerLip);
face.LandmarkPoses[MRFaceLandmark.RightEyebrow] = facePose.ApplyOffsetTo(rightEyebrow);
face.LandmarkPoses[MRFaceLandmark.LeftEyebrow] = facePose.ApplyOffsetTo(leftEyebrow);
face.LandmarkPoses[MRFaceLandmark.RightEye] = facePose.ApplyOffsetTo(rightEye);
face.LandmarkPoses[MRFaceLandmark.LeftEye] = facePose.ApplyOffsetTo(leftEye);
}
internal static void ToARFoundationFace(this ARFace face, XRFaceSubsystem xrFaceSubsystem, ref ARFoundationFace arFoundationFace)
{
if (arFoundationFace == null)
{
arFoundationFace = new ARFoundationFace(face.trackableId.ToMarsId());
}
arFoundationFace.pose = face.transform.GetWorldPose();
var indices = face.indices;
if (indices.Length > 0)
{
k_Vector3Buffer.Clear();
foreach (var vertex in face.vertices)
{
k_Vector3Buffer.Add(vertex);
}
arFoundationFace.Mesh.SetVertices(k_Vector3Buffer);
k_Vector3Buffer.Clear();
foreach (var normal in face.normals)
{
k_Vector3Buffer.Add(normal);
}
arFoundationFace.Mesh.SetNormals(k_Vector3Buffer);
k_Vector2Buffer.Clear();
foreach (var uv in face.uvs)
{
k_Vector2Buffer.Add(uv);
}
arFoundationFace.Mesh.SetUVs(0, k_Vector2Buffer);
k_IntBuffer.Clear();
foreach (var index in indices)
{
k_IntBuffer.Add(index);
}
arFoundationFace.Mesh.SetTriangles(k_IntBuffer, 0);
#if !UNITY_EDITOR
#if UNITY_IOS && INCLUDE_ARKIT_FACE_PLUGIN
// For iOS, we use ARKit Face Blendshapes to determine expressions
arFoundationFace.GenerateLandmarks();
arFoundationFace.CalculateExpressions(xrFaceSubsystem, face.trackableId);
#elif UNITY_ANDROID
// For Android, we use the position of the face landmarks to determine expressions
arFoundationFace.GenerateLandmarks();
arFoundationFace.CalculateExpressions(ARCoreFaceLandmarksExtensions.LandmarkPositions);
#endif
#endif
}
}
internal static void GenerateLandmarks(this ARFoundationFace arFoundationFace)
{
var landmarkTriangleIndices = ARKitLandmarkMeshIndices.instance.landmarkTriangleIndices;
for (var i = 0; i < k_MeshLandmarksCount; i++)
{
var point = PointFromTriangle(arFoundationFace.Mesh.vertices, arFoundationFace.Mesh.triangles, landmarkTriangleIndices[i]);
k_LandmarkPositions[i] = point;
}
MapToAuthorLandmarks(k_LandmarkPositions, arFoundationFace);
}
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace)
{
const float raiseMinimum = 0.20f;
var left = arFoundationFace.Expressions[MRFaceExpression.LeftEyebrowRaise];
var right = arFoundationFace.Expressions[MRFaceExpression.RightEyebrowRaise];
if (left < raiseMinimum || right < raiseMinimum)
{
return(0f);
}
return(Mathf.Clamp01((left + right) * 0.5f) * 1.2f - raiseMinimum);
}
internal static void CalculateExpressions(this ARFoundationFace arFoundationFace, Vector3[] landmarks)
{
var expressions = arFoundationFace.Expressions;
expressions[MRFaceExpression.LeftEyeClose] = k_LeftEyeClose.Update(landmarks);
expressions[MRFaceExpression.RightEyeClose] = k_RightEyeClose.Update(landmarks);
expressions[MRFaceExpression.LeftEyebrowRaise] = k_LeftBrowRaise.Update(landmarks);
expressions[MRFaceExpression.RightEyebrowRaise] = k_RightBrowRaise.Update(landmarks);
expressions[MRFaceExpression.BothEyebrowsRaise] = k_BothBrowsRaise.Update(landmarks);
expressions[MRFaceExpression.LeftLipCornerRaise] = k_LeftLipCornerRaise.Update(landmarks);
expressions[MRFaceExpression.RightLipCornerRaise] = k_RightLipCornerRaise.Update(landmarks);
expressions[MRFaceExpression.Smile] = k_Smile.Update(landmarks);
expressions[MRFaceExpression.MouthOpen] = k_MouthOpen.Update(landmarks);
}
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace)
{
const float smileMinimum = 0.15f;
var left = arFoundationFace.Expressions[MRFaceExpression.LeftLipCornerRaise];
var right = arFoundationFace.Expressions[MRFaceExpression.RightLipCornerRaise];
// if both corners aren't at least slightly raised, no smile.
if (left < smileMinimum || right < smileMinimum)
{
return(0f);
}
return(Mathf.Clamp01((left + right) * 0.5f) * 1.2f - smileMinimum);
}
internal static void CalculateExpressions(this ARFoundationFace arFoundationFace, XRFaceSubsystem xrFaceSubsystem, TrackableId faceId)
{
GetFaceBlendShapes(xrFaceSubsystem, faceId, k_BlendShapes);
var expressions = arFoundationFace.Expressions;
expressions[MRFaceExpression.LeftEyeClose] = k_LeftEyeClose.Update(k_BlendShapes);
expressions[MRFaceExpression.RightEyeClose] = k_RightEyeClose.Update(k_BlendShapes);
expressions[MRFaceExpression.LeftEyebrowRaise] = k_LeftBrowRaise.Update(k_BlendShapes);
expressions[MRFaceExpression.RightEyebrowRaise] = k_RightBrowRaise.Update(k_BlendShapes);
expressions[MRFaceExpression.BothEyebrowsRaise] = k_BothBrowsRaise.Update(k_BlendShapes);
expressions[MRFaceExpression.LeftLipCornerRaise] = k_LeftLipCornerRaise.Update(k_BlendShapes);
expressions[MRFaceExpression.RightLipCornerRaise] = k_RightLipCornerRaise.Update(k_BlendShapes);
expressions[MRFaceExpression.Smile] = k_Smile.Update(k_BlendShapes);
expressions[MRFaceExpression.MouthOpen] = k_MouthOpen.Update(k_BlendShapes);
}
internal float Update(ARFoundationFace arFoundationFace)
{
var time = Time.time;
m_Coefficient = Mathf.Lerp(m_PreviousCoefficient, GetRawCoefficient(arFoundationFace), m_SmoothingFilter);
if (m_Coefficient > m_Threshold && !m_Engaged)
{
m_Engaged = true;
if (Engaged != null && time - m_LastEngageEvent > m_EventCooldownInSeconds)
{
m_LastEngageEvent = time;
Engaged(m_Coefficient);
}
}
else if (m_Coefficient < m_Threshold && m_Engaged)
{
DisengageIfAppropriate(time);
}
m_PreviousCoefficient = m_Coefficient;
return(m_Coefficient);
}
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace)
{
return(arFoundationFace.Expressions[MRFaceExpression.RightEyeClose]);
}
internal abstract float GetRawCoefficient(ARFoundationFace arFoundationFace);
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace)
{
return(arFoundationFace.Expressions[MRFaceExpression.MouthOpen]);
}
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace)
{
return(arFoundationFace.Expressions[MRFaceExpression.LeftLipCornerRaise]);
}
