static void MapToAuthorLandmarks(Vector3[] vertexPositions, ARFoundationFace face) { var rotation = Quaternion.identity; var chin = new Pose(vertexPositions[(int)ARKitFaceMeshLandmark.ChinMiddle], rotation); var noseBridge = new Pose(vertexPositions[(int)ARKitFaceMeshLandmark.NoseBridge], rotation); var noseTip = new Pose(vertexPositions[(int)ARKitFaceMeshLandmark.NoseTip], rotation); var rightEyeOuter = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeOuter]; var rightEyeUpper = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeTop]; var rightEyeInner = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeInner]; var rightEyeLower = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyeBottom]; var rightEyeAverage = (rightEyeOuter + rightEyeUpper + rightEyeInner + rightEyeLower) * 0.25f; var rightEye = new Pose(rightEyeAverage, rotation); var leftEyeOuter = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeOuter]; var leftEyeUpper = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeTop]; var leftEyeInner = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeInner]; var leftEyeLower = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyeBottom]; var leftEyeAverage = (leftEyeOuter + leftEyeUpper + leftEyeInner + leftEyeLower) * 0.25f; var leftEye = new Pose(leftEyeAverage, rotation); var rightEyebrowInner = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyebrowInner]; var rightEyebrowOuter = vertexPositions[(int)ARKitFaceMeshLandmark.RightEyebrowOuter]; var rightEyebrowAverage = (rightEyebrowInner + rightEyebrowOuter) * 0.5f; var rightEyebrow = new Pose(rightEyebrowAverage, rotation); var leftEyebrowInner = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyebrowInner]; var leftEyebrowOuter = vertexPositions[(int)ARKitFaceMeshLandmark.LeftEyebrowOuter]; var leftEyebrowAverage = (leftEyebrowInner + leftEyebrowOuter) * 0.5f; var leftEyebrow = new Pose(leftEyebrowAverage, rotation); var upperLipLeft = vertexPositions[(int)ARKitFaceMeshLandmark.UpperLipLeft]; var upperLipRight = vertexPositions[(int)ARKitFaceMeshLandmark.UpperLipRight]; var upperLipAverage = (upperLipLeft + upperLipRight) * 0.5f; var upperLip = new Pose(upperLipAverage, rotation); var lowerLipLeft = vertexPositions[(int)ARKitFaceMeshLandmark.LowerLipLeft]; var lowerLipRight = vertexPositions[(int)ARKitFaceMeshLandmark.LowerLipRight]; var lowerLipAverage = (lowerLipLeft + lowerLipRight) * 0.5f; var lowerLip = new Pose(lowerLipAverage, rotation); var leftLipCorner = vertexPositions[(int)ARKitFaceMeshLandmark.LeftLipCorner]; var rightLipCorner = vertexPositions[(int)ARKitFaceMeshLandmark.RightLipCorner]; var mouthAverage = (upperLipAverage + lowerLipAverage + leftLipCorner + rightLipCorner) * 0.25f; var mouth = new Pose(mouthAverage, rotation); var facePose = face.pose; face.LandmarkPoses[MRFaceLandmark.Chin] = facePose.ApplyOffsetTo(chin); face.LandmarkPoses[MRFaceLandmark.NoseBridge] = facePose.ApplyOffsetTo(noseBridge); face.LandmarkPoses[MRFaceLandmark.NoseTip] = facePose.ApplyOffsetTo(noseTip); face.LandmarkPoses[MRFaceLandmark.Mouth] = facePose.ApplyOffsetTo(mouth); face.LandmarkPoses[MRFaceLandmark.UpperLip] = facePose.ApplyOffsetTo(upperLip); face.LandmarkPoses[MRFaceLandmark.LowerLip] = facePose.ApplyOffsetTo(lowerLip); face.LandmarkPoses[MRFaceLandmark.RightEyebrow] = facePose.ApplyOffsetTo(rightEyebrow); face.LandmarkPoses[MRFaceLandmark.LeftEyebrow] = facePose.ApplyOffsetTo(leftEyebrow); face.LandmarkPoses[MRFaceLandmark.RightEye] = facePose.ApplyOffsetTo(rightEye); face.LandmarkPoses[MRFaceLandmark.LeftEye] = facePose.ApplyOffsetTo(leftEye); }
internal static void ToARFoundationFace(this ARFace face, XRFaceSubsystem xrFaceSubsystem, ref ARFoundationFace arFoundationFace) { if (arFoundationFace == null) { arFoundationFace = new ARFoundationFace(face.trackableId.ToMarsId()); } arFoundationFace.pose = face.transform.GetWorldPose(); var indices = face.indices; if (indices.Length > 0) { k_Vector3Buffer.Clear(); foreach (var vertex in face.vertices) { k_Vector3Buffer.Add(vertex); } arFoundationFace.Mesh.SetVertices(k_Vector3Buffer); k_Vector3Buffer.Clear(); foreach (var normal in face.normals) { k_Vector3Buffer.Add(normal); } arFoundationFace.Mesh.SetNormals(k_Vector3Buffer); k_Vector2Buffer.Clear(); foreach (var uv in face.uvs) { k_Vector2Buffer.Add(uv); } arFoundationFace.Mesh.SetUVs(0, k_Vector2Buffer); k_IntBuffer.Clear(); foreach (var index in indices) { k_IntBuffer.Add(index); } arFoundationFace.Mesh.SetTriangles(k_IntBuffer, 0); #if !UNITY_EDITOR #if UNITY_IOS && INCLUDE_ARKIT_FACE_PLUGIN // For iOS, we use ARKit Face Blendshapes to determine expressions arFoundationFace.GenerateLandmarks(); arFoundationFace.CalculateExpressions(xrFaceSubsystem, face.trackableId); #elif UNITY_ANDROID // For Android, we use the position of the face landmarks to determine expressions arFoundationFace.GenerateLandmarks(); arFoundationFace.CalculateExpressions(ARCoreFaceLandmarksExtensions.LandmarkPositions); #endif #endif } }
internal static void GenerateLandmarks(this ARFoundationFace arFoundationFace) { var landmarkTriangleIndices = ARKitLandmarkMeshIndices.instance.landmarkTriangleIndices; for (var i = 0; i < k_MeshLandmarksCount; i++) { var point = PointFromTriangle(arFoundationFace.Mesh.vertices, arFoundationFace.Mesh.triangles, landmarkTriangleIndices[i]); k_LandmarkPositions[i] = point; } MapToAuthorLandmarks(k_LandmarkPositions, arFoundationFace); }
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace) { const float raiseMinimum = 0.20f; var left = arFoundationFace.Expressions[MRFaceExpression.LeftEyebrowRaise]; var right = arFoundationFace.Expressions[MRFaceExpression.RightEyebrowRaise]; if (left < raiseMinimum || right < raiseMinimum) { return(0f); } return(Mathf.Clamp01((left + right) * 0.5f) * 1.2f - raiseMinimum); }
internal static void CalculateExpressions(this ARFoundationFace arFoundationFace, Vector3[] landmarks) { var expressions = arFoundationFace.Expressions; expressions[MRFaceExpression.LeftEyeClose] = k_LeftEyeClose.Update(landmarks); expressions[MRFaceExpression.RightEyeClose] = k_RightEyeClose.Update(landmarks); expressions[MRFaceExpression.LeftEyebrowRaise] = k_LeftBrowRaise.Update(landmarks); expressions[MRFaceExpression.RightEyebrowRaise] = k_RightBrowRaise.Update(landmarks); expressions[MRFaceExpression.BothEyebrowsRaise] = k_BothBrowsRaise.Update(landmarks); expressions[MRFaceExpression.LeftLipCornerRaise] = k_LeftLipCornerRaise.Update(landmarks); expressions[MRFaceExpression.RightLipCornerRaise] = k_RightLipCornerRaise.Update(landmarks); expressions[MRFaceExpression.Smile] = k_Smile.Update(landmarks); expressions[MRFaceExpression.MouthOpen] = k_MouthOpen.Update(landmarks); }
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace) { const float smileMinimum = 0.15f; var left = arFoundationFace.Expressions[MRFaceExpression.LeftLipCornerRaise]; var right = arFoundationFace.Expressions[MRFaceExpression.RightLipCornerRaise]; // if both corners aren't at least slightly raised, no smile. if (left < smileMinimum || right < smileMinimum) { return(0f); } return(Mathf.Clamp01((left + right) * 0.5f) * 1.2f - smileMinimum); }
internal static void CalculateExpressions(this ARFoundationFace arFoundationFace, XRFaceSubsystem xrFaceSubsystem, TrackableId faceId) { GetFaceBlendShapes(xrFaceSubsystem, faceId, k_BlendShapes); var expressions = arFoundationFace.Expressions; expressions[MRFaceExpression.LeftEyeClose] = k_LeftEyeClose.Update(k_BlendShapes); expressions[MRFaceExpression.RightEyeClose] = k_RightEyeClose.Update(k_BlendShapes); expressions[MRFaceExpression.LeftEyebrowRaise] = k_LeftBrowRaise.Update(k_BlendShapes); expressions[MRFaceExpression.RightEyebrowRaise] = k_RightBrowRaise.Update(k_BlendShapes); expressions[MRFaceExpression.BothEyebrowsRaise] = k_BothBrowsRaise.Update(k_BlendShapes); expressions[MRFaceExpression.LeftLipCornerRaise] = k_LeftLipCornerRaise.Update(k_BlendShapes); expressions[MRFaceExpression.RightLipCornerRaise] = k_RightLipCornerRaise.Update(k_BlendShapes); expressions[MRFaceExpression.Smile] = k_Smile.Update(k_BlendShapes); expressions[MRFaceExpression.MouthOpen] = k_MouthOpen.Update(k_BlendShapes); }
internal float Update(ARFoundationFace arFoundationFace) { var time = Time.time; m_Coefficient = Mathf.Lerp(m_PreviousCoefficient, GetRawCoefficient(arFoundationFace), m_SmoothingFilter); if (m_Coefficient > m_Threshold && !m_Engaged) { m_Engaged = true; if (Engaged != null && time - m_LastEngageEvent > m_EventCooldownInSeconds) { m_LastEngageEvent = time; Engaged(m_Coefficient); } } else if (m_Coefficient < m_Threshold && m_Engaged) { DisengageIfAppropriate(time); } m_PreviousCoefficient = m_Coefficient; return(m_Coefficient); }
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace) { return(arFoundationFace.Expressions[MRFaceExpression.RightEyeClose]); }
internal abstract float GetRawCoefficient(ARFoundationFace arFoundationFace);
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace) { return(arFoundationFace.Expressions[MRFaceExpression.MouthOpen]); }
internal override float GetRawCoefficient(ARFoundationFace arFoundationFace) { return(arFoundationFace.Expressions[MRFaceExpression.LeftLipCornerRaise]); }