static XRFace XRFaceFromIMRFace(IMRFace mrFace)
{
var face = new XRFace
{
m_TrackableId = new TrackableId(mrFace.id.subId1, mrFace.id.subId2),
m_Pose = mrFace.pose,
m_TrackingState = TrackingState.Tracking,
#if ARSUBSYSTEMS_3_OR_NEWER
m_LeftEyePose = Pose.identity,
m_RightEyePose = Pose.identity,
#endif
};
#if ARSUBSYSTEMS_3_OR_NEWER
if (mrFace.LandmarkPoses.TryGetValue(MRFaceLandmark.LeftEye, out var pose))
{
face.m_LeftEyePose = new Pose(pose.position - mrFace.pose.position,
Quaternion.Inverse(mrFace.pose.rotation) * pose.rotation);
}
if (mrFace.LandmarkPoses.TryGetValue(MRFaceLandmark.RightEye, out pose))
{
face.m_RightEyePose = new Pose(pose.position - mrFace.pose.position,
Quaternion.Inverse(mrFace.pose.rotation) * pose.rotation);
}
#endif
return(face);
}
void OnFaceAdded(IMRFace face)
{
var id = face.id;
var landmarks = GetLandmarkSet(id.ToString());
landmarks.SetEnabled(true);
m_LandmarkSets[id] = landmarks;
UpdateLandmarks(face, landmarks);
}
void OnFaceUpdated(IMRFace face)
{
m_FaceEvents.Add(new FaceEvent
{
Time = GetCurrentTime(),
FaceData = new SerializedFaceData(face, m_LandmarkEnumValues, m_ExpressionEnumValues),
EventType = TrackableEventType.Updated
});
}
void FaceRemoved(IMRFace face)
{
if (!m_AddedFaces.Remove(face.id))
{
m_UpdatedFaces.Remove(face.id);
m_RemovedFaces.Add(face.id);
}
m_Faces.Remove(new TrackableId(face.id.subId1, face.id.subId2));
}
void FaceUpdated(IMRFace face)
{
if (m_Faces.TryGetValue(face.id, out var faceObj))
{
faceObj.transform.SetLocalPose(this.ApplyOffsetToPose(face.pose));
}
else
{
FaceAdded(face);
}
}
void FaceRemoved(IMRFace face)
{
// destroy face mesh
if (m_FaceMeshRenderer != null)
{
UnityObjectUtils.Destroy(m_FaceMeshRenderer.gameObject);
}
// reset reference
m_FaceMeshRenderer = null;
}
void OnFaceRemoved(IMRFace face)
{
LandmarkSet landmarks;
var id = face.id;
if (m_LandmarkSets.TryGetValue(id, out landmarks))
{
landmarks.SetEnabled(false);
m_LandmarkSets.Remove(id);
m_LandmarkSetPool.Enqueue(landmarks);
}
}
void FaceUpdated(IMRFace face)
{
if (m_AddedFaces.ContainsKey(face.id))
{
m_AddedFaces[face.id] = face;
}
else
{
m_UpdatedFaces[face.id] = face;
}
m_Faces[new TrackableId(face.id.subId1, face.id.subId2)] = face;
}
void FaceRemoved(IMRFace face)
{
var id = face.id;
var faceContainer = m_Faces[id];
m_FacePool.Enqueue(faceContainer);
m_Faces.Remove(id);
var faceObject = faceContainer.gameObject;
if (faceObject)
{
faceObject.SetActive(false);
}
}
void OnFaceUpdated(IMRFace face)
{
LandmarkSet landmarks;
var id = face.id;
if (!m_LandmarkSets.TryGetValue(id, out landmarks))
{
landmarks = GetLandmarkSet(id.ToString());
landmarks.SetEnabled(true);
m_LandmarkSets[id] = landmarks;
}
UpdateLandmarks(face, landmarks);
}
void FaceAdded(IMRFace face)
{
Debug.Assert(m_FaceMeshRenderer == null);
// spawn face mesh
var faceGO = Instantiate(m_FaceMeshPrefab);
faceGO.hideFlags = HideFlags.DontSave;
m_FaceMeshRenderer = faceGO.GetComponent <MeshRenderer>();
// get initial size and scale
var scaledSize = m_FaceMeshRenderer.bounds.size;
var initialScale = m_FaceMeshRenderer.transform.localScale;
// calculate unscaled size
m_OriginalUnscaledSize.x = initialScale.x > 0 ? scaledSize.x / initialScale.x : scaledSize.x;
m_OriginalUnscaledSize.y = initialScale.y > 0 ? scaledSize.y / initialScale.y : scaledSize.y;
m_OriginalUnscaledSize.z = initialScale.z > 0 ? scaledSize.z / initialScale.z : scaledSize.z;
FaceUpdated(face);
}
/// <summary>
/// Calculates the Bounds of a MRFace using landmark positions.
/// </summary>
/// <returns>The min Bounds tha contains all the present face landmarks</returns>
/// <param name="face">Face.</param>
public static Bounds GetBounds(this IMRFace face)
{
// face pose position as the bound's center
var center = face.pose.position;
// default size
var size = Vector3.one;
// calculate size from landmarks
if (face.LandmarkPoses != null)
{
// get extreme landmark positions
var lowestX = float.PositiveInfinity;
var highestX = float.NegativeInfinity;
var lowestY = float.PositiveInfinity;
var highestY = float.NegativeInfinity;
var lowestZ = float.PositiveInfinity;
var highestZ = float.NegativeInfinity;
foreach (var landmarkPose in face.LandmarkPoses)
{
var landmarkPosition = landmarkPose.Value.position;
lowestX = Mathf.Min(lowestX, landmarkPosition.x);
highestX = Mathf.Max(highestX, landmarkPosition.x);
lowestY = Mathf.Min(lowestY, landmarkPosition.y);
highestY = Mathf.Max(highestY, landmarkPosition.y);
lowestZ = Mathf.Min(lowestZ, landmarkPosition.z);
highestZ = Mathf.Max(highestZ, landmarkPosition.z);
}
// calculate bounding box using the extreme values
size.x = highestX - lowestX;
size.y = highestY - lowestY;
size.z = highestZ - lowestZ;
}
return(new Bounds(center, size));
}
/// <summary>
/// Constructs a <see cref="SerializedFaceData"/> from an <see cref="IMRFace"/>
/// </summary>
/// <param name="face">Face data to serialize</param>
/// <param name="landmarkEnumValues">Enumeration values to map 1:1 with serialized array of face landmark poses</param>
/// <param name="expressionEnumValues">Enumeration values to map 1:1 with serialized array of face expression coefficients</param>
public SerializedFaceData(IMRFace face, MRFaceLandmark[] landmarkEnumValues, MRFaceExpression[] expressionEnumValues)
{
m_ID = face.id;
m_Pose = face.pose;
m_Mesh = face.Mesh;
var landmarksCount = landmarkEnumValues.Length;
m_LandmarkPoses = new Pose[landmarksCount];
for (var i = 0; i < landmarksCount; i++)
{
m_LandmarkPoses[i] = face.LandmarkPoses[landmarkEnumValues[i]];
}
var expressionsCount = expressionEnumValues.Length;
m_ExpressionValues = new float[expressionsCount];
for (var i = 0; i < expressionsCount; i++)
{
m_ExpressionValues[i] = face.Expressions[expressionEnumValues[i]];
}
}
void FaceUpdated(IMRFace face)
{
Debug.Assert(m_FaceMeshRenderer != null);
m_FaceMeshRenderer.transform.SetPositionAndRotation(face.pose.position, face.pose.rotation);
// get current face's desired size (based on landmarks)
var targetSize = face.GetBounds().size;
// dynamic scale (not keeping aspect ratio)
m_DynamicScale.x = m_OriginalUnscaledSize.x > 0 ? targetSize.x / m_OriginalUnscaledSize.x : targetSize.x;
m_DynamicScale.y = m_OriginalUnscaledSize.y > 0 ? targetSize.y / m_OriginalUnscaledSize.y : targetSize.y;
m_DynamicScale.z = m_OriginalUnscaledSize.z > 0 ? targetSize.z / m_OriginalUnscaledSize.z : targetSize.z;
// get average scale from the axis (to keep aspect ratio)
var averageScale = (m_DynamicScale.x + m_DynamicScale.y + m_DynamicScale.z) / 3;
m_DynamicScale.Set(averageScale, averageScale, averageScale);
// apply dynamic scale and scale factor
m_FaceMeshRenderer.transform.localScale = m_DynamicScale * m_ScaleFactor;
}
void UpdateLandmarks(IMRFace face, LandmarkSet landmarkSet)
{
var points = landmarkSet.points;
var renderers = landmarkSet.renderers;
var landmarkPoses = face.LandmarkPoses;
var landmarkCount = points.Length;
var sphereScale = this.GetCameraScale() * m_Scale * Vector3.one;
for (var i = 0; i < landmarkCount; i++)
{
Pose pose;
if (landmarkPoses.TryGetValue((MRFaceLandmark)i, out pose))
{
var point = points[i];
point.SetWorldPose(this.ApplyOffsetToPose(pose));
point.localScale = sphereScale;
renderers[i].enabled = true;
}
else
{
renderers[i].enabled = false;
}
}
}
void FaceAdded(IMRFace face)
{
var faceContainer = GetFace();
m_Faces.Add(face.id, faceContainer);
faceContainer.gameObject.SetActive(true);
var faceTransform = faceContainer.transform;
faceTransform.SetLocalPose(this.ApplyOffsetToPose(face.pose));
var meshFilter = faceContainer.meshFilter;
if (meshFilter != null)
{
var faceMesh = face.Mesh;
if (faceMesh != null)
{
meshFilter.sharedMesh = faceMesh;
}
else
{
meshFilter.GetComponent <MeshRenderer>().enabled = false;
}
}
}
void FaceAdded(IMRFace face)
{
m_AddedFaces.Add(face.id, face);
m_Faces.Add(new TrackableId(face.id.subId1, face.id.subId2), face);
}
