public void ProcessFrame(PXCMCapture.Sample sample)
{
this.faceBox = null;
this.heartRate = null;
this.landmarks = null;
this.expressions = new Dictionary <string, float>();
if (this.faceData.Update().Succeeded())
{
var firstFace = this.faceData.QueryFaces().FirstOrDefault();
if (firstFace != null)
{
// face detection - the bounding rectangle of the face.
var localFaceBox = default(PXCMRectI32);
if (firstFace.QueryDetection()?.QueryBoundingRect(out localFaceBox) == true)
{
this.faceBox = localFaceBox;
}
// facial landmarks
var landmarks = firstFace.QueryLandmarks()?.QueryPoints(
out this.landmarks);
// heart rate.
this.heartRate = firstFace.QueryPulse()?.QueryHeartRate();
// facial expressions
var expressions = firstFace.QueryExpressions();
if (expressions != null)
{
foreach (Expression expressionType in Enum.GetValues(typeof(Expression)))
{
ExpressionResult expressionResult = default(ExpressionResult);
if (expressions.QueryExpression(expressionType, out expressionResult))
{
this.expressions[expressionType.GetName()] = expressionResult.intensity;
}
}
}
}
}
}
public void DisplayExpression(PXCMFaceData.ExpressionsData.FaceExpressionResult expressionResult, PXCMFaceData.ExpressionsData.FaceExpression faceExpression)
{
ExpressionGUIText.GetComponent <TextMesh> ().text = faceExpression.ToString() + "\n" + expressionResult.intensity.ToString();
}
private bool UpdateFace(AnimationTrigger animationTrigger)
{
animationTrigger.IsAnimationDataValid = false;
// Get the closest face
PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
{
// PXCMFaceData.QueryFaceByIndex(0) failed.
return false;
}
PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
if (pxcmLandmarksData == null)
{
// PXCMFaceData.Face.QueryLandmarks() failed.
return false;
}
Dictionary<PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary<PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
{
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
};
foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
{
PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);
if (landmarkPoint == null)
{
// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
return false;
}
landmarkPoints.Add(landmarkType, landmarkPoint);
}
PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
if (pxcmExpressionsData == null)
{
// PXCMFaceData.Face.QueryExpressions() failed.
return false;
}
animationTrigger.IsAnimationDataValid = true;
PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
if (pxcmCaptureManager != null)
{
PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
if (pxcmCaptureDevice != null)
{
mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
}
}
animationTrigger.Animations.Clear();
float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;
float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
}
else
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
}
// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));
PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
}
else
{
// Error querying expression: EXPRESSION_SMILE.
return false;
}
return true;
}
private bool UpdateFace(AnimationTrigger animationTrigger)
{
animationTrigger.IsAnimationDataValid = false;
// Get the closest face
PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
if (pxcmFace == null)
{
return(false);
}
if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
{
// PXCMFaceData.QueryFaceByIndex(0) failed.
return(false);
}
PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
if (pxcmLandmarksData == null)
{
// PXCMFaceData.Face.QueryLandmarks() failed.
return(false);
}
Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
{
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
};
foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
{
PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);
if (landmarkPoint == null)
{
// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
return(false);
}
landmarkPoints.Add(landmarkType, landmarkPoint);
}
PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
if (pxcmExpressionsData == null)
{
// PXCMFaceData.Face.QueryExpressions() failed.
return(false);
}
animationTrigger.IsAnimationDataValid = true;
PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
if (pxcmCaptureManager != null)
{
PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
if (pxcmCaptureDevice != null)
{
mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
}
}
animationTrigger.Animations.Clear();
float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;
float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
}
else
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
}
// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));
PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
}
else
{
// Error querying expression: EXPRESSION_SMILE.
return(false);
}
return(true);
}
