public override bool Process(Trigger trigger)
{
trigger.ErrorDetected = false;
if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Face))
{
trigger.ErrorDetected = true;
return(false);
}
AnimationTrigger animationTrigger = trigger as AnimationTrigger;
if (animationTrigger == null)
{
trigger.ErrorDetected = true;
return(false);
}
bool success = false;
if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.FaceModuleOutput != null)
{
success = UpdateFace(animationTrigger);
}
else
{
success = false;
}
return(success);
}
private bool UpdateFace(AnimationTrigger animationTrigger)
{
animationTrigger.IsAnimationDataValid = false;
// Get the closest face
PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
{
// PXCMFaceData.QueryFaceByIndex(0) failed.
return false;
}
PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
if (pxcmLandmarksData == null)
{
// PXCMFaceData.Face.QueryLandmarks() failed.
return false;
}
Dictionary<PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary<PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
{
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
};
foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
{
PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);
if (landmarkPoint == null)
{
// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
return false;
}
landmarkPoints.Add(landmarkType, landmarkPoint);
}
PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
if (pxcmExpressionsData == null)
{
// PXCMFaceData.Face.QueryExpressions() failed.
return false;
}
animationTrigger.IsAnimationDataValid = true;
PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
if (pxcmCaptureManager != null)
{
PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
if (pxcmCaptureDevice != null)
{
mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
}
}
animationTrigger.Animations.Clear();
float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;
float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
}
else
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
}
// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));
PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
}
else
{
// Error querying expression: EXPRESSION_SMILE.
return false;
}
return true;
}
private bool UpdateFace(AnimationTrigger animationTrigger)
{
animationTrigger.IsAnimationDataValid = false;
// Get the closest face
PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
if (pxcmFace == null)
{
return(false);
}
if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
{
// PXCMFaceData.QueryFaceByIndex(0) failed.
return(false);
}
PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
if (pxcmLandmarksData == null)
{
// PXCMFaceData.Face.QueryLandmarks() failed.
return(false);
}
Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
{
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
};
foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
{
PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);
if (landmarkPoint == null)
{
// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
return(false);
}
landmarkPoints.Add(landmarkType, landmarkPoint);
}
PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
if (pxcmExpressionsData == null)
{
// PXCMFaceData.Face.QueryExpressions() failed.
return(false);
}
animationTrigger.IsAnimationDataValid = true;
PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;
PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
if (pxcmCaptureManager != null)
{
PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
if (pxcmCaptureDevice != null)
{
mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
}
}
animationTrigger.Animations.Clear();
float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;
float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
}
else
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
}
// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));
PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
{
animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
}
else
{
// Error querying expression: EXPRESSION_SMILE.
return(false);
}
return(true);
}