Exemplo n.º 1
0
        public override bool Process(Trigger trigger)
        {
            trigger.ErrorDetected = false;
            if (!SenseToolkitManager.Instance.IsSenseOptionSet(SenseOption.SenseOptionID.Face))
            {
                trigger.ErrorDetected = true;
                return(false);
            }

            AnimationTrigger animationTrigger = trigger as AnimationTrigger;

            if (animationTrigger == null)
            {
                trigger.ErrorDetected = true;
                return(false);
            }

            bool success = false;

            if (SenseToolkitManager.Instance.Initialized && SenseToolkitManager.Instance.FaceModuleOutput != null)
            {
                success = UpdateFace(animationTrigger);
            }
            else
            {
                success = false;
            }

            return(success);
        }
		private bool UpdateFace(AnimationTrigger animationTrigger)
		{
			animationTrigger.IsAnimationDataValid = false;

			// Get the closest face
			PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);
			if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
			{
				// PXCMFaceData.QueryFaceByIndex(0) failed.
				return false;
			}
			
			PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
			if (pxcmLandmarksData == null)
			{
				// PXCMFaceData.Face.QueryLandmarks() failed.
				return false;
			}
			
			Dictionary<PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary<PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();
			PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[] 
				{
					PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
					PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
					PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
					PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
					PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
					PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
					PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
					PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
					PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
					PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
					PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
					PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
				};

			foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
			{
				PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);

				if (landmarkPoint == null)
				{
					// PXCMFaceData.LandmarksData.QueryPointIndex() failed.
					return false;
				}

				landmarkPoints.Add(landmarkType, landmarkPoint);
			}

			PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
			if (pxcmExpressionsData == null)
			{
				// PXCMFaceData.Face.QueryExpressions() failed.
				return false;
			}
			
			animationTrigger.IsAnimationDataValid = true;

			PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;

			PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();
			if (pxcmCaptureManager != null)
			{
				PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
				if (pxcmCaptureDevice != null)
				{
					mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
				}
			}
			
			animationTrigger.Animations.Clear();

			float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;

			float leftEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
			float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
			
			float leftEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
			float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);

			if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
			{
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);
				
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
			}
			else
			{
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);
				
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
			}

			// Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
			// Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
			animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));

			PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();
			
			if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
			{
				animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
			}
			else
			{
				// Error querying expression: EXPRESSION_SMILE.
				return false;
			}

			return true;
		}
Exemplo n.º 3
0
        private bool UpdateFace(AnimationTrigger animationTrigger)
        {
            animationTrigger.IsAnimationDataValid = false;

            // Get the closest face
            PXCMFaceData.Face pxcmFace = SenseToolkitManager.Instance.FaceModuleOutput.QueryFaceByIndex(0);

            if (pxcmFace == null)
            {
                return(false);
            }

            if (SenseToolkitManager.Instance.FaceModuleOutput.QueryNumberOfDetectedFaces() <= 0)
            {
                // PXCMFaceData.QueryFaceByIndex(0) failed.
                return(false);
            }

            PXCMFaceData.LandmarksData pxcmLandmarksData = pxcmFace.QueryLandmarks();
            if (pxcmLandmarksData == null)
            {
                // PXCMFaceData.Face.QueryLandmarks() failed.
                return(false);
            }

            Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint> landmarkPoints = new Dictionary <PXCMFaceData.LandmarkType, PXCMFaceData.LandmarkPoint>();

            PXCMFaceData.LandmarkType[] landmarkTypes = new PXCMFaceData.LandmarkType[]
            {
                PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP,
                PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP,
                PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER,
                PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER,
                PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER,
                PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER,
                PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER,
                PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER,
                PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP,
                PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM,
                PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP,
                PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM
            };

            foreach (PXCMFaceData.LandmarkType landmarkType in landmarkTypes)
            {
                PXCMFaceData.LandmarkPoint landmarkPoint = GetLandmarkPoint(pxcmLandmarksData, landmarkType);

                if (landmarkPoint == null)
                {
                    // PXCMFaceData.LandmarksData.QueryPointIndex() failed.
                    return(false);
                }

                landmarkPoints.Add(landmarkType, landmarkPoint);
            }

            PXCMFaceData.ExpressionsData pxcmExpressionsData = pxcmFace.QueryExpressions();
            if (pxcmExpressionsData == null)
            {
                // PXCMFaceData.Face.QueryExpressions() failed.
                return(false);
            }

            animationTrigger.IsAnimationDataValid = true;

            PXCMCapture.Device.MirrorMode mirrorMode = PXCMCapture.Device.MirrorMode.MIRROR_MODE_DISABLED;

            PXCMCaptureManager pxcmCaptureManager = SenseToolkitManager.Instance.SenseManager.QueryCaptureManager();

            if (pxcmCaptureManager != null)
            {
                PXCMCapture.Device pxcmCaptureDevice = pxcmCaptureManager.QueryDevice();
                if (pxcmCaptureDevice != null)
                {
                    mirrorMode = pxcmCaptureDevice.QueryMirrorMode();
                }
            }

            animationTrigger.Animations.Clear();

            float faceHeight = landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TIP].image.y - landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_NOSE_TOP].image.y;

            float leftEyebrowUp  = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_LEFT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_LEFT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
            float rightEyebrowUp = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYE_RIGHT_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYEBROW_RIGHT_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYEBROW_UP_INITIAL_DISTANCE, OPTIMAL_EYEBROW_UP_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);

            float leftEyeClose  = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_LEFT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);
            float rightEyeClose = GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_BOTTOM].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_EYELID_RIGHT_TOP].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, EYE_CLOSE_INITIAL_DISTANCE, OPTIMAL_EYE_CLOSE_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true);

            if (mirrorMode == PXCMCapture.Device.MirrorMode.MIRROR_MODE_HORIZONTAL)
            {
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), rightEyebrowUp);
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), leftEyebrowUp);

                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), rightEyeClose);
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), leftEyeClose);
            }
            else
            {
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_LEFT.ToString(), leftEyebrowUp);
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_BROW_RAISER_RIGHT.ToString(), rightEyebrowUp);

                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_LEFT.ToString(), leftEyeClose);
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_EYES_CLOSED_RIGHT.ToString(), rightEyeClose);
            }

            // Instead of LANDMARK_LOWER_LIP_CENTER, we need landmark 51 (lower lip upper center)
            // Instead of LANDMARK_UPPER_LIP_CENTER, we need landmark 47 (upper lip lower center)
            animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_MOUTH_OPEN.ToString(), GetNormalizedLandmarksDistance(landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_LOWER_LIP_CENTER].image.y, landmarkPoints[PXCMFaceData.LandmarkType.LANDMARK_UPPER_LIP_CENTER].image.y, OPTIMAL_RELATIVE_FACE_HEIGHT, faceHeight, MOUTH_OPEN_INITIAL_DISTANCE, OPTIMAL_MOUTH_OPEN_MAX_DISTANCE, NORMALIZE_MAX_FACIAL_EXPRESSION_VALUE, true));

            PXCMFaceData.ExpressionsData.FaceExpressionResult pxcmFaceExpressionResult = new PXCMFaceData.ExpressionsData.FaceExpressionResult();

            if (pxcmExpressionsData.QueryExpression(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE, out pxcmFaceExpressionResult))
            {
                animationTrigger.Animations.Add(PXCMFaceData.ExpressionsData.FaceExpression.EXPRESSION_SMILE.ToString(), (float)pxcmFaceExpressionResult.intensity);
            }
            else
            {
                // Error querying expression: EXPRESSION_SMILE.
                return(false);
            }

            return(true);
        }