void Update() { if (manager && manager.IsTracking()) { // set head position & rotation if (Head != null) { Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(); Head.localPosition = Vector3.Lerp(Head.localPosition, newPosition, 3 * Time.deltaTime); Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(); Head.localRotation = Quaternion.Slerp(Head.localRotation, newRotation, 3 * Time.deltaTime); } // get animation units int iAUCount = manager.GetAnimUnitsCount(); if (iAUCount >= 6) { // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(0); SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp); SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(1); SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2 = manager.GetAnimUnit(2); SetJointRotation(LipLeft, LipLeftAxis, fAU2, LipLeftNeutral, LipLeftStretched); SetJointRotation(LipRight, LipRightAxis, fAU2, LipRightNeutral, LipRightStretched); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3 = manager.GetAnimUnit(3); SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3, EyebrowLeftNeutral, EyebrowLeftLowered); SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3, EyebrowRightNeutral, EyebrowRightLowered); // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4 = manager.GetAnimUnit(4); SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4, LipCornerLeftNeutral, LipCornerLeftDepressed); SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4, LipCornerRightNeutral, LipCornerRightDepressed); // AU5 – Outer Brow Raiser // 0=neutral; -1=fully lowered as a very sad face; +1=raised as in an expression of deep surprise float fAU5 = manager.GetAnimUnit(5); SetJointRotation(OuterBrowLeft, OuterBrowLeftAxis, fAU5, OuterBrowLeftNeutral, OuterBrowLeftRaised); SetJointRotation(OuterBrowRight, OuterBrowRightAxis, fAU5, OuterBrowRightNeutral, OuterBrowRightRaised); } } }
void Update() { // get the face-tracking manager instance if (manager == null) { manager = FacetrackingManager.Instance; } if (manager && manager.IsTrackingFace()) { // set head position & rotation if (HeadTransform != null) { // head position Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement); if (smoothFactor != 0f) { HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime); } else { HeadTransform.localPosition = newPosition; } // head rotation Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); if (smoothFactor != 0f) { HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime); } else { HeadTransform.localRotation = newRotation; } } // apply animation units // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker); SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp); SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen); SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft); fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left; SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched); float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight); fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right; SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer); fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left; SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered); float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer); fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right; SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered); // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft); fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left; SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed); float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight); fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right; SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed); // AU6, AU7 – Eyelid closed // 0=neutral; -1=raised; +1=fully lowered float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed); fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left; SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered); SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised); float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed); fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right; SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered); SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, LowerEyelidRightRaised); } }
void Update() { // get the face-tracking manager instance if (manager == null) { manager = FacetrackingManager.Instance; } if (manager && manager.GetFaceTrackingID() != 0) { // set head position & rotation if (HeadTransform != null) { // head position Vector3 newPosition = manager.GetHeadPosition(mirroredHeadMovement); // head rotation Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); // rotational fix, provided by Richard Borys: // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z), Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero; addedRotation.x = newRotation.eulerAngles.x + addedRotation.x; addedRotation.y = newRotation.eulerAngles.y + addedRotation.y; addedRotation.z = newRotation.eulerAngles.z + addedRotation.z; newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z); // end of rotational fix if (smoothFactor != 0f) { HeadTransform.rotation = Quaternion.Slerp(HeadTransform.rotation, newRotation, smoothFactor * Time.deltaTime); } else { HeadTransform.rotation = newRotation; } // check for head pos overlay if (foregroundCamera) { // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } KinectManager kinectManager = KinectManager.Instance; if (kinectManager) { long userId = kinectManager.GetUserIdByIndex(manager.playerIndex); Vector3 posColorOverlay = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect); if (posColorOverlay != Vector3.zero) { newPosition = posColorOverlay; // if(overlayObj) // { // overlayObj.position = newPosition; // } } } } else { // move around the initial position newPosition += HeadInitialPosition; } // vertical offet if (verticalOffset != 0f) { // add the vertical offset Vector3 dirHead = new Vector3(0, verticalOffset, 0); dirHead = HeadTransform.InverseTransformDirection(dirHead); newPosition += dirHead; } // set the position if (smoothFactor != 0f) { HeadTransform.position = Vector3.Lerp(HeadTransform.position, newPosition, smoothFactor * Time.deltaTime); } else { HeadTransform.position = newPosition; } // scale factor if (HeadTransform.localScale.x != modelScaleFactor) { HeadTransform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor); } } // apply animation units // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker); SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp); SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen); SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft); fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left; SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched); float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight); fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right; SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer); fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left; SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered); float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer); fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right; SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered); // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft); fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left; SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed); float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight); fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right; SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed); // AU6, AU7 – Eyelid closed // 0=neutral; -1=raised; +1=fully lowered float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed); fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left; SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered); SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised); float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed); fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right; SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered); SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised); } else { // hide the model behind the camera if (HeadTransform && HeadTransform.position.z >= 0f) { HeadTransform.position = new Vector3(0f, 0f, -10f); } } }
void FixedUpdate() { // float translation = Time.deltaTime * 10; // get the face-tracking manager instance if (manager == null) { manager = FacetrackingManager.Instance; } if (manager && manager.IsTrackingFace()) { // set head position & rotation if (HeadTransform != null) { // head position Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement); if (smoothFactor != 0f) { HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime); } else { HeadTransform.localPosition = newPosition; } // head rotation Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); if (smoothFactor != 0f) { HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime); } else { HeadTransform.localRotation = newRotation; } } if (LeftEyeTransform != null) { // head position Vector3 newPosition = LeftEyeInitialPosition + manager.GetHeadPosition(mirroredHeadMovement); if (smoothFactor != 0f) { LeftEyeTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime); } else { LeftEyeTransform.localPosition = newPosition; } // head rotation Quaternion newRotation = LeftEyeInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); if (smoothFactor != 0f) { LeftEyeTransform.localRotation = Quaternion.Slerp(LeftEyeTransform.localRotation, newRotation, smoothFactor * Time.deltaTime); } else { LeftEyeTransform.localRotation = newRotation; } } if (RightEyeTransform != null) { // head position Vector3 newPosition = RightEyeInitialPosition + manager.GetHeadPosition(mirroredHeadMovement); if (smoothFactor != 0f) { RightEyeTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime); } else { RightEyeTransform.localPosition = newPosition; } // head rotation Quaternion newRotation = RightEyeInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); if (smoothFactor != 0f) { RightEyeTransform.localRotation = Quaternion.Slerp(RightEyeTransform.localRotation, newRotation, smoothFactor * Time.deltaTime); } else { RightEyeTransform.localRotation = newRotation; } } // apply animation units // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker); if (fAU0 > Value_Lippucker && isSadMouth == false && isSmillingMouth == false && isOpenMouth == false) { setBlendShape(100, 4); isKissingMouth = true; } else { setBlendShape(0, 4); isKissingMouth = false; } // Debug.Log (fAU0 .ToString ()); // setBlendShape1 (fAU0, 5); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen); if (fAU1 > Value_JawOpen && isSadMouth == false && isSmillingMouth == false && isKissingMouth == false) { setBlendShape(100, 8); isOpenMouth = true; } else { setBlendShape(0, 8); isOpenMouth = false; } // Debug.Log (fAU1 .ToString ()); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft); //fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left; float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight); // fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right; if (fAU2_left > Value_Lipstretcher && isSadMouth == false && isKissingMouth == false && isOpenMouth == false) { setBlendShape(100, 1); isSmillingMouth = true; } else { //setBlendShape (0, 4); setBlendShape(0, 1); isSmillingMouth = false; } // Debug.Log (fAU2_left .ToString ()); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer); //fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left; float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer); //fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right; if (fAU3_left < -0.1) { setBlendShape(100f, 3); } else if (fAU3_left > 0.1) { setBlendShape(100f, 7); } else { setBlendShape(0, 7); setBlendShape(0, 3); } // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft); // fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2-1) : fAU4_left; float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight); // Debug.Log (fAU4_left.ToString ()); if (fAU1 < 0.6 && fAU4_left > Value_LipCornerDepressor && isSmillingMouth == false && isKissingMouth == false) { setBlendShape(100f, 2); isSadMouth = true; } else { setBlendShape(0f, 2); isSadMouth = false; } Debug.Log(fAU4_left.ToString()); // AU6, AU7 – Eyelid closed // 0=neutral; -1=raised; +1=fully lowered float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed); fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left; float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed); fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right; /*if (blendShapeCount > 2) { * * if (blendOne<<100f &&fAU2_left>0f) { * blendOne=1000*fAU2_left; * skinnedMeshRenderer.SetBlendShapeWeight (1, blendOne); * //blendOne += blendSpeed; * } else { * //blendOneFinished = true; * } * * if ( blendTwo < 100f) { * skinnedMeshRenderer.SetBlendShapeWeight (2, blendTwo); * blendTwo += blendSpeed; * } * * }*/ } }