private static void SetAnimationCurvesForFace(Animator animator, GameObject rootObject, KeyValuePair <float, float>[] weights, AnimationClip animationClip, MoCapMixamoFacialExpression expression) { if (animator == null) { throw new Exception("Animator can not be null!"); } if (rootObject == null) { throw new Exception("Root object can not be null!"); } if (weights == null || weights.Length == 0) { throw new Exception("Weights can not be empty!"); } if (animationClip == null) { throw new Exception("Animation clip can not be null!"); } if (!MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(expression)) { throw new Exception("Invalid Mixamo facial expression!"); } var keyframes = new Keyframe[weights.Length]; for (var i = 0; i < weights.Length; i++) { keyframes[i] = new Keyframe(weights[i].Key, weights[i].Value); } var animationCurve = new AnimationCurve(keyframes); animationClip.SetCurve("Body", typeof(SkinnedMeshRenderer), string.Format("blendShape.Facial_Blends.{0}", expression), animationCurve); if (expression == MoCapMixamoFacialExpression.Blink_Left || expression == MoCapMixamoFacialExpression.Blink_Right) { animationClip.SetCurve("Eyelashes", typeof(SkinnedMeshRenderer), string.Format("blendShape.Facial_Blends.{0}", expression), animationCurve); } }
private static AnimationClip GetAnimationClip(Animator animator, GameObject rootObject, MoCapData data, bool processBodyTransforms, bool processFaceTransforms, bool processFaceBlendShapes) { var animationClip = new AnimationClip(); animationClip.name = Path.GetFileNameWithoutExtension(DataFile); animationClip.legacy = true; animationClip.wrapMode = WrapMode.Once; animationClip.frameRate = 25; animationClip.ClearCurves(); if (data.BodyFrames != null && data.BodyFrames.Length > 0) { if (processBodyTransforms) { var transforms = new TransformTime[(int)MoCapKinectBone.Count][]; for (var i = 0; i < (int)MoCapKinectBone.Count; i++) { transforms[i] = new TransformTime[data.BodyFrames.Length]; for (var j = 0; j < data.BodyFrames.Length; j++) { transforms[i][j].Time = data.BodyFrames[j].SkeletonTransforms[i].Time; transforms[i][j].Position = Vector3.Zero; transforms[i][j].Rotation = MoCapBoneMapper.LocalRotation(ref data.BodyFrames[j], (MoCapKinectBone)i); transforms[i][j].Scale = Vector3.One; } } foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone))) { if (MoCapBoneMapper.IsValidKinectBone(kinectBone)) { var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone); if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone)) { SetAnimationCurvesForBody(animator, rootObject, transforms[(int)kinectBone], animationClip, mecanimBone, false, true, false); } } } } } if (data.FaceFrames != null && data.FaceFrames.Length > 0) { if (processFaceTransforms) { var transforms = new TransformTime[data.FaceFrames.Length]; for (var i = 0; i < data.FaceFrames.Length; i++) { transforms[i].Time = data.FaceFrames[i].FaceTransform.Time; transforms[i].Position = Vector3.Zero; transforms[i].Rotation = new Quaternion(-data.FaceFrames[i].FaceTransform.Rotation.X, data.FaceFrames[i].FaceTransform.Rotation.Y, data.FaceFrames[i].FaceTransform.Rotation.Z, data.FaceFrames[i].FaceTransform.Rotation.W); transforms[i].Scale = Vector3.One; } SetAnimationCurvesForBody(animator, rootObject, transforms, animationClip, MoCapMecanimBone.Head, false, true, false); } if (processFaceBlendShapes) { var weights = new List <KeyValuePair <float, float> > [(int)MoCapMixamoFacialExpression.LastBlendShape]; foreach ( MoCapKinectFacialExpression kinectFacialExpression in Enum.GetValues(typeof(MoCapKinectFacialExpression))) { if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression)) { for (var j = 0; j < data.FaceFrames.Length; j++) { MoCapMixamoFacialExpression mixamoFacialExpression; float mixamoWeight; MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression, data.FaceFrames[j].ExpressionWeights[(int)kinectFacialExpression], out mixamoFacialExpression, out mixamoWeight); if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression)) { if (weights[(int)mixamoFacialExpression] == null) { weights[(int)mixamoFacialExpression] = new List <KeyValuePair <float, float> >(data.FaceFrames.Length); } weights[(int)mixamoFacialExpression].Add( new KeyValuePair <float, float>(data.FaceFrames[j].FaceTransform.Time, mixamoWeight)); } } } } foreach ( MoCapMixamoFacialExpression mixamoFacialExpression in Enum.GetValues(typeof(MoCapMixamoFacialExpression))) { if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression)) { if (weights[(int)mixamoFacialExpression] != null && weights[(int)mixamoFacialExpression].Count > 0) { SetAnimationCurvesForFace(animator, rootObject, weights[(int)mixamoFacialExpression].ToArray(), animationClip, mixamoFacialExpression); } } } } } animationClip.EnsureQuaternionContinuity(); return(animationClip); }
private void Update() { lock (Lock) { if (AttachedGameObject != null) { AttachedGameObject.transform.localPosition = VolatilePosition; AttachedGameObject.transform.localRotation = VolatileRotation; AttachedGameObject.transform.localScale = VolatileScale; cameraCurrentPosition = transform.localPosition; if (VoiceCommandQueue.Count > 0) { var voiceCommand = VoiceCommandQueue.Dequeue(); switch (voiceCommand) { case "forward": cameraTargetPosition += Vector3.forward * MOVEMENT_SCALE; break; case "backward": cameraTargetPosition += Vector3.back * MOVEMENT_SCALE; break; case "up": cameraTargetPosition += Vector3.up * MOVEMENT_SCALE; break; case "down": cameraTargetPosition += Vector3.down * MOVEMENT_SCALE; break; case "right": cameraTargetPosition += Vector3.right * MOVEMENT_SCALE; break; case "left": cameraTargetPosition += Vector3.left * MOVEMENT_SCALE; break; case "body": cameraTargetPosition = CAMERA_BODY_POSITION; break; case "face": cameraTargetPosition = CAMERA_FACE_POSITION; break; default: throw new NotImplementedException(); } } transform.localPosition = Vector3.MoveTowards(cameraCurrentPosition, cameraTargetPosition, MAX_DISTANCE_DELTA); if (VisemeQueue.Count > 0) { var viseme = VisemeQueue.Dequeue(); AttachedAnimator.SetInteger("VisemeID", (int)viseme); } while (KinectBodyQueue.Count > KINECT_BODY_QUEUE_SIZE) { KinectBodyQueue.Dequeue(); } if (KinectBodyQueue.Count > 0) { var kinectBody = MoCapBodyFrame.Average(KinectBodyQueue.ToArray()); if (kinectBody.SkeletonTransforms != null && kinectBody.SkeletonTransforms.Length > 0) { foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone))) { if (MoCapBoneMapper.IsValidKinectBone(kinectBone)) { var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone); if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone)) { var humanBodyBone = (HumanBodyBones)mecanimBone; var mecanimBoneName = mecanimBone.ToString().ToLower(); var boneTransform = AttachedAnimator.GetBoneTransform(humanBodyBone); if (boneTransform != null) { var localRotation = MoCapBoneMapper.LocalRotation(ref kinectBody, kinectBone); boneTransform.localRotation = new Quaternion(localRotation.X, localRotation.Y, localRotation.Z, localRotation.W); } } } } } } while (KinectFaceQueue.Count > KINECT_FACE_QUEUE_SIZE) { KinectFaceQueue.Dequeue(); } if (KinectFaceQueue.Count > 0) { MoCapFaceFrame kinectFace; kinectFace = MoCapFaceFrame.Average(KinectFaceQueue.ToArray()); var headTransform = AttachedAnimator.GetBoneTransform(HumanBodyBones.Head); var headRotation = new Quaternion(-kinectFace.FaceTransform.Rotation.X, kinectFace.FaceTransform.Rotation.Y, kinectFace.FaceTransform.Rotation.Z, kinectFace.FaceTransform.Rotation.W); headTransform.localRotation = headRotation; foreach ( MoCapKinectFacialExpression kinectFacialExpression in Enum.GetValues(typeof(MoCapKinectFacialExpression))) { if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression)) { var kinectWeight = kinectFace.ExpressionWeights[(int)kinectFacialExpression]; MoCapMixamoFacialExpression mixamoFacialExpression; float mixamoWeight; MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression, kinectWeight, out mixamoFacialExpression, out mixamoWeight); if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression)) { AttachedBodySkinnedMeshRenderer.SetBlendShapeWeight((int)mixamoFacialExpression, mixamoWeight); if (mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Left || mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Right) { AttachedEyelashesSkinnedMeshRenderer.SetBlendShapeWeight( (int)mixamoFacialExpression, mixamoWeight); } } } } } } } }