Example #1
0
    private static void SetAnimationCurvesForFace(Animator animator, GameObject rootObject,
                                                  KeyValuePair <float, float>[] weights, AnimationClip animationClip, MoCapMixamoFacialExpression expression)
    {
        if (animator == null)
        {
            throw new Exception("Animator can not be null!");
        }
        if (rootObject == null)
        {
            throw new Exception("Root object can not be null!");
        }
        if (weights == null || weights.Length == 0)
        {
            throw new Exception("Weights can not be empty!");
        }
        if (animationClip == null)
        {
            throw new Exception("Animation clip can not be null!");
        }
        if (!MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(expression))
        {
            throw new Exception("Invalid Mixamo facial expression!");
        }

        var keyframes = new Keyframe[weights.Length];

        for (var i = 0; i < weights.Length; i++)
        {
            keyframes[i] = new Keyframe(weights[i].Key, weights[i].Value);
        }

        var animationCurve = new AnimationCurve(keyframes);

        animationClip.SetCurve("Body", typeof(SkinnedMeshRenderer),
                               string.Format("blendShape.Facial_Blends.{0}", expression), animationCurve);

        if (expression == MoCapMixamoFacialExpression.Blink_Left ||
            expression == MoCapMixamoFacialExpression.Blink_Right)
        {
            animationClip.SetCurve("Eyelashes", typeof(SkinnedMeshRenderer),
                                   string.Format("blendShape.Facial_Blends.{0}", expression), animationCurve);
        }
    }
Example #2
0
    private static AnimationClip GetAnimationClip(Animator animator, GameObject rootObject, MoCapData data,
                                                  bool processBodyTransforms, bool processFaceTransforms, bool processFaceBlendShapes)
    {
        var animationClip = new AnimationClip();

        animationClip.name      = Path.GetFileNameWithoutExtension(DataFile);
        animationClip.legacy    = true;
        animationClip.wrapMode  = WrapMode.Once;
        animationClip.frameRate = 25;

        animationClip.ClearCurves();

        if (data.BodyFrames != null && data.BodyFrames.Length > 0)
        {
            if (processBodyTransforms)
            {
                var transforms = new TransformTime[(int)MoCapKinectBone.Count][];

                for (var i = 0; i < (int)MoCapKinectBone.Count; i++)
                {
                    transforms[i] = new TransformTime[data.BodyFrames.Length];

                    for (var j = 0; j < data.BodyFrames.Length; j++)
                    {
                        transforms[i][j].Time     = data.BodyFrames[j].SkeletonTransforms[i].Time;
                        transforms[i][j].Position = Vector3.Zero;
                        transforms[i][j].Rotation = MoCapBoneMapper.LocalRotation(ref data.BodyFrames[j],
                                                                                  (MoCapKinectBone)i);
                        transforms[i][j].Scale = Vector3.One;
                    }
                }

                foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
                {
                    if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
                    {
                        var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
                        if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
                        {
                            SetAnimationCurvesForBody(animator, rootObject, transforms[(int)kinectBone], animationClip,
                                                      mecanimBone, false, true, false);
                        }
                    }
                }
            }
        }

        if (data.FaceFrames != null && data.FaceFrames.Length > 0)
        {
            if (processFaceTransforms)
            {
                var transforms = new TransformTime[data.FaceFrames.Length];

                for (var i = 0; i < data.FaceFrames.Length; i++)
                {
                    transforms[i].Time     = data.FaceFrames[i].FaceTransform.Time;
                    transforms[i].Position = Vector3.Zero;
                    transforms[i].Rotation = new Quaternion(-data.FaceFrames[i].FaceTransform.Rotation.X,
                                                            data.FaceFrames[i].FaceTransform.Rotation.Y, data.FaceFrames[i].FaceTransform.Rotation.Z,
                                                            data.FaceFrames[i].FaceTransform.Rotation.W);
                    transforms[i].Scale = Vector3.One;
                }

                SetAnimationCurvesForBody(animator, rootObject, transforms, animationClip, MoCapMecanimBone.Head, false,
                                          true, false);
            }

            if (processFaceBlendShapes)
            {
                var weights =
                    new List <KeyValuePair <float, float> > [(int)MoCapMixamoFacialExpression.LastBlendShape];

                foreach (
                    MoCapKinectFacialExpression kinectFacialExpression in
                    Enum.GetValues(typeof(MoCapKinectFacialExpression)))
                {
                    if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
                    {
                        for (var j = 0; j < data.FaceFrames.Length; j++)
                        {
                            MoCapMixamoFacialExpression mixamoFacialExpression;
                            float mixamoWeight;
                            MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression,
                                                                      data.FaceFrames[j].ExpressionWeights[(int)kinectFacialExpression],
                                                                      out mixamoFacialExpression, out mixamoWeight);

                            if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
                            {
                                if (weights[(int)mixamoFacialExpression] == null)
                                {
                                    weights[(int)mixamoFacialExpression] =
                                        new List <KeyValuePair <float, float> >(data.FaceFrames.Length);
                                }

                                weights[(int)mixamoFacialExpression].Add(
                                    new KeyValuePair <float, float>(data.FaceFrames[j].FaceTransform.Time, mixamoWeight));
                            }
                        }
                    }
                }

                foreach (
                    MoCapMixamoFacialExpression mixamoFacialExpression in
                    Enum.GetValues(typeof(MoCapMixamoFacialExpression)))
                {
                    if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
                    {
                        if (weights[(int)mixamoFacialExpression] != null &&
                            weights[(int)mixamoFacialExpression].Count > 0)
                        {
                            SetAnimationCurvesForFace(animator, rootObject,
                                                      weights[(int)mixamoFacialExpression].ToArray(), animationClip, mixamoFacialExpression);
                        }
                    }
                }
            }
        }

        animationClip.EnsureQuaternionContinuity();

        return(animationClip);
    }
Example #3
0
    private void Update()
    {
        lock (Lock)
        {
            if (AttachedGameObject != null)
            {
                AttachedGameObject.transform.localPosition = VolatilePosition;
                AttachedGameObject.transform.localRotation = VolatileRotation;
                AttachedGameObject.transform.localScale    = VolatileScale;

                cameraCurrentPosition = transform.localPosition;
                if (VoiceCommandQueue.Count > 0)
                {
                    var voiceCommand = VoiceCommandQueue.Dequeue();

                    switch (voiceCommand)
                    {
                    case "forward":
                        cameraTargetPosition += Vector3.forward * MOVEMENT_SCALE;
                        break;

                    case "backward":
                        cameraTargetPosition += Vector3.back * MOVEMENT_SCALE;
                        break;

                    case "up":
                        cameraTargetPosition += Vector3.up * MOVEMENT_SCALE;
                        break;

                    case "down":
                        cameraTargetPosition += Vector3.down * MOVEMENT_SCALE;
                        break;

                    case "right":
                        cameraTargetPosition += Vector3.right * MOVEMENT_SCALE;
                        break;

                    case "left":
                        cameraTargetPosition += Vector3.left * MOVEMENT_SCALE;
                        break;

                    case "body":
                        cameraTargetPosition = CAMERA_BODY_POSITION;
                        break;

                    case "face":
                        cameraTargetPosition = CAMERA_FACE_POSITION;
                        break;

                    default:
                        throw new NotImplementedException();
                    }
                }
                transform.localPosition = Vector3.MoveTowards(cameraCurrentPosition, cameraTargetPosition,
                                                              MAX_DISTANCE_DELTA);

                if (VisemeQueue.Count > 0)
                {
                    var viseme = VisemeQueue.Dequeue();

                    AttachedAnimator.SetInteger("VisemeID", (int)viseme);
                }

                while (KinectBodyQueue.Count > KINECT_BODY_QUEUE_SIZE)
                {
                    KinectBodyQueue.Dequeue();
                }
                if (KinectBodyQueue.Count > 0)
                {
                    var kinectBody = MoCapBodyFrame.Average(KinectBodyQueue.ToArray());

                    if (kinectBody.SkeletonTransforms != null && kinectBody.SkeletonTransforms.Length > 0)
                    {
                        foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
                        {
                            if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
                            {
                                var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);

                                if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
                                {
                                    var humanBodyBone   = (HumanBodyBones)mecanimBone;
                                    var mecanimBoneName = mecanimBone.ToString().ToLower();
                                    var boneTransform   = AttachedAnimator.GetBoneTransform(humanBodyBone);

                                    if (boneTransform != null)
                                    {
                                        var localRotation = MoCapBoneMapper.LocalRotation(ref kinectBody,
                                                                                          kinectBone);

                                        boneTransform.localRotation = new Quaternion(localRotation.X, localRotation.Y,
                                                                                     localRotation.Z, localRotation.W);
                                    }
                                }
                            }
                        }
                    }
                }

                while (KinectFaceQueue.Count > KINECT_FACE_QUEUE_SIZE)
                {
                    KinectFaceQueue.Dequeue();
                }
                if (KinectFaceQueue.Count > 0)
                {
                    MoCapFaceFrame kinectFace;

                    kinectFace = MoCapFaceFrame.Average(KinectFaceQueue.ToArray());

                    var headTransform = AttachedAnimator.GetBoneTransform(HumanBodyBones.Head);

                    var headRotation = new Quaternion(-kinectFace.FaceTransform.Rotation.X,
                                                      kinectFace.FaceTransform.Rotation.Y, kinectFace.FaceTransform.Rotation.Z,
                                                      kinectFace.FaceTransform.Rotation.W);

                    headTransform.localRotation = headRotation;

                    foreach (
                        MoCapKinectFacialExpression kinectFacialExpression in
                        Enum.GetValues(typeof(MoCapKinectFacialExpression)))
                    {
                        if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
                        {
                            var kinectWeight = kinectFace.ExpressionWeights[(int)kinectFacialExpression];

                            MoCapMixamoFacialExpression mixamoFacialExpression;
                            float mixamoWeight;
                            MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression, kinectWeight,
                                                                      out mixamoFacialExpression, out mixamoWeight);

                            if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
                            {
                                AttachedBodySkinnedMeshRenderer.SetBlendShapeWeight((int)mixamoFacialExpression,
                                                                                    mixamoWeight);

                                if (mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Left ||
                                    mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Right)
                                {
                                    AttachedEyelashesSkinnedMeshRenderer.SetBlendShapeWeight(
                                        (int)mixamoFacialExpression, mixamoWeight);
                                }
                            }
                        }
                    }
                }
            }
        }
    }