private static AnimationClip GetAnimationClip(Animator animator, GameObject rootObject, MoCapData data,
bool processBodyTransforms, bool processFaceTransforms, bool processFaceBlendShapes)
{
var animationClip = new AnimationClip();
animationClip.name = Path.GetFileNameWithoutExtension(DataFile);
animationClip.legacy = true;
animationClip.wrapMode = WrapMode.Once;
animationClip.frameRate = 25;
animationClip.ClearCurves();
if (data.BodyFrames != null && data.BodyFrames.Length > 0)
{
if (processBodyTransforms)
{
var transforms = new TransformTime[(int)MoCapKinectBone.Count][];
for (var i = 0; i < (int)MoCapKinectBone.Count; i++)
{
transforms[i] = new TransformTime[data.BodyFrames.Length];
for (var j = 0; j < data.BodyFrames.Length; j++)
{
transforms[i][j].Time = data.BodyFrames[j].SkeletonTransforms[i].Time;
transforms[i][j].Position = Vector3.Zero;
transforms[i][j].Rotation = MoCapBoneMapper.LocalRotation(ref data.BodyFrames[j],
(MoCapKinectBone)i);
transforms[i][j].Scale = Vector3.One;
}
}
foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
{
if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
{
var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
{
SetAnimationCurvesForBody(animator, rootObject, transforms[(int)kinectBone], animationClip,
mecanimBone, false, true, false);
}
}
}
}
}
if (data.FaceFrames != null && data.FaceFrames.Length > 0)
{
if (processFaceTransforms)
{
var transforms = new TransformTime[data.FaceFrames.Length];
for (var i = 0; i < data.FaceFrames.Length; i++)
{
transforms[i].Time = data.FaceFrames[i].FaceTransform.Time;
transforms[i].Position = Vector3.Zero;
transforms[i].Rotation = new Quaternion(-data.FaceFrames[i].FaceTransform.Rotation.X,
data.FaceFrames[i].FaceTransform.Rotation.Y, data.FaceFrames[i].FaceTransform.Rotation.Z,
data.FaceFrames[i].FaceTransform.Rotation.W);
transforms[i].Scale = Vector3.One;
}
SetAnimationCurvesForBody(animator, rootObject, transforms, animationClip, MoCapMecanimBone.Head, false,
true, false);
}
if (processFaceBlendShapes)
{
var weights =
new List <KeyValuePair <float, float> > [(int)MoCapMixamoFacialExpression.LastBlendShape];
foreach (
MoCapKinectFacialExpression kinectFacialExpression in
Enum.GetValues(typeof(MoCapKinectFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
{
for (var j = 0; j < data.FaceFrames.Length; j++)
{
MoCapMixamoFacialExpression mixamoFacialExpression;
float mixamoWeight;
MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression,
data.FaceFrames[j].ExpressionWeights[(int)kinectFacialExpression],
out mixamoFacialExpression, out mixamoWeight);
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int)mixamoFacialExpression] == null)
{
weights[(int)mixamoFacialExpression] =
new List <KeyValuePair <float, float> >(data.FaceFrames.Length);
}
weights[(int)mixamoFacialExpression].Add(
new KeyValuePair <float, float>(data.FaceFrames[j].FaceTransform.Time, mixamoWeight));
}
}
}
}
foreach (
MoCapMixamoFacialExpression mixamoFacialExpression in
Enum.GetValues(typeof(MoCapMixamoFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int)mixamoFacialExpression] != null &&
weights[(int)mixamoFacialExpression].Count > 0)
{
SetAnimationCurvesForFace(animator, rootObject,
weights[(int)mixamoFacialExpression].ToArray(), animationClip, mixamoFacialExpression);
}
}
}
}
}
animationClip.EnsureQuaternionContinuity();
return(animationClip);
}
private static void SetAnimationCurvesForBody(Animator animator, GameObject rootObject, TransformTime[] transforms,
AnimationClip animationClip, MoCapMecanimBone bone, bool setPosition, bool setRotation, bool setScale)
{
if (animator == null)
{
throw new Exception("Animator can not be null!");
}
if (rootObject == null)
{
throw new Exception("Root object can not be null!");
}
if (transforms == null || transforms.Length == 0)
{
throw new Exception("Transforms can not be empty!");
}
if (animationClip == null)
{
throw new Exception("Animation clip can not be null!");
}
if (!MoCapBoneMapper.IsValidMecanimBone(bone))
{
throw new Exception("Invalid Mecanim bone!");
}
var relativeTransform = animator.GetBoneTransform((HumanBodyBones)bone);
var relativePath = AnimationUtility.CalculateTransformPath(relativeTransform, rootObject.transform);
var keyframesTransformPositionX = new Keyframe[transforms.Length];
var keyframesTransformPositionY = new Keyframe[transforms.Length];
var keyframesTransformPositionZ = new Keyframe[transforms.Length];
var keyframesTransformRotationX = new Keyframe[transforms.Length];
var keyframesTransformRotationY = new Keyframe[transforms.Length];
var keyframesTransformRotationZ = new Keyframe[transforms.Length];
var keyframesTransformRotationW = new Keyframe[transforms.Length];
var keyframesTransformScaleX = new Keyframe[transforms.Length];
var keyframesTransformScaleY = new Keyframe[transforms.Length];
var keyframesTransformScaleZ = new Keyframe[transforms.Length];
for (var i = 0; i < transforms.Length; i++)
{
var transform = transforms[i];
keyframesTransformPositionX[i] = new Keyframe(transform.Time, transform.Position.X);
keyframesTransformPositionY[i] = new Keyframe(transform.Time, transform.Position.Y);
keyframesTransformPositionZ[i] = new Keyframe(transform.Time, transform.Position.Z);
keyframesTransformRotationX[i] = new Keyframe(transform.Time, transform.Rotation.X);
keyframesTransformRotationY[i] = new Keyframe(transform.Time, transform.Rotation.Y);
keyframesTransformRotationZ[i] = new Keyframe(transform.Time, transform.Rotation.Z);
keyframesTransformRotationW[i] = new Keyframe(transform.Time, transform.Rotation.W);
keyframesTransformScaleX[i] = new Keyframe(transform.Time, transform.Scale.X);
keyframesTransformScaleY[i] = new Keyframe(transform.Time, transform.Scale.Y);
keyframesTransformScaleZ[i] = new Keyframe(transform.Time, transform.Scale.Z);
}
var animationCurvePositionX = new AnimationCurve(keyframesTransformPositionX);
var animationCurvePositionY = new AnimationCurve(keyframesTransformPositionY);
var animationCurvePositionZ = new AnimationCurve(keyframesTransformPositionZ);
if (setPosition)
{
animationClip.SetCurve(relativePath, typeof(Transform), "localPosition.x", animationCurvePositionX);
animationClip.SetCurve(relativePath, typeof(Transform), "localPosition.y", animationCurvePositionY);
animationClip.SetCurve(relativePath, typeof(Transform), "localPosition.z", animationCurvePositionZ);
}
var animationCurveRotationX = new AnimationCurve(keyframesTransformRotationX);
var animationCurveRotationY = new AnimationCurve(keyframesTransformRotationY);
var animationCurveRotationZ = new AnimationCurve(keyframesTransformRotationZ);
var animationCurveRotationW = new AnimationCurve(keyframesTransformRotationW);
if (setRotation)
{
animationClip.SetCurve(relativePath, typeof(Transform), "localRotation.x", animationCurveRotationX);
animationClip.SetCurve(relativePath, typeof(Transform), "localRotation.y", animationCurveRotationY);
animationClip.SetCurve(relativePath, typeof(Transform), "localRotation.z", animationCurveRotationZ);
animationClip.SetCurve(relativePath, typeof(Transform), "localRotation.w", animationCurveRotationW);
}
var animationCurveScaleX = new AnimationCurve(keyframesTransformScaleX);
var animationCurveScaleY = new AnimationCurve(keyframesTransformScaleY);
var animationCurveScaleZ = new AnimationCurve(keyframesTransformScaleZ);
if (setScale)
{
animationClip.SetCurve(relativePath, typeof(Transform), "localScale.x", animationCurveScaleX);
animationClip.SetCurve(relativePath, typeof(Transform), "localScale.y", animationCurveScaleY);
animationClip.SetCurve(relativePath, typeof(Transform), "localScale.z", animationCurveScaleZ);
}
}
private void Update()
{
lock (Lock)
{
if (AttachedGameObject != null)
{
AttachedGameObject.transform.localPosition = VolatilePosition;
AttachedGameObject.transform.localRotation = VolatileRotation;
AttachedGameObject.transform.localScale = VolatileScale;
cameraCurrentPosition = transform.localPosition;
if (VoiceCommandQueue.Count > 0)
{
var voiceCommand = VoiceCommandQueue.Dequeue();
switch (voiceCommand)
{
case "forward":
cameraTargetPosition += Vector3.forward * MOVEMENT_SCALE;
break;
case "backward":
cameraTargetPosition += Vector3.back * MOVEMENT_SCALE;
break;
case "up":
cameraTargetPosition += Vector3.up * MOVEMENT_SCALE;
break;
case "down":
cameraTargetPosition += Vector3.down * MOVEMENT_SCALE;
break;
case "right":
cameraTargetPosition += Vector3.right * MOVEMENT_SCALE;
break;
case "left":
cameraTargetPosition += Vector3.left * MOVEMENT_SCALE;
break;
case "body":
cameraTargetPosition = CAMERA_BODY_POSITION;
break;
case "face":
cameraTargetPosition = CAMERA_FACE_POSITION;
break;
default:
throw new NotImplementedException();
}
}
transform.localPosition = Vector3.MoveTowards(cameraCurrentPosition, cameraTargetPosition,
MAX_DISTANCE_DELTA);
if (VisemeQueue.Count > 0)
{
var viseme = VisemeQueue.Dequeue();
AttachedAnimator.SetInteger("VisemeID", (int)viseme);
}
while (KinectBodyQueue.Count > KINECT_BODY_QUEUE_SIZE)
{
KinectBodyQueue.Dequeue();
}
if (KinectBodyQueue.Count > 0)
{
var kinectBody = MoCapBodyFrame.Average(KinectBodyQueue.ToArray());
if (kinectBody.SkeletonTransforms != null && kinectBody.SkeletonTransforms.Length > 0)
{
foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
{
if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
{
var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
{
var humanBodyBone = (HumanBodyBones)mecanimBone;
var mecanimBoneName = mecanimBone.ToString().ToLower();
var boneTransform = AttachedAnimator.GetBoneTransform(humanBodyBone);
if (boneTransform != null)
{
var localRotation = MoCapBoneMapper.LocalRotation(ref kinectBody,
kinectBone);
boneTransform.localRotation = new Quaternion(localRotation.X, localRotation.Y,
localRotation.Z, localRotation.W);
}
}
}
}
}
}
while (KinectFaceQueue.Count > KINECT_FACE_QUEUE_SIZE)
{
KinectFaceQueue.Dequeue();
}
if (KinectFaceQueue.Count > 0)
{
MoCapFaceFrame kinectFace;
kinectFace = MoCapFaceFrame.Average(KinectFaceQueue.ToArray());
var headTransform = AttachedAnimator.GetBoneTransform(HumanBodyBones.Head);
var headRotation = new Quaternion(-kinectFace.FaceTransform.Rotation.X,
kinectFace.FaceTransform.Rotation.Y, kinectFace.FaceTransform.Rotation.Z,
kinectFace.FaceTransform.Rotation.W);
headTransform.localRotation = headRotation;
foreach (
MoCapKinectFacialExpression kinectFacialExpression in
Enum.GetValues(typeof(MoCapKinectFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
{
var kinectWeight = kinectFace.ExpressionWeights[(int)kinectFacialExpression];
MoCapMixamoFacialExpression mixamoFacialExpression;
float mixamoWeight;
MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression, kinectWeight,
out mixamoFacialExpression, out mixamoWeight);
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
AttachedBodySkinnedMeshRenderer.SetBlendShapeWeight((int)mixamoFacialExpression,
mixamoWeight);
if (mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Left ||
mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Right)
{
AttachedEyelashesSkinnedMeshRenderer.SetBlendShapeWeight(
(int)mixamoFacialExpression, mixamoWeight);
}
}
}
}
}
}
}
}