private static void SetAnimationCurvesForFace(Animator animator, GameObject rootObject,
KeyValuePair <float, float>[] weights, AnimationClip animationClip, MoCapMixamoFacialExpression expression)
{
if (animator == null)
{
throw new Exception("Animator can not be null!");
}
if (rootObject == null)
{
throw new Exception("Root object can not be null!");
}
if (weights == null || weights.Length == 0)
{
throw new Exception("Weights can not be empty!");
}
if (animationClip == null)
{
throw new Exception("Animation clip can not be null!");
}
if (!MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(expression))
{
throw new Exception("Invalid Mixamo facial expression!");
}
var keyframes = new Keyframe[weights.Length];
for (var i = 0; i < weights.Length; i++)
{
keyframes[i] = new Keyframe(weights[i].Key, weights[i].Value);
}
var animationCurve = new AnimationCurve(keyframes);
animationClip.SetCurve("Body", typeof(SkinnedMeshRenderer),
string.Format("blendShape.Facial_Blends.{0}", expression), animationCurve);
if (expression == MoCapMixamoFacialExpression.Blink_Left ||
expression == MoCapMixamoFacialExpression.Blink_Right)
{
animationClip.SetCurve("Eyelashes", typeof(SkinnedMeshRenderer),
string.Format("blendShape.Facial_Blends.{0}", expression), animationCurve);
}
}
private static AnimationClip GetAnimationClip(Animator animator, GameObject rootObject, MoCapData data,
bool processBodyTransforms, bool processFaceTransforms, bool processFaceBlendShapes)
{
var animationClip = new AnimationClip();
animationClip.name = Path.GetFileNameWithoutExtension(DataFile);
animationClip.legacy = true;
animationClip.wrapMode = WrapMode.Once;
animationClip.frameRate = 25;
animationClip.ClearCurves();
if (data.BodyFrames != null && data.BodyFrames.Length > 0)
{
if (processBodyTransforms)
{
var transforms = new TransformTime[(int)MoCapKinectBone.Count][];
for (var i = 0; i < (int)MoCapKinectBone.Count; i++)
{
transforms[i] = new TransformTime[data.BodyFrames.Length];
for (var j = 0; j < data.BodyFrames.Length; j++)
{
transforms[i][j].Time = data.BodyFrames[j].SkeletonTransforms[i].Time;
transforms[i][j].Position = Vector3.Zero;
transforms[i][j].Rotation = MoCapBoneMapper.LocalRotation(ref data.BodyFrames[j],
(MoCapKinectBone)i);
transforms[i][j].Scale = Vector3.One;
}
}
foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
{
if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
{
var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
{
SetAnimationCurvesForBody(animator, rootObject, transforms[(int)kinectBone], animationClip,
mecanimBone, false, true, false);
}
}
}
}
}
if (data.FaceFrames != null && data.FaceFrames.Length > 0)
{
if (processFaceTransforms)
{
var transforms = new TransformTime[data.FaceFrames.Length];
for (var i = 0; i < data.FaceFrames.Length; i++)
{
transforms[i].Time = data.FaceFrames[i].FaceTransform.Time;
transforms[i].Position = Vector3.Zero;
transforms[i].Rotation = new Quaternion(-data.FaceFrames[i].FaceTransform.Rotation.X,
data.FaceFrames[i].FaceTransform.Rotation.Y, data.FaceFrames[i].FaceTransform.Rotation.Z,
data.FaceFrames[i].FaceTransform.Rotation.W);
transforms[i].Scale = Vector3.One;
}
SetAnimationCurvesForBody(animator, rootObject, transforms, animationClip, MoCapMecanimBone.Head, false,
true, false);
}
if (processFaceBlendShapes)
{
var weights =
new List <KeyValuePair <float, float> > [(int)MoCapMixamoFacialExpression.LastBlendShape];
foreach (
MoCapKinectFacialExpression kinectFacialExpression in
Enum.GetValues(typeof(MoCapKinectFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
{
for (var j = 0; j < data.FaceFrames.Length; j++)
{
MoCapMixamoFacialExpression mixamoFacialExpression;
float mixamoWeight;
MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression,
data.FaceFrames[j].ExpressionWeights[(int)kinectFacialExpression],
out mixamoFacialExpression, out mixamoWeight);
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int)mixamoFacialExpression] == null)
{
weights[(int)mixamoFacialExpression] =
new List <KeyValuePair <float, float> >(data.FaceFrames.Length);
}
weights[(int)mixamoFacialExpression].Add(
new KeyValuePair <float, float>(data.FaceFrames[j].FaceTransform.Time, mixamoWeight));
}
}
}
}
foreach (
MoCapMixamoFacialExpression mixamoFacialExpression in
Enum.GetValues(typeof(MoCapMixamoFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int)mixamoFacialExpression] != null &&
weights[(int)mixamoFacialExpression].Count > 0)
{
SetAnimationCurvesForFace(animator, rootObject,
weights[(int)mixamoFacialExpression].ToArray(), animationClip, mixamoFacialExpression);
}
}
}
}
}
animationClip.EnsureQuaternionContinuity();
return(animationClip);
}
private void Update()
{
lock (Lock)
{
if (AttachedGameObject != null)
{
AttachedGameObject.transform.localPosition = VolatilePosition;
AttachedGameObject.transform.localRotation = VolatileRotation;
AttachedGameObject.transform.localScale = VolatileScale;
cameraCurrentPosition = transform.localPosition;
if (VoiceCommandQueue.Count > 0)
{
var voiceCommand = VoiceCommandQueue.Dequeue();
switch (voiceCommand)
{
case "forward":
cameraTargetPosition += Vector3.forward * MOVEMENT_SCALE;
break;
case "backward":
cameraTargetPosition += Vector3.back * MOVEMENT_SCALE;
break;
case "up":
cameraTargetPosition += Vector3.up * MOVEMENT_SCALE;
break;
case "down":
cameraTargetPosition += Vector3.down * MOVEMENT_SCALE;
break;
case "right":
cameraTargetPosition += Vector3.right * MOVEMENT_SCALE;
break;
case "left":
cameraTargetPosition += Vector3.left * MOVEMENT_SCALE;
break;
case "body":
cameraTargetPosition = CAMERA_BODY_POSITION;
break;
case "face":
cameraTargetPosition = CAMERA_FACE_POSITION;
break;
default:
throw new NotImplementedException();
}
}
transform.localPosition = Vector3.MoveTowards(cameraCurrentPosition, cameraTargetPosition,
MAX_DISTANCE_DELTA);
if (VisemeQueue.Count > 0)
{
var viseme = VisemeQueue.Dequeue();
AttachedAnimator.SetInteger("VisemeID", (int)viseme);
}
while (KinectBodyQueue.Count > KINECT_BODY_QUEUE_SIZE)
{
KinectBodyQueue.Dequeue();
}
if (KinectBodyQueue.Count > 0)
{
var kinectBody = MoCapBodyFrame.Average(KinectBodyQueue.ToArray());
if (kinectBody.SkeletonTransforms != null && kinectBody.SkeletonTransforms.Length > 0)
{
foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
{
if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
{
var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
{
var humanBodyBone = (HumanBodyBones)mecanimBone;
var mecanimBoneName = mecanimBone.ToString().ToLower();
var boneTransform = AttachedAnimator.GetBoneTransform(humanBodyBone);
if (boneTransform != null)
{
var localRotation = MoCapBoneMapper.LocalRotation(ref kinectBody,
kinectBone);
boneTransform.localRotation = new Quaternion(localRotation.X, localRotation.Y,
localRotation.Z, localRotation.W);
}
}
}
}
}
}
while (KinectFaceQueue.Count > KINECT_FACE_QUEUE_SIZE)
{
KinectFaceQueue.Dequeue();
}
if (KinectFaceQueue.Count > 0)
{
MoCapFaceFrame kinectFace;
kinectFace = MoCapFaceFrame.Average(KinectFaceQueue.ToArray());
var headTransform = AttachedAnimator.GetBoneTransform(HumanBodyBones.Head);
var headRotation = new Quaternion(-kinectFace.FaceTransform.Rotation.X,
kinectFace.FaceTransform.Rotation.Y, kinectFace.FaceTransform.Rotation.Z,
kinectFace.FaceTransform.Rotation.W);
headTransform.localRotation = headRotation;
foreach (
MoCapKinectFacialExpression kinectFacialExpression in
Enum.GetValues(typeof(MoCapKinectFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
{
var kinectWeight = kinectFace.ExpressionWeights[(int)kinectFacialExpression];
MoCapMixamoFacialExpression mixamoFacialExpression;
float mixamoWeight;
MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression, kinectWeight,
out mixamoFacialExpression, out mixamoWeight);
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
AttachedBodySkinnedMeshRenderer.SetBlendShapeWeight((int)mixamoFacialExpression,
mixamoWeight);
if (mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Left ||
mixamoFacialExpression == MoCapMixamoFacialExpression.Blink_Right)
{
AttachedEyelashesSkinnedMeshRenderer.SetBlendShapeWeight(
(int)mixamoFacialExpression, mixamoWeight);
}
}
}
}
}
}
}
}