public static MoCapFaceFrame Average(params MoCapFaceFrame[] values)
{
if (values == null || values.Length == 0)
throw new Exception("Empty values list!");
var transforms = new TransformTime[values.Length];
for (var i = 0; i < values.Length; i++)
transforms[i] = values[i].FaceTransform;
var weights = new float[(int) MoCapKinectFacialExpression.Count][];
for (var i = 0; i < (int) MoCapKinectFacialExpression.Count; i++)
{
weights[i] = new float[values.Length];
for (var j = 0; j < values.Length; j++)
weights[i][j] = values[j].ExpressionWeights[i];
}
var ret = new MoCapFaceFrame
{
ExpressionWeights = new float[(int) MoCapKinectFacialExpression.Count]
};
ret.FaceTransform = TransformTime.Average(transforms);
for (var i = 0; i < (int) MoCapKinectFacialExpression.Count; i++)
ret.ExpressionWeights[i] = MathHelper.Average(weights[i]);
return ret;
}
private void HighDefinitionFaceFrameReader_FrameArrived(object sender, HighDefinitionFaceFrameArrivedEventArgs e)
{
using (var frame = e.FrameReference.AcquireFrame())
{
if (frame != null && frame.IsTrackingIdValid && frame.IsFaceTracked &&
frame.FaceAlignmentQuality == FaceAlignmentQuality.High)
{
frame.GetAndRefreshFaceAlignmentResult(FaceAlignment);
var command = new CommandMessage
{
CommandType = CommandType.KinectFace,
KinectFace =
new MoCapFaceFrame
{
ExpressionWeights = new float[(int)MoCapKinectFacialExpression.Count]
}
};
var time = (float)(DateTime.Now - MoCapRecordStartTime).TotalSeconds;
var rotationX = FaceAlignment.FaceOrientation.X;
var rotationY = FaceAlignment.FaceOrientation.Y;
var rotationZ = FaceAlignment.FaceOrientation.Z;
var rotationW = FaceAlignment.FaceOrientation.W;
var transform = new TransformTime();
transform.Time = time;
transform.Position = Vector3.Zero;
transform.Rotation = new Quaternion(rotationX, rotationY, rotationZ, rotationW);
transform.Scale = Vector3.One;
transform.Rotation = FaceDESP.Predict(transform.Rotation);
command.KinectFace.FaceTransform = transform;
foreach (FaceShapeAnimations faceShapeAnimation in Enum.GetValues(typeof(FaceShapeAnimations)))
{
float weight;
if (FaceAlignment.AnimationUnits.TryGetValue(faceShapeAnimation, out weight))
{
FaceExpressionDESP[(int)faceShapeAnimation].Update(weight);
command.KinectFace.ExpressionWeights[(int)faceShapeAnimation] =
FaceExpressionDESP[(int)faceShapeAnimation].Predict(1);
}
}
if (IsRecording)
{
FaceFrames.Add(command.KinectFace);
}
BeginWriteCommand(command);
}
}
}
public static MoCapBodyFrame Average(params MoCapBodyFrame[] values)
{
if (values == null || values.Length == 0)
throw new Exception("Empty values list!");
var transforms = new TransformTime[(int) MoCapKinectBone.Count][];
for (var i = 0; i < (int) MoCapKinectBone.Count; i++)
{
transforms[i] = new TransformTime[values.Length];
for (var j = 0; j < values.Length; j++)
transforms[i][j] = values[j].SkeletonTransforms[i];
}
var ret = new MoCapBodyFrame
{
SkeletonTransforms = new TransformTime[(int) MoCapKinectBone.Count]
};
for (var i = 0; i < (int) MoCapKinectBone.Count; i++)
ret.SkeletonTransforms[i] = TransformTime.Average(transforms[i]);
return ret;
}
private void HighDefinitionFaceFrameReader_FrameArrived(object sender, HighDefinitionFaceFrameArrivedEventArgs e)
{
using (var frame = e.FrameReference.AcquireFrame())
{
if (frame != null && frame.IsTrackingIdValid && frame.IsFaceTracked &&
frame.FaceAlignmentQuality == FaceAlignmentQuality.High)
{
frame.GetAndRefreshFaceAlignmentResult(FaceAlignment);
var command = new CommandMessage
{
CommandType = CommandType.KinectFace,
KinectFace =
new MoCapFaceFrame
{
ExpressionWeights = new float[(int) MoCapKinectFacialExpression.Count]
}
};
var time = (float) (DateTime.Now - MoCapRecordStartTime).TotalSeconds;
var rotationX = FaceAlignment.FaceOrientation.X;
var rotationY = FaceAlignment.FaceOrientation.Y;
var rotationZ = FaceAlignment.FaceOrientation.Z;
var rotationW = FaceAlignment.FaceOrientation.W;
var transform = new TransformTime();
transform.Time = time;
transform.Position = Vector3.Zero;
transform.Rotation = new Quaternion(rotationX, rotationY, rotationZ, rotationW);
transform.Scale = Vector3.One;
transform.Rotation = FaceDESP.Predict(transform.Rotation);
command.KinectFace.FaceTransform = transform;
foreach (FaceShapeAnimations faceShapeAnimation in Enum.GetValues(typeof (FaceShapeAnimations)))
{
float weight;
if (FaceAlignment.AnimationUnits.TryGetValue(faceShapeAnimation, out weight))
{
FaceExpressionDESP[(int) faceShapeAnimation].Update(weight);
command.KinectFace.ExpressionWeights[(int) faceShapeAnimation] =
FaceExpressionDESP[(int) faceShapeAnimation].Predict(1);
}
}
if (IsRecording)
FaceFrames.Add(command.KinectFace);
BeginWriteCommand(command);
}
}
}
private void BodyFrameReader_FrameArrived(object sender, BodyFrameArrivedEventArgs e)
{
using (var frame = e.FrameReference.AcquireFrame())
{
if (frame != null)
{
if (Bodies == null)
Bodies = new Body[frame.BodyCount];
frame.GetAndRefreshBodyData(Bodies);
foreach (var body in Bodies)
if (body.IsTracked)
{
HighDefinitionFaceFrameSource.TrackingId = body.TrackingId;
var command = new CommandMessage
{
CommandType = CommandType.KinectBody,
KinectBody =
new MoCapBodyFrame
{
SkeletonTransforms = new TransformTime[(int) MoCapKinectBone.Count]
}
};
foreach (JointType jointType in Enum.GetValues(typeof (JointType)))
{
Joint joint;
if (body.Joints.TryGetValue(jointType, out joint))
if (joint.TrackingState != TrackingState.NotTracked)
{
JointOrientation jointOrientation;
if (body.JointOrientations.TryGetValue(jointType, out jointOrientation))
{
var time = (float) (DateTime.Now - MoCapRecordStartTime).TotalSeconds;
//var positionX = joint.Position.X;
//var positionY = joint.Position.Y;
//var positionZ = joint.Position.Z;
var rotationX = jointOrientation.Orientation.X;
var rotationY = jointOrientation.Orientation.Y;
var rotationZ = jointOrientation.Orientation.Z;
var rotationW = jointOrientation.Orientation.W;
var transform = new TransformTime();
transform.Time = time;
transform.Position = Vector3.Zero;
transform.Rotation = new Quaternion(rotationX, rotationY, rotationZ, rotationW);
transform.Scale = Vector3.One;
transform.Rotation = BodyDESP[(int) jointType].Predict(transform.Rotation);
command.KinectBody.SkeletonTransforms[(int) jointType] = transform;
}
}
}
if (IsRecording)
BodyFrames.Add(command.KinectBody);
BeginWriteCommand(command);
break;
}
}
}
}
private static AnimationClip GetAnimationClip(Animator animator, GameObject rootObject, MoCapData data,
bool processBodyTransforms, bool processFaceTransforms, bool processFaceBlendShapes)
{
var animationClip = new AnimationClip();
animationClip.name = Path.GetFileNameWithoutExtension(DataFile);
animationClip.legacy = true;
animationClip.wrapMode = WrapMode.Once;
animationClip.frameRate = 25;
animationClip.ClearCurves();
if (data.BodyFrames != null && data.BodyFrames.Length > 0)
{
if (processBodyTransforms)
{
var transforms = new TransformTime[(int)MoCapKinectBone.Count][];
for (var i = 0; i < (int)MoCapKinectBone.Count; i++)
{
transforms[i] = new TransformTime[data.BodyFrames.Length];
for (var j = 0; j < data.BodyFrames.Length; j++)
{
transforms[i][j].Time = data.BodyFrames[j].SkeletonTransforms[i].Time;
transforms[i][j].Position = Vector3.Zero;
transforms[i][j].Rotation = MoCapBoneMapper.LocalRotation(ref data.BodyFrames[j],
(MoCapKinectBone)i);
transforms[i][j].Scale = Vector3.One;
}
}
foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof(MoCapKinectBone)))
{
if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
{
var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
{
SetAnimationCurvesForBody(animator, rootObject, transforms[(int)kinectBone], animationClip,
mecanimBone, false, true, false);
}
}
}
}
}
if (data.FaceFrames != null && data.FaceFrames.Length > 0)
{
if (processFaceTransforms)
{
var transforms = new TransformTime[data.FaceFrames.Length];
for (var i = 0; i < data.FaceFrames.Length; i++)
{
transforms[i].Time = data.FaceFrames[i].FaceTransform.Time;
transforms[i].Position = Vector3.Zero;
transforms[i].Rotation = new Quaternion(-data.FaceFrames[i].FaceTransform.Rotation.X,
data.FaceFrames[i].FaceTransform.Rotation.Y, data.FaceFrames[i].FaceTransform.Rotation.Z,
data.FaceFrames[i].FaceTransform.Rotation.W);
transforms[i].Scale = Vector3.One;
}
SetAnimationCurvesForBody(animator, rootObject, transforms, animationClip, MoCapMecanimBone.Head, false,
true, false);
}
if (processFaceBlendShapes)
{
var weights =
new List <KeyValuePair <float, float> > [(int)MoCapMixamoFacialExpression.LastBlendShape];
foreach (
MoCapKinectFacialExpression kinectFacialExpression in
Enum.GetValues(typeof(MoCapKinectFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
{
for (var j = 0; j < data.FaceFrames.Length; j++)
{
MoCapMixamoFacialExpression mixamoFacialExpression;
float mixamoWeight;
MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression,
data.FaceFrames[j].ExpressionWeights[(int)kinectFacialExpression],
out mixamoFacialExpression, out mixamoWeight);
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int)mixamoFacialExpression] == null)
{
weights[(int)mixamoFacialExpression] =
new List <KeyValuePair <float, float> >(data.FaceFrames.Length);
}
weights[(int)mixamoFacialExpression].Add(
new KeyValuePair <float, float>(data.FaceFrames[j].FaceTransform.Time, mixamoWeight));
}
}
}
}
foreach (
MoCapMixamoFacialExpression mixamoFacialExpression in
Enum.GetValues(typeof(MoCapMixamoFacialExpression)))
{
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int)mixamoFacialExpression] != null &&
weights[(int)mixamoFacialExpression].Count > 0)
{
SetAnimationCurvesForFace(animator, rootObject,
weights[(int)mixamoFacialExpression].ToArray(), animationClip, mixamoFacialExpression);
}
}
}
}
}
animationClip.EnsureQuaternionContinuity();
return(animationClip);
}
private static AnimationClip GetAnimationClip(Animator animator, GameObject rootObject, MoCapData data,
bool processBodyTransforms, bool processFaceTransforms, bool processFaceBlendShapes)
{
var animationClip = new AnimationClip();
animationClip.name = Path.GetFileNameWithoutExtension(DataFile);
animationClip.legacy = true;
animationClip.wrapMode = WrapMode.Once;
animationClip.frameRate = 25;
animationClip.ClearCurves();
if (data.BodyFrames != null && data.BodyFrames.Length > 0)
{
if (processBodyTransforms)
{
var transforms = new TransformTime[(int) MoCapKinectBone.Count][];
for (var i = 0; i < (int) MoCapKinectBone.Count; i++)
{
transforms[i] = new TransformTime[data.BodyFrames.Length];
for (var j = 0; j < data.BodyFrames.Length; j++)
{
transforms[i][j].Time = data.BodyFrames[j].SkeletonTransforms[i].Time;
transforms[i][j].Position = Vector3.Zero;
transforms[i][j].Rotation = MoCapBoneMapper.LocalRotation(ref data.BodyFrames[j],
(MoCapKinectBone) i);
transforms[i][j].Scale = Vector3.One;
}
}
foreach (MoCapKinectBone kinectBone in Enum.GetValues(typeof (MoCapKinectBone)))
if (MoCapBoneMapper.IsValidKinectBone(kinectBone))
{
var mecanimBone = MoCapBoneMapper.Kinect2Mecanim(kinectBone);
if (MoCapBoneMapper.IsValidMecanimBone(mecanimBone))
SetAnimationCurvesForBody(animator, rootObject, transforms[(int) kinectBone], animationClip,
mecanimBone, false, true, false);
}
}
}
if (data.FaceFrames != null && data.FaceFrames.Length > 0)
{
if (processFaceTransforms)
{
var transforms = new TransformTime[data.FaceFrames.Length];
for (var i = 0; i < data.FaceFrames.Length; i++)
{
transforms[i].Time = data.FaceFrames[i].FaceTransform.Time;
transforms[i].Position = Vector3.Zero;
transforms[i].Rotation = new Quaternion(-data.FaceFrames[i].FaceTransform.Rotation.X,
data.FaceFrames[i].FaceTransform.Rotation.Y, data.FaceFrames[i].FaceTransform.Rotation.Z,
data.FaceFrames[i].FaceTransform.Rotation.W);
transforms[i].Scale = Vector3.One;
}
SetAnimationCurvesForBody(animator, rootObject, transforms, animationClip, MoCapMecanimBone.Head, false,
true, false);
}
if (processFaceBlendShapes)
{
var weights =
new List<KeyValuePair<float, float>>[(int) MoCapMixamoFacialExpression.LastBlendShape];
foreach (
MoCapKinectFacialExpression kinectFacialExpression in
Enum.GetValues(typeof (MoCapKinectFacialExpression)))
if (MoCapFacialExpressionMapper.IsValidKinectFacialExpression(kinectFacialExpression))
{
for (var j = 0; j < data.FaceFrames.Length; j++)
{
MoCapMixamoFacialExpression mixamoFacialExpression;
float mixamoWeight;
MoCapFacialExpressionMapper.Kinect2Mixamo(kinectFacialExpression,
data.FaceFrames[j].ExpressionWeights[(int) kinectFacialExpression],
out mixamoFacialExpression, out mixamoWeight);
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
{
if (weights[(int) mixamoFacialExpression] == null)
weights[(int) mixamoFacialExpression] =
new List<KeyValuePair<float, float>>(data.FaceFrames.Length);
weights[(int) mixamoFacialExpression].Add(
new KeyValuePair<float, float>(data.FaceFrames[j].FaceTransform.Time, mixamoWeight));
}
}
}
foreach (
MoCapMixamoFacialExpression mixamoFacialExpression in
Enum.GetValues(typeof (MoCapMixamoFacialExpression)))
if (MoCapFacialExpressionMapper.IsValidMixamoFacialExpression(mixamoFacialExpression))
if (weights[(int) mixamoFacialExpression] != null &&
weights[(int) mixamoFacialExpression].Count > 0)
SetAnimationCurvesForFace(animator, rootObject,
weights[(int) mixamoFacialExpression].ToArray(), animationClip, mixamoFacialExpression);
}
}
animationClip.EnsureQuaternionContinuity();
return animationClip;
}
private static void SetAnimationCurvesForBody(Animator animator, GameObject rootObject, TransformTime[] transforms,
AnimationClip animationClip, MoCapMecanimBone bone, bool setPosition, bool setRotation, bool setScale)
{
if (animator == null)
throw new Exception("Animator can not be null!");
if (rootObject == null)
throw new Exception("Root object can not be null!");
if (transforms == null || transforms.Length == 0)
throw new Exception("Transforms can not be empty!");
if (animationClip == null)
throw new Exception("Animation clip can not be null!");
if (!MoCapBoneMapper.IsValidMecanimBone(bone))
throw new Exception("Invalid Mecanim bone!");
var relativeTransform = animator.GetBoneTransform((HumanBodyBones) bone);
var relativePath = AnimationUtility.CalculateTransformPath(relativeTransform, rootObject.transform);
var keyframesTransformPositionX = new Keyframe[transforms.Length];
var keyframesTransformPositionY = new Keyframe[transforms.Length];
var keyframesTransformPositionZ = new Keyframe[transforms.Length];
var keyframesTransformRotationX = new Keyframe[transforms.Length];
var keyframesTransformRotationY = new Keyframe[transforms.Length];
var keyframesTransformRotationZ = new Keyframe[transforms.Length];
var keyframesTransformRotationW = new Keyframe[transforms.Length];
var keyframesTransformScaleX = new Keyframe[transforms.Length];
var keyframesTransformScaleY = new Keyframe[transforms.Length];
var keyframesTransformScaleZ = new Keyframe[transforms.Length];
for (var i = 0; i < transforms.Length; i++)
{
var transform = transforms[i];
keyframesTransformPositionX[i] = new Keyframe(transform.Time, transform.Position.X);
keyframesTransformPositionY[i] = new Keyframe(transform.Time, transform.Position.Y);
keyframesTransformPositionZ[i] = new Keyframe(transform.Time, transform.Position.Z);
keyframesTransformRotationX[i] = new Keyframe(transform.Time, transform.Rotation.X);
keyframesTransformRotationY[i] = new Keyframe(transform.Time, transform.Rotation.Y);
keyframesTransformRotationZ[i] = new Keyframe(transform.Time, transform.Rotation.Z);
keyframesTransformRotationW[i] = new Keyframe(transform.Time, transform.Rotation.W);
keyframesTransformScaleX[i] = new Keyframe(transform.Time, transform.Scale.X);
keyframesTransformScaleY[i] = new Keyframe(transform.Time, transform.Scale.Y);
keyframesTransformScaleZ[i] = new Keyframe(transform.Time, transform.Scale.Z);
}
var animationCurvePositionX = new AnimationCurve(keyframesTransformPositionX);
var animationCurvePositionY = new AnimationCurve(keyframesTransformPositionY);
var animationCurvePositionZ = new AnimationCurve(keyframesTransformPositionZ);
if (setPosition)
{
animationClip.SetCurve(relativePath, typeof (Transform), "localPosition.x", animationCurvePositionX);
animationClip.SetCurve(relativePath, typeof (Transform), "localPosition.y", animationCurvePositionY);
animationClip.SetCurve(relativePath, typeof (Transform), "localPosition.z", animationCurvePositionZ);
}
var animationCurveRotationX = new AnimationCurve(keyframesTransformRotationX);
var animationCurveRotationY = new AnimationCurve(keyframesTransformRotationY);
var animationCurveRotationZ = new AnimationCurve(keyframesTransformRotationZ);
var animationCurveRotationW = new AnimationCurve(keyframesTransformRotationW);
if (setRotation)
{
animationClip.SetCurve(relativePath, typeof (Transform), "localRotation.x", animationCurveRotationX);
animationClip.SetCurve(relativePath, typeof (Transform), "localRotation.y", animationCurveRotationY);
animationClip.SetCurve(relativePath, typeof (Transform), "localRotation.z", animationCurveRotationZ);
animationClip.SetCurve(relativePath, typeof (Transform), "localRotation.w", animationCurveRotationW);
}
var animationCurveScaleX = new AnimationCurve(keyframesTransformScaleX);
var animationCurveScaleY = new AnimationCurve(keyframesTransformScaleY);
var animationCurveScaleZ = new AnimationCurve(keyframesTransformScaleZ);
if (setScale)
{
animationClip.SetCurve(relativePath, typeof (Transform), "localScale.x", animationCurveScaleX);
animationClip.SetCurve(relativePath, typeof (Transform), "localScale.y", animationCurveScaleY);
animationClip.SetCurve(relativePath, typeof (Transform), "localScale.z", animationCurveScaleZ);
}
}
private void BodyFrameReader_FrameArrived(object sender, BodyFrameArrivedEventArgs e)
{
using (var frame = e.FrameReference.AcquireFrame())
{
if (frame != null)
{
if (Bodies == null)
{
Bodies = new Body[frame.BodyCount];
}
frame.GetAndRefreshBodyData(Bodies);
foreach (var body in Bodies)
{
if (body.IsTracked)
{
HighDefinitionFaceFrameSource.TrackingId = body.TrackingId;
var command = new CommandMessage
{
CommandType = CommandType.KinectBody,
KinectBody =
new MoCapBodyFrame
{
SkeletonTransforms = new TransformTime[(int)MoCapKinectBone.Count]
}
};
foreach (JointType jointType in Enum.GetValues(typeof(JointType)))
{
Joint joint;
if (body.Joints.TryGetValue(jointType, out joint))
{
if (joint.TrackingState != TrackingState.NotTracked)
{
JointOrientation jointOrientation;
if (body.JointOrientations.TryGetValue(jointType, out jointOrientation))
{
var time = (float)(DateTime.Now - MoCapRecordStartTime).TotalSeconds;
//var positionX = joint.Position.X;
//var positionY = joint.Position.Y;
//var positionZ = joint.Position.Z;
var rotationX = jointOrientation.Orientation.X;
var rotationY = jointOrientation.Orientation.Y;
var rotationZ = jointOrientation.Orientation.Z;
var rotationW = jointOrientation.Orientation.W;
var transform = new TransformTime();
transform.Time = time;
transform.Position = Vector3.Zero;
transform.Rotation = new Quaternion(rotationX, rotationY, rotationZ, rotationW);
transform.Scale = Vector3.One;
transform.Rotation = BodyDESP[(int)jointType].Predict(transform.Rotation);
command.KinectBody.SkeletonTransforms[(int)jointType] = transform;
}
}
}
}
if (IsRecording)
{
BodyFrames.Add(command.KinectBody);
}
BeginWriteCommand(command);
break;
}
}
}
}
}
