/**
* Adds all possible blendshape targets of the game object
* curGameObject and its sub-objects (recursively) to the
* list of target blendshapes.
*/
public static void GetGameObjectBlendshapes(GameObject curGameObject, ArrayList blendshape_infos) {
if (blendshape_infos != null) blendshape_infos.Clear();
// Iterate over game object itself and over children and add blendshapes
Transform [] children = curGameObject.GetComponentsInChildren<Transform>();
foreach (Transform child in children)
{
string transformPath = CalculateTransformPath(child, curGameObject.transform);
SkinnedMeshRenderer meshRenderer = (SkinnedMeshRenderer)child.GetComponent(typeof(SkinnedMeshRenderer));
if (meshRenderer != null)
{
if (meshRenderer.sharedMesh != null)
{
//Debug.Log("Number of blend shapes: " + meshRenderer.sharedMesh.blendShapeCount);
for (int blend_shape_nr = 0; blend_shape_nr < meshRenderer.sharedMesh.blendShapeCount; blend_shape_nr++) {
string blend_shape_name = meshRenderer.sharedMesh.GetBlendShapeName(blend_shape_nr);
//Debug.Log("Path: '" + transformPath + "', blend shape: '" + blend_shape_name + "'");
if (blendshape_infos != null) {
BlendshapeInfo blendshape_info = new BlendshapeInfo();
blendshape_info.m_path = transformPath;
blendshape_info.m_name = blend_shape_name;
blendshape_info.m_index = blend_shape_nr;
blendshape_info.m_mesh_renderer = meshRenderer;
blendshape_infos.Add (blendshape_info);
}
}
}
}
}
}
/// <summary>
/// Applies the blendshapes given an influence factor
/// </summary>
/// <returns><c>true</c>, if blendshapes could be applied, <c>false</c> otherwise.</returns>
/// <param name="transformationsToSet">The set of blendshapes to apply</param>
/// <param name="influence">The factor to apply the blendshapes. 0.0 means that they are not used at all and the current
/// animations are not changed. 1.0 means that they are fully used.</param>
protected bool ApplyBlendshapes(BlendshapeValue [] blendshapesToSetToSet, float influence)
{
if (blendshapesToSetToSet.Length != m_GameObjectBlendshapes.Count)
{
return(false);
}
for (int index = 0; index < m_GameObjectBlendshapes.Count; index++)
{
// Apply the value for this target
if (blendshapesToSetToSet[index] != null)
{
BlendshapeInfo bs_info = m_GameObjectBlendshapes[index] as BlendshapeInfo;
float originalValue = bs_info.m_mesh_renderer.GetBlendShapeWeight(bs_info.m_index);
float newValue = (float)blendshapesToSetToSet[index].m_value;
bs_info.m_mesh_renderer.SetBlendShapeWeight(bs_info.m_index, influence * newValue + (1.0f - influence) * originalValue);
}
}
return(true);
}
/**
* Adds all possible blendshape targets of the game object
* curGameObject and its sub-objects (recursively) to the
* list of target blendshapes.
*/
public static void getGameObjectBlendshapes(GameObject curGameObject,
ArrayList blendshape_infos)
{
if (blendshape_infos != null)
{
blendshape_infos.Clear();
}
// Iterate over game object itself and over children and add blendshapes
Transform [] children = curGameObject.GetComponentsInChildren <Transform>();
foreach (Transform child in children)
{
string transformPath = calculateTransformPath(child, curGameObject.transform);
SkinnedMeshRenderer meshRenderer = (SkinnedMeshRenderer)child.GetComponent(typeof(SkinnedMeshRenderer));
if (meshRenderer != null)
{
if (meshRenderer.sharedMesh != null)
{
//Debug.Log("Number of blend shapes: " + meshRenderer.sharedMesh.blendShapeCount);
for (int blend_shape_nr = 0; blend_shape_nr < meshRenderer.sharedMesh.blendShapeCount; blend_shape_nr++)
{
string blend_shape_name = meshRenderer.sharedMesh.GetBlendShapeName(blend_shape_nr);
//Debug.Log("Path: '" + transformPath + "', blend shape: '" + blend_shape_name + "'");
if (blendshape_infos != null)
{
BlendshapeInfo blendshape_info = new BlendshapeInfo();
blendshape_info.m_path = transformPath;
blendshape_info.m_name = blend_shape_name;
blendshape_info.m_index = blend_shape_nr;
blendshape_info.m_mesh_renderer = meshRenderer;
blendshape_infos.Add(blendshape_info);
}
}
}
}
}
}
/**
* @brief Stores the current clip into a Unity .anim file
* Important: If you use your own character with blend shapes, you have to make sure it is
* set to 'legacy' animation type. You can do this by the following steps in Unity3D:
* 1. In the 'project' window in the Assets hierarchy, click on your fbx model
* 2. In the Inspector, you should see now the Import Settings of your model.
* 3. Select in these Import Settings in the 'Rig' tab for the 'Animation Type' the value 'Legacy'
*/
public void SaveClipAsAnim(string path)
{
bool have_t_pose = false;
if (m_tpose != null && m_tpose.m_joints != null)
{
if (m_tpose.m_joints.Count != targetTransformations.Count)
{
Debug.LogError("tpose and model do not have the same number of transformations (" + m_tpose.m_joints.Count + "!="
+ targetTransformations.Count + ")");
return;
}
have_t_pose = true;
}
else
{
Debug.LogWarning("tpose missing");
}
AnimationClip animation_clip = new AnimationClip();
// Not all of them might actually be used, since not all of them might be a target
AnimationCurve [] translation_x_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] translation_y_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] translation_z_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] rotation_x_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] rotation_y_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] rotation_z_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] rotation_w_curves = new AnimationCurve[targetTransformations.Count];
AnimationCurve [] bs_curves = new AnimationCurve[targetBlendShapeInfos.Count];
Debug.Log("nr of clip states: " + m_clip.num_states());
for (int frame_nr = 0; frame_nr < m_clip.num_states(); frame_nr++)
{
// get frame time
float time = (float)(m_clip[frame_nr].timestamp() - m_clip[0].timestamp()) * 0.001f; // time is in ms
// evaluate transformation
TransformationValue [] transformation_values = null;
if (have_t_pose)
{
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_fs_Rig, m_clip[frame_nr], m_tpose.m_joints);
}
else
{
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_fs_Rig, m_clip[frame_nr], targetTransformations);
}
if (transformation_values.Length == targetTransformations.Count)
{
for (int index = 0; index < transformation_values.Length; index++)
{
// Apply the value for this target
if (transformation_values[index] != null)
{
// Apply the translation value for this target
if (translation_x_curves[index] == null)
{
translation_x_curves[index] = new AnimationCurve();
translation_y_curves[index] = new AnimationCurve();
translation_z_curves[index] = new AnimationCurve();
}
translation_x_curves[index].AddKey(time, transformation_values[index].m_translation.x);
translation_y_curves[index].AddKey(time, transformation_values[index].m_translation.y);
translation_z_curves[index].AddKey(time, transformation_values[index].m_translation.z);
if (rotation_x_curves[index] == null)
{
rotation_x_curves[index] = new AnimationCurve();
rotation_y_curves[index] = new AnimationCurve();
rotation_z_curves[index] = new AnimationCurve();
rotation_w_curves[index] = new AnimationCurve();
}
// Add to curve for the animation
rotation_x_curves[index].AddKey(time, transformation_values[index].m_rotation.x);
rotation_y_curves[index].AddKey(time, transformation_values[index].m_rotation.y);
rotation_z_curves[index].AddKey(time, transformation_values[index].m_rotation.z);
rotation_w_curves[index].AddKey(time, transformation_values[index].m_rotation.w);
}
}
}
else
{
Debug.LogError("Cannot create transformation as evaluated shape size is incorrect");
}
// evaluate blendshapes
BlendshapeValue [] blendshape_values = Utils.evaluate_target_blendshapes(m_retargeting, m_clip.rig(), m_clip[frame_nr], targetBlendShapeInfos);
if (blendshape_values.Length == targetBlendShapeInfos.Count)
{
for (int index = 0; index < targetBlendShapeInfos.Count; index++)
{
// Apply the value for this target
if (blendshape_values[index] != null)
{
if (bs_curves[index] == null)
{
bs_curves[index] = new AnimationCurve();
}
bs_curves[index].AddKey(time, (float)blendshape_values[index].m_value);
}
}
}
else
{
Debug.LogError("Cannot create blendshapes as evaluated shape size is incorrect");
}
}
// Set all transformation curves for all transformation that are animated
for (int target_nr = 0; target_nr < targetTransformations.Count; target_nr++)
{
// Extract path:
string path_to_transformation = ((TransformationInformation)targetTransformations[target_nr]).transformPath;
// Apply translation curve, if there is one
if (translation_x_curves[target_nr] != null)
{
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localPosition.x", translation_x_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localPosition.y", translation_y_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localPosition.z", translation_z_curves[target_nr]);
}
// Apply rotation curve, if there is one
if (rotation_x_curves[target_nr] != null)
{
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.x", rotation_x_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.y", rotation_y_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.z", rotation_z_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.w", rotation_w_curves[target_nr]);
}
}
// Without this, there are some weird jumps (rotation) in the animation:
animation_clip.EnsureQuaternionContinuity();
// Set all blendshape curves for all blendshapes that are animated
for (int i = 0; i < targetBlendShapeInfos.Count; i++)
{
if (bs_curves[i] != null)
{
BlendshapeInfo bs_info = (BlendshapeInfo)(targetBlendShapeInfos[i]);
// Debug.Log("bs_curves[" + i + "].length=" + bs_curves[i].length);
string bs_path = bs_info.m_path;
string bs_name = bs_info.m_name;
animation_clip.SetCurve(bs_path, typeof(SkinnedMeshRenderer), "blendShape." + bs_name, bs_curves[i]);
}
}
// animation clip asset
string animation_name = Path.GetFileNameWithoutExtension(path);
animation_clip.name = animation_name;
#if UNITY_EDITOR
AssetDatabase.CreateAsset(animation_clip, path);
#endif
// Add the clip to the animations of the attached game object
Animation my_animation = gameObject.animation;
if (!my_animation)
{
gameObject.AddComponent(typeof(Animation));
my_animation = gameObject.animation;
}
if (my_animation)
{
my_animation.AddClip(animation_clip, animation_name);
my_animation.clip = animation_clip;
my_animation[animation_name].blendMode = AnimationBlendMode.Blend;
}
else
{
print("Error: could not get animation object");
}
Debug.Log("Wrote animation with length " + (1000.0 * animation_clip.length) + " milliseconds");
}
/**
* @brief Stores the current clip into a Unity .anim file
* Important: If you use your own character with blend shapes, you have to make sure it is
* set to 'legacy' animation type. You can do this by the following steps in Unity3D:
* 1. In the 'project' window in the Assets hierarchy, click on your fbx model
* 2. In the Inspector, you should see now the Import Settings of your model.
* 3. Select in these Import Settings in the 'Rig' tab for the 'Animation Type' the value 'Legacy'
*/
public AnimationClip GetClipAsAnim(string path)
{
bool have_t_pose = false;
if (m_TPose != null && m_TPose.m_joints != null)
{
if (m_TPose.m_joints.Count != m_GameObjectTransformations.Count)
{
Debug.LogError("tpose and model do not have the same number of transformations (" + m_TPose.m_joints.Count + "!="
+ m_GameObjectTransformations.Count + ")");
return(null);
}
have_t_pose = true;
}
else
{
Debug.LogWarning("tpose missing");
}
AnimationClip animation_clip = new AnimationClip();
// Not all of them might actually be used, since not all of them might be a target
AnimationCurve [] translation_x_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] translation_y_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] translation_z_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] rotation_x_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] rotation_y_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] rotation_z_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] rotation_w_curves = new AnimationCurve[m_GameObjectTransformations.Count];
AnimationCurve [] bs_curves = new AnimationCurve[m_GameObjectBlendshapes.Count];
Clip clip = m_clip.Duplicate();
if (m_normalize_headpose)
{
clip.NormalizeHeadPoseAllClip();
}
Debug.Log("nr of clip states: " + clip.NumStates());
for (int frame_nr = 0; frame_nr < clip.NumStates(); frame_nr++)
{
if (!clip[frame_nr].TrackingSuccessful())
{
Debug.Log("skipping clip state");
continue;
}
// get frame time
float time = (float)(clip[frame_nr].Timestamp() - clip[0].Timestamp()) * 0.001f; // time is in ms
// evaluate transformation
TransformationValue [] transformation_values = null;
if (have_t_pose)
{
transformation_values = Utils.EvaluateTargetTransformations(m_Retargeting, clip.Rig(), clip[frame_nr], m_TPose.m_joints);
}
else
{
transformation_values = Utils.EvaluateTargetTransformations(m_Retargeting, clip.Rig(), clip[frame_nr], m_GameObjectTransformations);
}
int key_index = -1;
if (transformation_values.Length == m_GameObjectTransformations.Count)
{
for (int index = 0; index < transformation_values.Length; index++)
{
// Apply the value for this target
if (transformation_values[index] != null)
{
// Apply the translation value for this target
if (translation_x_curves[index] == null)
{
translation_x_curves[index] = new AnimationCurve();
translation_y_curves[index] = new AnimationCurve();
translation_z_curves[index] = new AnimationCurve();
}
key_index = translation_x_curves[index].AddKey(time, transformation_values[index].m_translation.x);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
key_index = translation_y_curves[index].AddKey(time, transformation_values[index].m_translation.y);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
key_index = translation_z_curves[index].AddKey(time, transformation_values[index].m_translation.z);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
if (rotation_x_curves[index] == null)
{
rotation_x_curves[index] = new AnimationCurve();
rotation_y_curves[index] = new AnimationCurve();
rotation_z_curves[index] = new AnimationCurve();
rotation_w_curves[index] = new AnimationCurve();
}
// Add to curve for the animation
key_index = rotation_x_curves[index].AddKey(time, transformation_values[index].m_rotation.x);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
key_index = rotation_y_curves[index].AddKey(time, transformation_values[index].m_rotation.y);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
key_index = rotation_z_curves[index].AddKey(time, transformation_values[index].m_rotation.z);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
key_index = rotation_w_curves[index].AddKey(time, transformation_values[index].m_rotation.w);
if (key_index < 0)
{
Debug.LogError("Could not add key at time " + time);
}
}
}
}
else
{
Debug.LogError("Cannot create transformation as evaluated shape size is incorrect");
}
// evaluate blendshapes
BlendshapeValue [] blendshape_values = Utils.EvaluateTargetBlendshapes(m_Retargeting, clip.Rig(), clip[frame_nr], m_GameObjectBlendshapes);
if (blendshape_values.Length == m_GameObjectBlendshapes.Count)
{
for (int index = 0; index < m_GameObjectBlendshapes.Count; index++)
{
// Apply the value for this target
if (blendshape_values[index] != null)
{
if (bs_curves[index] == null)
{
bs_curves[index] = new AnimationCurve();
}
bs_curves[index].AddKey(time, (float)blendshape_values[index].m_value);
}
}
}
else
{
Debug.LogError("Cannot create blendshapes as evaluated shape size is incorrect");
}
}
// Set all transformation curves for all transformations that are animated
for (int target_nr = 0; target_nr < m_GameObjectTransformations.Count; target_nr++)
{
// Extract path:
string path_to_transformation = ((TransformationInformation)m_GameObjectTransformations[target_nr]).transformPath;
// Apply translation curve, if there is one
if (translation_x_curves[target_nr] != null)
{
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localPosition.x", translation_x_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localPosition.y", translation_y_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localPosition.z", translation_z_curves[target_nr]);
}
// Apply rotation curve, if there is one
if (rotation_x_curves[target_nr] != null)
{
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.x", rotation_x_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.y", rotation_y_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.z", rotation_z_curves[target_nr]);
animation_clip.SetCurve(path_to_transformation, typeof(Transform), "localRotation.w", rotation_w_curves[target_nr]);
}
}
// Without this, there are some weird jumps (rotation) in the animation:
animation_clip.EnsureQuaternionContinuity();
// Set all blendshape curves for all blendshapes that are animated
for (int i = 0; i < m_GameObjectBlendshapes.Count; i++)
{
if (bs_curves[i] != null)
{
BlendshapeInfo bs_info = (BlendshapeInfo)(m_GameObjectBlendshapes[i]);
// Debug.Log("bs_curves[" + i + "].length=" + bs_curves[i].length);
string bs_path = bs_info.m_path;
string bs_name = bs_info.m_name;
animation_clip.SetCurve(bs_path, typeof(SkinnedMeshRenderer), "blendShape." + bs_name, bs_curves[i]);
}
}
Debug.Log("Animation clip = " + animation_clip.length);
// animation clip asset
string animation_name = Path.GetFileNameWithoutExtension(path);
animation_clip.name = animation_name;
AnimationEvent animation_event = new AnimationEvent();
animation_event.functionName = "AnimationClipEventCallback";
animation_event.time = animation_clip.length;
#if UNITY_EDITOR
AnimationEvent[] animation_events = { animation_event };
AnimationUtility.SetAnimationEvents(animation_clip, animation_events);
AssetDatabase.CreateAsset(animation_clip, path);
#endif
Debug.Log("Wrote animation with length " + (1000.0 * animation_clip.length) + " milliseconds");
return(animation_clip);
}
void Update()
{
// ten kod kalkuluje bazowa pozycje glowy
// trzeba jeszcze dorobic zeby przechwytywal trigerra
// i wtedy zmieniał isBaseCalculated na false
// i resetował timer i kalkulował jeszcze raz
// pozniej trzeba jeszcze dodac zeby kazda metoda zwracała
// to co zwraca minus base
if (!isBaseCalculated)
{
timer += Time.deltaTime;
if (timer < baseCalculationTime)
{
if (getXHeadRotation() != -1)
{
baseX += getXHeadRotation();
ticksX++;
}
if (getYHeadRotation() != -1)
{
baseY += getYHeadRotation();
ticksY++;
}
}
else
{
baseX /= ticksX;
baseY /= ticksY;
isBaseCalculated = true;
}
}
new_data = false;
m_mutex.WaitOne();
// check if there are new blendshape names, and if so update
if (newBlendShapeNamesArrived())
{
m_rig = getRigFromBlendShapeNames(getBlendShapeNames());
}
// get most recent tracking data
while (m_data_parser.CountAvailableTracks() > 0)
{
m_current_track = m_data_parser.Dequeue();
new_data = true;
}
// check that we have a rig (set up from blendshape names) with the same number of blendshapes as what we receive
if (m_rig != null && m_current_track != null)
{
int n_track_coefficients = m_current_track.n_coefficients();
int n_rig_coefficients = m_rig.num_shapes();
if (n_track_coefficients != n_rig_coefficients)
{
Debug.LogWarning("number of coefficients of rig and tracking state have changed: " + n_track_coefficients + " vs " + n_rig_coefficients);
// clear the current rig as it is not usable with the data coming from faceshift
m_rig = null;
m_current_track = null;
// get again the blendshapes from fs studio
askForBlendshapeNames();
}
}
m_mutex.ReleaseMutex();
if (m_rig != null && m_current_track != null && m_retargeting != null)
{
if (new_data && m_current_track.TrackSuccess)
{
// create a rig state
RigState state = new RigState(m_rig);
state.set_timestamp(m_current_track.TimeStamp);
state.set_bone_translation(0, m_current_track.HeadTranslation());
state.set_bone_rotation(0, m_current_track.HeadRotation());
state.set_bone_rotation(1, m_current_track.LeftEyeRotation());
state.set_bone_rotation(2, m_current_track.RightEyeRotation());
for (int i = 0; i < m_rig.num_shapes(); i++)
{
state.set_blendshape_coefficient(i, m_current_track.Coefficient [i]);
}
// evaluate joint transformations
TransformationValue [] transformation_values = null;
if (m_tpose != null && m_tpose.m_joints.Count == m_game_object_transformations.Count)
{
// evaluate using tpose
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_rig, state, m_tpose.m_joints);
}
else
{
// evaluate using state from start of application
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_rig, state, m_game_object_transformations);
}
if (transformation_values.Length == m_game_object_transformations.Count)
{
for (int index = 0; index < transformation_values.Length; index++)
{
// Apply the value for this target
if (transformation_values [index] != null)
{
TransformationInformation joint = m_game_object_transformations [index] as TransformationInformation;
joint.transform.localRotation = transformation_values [index].m_rotation;
joint.transform.localPosition = transformation_values [index].m_translation;
}
}
}
else
{
Debug.LogError("Cannot create transformation as evaluated shape size is incorrect");
}
// evaluate blendshape valuesf
BlendshapeValue [] values = Utils.evaluate_target_blendshapes(m_retargeting, m_rig, state, m_game_object_blendshapes);
if (values.Length == m_game_object_blendshapes.Count)
{
for (int index = 0; index < m_game_object_blendshapes.Count; index++)
{
BlendshapeInfo bs_info = m_game_object_blendshapes [index] as BlendshapeInfo;
// Apply the value for this target
if (bs_info != null && values [index] != null)
{
bs_info.m_mesh_renderer.SetBlendShapeWeight(bs_info.m_index, (float)values [index].m_value);
}
}
}
else
{
Debug.LogError("Cannot create blendshapes as evaluated shape size is incorrect");
}
}
}
}
