//! duplicate this rig (deep copy)
public Rig Duplicate() {
Rig rig = new Rig ();
for (int i = 0; i < NumShapes(); i++) rig.AddShape(ShapeName(i));
for (int i = 0; i < NumBones() ; i++) rig.AddBone(BoneName(i));
return rig;
}
/**
* @brief Reads an fsb file and returns a clip constructed from the the content of the fsb file.
* @param[in] data The binary data from the fsb file.
* @return A clip with the track data imported from the fsb file, null if there was some error during loading.
*/
public static Clip Read(byte[] data) {
DataParser parser = new DataParser();
parser.ExtractData(data, false, 0.0);
if (parser.Valid()) {
Rig rig = new Rig();
// add hardcoded bone names of fsb file
rig.AddBone("Neck");
rig.AddBone("EyeLeft");
rig.AddBone("EyeRight");
for (int i = 0; i < parser.fsBlendShapeNames.Length; i++) {
rig.AddShape(parser.fsBlendShapeNames[i]);
}
Clip clip = new Clip(rig);
while (parser.CountAvailableTracks() > 0) {
TrackData track_data = parser.Dequeue();
if (!track_data.TrackSuccess) {
// do not save the state if the tracking data is not valid
Debug.LogWarning("this frame doesn't contain valid tracking data");
continue;
}
RigState state = clip.NewState(track_data.TimeStamp);
state.SetTrackingSuccessful(track_data.TrackSuccess);
if (rig.NumShapes() != track_data.n_coefficients()) {
Debug.LogError("num blendshapes do not agree in file with " + rig.NumShapes() +
" in rig and " + track_data.n_coefficients() + " in state");
return null;
}
// Assume the head translation to be joint 0 (Neck)
state.SetBoneTranslation(0, track_data.HeadTranslation());
// bone indices same as the order when added to the rig
state.SetBoneRotation(0, track_data.HeadRotation());
state.SetBoneRotation(1, track_data.LeftEyeRotation());
state.SetBoneRotation(2, track_data.RightEyeRotation());
for (int i = 0; i < track_data.n_coefficients(); i++) {
state.SetBlendshapeCoefficient(i, track_data.Coefficient[i]);
}
}
return clip;
} else {
Debug.LogError("cannot parse fsb file");
}
return null;
}
//! @returns a rig build from a set of blendshape names
static public Rig GetRigFromBlendShapeNames(string [] blendshape_names) {
Rig rig = new Rig();
rig.AddBone("Neck");
rig.AddBone("EyeLeft");
rig.AddBone("EyeRight");
for (int i = 0; i < blendshape_names.Length; i++) {
rig.AddShape(blendshape_names[i]);
}
return rig;
}
void Update () {
bool new_data = false;
m_mutex.WaitOne();
// check if there are new blendshape names, and if so update
if (NewBlendShapeNamesArrived()) {
m_rig = Rig.GetRigFromBlendShapeNames(GetBlendShapeNames());
}
// get most recent tracking data
while (m_data_parser.CountAvailableTracks() > 0) {
m_current_track = m_data_parser.Dequeue();
new_data = true;
}
// check that we have a rig (set up from blendshape names) with the same number of blendshapes as what we receive
if (m_rig != null && m_current_track != null) {
int n_track_coefficients = m_current_track.n_coefficients();
int n_rig_coefficients = m_rig.NumShapes();
if (n_track_coefficients != n_rig_coefficients) {
Debug.LogWarning("number of coefficients of rig and tracking state have changed: " + n_track_coefficients + " vs " + n_rig_coefficients);
// clear the current rig as it is not usable with the data coming from faceshift
m_rig = null;
m_current_track = null;
// get again the blendshapes from fs studio
AskForBlendshapeNames();
}
}
m_mutex.ReleaseMutex();
if (m_rig != null && m_current_track != null && m_Retargeting != null) {
if (new_data && m_current_track.TrackSuccess) {
// create a rig state
RigState state = new RigState(m_rig);
state.SetTimestamp(m_current_track.TimeStamp);
state.SetBoneTranslation(0, m_current_track.HeadTranslation());
state.SetBoneRotation(0, m_current_track.HeadRotation());
state.SetBoneRotation(1, m_current_track.LeftEyeRotation());
state.SetBoneRotation(2, m_current_track.RightEyeRotation());
for (int i = 0; i < m_rig.NumShapes(); i++) {
state.SetBlendshapeCoefficient(i, m_current_track.Coefficient[i]);
}
base.UpdateAnimation(m_rig, state);
}
}
}
//! @brief Loads a faceshift clip
public void LoadFSB(string path)
{
if (!File.Exists(path))
{
#if UNITY_EDITOR
EditorUtility.DisplayDialog("Clip loading failed", "File " + path + " does exist", "Ok");
#endif
return;
}
Clip new_clip = FsbReader.Read(path);
if (new_clip == null)
{
Debug.LogError("could not read clip from file " + path);
#if UNITY_EDITOR
EditorUtility.DisplayDialog("Load failed", "File " + path + " does not contain a clip", "Ok");
#endif
return;
}
Rig clip_rig = new_clip.Rig();
if (IsSourceRigDifferent(clip_rig))
{
if ((cachedRigLoaded) && (!SourceRigChangedAskToContinue()))
{
// The user wants to cancel
return;
}
}
// always save new rig from the fsb file to have the same order
// Apply new rig
m_fs_Rig = clip_rig;
// Cache new rig
SaveSourceRig();
m_clip = new_clip;
m_clip_path = path;
}
/**
* Evaluates the target blendshape values based on the state of a rig and a retargeting configuration.
* If a blendshape is not affected by the retargeting then the value of this blendshape is null.
*/
public static BlendshapeValue [] evaluate_target_blendshapes(ClipRetargeting retargeting,
Rig rig,
RigState state,
ArrayList target_blendshapes)
{
int n_target_blendshapes = target_blendshapes.Count;
BlendshapeValue [] values = new BlendshapeValue[n_target_blendshapes];
// We iterate over the targets and accumulate all sources to them
for (int index = 0; index < n_target_blendshapes; index++)
{
double value = 0.0;
int value_count = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.get_number_of_blendshape_mappings(); mapping_nr++)
{
string mapping_target = retargeting.get_blendshape_mapping_destination(mapping_nr);
if (!mapping_target.Equals(((BlendshapeInfo)target_blendshapes[index]).m_name))
{
continue;
}
string mapping_src = retargeting.get_blendshape_mapping_source(mapping_nr);
int src_index = rig.shape_index(mapping_src);
if (src_index >= 0)
{
double mapping_weight = retargeting.get_blendshape_mapping_weight(mapping_nr);
value += state.blendshape_coefficient(src_index) * mapping_weight;
value_count++;
}
else
{
Debug.Log("Could not find source blend shape '" + mapping_src);
}
}
// Apply the value for this target
if (value_count > 0)
{
values[index] = new BlendshapeValue(value);
}
}
return(values);
}
//! @brief Loads a faceshift clip
public void LoadFSB(string path)
{
if (!File.Exists(path))
{
#if UNITY_EDITOR
EditorUtility.DisplayDialog("Clip loading failed", "File " + path + " does exist", "Ok");
#endif
return;
}
Clip new_clip = FsbReader.read(path);
if (new_clip == null)
{
Debug.LogError("could not read clip from file " + path);
#if UNITY_EDITOR
EditorUtility.DisplayDialog("Load failed", "File " + path + " does not contain a clip", "Ok");
#endif
return;
}
Rig clip_rig = new_clip.rig();
if (!clip_rig.Equals(m_fs_Rig))
{
if ((cachedRigLoaded) && (!source_rig_changed_ask_to_continue()))
{
// The user wants to cancel
return;
}
// Apply new rig
m_fs_Rig = clip_rig;
// Cache new rig
SaveSourceRig();
}
m_clip = new_clip;
m_clip_path = path;
}
/// <summary>
/// Performs faceshift live tracking animation given a rig and a state.
/// </summary>
public void UpdateAnimation(Rig rig, RigState state)
{
if (rig == null || state == null)
{
return;
}
// evaluate joint transformations
TransformationValue [] transformationsTracking = null;
if (m_TPose != null && m_TPose.m_joints.Count == m_GameObjectTransformations.Count)
{
// evaluate using tpose
transformationsTracking = Utils.EvaluateTargetTransformations(m_Retargeting, rig, state, m_TPose.m_joints);
}
else
{
// evaluate using state from start of application
transformationsTracking = Utils.EvaluateTargetTransformations(m_Retargeting, rig, state, m_GameObjectTransformations);
}
if (transformationsTracking != null)
{
if (!ApplyTransformations(transformationsTracking, 1.0f))
{
Debug.LogError("Cannot apply tracking transformations as evaluated shape size is incorrect");
}
}
if (state != null)
{
BlendshapeValue [] blendshapesTracking = Utils.EvaluateTargetBlendshapes(m_Retargeting, rig, state, m_GameObjectBlendshapes);
if (!ApplyBlendshapes(blendshapesTracking, 1.0f))
{
Debug.LogError("Cannot apply tracking blendshapes as the number of blendshapes is not the same");
}
}
}
/**
* @brief Loads an FST retargeting file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromAsset(TextAsset asset)
{
Rig new_rig = new Rig();
if (!new_rig.load_from_fst(asset.bytes))
{
return(false);
}
if (!new_rig.Equals(m_fs_Rig))
{
if ((cachedRigLoaded) && (!source_rig_changed_ask_to_continue()))
{
// The user wants to cancel
return(false);
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.load(asset.bytes);
if (retargeting != null && !retargeting.isEmpty())
{
m_retargeting = retargeting;
m_retargeting_asset = asset;
return(true);
}
else
{
return(false);
}
}
/**
* @brief Loads an FST file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromFile(string path)
{
Rig new_rig = new Rig();
if (!new_rig.LoadFromFST(path))
{
return(false);
}
if (IsSourceRigDifferent(new_rig))
{
if (cachedRigLoaded && !SourceRigChangedAskToContinue())
{
// The user wants to cancel
return(false);
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.Load(path);
if (retargeting != null && !retargeting.IsEmpty())
{
m_Retargeting = retargeting;
m_RetargetingAsset = null; // clear asset as we use a file
return(true);
}
else
{
return(false);
}
}
/**
* @brief Loads an FST file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromFile(string path)
{
Rig new_rig = new Rig();
if (!new_rig.load_from_fst(path))
{
return(false);
}
if (!new_rig.Equals(m_fs_Rig))
{
if ((cachedRigLoaded) && (!source_rig_changed_ask_to_continue()))
{
// The user wants to cancel
return(false);
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.load(path);
if (retargeting != null && !retargeting.isEmpty())
{
m_retargeting = retargeting;
m_retargeting_asset = null; // clear asset as we use a file
return(true);
}
else
{
return(false);
}
}
/**
* Evaluates the target transformation based on the state of a rig and a retargeting configuration.
* If a transform is not affected by the retargeting then the value of the transform is null.
*/
public static TransformationValue [] EvaluateTargetTransformations(ClipRetargeting retargeting,
Rig rig,
RigState state,
ArrayList target_transformations) {
if (retargeting == null || rig == null || state == null || target_transformations == null) {
Debug.LogError("cannot evaluat target transformations as one or more object is null");
return null;
}
int n_target_transformations = target_transformations.Count;
TransformationValue [] values = new TransformationValue[n_target_transformations];
// We iterate over the target transformations and accumulate all sources to them
for (int target_nr = 0; target_nr < target_transformations.Count; target_nr++) {
// get original rotation and translation from tpose
TransformationInformation tpose_joint = target_transformations[target_nr] as TransformationInformation;
if (tpose_joint == null) {
Debug.LogError("joint " + target_nr + " is null");
continue;
}
Vector3 tpose_translation = tpose_joint.localPosition;
Quaternion tpose_local_rotation = tpose_joint.localRotation;
Quaternion tpose_parent_global_rotation = tpose_joint.parentRotation;
Quaternion fs_joint_rotation_local_from_t_pose = Quaternion.identity;
Vector3 fs_joint_translation_local_from_t_pose = new Vector3(0, 0, 0);
// Sum the translation to apply
int value_count_trans = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.GetNumberOfTranslationMappings(); mapping_nr++) {
string mapping_target = retargeting.GetTranslationMappingDestination (mapping_nr);
if (!mapping_target.Equals(tpose_joint.transformName)) {
continue;
}
string mapping_src = retargeting.GetTranslationMappingSource (mapping_nr);
int src_index = rig.BoneIndex (mapping_src);
if (src_index >= 0) {
double mapping_weight = retargeting.GetTranslationMappingWeight (mapping_nr);
fs_joint_translation_local_from_t_pose += state.BoneTranslation (src_index) * (float)mapping_weight;
value_count_trans++;
} else {
Debug.Log ("Could not find source index for '" + mapping_src + "'");
}
}
// Convert translation to global translation
//Vector3 unity_joint_translation_local = fs_joint_translation_local_from_t_pose + tpose_translation;
Vector3 unity_joint_translation_local =
Quaternion.Inverse(tpose_parent_global_rotation) * fs_joint_translation_local_from_t_pose
+ tpose_translation;
// Sum the rotations to apply
int value_count_rot = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.GetNumberOfRotationMappings(); mapping_nr++) {
string mapping_target = retargeting.GetRotationMappingDestination (mapping_nr);
if (!mapping_target.Equals (tpose_joint.transformName)) {
continue;
}
string mapping_src = retargeting.GetRotationMappingSource (mapping_nr);
int src_index = rig.BoneIndex (mapping_src);
if (src_index >= 0) {
double mapping_weight = retargeting.GetRotationMappingWeight (mapping_nr);
// use slerp for weighting
fs_joint_rotation_local_from_t_pose = Quaternion.Slerp (Quaternion.identity, state.BoneRotation (src_index), (float)mapping_weight);
// TODO: here we should accumulate if there are more than one sources (like with blendshapes)
value_count_rot++;
} else {
Debug.Log ("Could not find source rotation for '" + mapping_src);
}
}
// Convert to local unity rotation
Quaternion unity_joint_rotation_local =
Quaternion.Inverse(tpose_parent_global_rotation)
* fs_joint_rotation_local_from_t_pose
* tpose_parent_global_rotation
* tpose_local_rotation; // The initial local rotation;
if (value_count_trans > 0 || value_count_rot > 0) {
values[target_nr] = new TransformationValue(unity_joint_rotation_local, unity_joint_translation_local);
}
}
return values;
}
public bool Equals(Rig other) {
if ((m_shapes.Count != other.m_shapes.Count) || (m_bones.Count != other.m_bones.Count)) {
return false;
}
// Since we do not have duplicates and we only reach here if we have the same
// number of bones and shapes in both rigs, we can check only in one direction with Contains() tests.
// The order of the entries does not have to be the same
for (int i = 0; i < m_shapes.Count; i++) {
if (!m_shapes.Contains(other.m_shapes[i])) {
return false;
}
}
for (int i = 0; i < m_bones.Count; i++) {
if (!m_bones.Contains(other.m_bones[i])) {
return false;
}
}
return true;
}
/**
* Evaluates the target transformation based on the state of a rig and a retargeting configuration.
* If a transform is not affected by the retargeting then the value of the transform is null.
*/
public static TransformationValue [] evaluate_target_transformations(ClipRetargeting retargeting,
Rig rig,
RigState state,
ArrayList target_transformations)
{
if (retargeting == null || rig == null || state == null || target_transformations == null)
{
Debug.LogError("cannot evaluat target transformations as one or more object is null");
return(null);
}
int n_target_transformations = target_transformations.Count;
TransformationValue [] values = new TransformationValue[n_target_transformations];
// We iterate over the target transformations and accumulate all sources to them
for (int target_nr = 0; target_nr < target_transformations.Count; target_nr++)
{
// get original rotation and translation from tpose
TransformationInformation tpose_joint = target_transformations[target_nr] as TransformationInformation;
if (tpose_joint == null)
{
Debug.LogError("joint " + target_nr + " is null");
continue;
}
Vector3 tpose_translation = tpose_joint.localPosition;
Quaternion tpose_local_rotation = tpose_joint.localRotation;
Quaternion tpose_parent_global_rotation = tpose_joint.parentRotation;
Quaternion fs_joint_rotation_local_from_t_pose = Quaternion.identity;
Vector3 fs_joint_translation_local_from_t_pose = new Vector3(0, 0, 0);
// Sum the translation to apply
int value_count_trans = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.get_number_of_translation_mappings(); mapping_nr++)
{
string mapping_target = retargeting.get_translation_mapping_destination(mapping_nr);
if (!mapping_target.Equals(tpose_joint.transformName))
{
continue;
}
string mapping_src = retargeting.get_translation_mapping_source(mapping_nr);
int src_index = rig.bone_index(mapping_src);
if (src_index >= 0)
{
double mapping_weight = retargeting.get_translation_mapping_weight(mapping_nr);
fs_joint_translation_local_from_t_pose += state.bone_translation(src_index) * (float)mapping_weight;
value_count_trans++;
}
else
{
Debug.Log("Could not find source index for '" + mapping_src + "'");
}
}
// Convert translation to global translation
//Vector3 unity_joint_translation_local = fs_joint_translation_local_from_t_pose + tpose_translation;
Vector3 unity_joint_translation_local =
Quaternion.Inverse(tpose_parent_global_rotation) * fs_joint_translation_local_from_t_pose
+ tpose_translation;
// Sum the rotations to apply
int value_count_rot = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.get_number_of_rotation_mappings(); mapping_nr++)
{
string mapping_target = retargeting.get_rotation_mapping_destination(mapping_nr);
if (!mapping_target.Equals(tpose_joint.transformName))
{
continue;
}
string mapping_src = retargeting.get_rotation_mapping_source(mapping_nr);
int src_index = rig.bone_index(mapping_src);
if (src_index >= 0)
{
double mapping_weight = retargeting.get_rotation_mapping_weight(mapping_nr);
// use slerp for weighting
fs_joint_rotation_local_from_t_pose = Quaternion.Slerp(Quaternion.identity, state.bone_rotation(src_index), (float)mapping_weight);
// TODO: here we should accumulate if there are more than one sources (like with blendshapes)
value_count_rot++;
}
else
{
Debug.Log("Could not find source rotation for '" + mapping_src);
}
}
// Convert to local unity rotation
Quaternion unity_joint_rotation_local =
Quaternion.Inverse(tpose_parent_global_rotation)
* fs_joint_rotation_local_from_t_pose
* tpose_parent_global_rotation
* tpose_local_rotation; // The initial local rotation;
if (value_count_trans > 0 || value_count_rot > 0)
{
values[target_nr] = new TransformationValue(unity_joint_rotation_local, unity_joint_translation_local);
}
}
return(values);
}
/**
* @brief Loads an FST retargeting file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
new public bool LoadRetargetingFromAsset(TextAsset asset) {
Rig new_rig = new Rig();
if (!new_rig.LoadFromFST(asset.bytes)) return false;
if (IsSourceRigDifferent(new_rig)) {
if (cachedRigLoaded && !SourceRigChangedAskToContinue()) {
// The user wants to cancel
return false;
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
return base.LoadRetargetingFromAsset(asset);
}
private bool IsSourceRigDifferent(Rig newRig) {
return !newRig.Equals(m_fs_Rig);
}
/**
* @brief Reads an fsb file and returns a clip constructed from the the content of the fsb file.
* @param[in] data The binary data from the fsb file.
* @return A clip with the track data imported from the fsb file, null if there was some error during loading.
*/
public static Clip Read(byte[] data)
{
DataParser parser = new DataParser();
parser.ExtractData(data, false, 0.0);
if (parser.Valid())
{
Rig rig = new Rig();
// add hardcoded bone names of fsb file
rig.AddBone("Neck");
rig.AddBone("EyeLeft");
rig.AddBone("EyeRight");
for (int i = 0; i < parser.fsBlendShapeNames.Length; i++)
{
rig.AddShape(parser.fsBlendShapeNames[i]);
}
Clip clip = new Clip(rig);
while (parser.CountAvailableTracks() > 0)
{
TrackData track_data = parser.Dequeue();
if (!track_data.TrackSuccess)
{
// do not save the state if the tracking data is not valid
Debug.LogWarning("this frame doesn't contain valid tracking data");
continue;
}
RigState state = clip.NewState(track_data.TimeStamp);
state.SetTrackingSuccessful(track_data.TrackSuccess);
if (rig.NumShapes() != track_data.n_coefficients())
{
Debug.LogError("num blendshapes do not agree in file with " + rig.NumShapes() +
" in rig and " + track_data.n_coefficients() + " in state");
return(null);
}
// Assume the head translation to be joint 0 (Neck)
state.SetBoneTranslation(0, track_data.HeadTranslation());
// bone indices same as the order when added to the rig
state.SetBoneRotation(0, track_data.HeadRotation());
state.SetBoneRotation(1, track_data.LeftEyeRotation());
state.SetBoneRotation(2, track_data.RightEyeRotation());
for (int i = 0; i < track_data.n_coefficients(); i++)
{
state.SetBlendshapeCoefficient(i, track_data.Coefficient[i]);
}
}
return(clip);
}
else
{
Debug.LogError("cannot parse fsb file");
}
return(null);
}
/**
* @brief Create a new clip with a specific rig.
* @param[in] rig The rig of the clip.
*/
public Clip(Rig rig)
{
m_rig = rig;
m_states = new List <RigState>();
}
public Faceshift()
{
mutexOnTargetBlendshapeList = new Mutex();
m_fs_Rig = new Rig();
cachedRigLoaded = LoadCachedSourceRig();
}
/**
* @brief Create a new clip with a specific rig.
* @param[in] rig The rig of the clip.
*/
public Clip(Rig rig) {
m_rig = rig;
m_states = new List<RigState>();
}
void Update()
{
// ten kod kalkuluje bazowa pozycje glowy
// trzeba jeszcze dorobic zeby przechwytywal trigerra
// i wtedy zmieniał isBaseCalculated na false
// i resetował timer i kalkulował jeszcze raz
// pozniej trzeba jeszcze dodac zeby kazda metoda zwracała
// to co zwraca minus base
if (!isBaseCalculated)
{
timer += Time.deltaTime;
if (timer < baseCalculationTime)
{
if (getXHeadRotation() != -1)
{
baseX += getXHeadRotation();
ticksX++;
}
if (getYHeadRotation() != -1)
{
baseY += getYHeadRotation();
ticksY++;
}
}
else
{
baseX /= ticksX;
baseY /= ticksY;
isBaseCalculated = true;
}
}
new_data = false;
m_mutex.WaitOne();
// check if there are new blendshape names, and if so update
if (newBlendShapeNamesArrived())
{
m_rig = getRigFromBlendShapeNames(getBlendShapeNames());
}
// get most recent tracking data
while (m_data_parser.CountAvailableTracks() > 0)
{
m_current_track = m_data_parser.Dequeue();
new_data = true;
}
// check that we have a rig (set up from blendshape names) with the same number of blendshapes as what we receive
if (m_rig != null && m_current_track != null)
{
int n_track_coefficients = m_current_track.n_coefficients();
int n_rig_coefficients = m_rig.num_shapes();
if (n_track_coefficients != n_rig_coefficients)
{
Debug.LogWarning("number of coefficients of rig and tracking state have changed: " + n_track_coefficients + " vs " + n_rig_coefficients);
// clear the current rig as it is not usable with the data coming from faceshift
m_rig = null;
m_current_track = null;
// get again the blendshapes from fs studio
askForBlendshapeNames();
}
}
m_mutex.ReleaseMutex();
if (m_rig != null && m_current_track != null && m_retargeting != null)
{
if (new_data && m_current_track.TrackSuccess)
{
// create a rig state
RigState state = new RigState(m_rig);
state.set_timestamp(m_current_track.TimeStamp);
state.set_bone_translation(0, m_current_track.HeadTranslation());
state.set_bone_rotation(0, m_current_track.HeadRotation());
state.set_bone_rotation(1, m_current_track.LeftEyeRotation());
state.set_bone_rotation(2, m_current_track.RightEyeRotation());
for (int i = 0; i < m_rig.num_shapes(); i++)
{
state.set_blendshape_coefficient(i, m_current_track.Coefficient [i]);
}
// evaluate joint transformations
TransformationValue [] transformation_values = null;
if (m_tpose != null && m_tpose.m_joints.Count == m_game_object_transformations.Count)
{
// evaluate using tpose
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_rig, state, m_tpose.m_joints);
}
else
{
// evaluate using state from start of application
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_rig, state, m_game_object_transformations);
}
if (transformation_values.Length == m_game_object_transformations.Count)
{
for (int index = 0; index < transformation_values.Length; index++)
{
// Apply the value for this target
if (transformation_values [index] != null)
{
TransformationInformation joint = m_game_object_transformations [index] as TransformationInformation;
joint.transform.localRotation = transformation_values [index].m_rotation;
joint.transform.localPosition = transformation_values [index].m_translation;
}
}
}
else
{
Debug.LogError("Cannot create transformation as evaluated shape size is incorrect");
}
// evaluate blendshape valuesf
BlendshapeValue [] values = Utils.evaluate_target_blendshapes(m_retargeting, m_rig, state, m_game_object_blendshapes);
if (values.Length == m_game_object_blendshapes.Count)
{
for (int index = 0; index < m_game_object_blendshapes.Count; index++)
{
BlendshapeInfo bs_info = m_game_object_blendshapes [index] as BlendshapeInfo;
// Apply the value for this target
if (bs_info != null && values [index] != null)
{
bs_info.m_mesh_renderer.SetBlendShapeWeight(bs_info.m_index, (float)values [index].m_value);
}
}
}
else
{
Debug.LogError("Cannot create blendshapes as evaluated shape size is incorrect");
}
}
}
}
/**
* @brief Loads an FST file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromFile(string path)
{
Rig new_rig = new Rig();
if (!new_rig.LoadFromFST(path)) return false;
if (IsSourceRigDifferent(new_rig)) {
if (cachedRigLoaded && !SourceRigChangedAskToContinue()) {
// The user wants to cancel
return false;
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.Load(path);
if (retargeting != null && !retargeting.IsEmpty()) {
m_Retargeting = retargeting;
m_RetargetingAsset = null; // clear asset as we use a file
return true;
} else {
return false;
}
}
/**
* Evaluates the target blendshape values based on the state of a rig and a retargeting configuration.
* If a blendshape is not affected by the retargeting then the value of this blendshape is null.
*/
public static BlendshapeValue [] EvaluateTargetBlendshapes(ClipRetargeting retargeting,
Rig rig,
RigState state,
ArrayList target_blendshapes) {
int n_target_blendshapes = target_blendshapes.Count;
BlendshapeValue [] values = new BlendshapeValue[n_target_blendshapes];
// We iterate over the targets and accumulate all sources to them
for (int index = 0; index < n_target_blendshapes; index++) {
double value = 0.0;
int value_count = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.GetNumberOfBlendshapeMappings(); mapping_nr++) {
string mapping_target = retargeting.GetBlendshapeMappingDestination(mapping_nr);
if (!mapping_target.Equals(((BlendshapeInfo)target_blendshapes[index]).m_name)) {
continue;
}
string mapping_src = retargeting.GetBlendshapeMappingSource(mapping_nr);
int src_index = rig.ShapeIndex(mapping_src);
if (src_index >= 0) {
double mapping_weight = retargeting.GetBlendshapeMappingWeight(mapping_nr);
value += state.BlendshapeCoefficient(src_index) * mapping_weight;
value_count++;
} else {
Debug.Log("Could not find source blend shape '" + mapping_src);
}
}
// Apply the value for this target
if (value_count > 0) {
values[index] = new BlendshapeValue(value);
}
}
return values;
}
/**
* @brief Create track state with shapes and bones.
* @param[in] rig The rig for which to create the state.
*/
public RigState(Rig rig)
{
m_blendshape_coefficients = new double[rig.NumShapes()];
m_bone_translations = new Vector3[rig.NumBones()];
m_bone_rotations = new Quaternion[rig.NumBones()];
}
//! @brief Loads a faceshift clip
public void LoadFSB(string path) {
if (!File.Exists (path)) {
#if UNITY_EDITOR
EditorUtility.DisplayDialog("Clip loading failed", "File " + path + " does exist", "Ok");
#endif
return;
}
Clip new_clip = FsbReader.Read(path);
if (new_clip == null) {
Debug.LogError("could not read clip from file " + path);
#if UNITY_EDITOR
EditorUtility.DisplayDialog("Load failed", "File " + path + " does not contain a clip", "Ok");
#endif
return;
}
Rig clip_rig = new_clip.Rig();
if (IsSourceRigDifferent(clip_rig)) {
if ((cachedRigLoaded) && (!SourceRigChangedAskToContinue ())) {
// The user wants to cancel
return;
}
}
// always save new rig from the fsb file to have the same order
// Apply new rig
m_fs_Rig = clip_rig;
// Cache new rig
SaveSourceRig();
m_clip = new_clip;
m_clip_path = path;
}
private bool IsSourceRigDifferent(Rig newRig)
{
return(!newRig.Equals(m_fs_Rig));
}
/**
* @brief Create track state with shapes and bones.
* @param[in] rig The rig for which to create the state.
*/
public RigState(Rig rig) {
m_blendshape_coefficients = new double[rig.NumShapes()];
m_bone_translations = new Vector3[rig.NumBones()];
m_bone_rotations = new Quaternion[rig.NumBones()];
}
public Faceshift()
{
mutexOnTargetBlendshapeList = new Mutex();
m_fs_Rig = new Rig();
cachedRigLoaded = LoadCachedSourceRig();
}
private void ClearCachedRig() {
m_fs_Rig = new Rig();
}
//! @brief Save retargeting to file.
public bool Save(string filename, Rig sourceRig)
{
StreamWriter stream = File.CreateText(filename);
//////////////
/// Blend shapes
// Get list of all blend shapes
List <string> possible_source_blend_shapes = null;
if (sourceRig != null)
{
// We also store the blend shapes that are not connected
possible_source_blend_shapes = new List <string>();
for (int i = 0; i < sourceRig.NumShapes(); i++)
{
possible_source_blend_shapes.Add(sourceRig.ShapeName(i));
}
}
// write blendshapes
foreach (Mapping entry in m_blendshape_mapping)
{
stream.WriteLine("bs = " + entry.source + " = " + entry.destination + " = " + entry.weight);
possible_source_blend_shapes.Remove(entry.source);
}
// write not mapped blendshapes
foreach (string entry in possible_source_blend_shapes)
{
stream.WriteLine("bs = " + entry);
}
//////////////
/// Rotations
// Get list of all rotations
List <string> possible_source_rotations = null;
if (sourceRig != null)
{
// We also store the rotations that are not connected
possible_source_rotations = new List <string>();
for (int i = 0; i < sourceRig.NumBones(); i++)
{
possible_source_rotations.Add(sourceRig.BoneName(i));
}
}
// write rotations
foreach (Mapping entry in m_rotation_mapping)
{
stream.WriteLine("rotation = " + entry.source + " = " + entry.destination + " = " + entry.weight);
possible_source_rotations.Remove(entry.source);
}
// write not mapped rotations
foreach (string entry in possible_source_rotations)
{
stream.WriteLine("rotation = " + entry);
}
//////////////
/// Translations
// Get list of all translations
List <string> possible_source_translations = null;
if (sourceRig != null)
{
// We also store the translations that are not connected
possible_source_translations = new List <string>();
for (int i = 0; i < sourceRig.NumBones(); i++)
{
possible_source_translations.Add(sourceRig.BoneName(i));
}
}
// write translations
foreach (Mapping entry in m_translation_mapping)
{
stream.WriteLine("translation = " + entry.source + " = " + entry.destination + " = " + entry.weight);
possible_source_translations.Remove(entry.source);
}
// write not mapped translations
foreach (string entry in possible_source_translations)
{
stream.WriteLine("translation = " + entry);
}
stream.Close();
return(true);
}
private void ClearCachedRig()
{
m_fs_Rig = new Rig();
}
/**
* @brief Reads an fsb file and returns a clip constructed from the the content of the fsb file.
* @param[in] filename The filename of the fsb file.
* @return A clip with the track data imported from the fsb file, null if there was some error during loading.
*/
public static Clip read(string filename)
{
if (!File.Exists(filename))
{
return(null);
}
FileStream fileReader = File.OpenRead(filename);
int size = (int)fileReader.Length;
byte [] data = new byte[size];
int size_read = fileReader.Read(data, 0, size);
if (size != size_read)
{
Debug.LogError("Reading file unsuccessful: read " + size_read + " bytes instead of " + size + " bytes");
return(null);
}
DataParser parser = new DataParser();
parser.ExtractData(data);
if (parser.Valid())
{
Rig rig = new Rig();
// add hardcoded bone names of fsb file
rig.add_bone("Neck");
rig.add_bone("EyeLeft");
rig.add_bone("EyeRight");
for (int i = 0; i < parser.fsBlendShapeNames.Length; i++)
{
rig.add_shape(parser.fsBlendShapeNames[i]);
}
Clip clip = new Clip(rig);
while (parser.CountAvailableTracks() > 0)
{
TrackData track_data = parser.Dequeue();
if (!track_data.TrackSuccess)
{
// do not save the state if the tracking data is not valid
Debug.LogWarning("this frame doesn't contain valid tracking data");
continue;
}
RigState state = clip.new_state(track_data.TimeStamp);
if (rig.num_shapes() != track_data.n_coefficients())
{
Debug.LogError("num blendshapes do not agree in file with " + rig.num_shapes() +
" in rig and " + track_data.n_coefficients() + " in state");
return(null);
}
// Assume the head translation to be joint 0 (Neck)
state.set_bone_translation(0, track_data.HeadTranslation());
// bone indices same as the order when added to the rig
state.set_bone_rotation(0, track_data.HeadRotation());
state.set_bone_rotation(1, track_data.LeftEyeRotation());
state.set_bone_rotation(2, track_data.RightEyeRotation());
for (int i = 0; i < track_data.n_coefficients(); i++)
{
state.set_blendshape_coefficient(i, track_data.Coefficient[i]);
}
}
return(clip);
}
else
{
Debug.LogError("cannot parse fsb file");
}
return(null);
}
/// <summary>
/// Performs faceshift live tracking animation given a rig and a state.
/// </summary>
public void UpdateAnimation(Rig rig, RigState state) {
if (rig == null || state == null) return;
// evaluate joint transformations
TransformationValue [] transformationsTracking = null;
if (m_TPose != null && m_TPose.m_joints.Count == m_GameObjectTransformations.Count) {
// evaluate using tpose
transformationsTracking = Utils.EvaluateTargetTransformations(m_Retargeting, rig, state, m_TPose.m_joints);
} else {
// evaluate using state from start of application
transformationsTracking = Utils.EvaluateTargetTransformations(m_Retargeting, rig, state, m_GameObjectTransformations);
}
if (transformationsTracking != null) {
if (!ApplyTransformations(transformationsTracking, 1.0f)) {
Debug.LogError("Cannot apply tracking transformations as evaluated shape size is incorrect");
}
}
if (state != null) {
BlendshapeValue [] blendshapesTracking = Utils.EvaluateTargetBlendshapes(m_Retargeting, rig, state, m_GameObjectBlendshapes);
if (!ApplyBlendshapes(blendshapesTracking, 1.0f)) {
Debug.LogError("Cannot apply tracking blendshapes as the number of blendshapes is not the same");
}
}
}