public Faceshift()
{
mutexOnTargetBlendshapeList = new Mutex();
m_retargeting = new ClipRetargeting();
targetBlendShapeInfos = new ArrayList();
targetTransformations = new ArrayList();
m_fs_Rig = new Rig();
cachedRigLoaded = LoadCachedSourceRig();
if (!cachedRigLoaded)
{
// There was no rig cached. So we assume this was the first start
showFirstStartMessage = true;
}
}
public bool LoadRetargetingFromAsset(TextAsset asset)
{
if (asset == null)
{
return(false);
}
ClipRetargeting retargeting = ClipRetargeting.Load(asset.bytes, /*fromAsset=*/ true);
if (retargeting == null || retargeting.IsEmpty())
{
ResetRetargeting();
return(false);
}
m_Retargeting = retargeting;
m_RetargetingAsset = asset;
return(true);
}
/**
* Evaluates the target blendshape values based on the state of a rig and a retargeting configuration.
* If a blendshape is not affected by the retargeting then the value of this blendshape is null.
*/
public static BlendshapeValue [] evaluate_target_blendshapes(ClipRetargeting retargeting,
Rig rig,
RigState state,
ArrayList target_blendshapes)
{
int n_target_blendshapes = target_blendshapes.Count;
BlendshapeValue [] values = new BlendshapeValue[n_target_blendshapes];
// We iterate over the targets and accumulate all sources to them
for (int index = 0; index < n_target_blendshapes; index++)
{
double value = 0.0;
int value_count = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.get_number_of_blendshape_mappings(); mapping_nr++)
{
string mapping_target = retargeting.get_blendshape_mapping_destination(mapping_nr);
if (!mapping_target.Equals(((BlendshapeInfo)target_blendshapes[index]).m_name))
{
continue;
}
string mapping_src = retargeting.get_blendshape_mapping_source(mapping_nr);
int src_index = rig.shape_index(mapping_src);
if (src_index >= 0)
{
double mapping_weight = retargeting.get_blendshape_mapping_weight(mapping_nr);
value += state.blendshape_coefficient(src_index) * mapping_weight;
value_count++;
}
else
{
Debug.Log("Could not find source blend shape '" + mapping_src);
}
}
// Apply the value for this target
if (value_count > 0)
{
values[index] = new BlendshapeValue(value);
}
}
return(values);
}
void Start()
{
m_mutex = new Mutex();
m_data_parser = new DataParser();
m_current_track = new TrackData();
m_connection = new NetworkConnection(this);
if (!m_connection.Start(hostName, port, true))
{
Debug.LogError("could not access faceshift over the network on " + hostName + ":" + port + " using the TCP/IP protocol");
}
// get the blendshapes from fs studio
askForBlendshapeNames();
if (RetargetingAsset != null)
{
m_retargeting = ClipRetargeting.load(RetargetingAsset.bytes);
if (m_retargeting == null)
{
Debug.LogError("could not load retargeting from asset");
}
}
if (TPoseAsset != null)
{
m_tpose = new TPose();
if (!m_tpose.loadFromBytes(TPoseAsset.bytes))
{
Debug.LogError("could not load tpose from asset");
}
}
// get game object blendshapes and transformations
Utils.getGameObjectBlendshapes(gameObject, m_game_object_blendshapes);
Utils.getGameObjectTransformations(gameObject, m_game_object_transformations);
}
/**
* @brief Loads an FST retargeting file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromAsset(TextAsset asset)
{
Rig new_rig = new Rig();
if (!new_rig.load_from_fst(asset.bytes))
{
return(false);
}
if (!new_rig.Equals(m_fs_Rig))
{
if ((cachedRigLoaded) && (!source_rig_changed_ask_to_continue()))
{
// The user wants to cancel
return(false);
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.load(asset.bytes);
if (retargeting != null && !retargeting.isEmpty())
{
m_retargeting = retargeting;
m_retargeting_asset = asset;
return(true);
}
else
{
return(false);
}
}
/**
* @brief Loads an FST file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromFile(string path)
{
Rig new_rig = new Rig();
if (!new_rig.load_from_fst(path))
{
return(false);
}
if (!new_rig.Equals(m_fs_Rig))
{
if ((cachedRigLoaded) && (!source_rig_changed_ask_to_continue()))
{
// The user wants to cancel
return(false);
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.load(path);
if (retargeting != null && !retargeting.isEmpty())
{
m_retargeting = retargeting;
m_retargeting_asset = null; // clear asset as we use a file
return(true);
}
else
{
return(false);
}
}
/**
* @brief Loads an FST file from faceshift which contains the retargeting information
* @param[in] path The path to the file
* @param onlyRig Should only the rig be loaded (true) or also the mapping (false)
*/
public bool LoadRetargetingFromFile(string path)
{
Rig new_rig = new Rig();
if (!new_rig.LoadFromFST(path))
{
return(false);
}
if (IsSourceRigDifferent(new_rig))
{
if (cachedRigLoaded && !SourceRigChangedAskToContinue())
{
// The user wants to cancel
return(false);
}
// Apply new rig
m_fs_Rig = new_rig;
// Cache new rig
SaveSourceRig();
}
ClipRetargeting retargeting = ClipRetargeting.Load(path);
if (retargeting != null && !retargeting.IsEmpty())
{
m_Retargeting = retargeting;
m_RetargetingAsset = null; // clear asset as we use a file
return(true);
}
else
{
return(false);
}
}
/**
* @brief Reads an fst file and returns a clip retargeting constructed from the content of the fst file.
* @param[in] filename The filename of the fst file.
* @return A clip retargeting with the retargeting info imported from the fst file, null if there was some error during loading.
*/
public static ClipRetargeting Load(byte [] data, bool fromAsset)
{
string text = System.Text.Encoding.UTF8.GetString(data);
ClipRetargeting retargeting = new ClipRetargeting();
string[] lines = text.Split('\n');
for (int i = 0; i < lines.Length; i++)
{
string[] elements = lines[i].Split('=');
if (elements.Length > 1)
{
string label = elements[0].Trim().ToLower();
switch (label)
{
case "name": break;
case "scale": break;
case "bs":
case "joint":
case "rotation":
case "translation": {
// format: bs/joint = src_name = target_name [= weight]
bool is_bs = (label == "bs");
double weight = 1;
string src = "";
string target = "";
if (elements.Length > 2)
{
// we require src and target for a mapping
src = elements[1].Trim();
target = elements[2].Trim();
if (elements.Length > 3)
{
weight = Convert.ToDouble(elements[3].Trim());
}
if (is_bs)
{
// automatically map to the 100 scale
if (!fromAsset)
{
weight *= 100;
}
retargeting.AddBlendshapeMapping(src, target, weight);
}
else
{
// automatically rename fs joint names
if (src == "jointNeck" || src == "neck")
{
src = "Neck";
}
if (src == "jointEyeLeft" || src == "eyeLeft")
{
src = "EyeLeft";
}
if (src == "jointEyeRight" || src == "eyeRight")
{
src = "EyeRight";
}
if (label.Equals("translation"))
{
retargeting.AddTranslationMapping(src, target, weight);
}
else
{
retargeting.AddRotationMapping(src, target, weight);
}
}
}
} break;
default: {
} break;
}
}
}
return(retargeting);
}
public void ClearRetargeting()
{
m_retargeting = new ClipRetargeting();
m_retargeting_asset = null;
}
/**
* Evaluates the target transformation based on the state of a rig and a retargeting configuration.
* If a transform is not affected by the retargeting then the value of the transform is null.
*/
public static TransformationValue [] evaluate_target_transformations(ClipRetargeting retargeting,
Rig rig,
RigState state,
ArrayList target_transformations)
{
if (retargeting == null || rig == null || state == null || target_transformations == null)
{
Debug.LogError("cannot evaluat target transformations as one or more object is null");
return(null);
}
int n_target_transformations = target_transformations.Count;
TransformationValue [] values = new TransformationValue[n_target_transformations];
// We iterate over the target transformations and accumulate all sources to them
for (int target_nr = 0; target_nr < target_transformations.Count; target_nr++)
{
// get original rotation and translation from tpose
TransformationInformation tpose_joint = target_transformations[target_nr] as TransformationInformation;
if (tpose_joint == null)
{
Debug.LogError("joint " + target_nr + " is null");
continue;
}
Vector3 tpose_translation = tpose_joint.localPosition;
Quaternion tpose_local_rotation = tpose_joint.localRotation;
Quaternion tpose_parent_global_rotation = tpose_joint.parentRotation;
Quaternion fs_joint_rotation_local_from_t_pose = Quaternion.identity;
Vector3 fs_joint_translation_local_from_t_pose = new Vector3(0, 0, 0);
// Sum the translation to apply
int value_count_trans = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.get_number_of_translation_mappings(); mapping_nr++)
{
string mapping_target = retargeting.get_translation_mapping_destination(mapping_nr);
if (!mapping_target.Equals(tpose_joint.transformName))
{
continue;
}
string mapping_src = retargeting.get_translation_mapping_source(mapping_nr);
int src_index = rig.bone_index(mapping_src);
if (src_index >= 0)
{
double mapping_weight = retargeting.get_translation_mapping_weight(mapping_nr);
fs_joint_translation_local_from_t_pose += state.bone_translation(src_index) * (float)mapping_weight;
value_count_trans++;
}
else
{
Debug.Log("Could not find source index for '" + mapping_src + "'");
}
}
// Convert translation to global translation
//Vector3 unity_joint_translation_local = fs_joint_translation_local_from_t_pose + tpose_translation;
Vector3 unity_joint_translation_local =
Quaternion.Inverse(tpose_parent_global_rotation) * fs_joint_translation_local_from_t_pose
+ tpose_translation;
// Sum the rotations to apply
int value_count_rot = 0;
for (int mapping_nr = 0; mapping_nr < retargeting.get_number_of_rotation_mappings(); mapping_nr++)
{
string mapping_target = retargeting.get_rotation_mapping_destination(mapping_nr);
if (!mapping_target.Equals(tpose_joint.transformName))
{
continue;
}
string mapping_src = retargeting.get_rotation_mapping_source(mapping_nr);
int src_index = rig.bone_index(mapping_src);
if (src_index >= 0)
{
double mapping_weight = retargeting.get_rotation_mapping_weight(mapping_nr);
// use slerp for weighting
fs_joint_rotation_local_from_t_pose = Quaternion.Slerp(Quaternion.identity, state.bone_rotation(src_index), (float)mapping_weight);
// TODO: here we should accumulate if there are more than one sources (like with blendshapes)
value_count_rot++;
}
else
{
Debug.Log("Could not find source rotation for '" + mapping_src);
}
}
// Convert to local unity rotation
Quaternion unity_joint_rotation_local =
Quaternion.Inverse(tpose_parent_global_rotation)
* fs_joint_rotation_local_from_t_pose
* tpose_parent_global_rotation
* tpose_local_rotation; // The initial local rotation;
if (value_count_trans > 0 || value_count_rot > 0)
{
values[target_nr] = new TransformationValue(unity_joint_rotation_local, unity_joint_translation_local);
}
}
return(values);
}
public void ClearRetargeting()
{
m_Retargeting = new ClipRetargeting();
m_RetargetingAsset = null;
ClearCachedRig();
}
public void ResetRetargeting()
{
m_Retargeting = null;
m_RetargetingAsset = null;
}
