public void setFromTrack(TrackData track)
{
m_timestamp = track.m_timestamp;
m_trackSuccess = track.m_trackSuccess;
m_headRotation = track.m_headRotation;
m_headTranslation = track.m_headTranslation;
m_coefficients = track.m_coefficients;
m_leftEyeTheta = track.m_leftEyeTheta;
m_leftEyePhi = track.m_leftEyePhi;
m_rightEyeTheta = track.m_rightEyeTheta;
m_rightEyePhi = track.m_rightEyePhi;
m_markersPositions = track.m_markersPositions;
m_presence = track.m_presence;
}
public void setFromTrack(TrackData track) {
m_timestamp = track.m_timestamp;
m_trackSuccess = track.m_trackSuccess;
m_headRotation = track.m_headRotation;
m_headTranslation = track.m_headTranslation;
m_coefficients = track.m_coefficients;
m_leftEyeTheta = track.m_leftEyeTheta;
m_leftEyePhi = track.m_leftEyePhi;
m_rightEyeTheta = track.m_rightEyeTheta;
m_rightEyePhi = track.m_rightEyePhi;
m_markersPositions = track.m_markersPositions;
m_presence = track.m_presence;
}
void Start () {
m_mutex = new Mutex();
m_data_parser = new DataParser();
m_current_track = new TrackData();
m_connection = new NetworkConnection(this);
if( !m_connection.Start(hostName,port,true) ) {
Debug.LogError("could not access faceshift over the network on " + hostName + ":" + port + " using the TCP/IP protocol");
}
// get the blendshapes from fs studio
AskForBlendshapeNames ();
Init();
}
void Update () {
bool new_data = false;
m_mutex.WaitOne();
// check if there are new blendshape names, and if so update
if (NewBlendShapeNamesArrived()) {
m_rig = Rig.GetRigFromBlendShapeNames(GetBlendShapeNames());
}
// get most recent tracking data
while (m_data_parser.CountAvailableTracks() > 0) {
m_current_track = m_data_parser.Dequeue();
new_data = true;
}
// check that we have a rig (set up from blendshape names) with the same number of blendshapes as what we receive
if (m_rig != null && m_current_track != null) {
int n_track_coefficients = m_current_track.n_coefficients();
int n_rig_coefficients = m_rig.NumShapes();
if (n_track_coefficients != n_rig_coefficients) {
Debug.LogWarning("number of coefficients of rig and tracking state have changed: " + n_track_coefficients + " vs " + n_rig_coefficients);
// clear the current rig as it is not usable with the data coming from faceshift
m_rig = null;
m_current_track = null;
// get again the blendshapes from fs studio
AskForBlendshapeNames();
}
}
m_mutex.ReleaseMutex();
if (m_rig != null && m_current_track != null && m_Retargeting != null) {
if (new_data && m_current_track.TrackSuccess) {
// create a rig state
RigState state = new RigState(m_rig);
state.SetTimestamp(m_current_track.TimeStamp);
state.SetBoneTranslation(0, m_current_track.HeadTranslation());
state.SetBoneRotation(0, m_current_track.HeadRotation());
state.SetBoneRotation(1, m_current_track.LeftEyeRotation());
state.SetBoneRotation(2, m_current_track.RightEyeRotation());
for (int i = 0; i < m_rig.NumShapes(); i++) {
state.SetBlendshapeCoefficient(i, m_current_track.Coefficient[i]);
}
base.UpdateAnimation(m_rig, state);
}
}
}
void Start()
{
m_mutex = new Mutex();
m_data_parser = new DataParser();
m_current_track = new TrackData();
m_connection = new NetworkConnection(this);
if (!m_connection.Start(hostName, port, true))
{
Debug.LogError("could not access faceshift over the network on " + hostName + ":" + port + " using the TCP/IP protocol");
}
// get the blendshapes from fs studio
AskForBlendshapeNames();
Init();
}
void Start()
{
m_mutex = new Mutex();
m_data_parser = new DataParser();
m_current_track = new TrackData();
m_connection = new NetworkConnection(this);
if (!m_connection.Start(hostName, port, true))
{
Debug.LogError("could not access faceshift over the network on " + hostName + ":" + port + " using the TCP/IP protocol");
}
// get the blendshapes from fs studio
askForBlendshapeNames();
if (RetargetingAsset != null)
{
m_retargeting = ClipRetargeting.load(RetargetingAsset.bytes);
if (m_retargeting == null)
{
Debug.LogError("could not load retargeting from asset");
}
}
if (TPoseAsset != null)
{
m_tpose = new TPose();
if (!m_tpose.loadFromBytes(TPoseAsset.bytes))
{
Debug.LogError("could not load tpose from asset");
}
}
// get game object blendshapes and transformations
Utils.getGameObjectBlendshapes(gameObject, m_game_object_blendshapes);
Utils.getGameObjectTransformations(gameObject, m_game_object_transformations);
}
public void ExtractData(byte[] data)
{
m_crawler.Append(data);
while (m_crawler.DataAvailable() && m_valid)
{
// initialise variables
// when magic number == 33433, we get
// BLOCK 101
double timestamp = 0.0;
bool trackSuccess = false;
// BLOCK 102
Quaternion headRotation = Quaternion.identity;
Vector3 headTranslation = Vector3.zero;
// BLOCK 103
float[] coefficients = new float[0];
// BLOCK 104
float leftEyeTheta = 0.0f;
float leftEyePhi = 0.0f;
float rightEyeTheta = 0.0f;
float rightEyePhi = 0.0f;
// BLOCK 105
List <Vector3> markersPositions = new List <Vector3>();
markersPositions.Capacity = 14;
// when magic number == 55355, we get
int presence = 2;
ushort magicNumber = m_crawler.ExtractUshort();
/*ushort version = */ m_crawler.ExtractUshort();
uint messageSize = m_crawler.ExtractUint();
int pos = m_crawler.Pos();
if (messageSize < 0)
{
m_valid = false;
}
if (magicNumber == 55355)
{
presence = m_crawler.ExtractInt32();
Debug.Log("presence message received: " + presence);
}
else if (magicNumber == 33633)
{
// Receiving blend shape names
ushort numberOfBlendShapes = m_crawler.ExtractUshort();
fsBlendShapeNames = new string[numberOfBlendShapes];
for (ushort j = 0; j < numberOfBlendShapes; j++)
{
fsBlendShapeNames[j] = m_crawler.ExtractString();
}
newBlendShapeNames = true;
}
else if (magicNumber == 33433)
{
ushort nbBlocks = m_crawler.ExtractUshort();
for (int i = 0; i < nbBlocks; i++)
{
ushort blockID = m_crawler.ExtractUshort();
/*ushort blockVersion =*/ m_crawler.ExtractUshort();
uint blockSize = m_crawler.ExtractUint();
int pos_start = m_crawler.Pos();
switch (blockID)
{
case 101: {
timestamp = m_crawler.ExtractDouble();
trackSuccess = m_crawler.ExtractBool();
} break;
case 102: {
headRotation = m_crawler.ExtractQuaternion();
headTranslation = m_crawler.ExtractVector3();
} break;
case 103: {
uint nbCoefficients = m_crawler.ExtractUint();
coefficients = m_crawler.ExtractCoefficients(nbCoefficients);
} break;
case 104: {
leftEyeTheta = m_crawler.ExtractFloat();
leftEyePhi = m_crawler.ExtractFloat();
rightEyeTheta = m_crawler.ExtractFloat();
rightEyePhi = m_crawler.ExtractFloat();
} break;
case 105: {
ushort nbMarkers = m_crawler.ExtractUshort();
markersPositions.AddRange(m_crawler.ExtractMarkersPositions(nbMarkers));
} break;
case 110: {
presence = m_crawler.ExtractUshort();
} break;
default: {
Debug.LogWarning("Unknown block " + blockID);
// do not make message invalid as it may just be an unknown block
m_crawler.Seek(m_crawler.Pos() + (int)blockSize);
} break;
}
int pos_end = m_crawler.Pos();
if (pos_end - pos_start != blockSize)
{
m_valid = false;
}
}
TrackData trackdata = new TrackData(timestamp,
trackSuccess,
headRotation,
headTranslation,
coefficients,
leftEyeTheta,
leftEyePhi,
rightEyeTheta,
rightEyePhi,
markersPositions,
presence);
m_queueOfTrackData.Enqueue(trackdata);
}
else
{
Debug.Log("unknown message block " + magicNumber);
// step to next block
m_crawler.Seek(pos + (int)messageSize);
}
if (!m_valid || m_crawler.Pos() != pos + (int)messageSize)
{
Debug.LogError("invalid data received");
m_valid = false;
}
}
// get rid of unnecessary data in crawler
m_crawler.Trim();
}
/**
* @brief Reads an fsb file and returns a clip constructed from the the content of the fsb file.
* @param[in] filename The filename of the fsb file.
* @return A clip with the track data imported from the fsb file, null if there was some error during loading.
*/
public static Clip read(string filename)
{
if (!File.Exists(filename))
{
return(null);
}
FileStream fileReader = File.OpenRead(filename);
int size = (int)fileReader.Length;
byte [] data = new byte[size];
int size_read = fileReader.Read(data, 0, size);
if (size != size_read)
{
Debug.LogError("Reading file unsuccessful: read " + size_read + " bytes instead of " + size + " bytes");
return(null);
}
DataParser parser = new DataParser();
parser.ExtractData(data);
if (parser.Valid())
{
Rig rig = new Rig();
// add hardcoded bone names of fsb file
rig.add_bone("Neck");
rig.add_bone("EyeLeft");
rig.add_bone("EyeRight");
for (int i = 0; i < parser.fsBlendShapeNames.Length; i++)
{
rig.add_shape(parser.fsBlendShapeNames[i]);
}
Clip clip = new Clip(rig);
while (parser.CountAvailableTracks() > 0)
{
TrackData track_data = parser.Dequeue();
if (!track_data.TrackSuccess)
{
// do not save the state if the tracking data is not valid
Debug.LogWarning("this frame doesn't contain valid tracking data");
continue;
}
RigState state = clip.new_state(track_data.TimeStamp);
if (rig.num_shapes() != track_data.n_coefficients())
{
Debug.LogError("num blendshapes do not agree in file with " + rig.num_shapes() +
" in rig and " + track_data.n_coefficients() + " in state");
return(null);
}
// Assume the head translation to be joint 0 (Neck)
state.set_bone_translation(0, track_data.HeadTranslation());
// bone indices same as the order when added to the rig
state.set_bone_rotation(0, track_data.HeadRotation());
state.set_bone_rotation(1, track_data.LeftEyeRotation());
state.set_bone_rotation(2, track_data.RightEyeRotation());
for (int i = 0; i < track_data.n_coefficients(); i++)
{
state.set_blendshape_coefficient(i, track_data.Coefficient[i]);
}
}
return(clip);
}
else
{
Debug.LogError("cannot parse fsb file");
}
return(null);
}
/**
* @brief Reads an fsb file and returns a clip constructed from the the content of the fsb file.
* @param[in] data The binary data from the fsb file.
* @return A clip with the track data imported from the fsb file, null if there was some error during loading.
*/
public static Clip Read(byte[] data)
{
DataParser parser = new DataParser();
parser.ExtractData(data, false, 0.0);
if (parser.Valid())
{
Rig rig = new Rig();
// add hardcoded bone names of fsb file
rig.AddBone("Neck");
rig.AddBone("EyeLeft");
rig.AddBone("EyeRight");
for (int i = 0; i < parser.fsBlendShapeNames.Length; i++)
{
rig.AddShape(parser.fsBlendShapeNames[i]);
}
Clip clip = new Clip(rig);
while (parser.CountAvailableTracks() > 0)
{
TrackData track_data = parser.Dequeue();
if (!track_data.TrackSuccess)
{
// do not save the state if the tracking data is not valid
Debug.LogWarning("this frame doesn't contain valid tracking data");
continue;
}
RigState state = clip.NewState(track_data.TimeStamp);
state.SetTrackingSuccessful(track_data.TrackSuccess);
if (rig.NumShapes() != track_data.n_coefficients())
{
Debug.LogError("num blendshapes do not agree in file with " + rig.NumShapes() +
" in rig and " + track_data.n_coefficients() + " in state");
return(null);
}
// Assume the head translation to be joint 0 (Neck)
state.SetBoneTranslation(0, track_data.HeadTranslation());
// bone indices same as the order when added to the rig
state.SetBoneRotation(0, track_data.HeadRotation());
state.SetBoneRotation(1, track_data.LeftEyeRotation());
state.SetBoneRotation(2, track_data.RightEyeRotation());
for (int i = 0; i < track_data.n_coefficients(); i++)
{
state.SetBlendshapeCoefficient(i, track_data.Coefficient[i]);
}
}
return(clip);
}
else
{
Debug.LogError("cannot parse fsb file");
}
return(null);
}
public TrackData(TrackData track)
{
setFromTrack(track);
}
public TrackData(TrackData track) {
setFromTrack(track);
}
public void ExtractData(byte[] data, bool manuallySetTimestamp, double timestamp) {
m_crawler.Append(data);
while (m_crawler.DataAvailable() && m_valid) {
// initialise variables
// when magic number == 33433, we get
// BLOCK 101
if (!manuallySetTimestamp) {
timestamp = 0.0;
}
bool trackSuccess = false;
// BLOCK 102
Quaternion headRotation = Quaternion.identity;
Vector3 headTranslation = Vector3.zero;
// BLOCK 103
float[] coefficients = new float[0];
// BLOCK 104
float leftEyeTheta = 0.0f;
float leftEyePhi = 0.0f;
float rightEyeTheta = 0.0f;
float rightEyePhi = 0.0f;
// BLOCK 105
List<Vector3> markersPositions = new List<Vector3>();
markersPositions.Capacity = 14;
// when magic number == 55355, we get
int presence = 2;
ushort magicNumber = m_crawler.ExtractUshort();
/*ushort version = */m_crawler.ExtractUshort();
uint messageSize = m_crawler.ExtractUint();
int pos = m_crawler.Pos();
if (messageSize < 0) m_valid = false;
if (magicNumber == 55355) {
presence = m_crawler.ExtractInt32();
Debug.Log("presence message received: " + presence);
} else if (magicNumber == 33633) {
// Receiving blend shape names
ushort numberOfBlendShapes = m_crawler.ExtractUshort();
fsBlendShapeNames = new string[numberOfBlendShapes];
for (ushort j = 0; j < numberOfBlendShapes; j++) {
fsBlendShapeNames[j] = m_crawler.ExtractString();
}
newBlendShapeNames = true;
} else if (magicNumber == 33433) {
ushort nbBlocks = m_crawler.ExtractUshort();
for (int i = 0; i < nbBlocks; i++) {
ushort blockID = m_crawler.ExtractUshort();
/*ushort blockVersion =*/ m_crawler.ExtractUshort();
uint blockSize = m_crawler.ExtractUint();
int pos_start = m_crawler.Pos();
switch (blockID) {
case 101: {
double timestampFromData = m_crawler.ExtractDouble();
if (!manuallySetTimestamp) {
timestamp = timestampFromData;
}
trackSuccess = m_crawler.ExtractBool();
} break;
case 102: {
headRotation = m_crawler.ExtractQuaternion();
headTranslation = m_crawler.ExtractVector3();
} break;
case 103: {
uint nbCoefficients = m_crawler.ExtractUint();
coefficients = m_crawler.ExtractCoefficients(nbCoefficients);
} break;
case 104: {
leftEyeTheta = m_crawler.ExtractFloat();
leftEyePhi = m_crawler.ExtractFloat();
rightEyeTheta = m_crawler.ExtractFloat();
rightEyePhi = m_crawler.ExtractFloat();
} break;
case 105: {
ushort nbMarkers = m_crawler.ExtractUshort();
markersPositions.AddRange( m_crawler.ExtractMarkersPositions(nbMarkers) );
} break;
case 110: {
presence = m_crawler.ExtractUshort();
} break;
default: {
//Debug.LogWarning("Unknown block " + blockID);
// do not make message invalid as it may just be an unknown block
m_crawler.Seek(m_crawler.Pos() + (int)blockSize);
} break;
}
int pos_end = m_crawler.Pos();
if (pos_end - pos_start != blockSize) m_valid = false;
}
TrackData trackdata = new TrackData( timestamp,
trackSuccess,
headRotation,
headTranslation,
coefficients,
leftEyeTheta,
leftEyePhi,
rightEyeTheta,
rightEyePhi,
markersPositions,
presence);
m_queueOfTrackData.Enqueue(trackdata);
} else {
Debug.Log("unknown message block " + magicNumber);
// step to next block
m_crawler.Seek(pos + (int)messageSize);
}
if (!m_valid || m_crawler.Pos() != pos + (int)messageSize) {
Debug.LogError("invalid data received");
m_valid = false;
}
}
// get rid of unnecessary data in crawler
m_crawler.Trim();
}
void Update()
{
// ten kod kalkuluje bazowa pozycje glowy
// trzeba jeszcze dorobic zeby przechwytywal trigerra
// i wtedy zmieniał isBaseCalculated na false
// i resetował timer i kalkulował jeszcze raz
// pozniej trzeba jeszcze dodac zeby kazda metoda zwracała
// to co zwraca minus base
if (!isBaseCalculated)
{
timer += Time.deltaTime;
if (timer < baseCalculationTime)
{
if (getXHeadRotation() != -1)
{
baseX += getXHeadRotation();
ticksX++;
}
if (getYHeadRotation() != -1)
{
baseY += getYHeadRotation();
ticksY++;
}
}
else
{
baseX /= ticksX;
baseY /= ticksY;
isBaseCalculated = true;
}
}
new_data = false;
m_mutex.WaitOne();
// check if there are new blendshape names, and if so update
if (newBlendShapeNamesArrived())
{
m_rig = getRigFromBlendShapeNames(getBlendShapeNames());
}
// get most recent tracking data
while (m_data_parser.CountAvailableTracks() > 0)
{
m_current_track = m_data_parser.Dequeue();
new_data = true;
}
// check that we have a rig (set up from blendshape names) with the same number of blendshapes as what we receive
if (m_rig != null && m_current_track != null)
{
int n_track_coefficients = m_current_track.n_coefficients();
int n_rig_coefficients = m_rig.num_shapes();
if (n_track_coefficients != n_rig_coefficients)
{
Debug.LogWarning("number of coefficients of rig and tracking state have changed: " + n_track_coefficients + " vs " + n_rig_coefficients);
// clear the current rig as it is not usable with the data coming from faceshift
m_rig = null;
m_current_track = null;
// get again the blendshapes from fs studio
askForBlendshapeNames();
}
}
m_mutex.ReleaseMutex();
if (m_rig != null && m_current_track != null && m_retargeting != null)
{
if (new_data && m_current_track.TrackSuccess)
{
// create a rig state
RigState state = new RigState(m_rig);
state.set_timestamp(m_current_track.TimeStamp);
state.set_bone_translation(0, m_current_track.HeadTranslation());
state.set_bone_rotation(0, m_current_track.HeadRotation());
state.set_bone_rotation(1, m_current_track.LeftEyeRotation());
state.set_bone_rotation(2, m_current_track.RightEyeRotation());
for (int i = 0; i < m_rig.num_shapes(); i++)
{
state.set_blendshape_coefficient(i, m_current_track.Coefficient [i]);
}
// evaluate joint transformations
TransformationValue [] transformation_values = null;
if (m_tpose != null && m_tpose.m_joints.Count == m_game_object_transformations.Count)
{
// evaluate using tpose
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_rig, state, m_tpose.m_joints);
}
else
{
// evaluate using state from start of application
transformation_values = Utils.evaluate_target_transformations(m_retargeting, m_rig, state, m_game_object_transformations);
}
if (transformation_values.Length == m_game_object_transformations.Count)
{
for (int index = 0; index < transformation_values.Length; index++)
{
// Apply the value for this target
if (transformation_values [index] != null)
{
TransformationInformation joint = m_game_object_transformations [index] as TransformationInformation;
joint.transform.localRotation = transformation_values [index].m_rotation;
joint.transform.localPosition = transformation_values [index].m_translation;
}
}
}
else
{
Debug.LogError("Cannot create transformation as evaluated shape size is incorrect");
}
// evaluate blendshape valuesf
BlendshapeValue [] values = Utils.evaluate_target_blendshapes(m_retargeting, m_rig, state, m_game_object_blendshapes);
if (values.Length == m_game_object_blendshapes.Count)
{
for (int index = 0; index < m_game_object_blendshapes.Count; index++)
{
BlendshapeInfo bs_info = m_game_object_blendshapes [index] as BlendshapeInfo;
// Apply the value for this target
if (bs_info != null && values [index] != null)
{
bs_info.m_mesh_renderer.SetBlendShapeWeight(bs_info.m_index, (float)values [index].m_value);
}
}
}
else
{
Debug.LogError("Cannot create blendshapes as evaluated shape size is incorrect");
}
}
}
}
