// checks if FacetrackingManager is initialized or not
private IEnumerator CheckFacetrackingManager()
{
// wait for 2 seconds
yield return(new WaitForSeconds(2f));
string sStatusMsg = string.Empty;
FacetrackingManager facetrackingManager = FacetrackingManager.Instance;
if (!facetrackingManager)
{
sStatusMsg = "FacetrackingManager is missing!";
}
else if (!facetrackingManager.IsFaceTrackingInitialized())
{
sStatusMsg = "FacetrackingManager not initialized! Check the log-file for details.";
}
else
{
LogToConsole("FacetrackingManager is ready.");
}
if (sStatusMsg.Length > 0)
{
LogErrorToConsole(sStatusMsg);
}
}
void Update()
{
if (!manager)
{
manager = KinectManager.Instance;
}
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
// get the face points
if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
long userId = manager.GetUserIdByIndex(playerIndex);
if (userId == 0)
{
return;
}
if (faceVertices == null)
{
int iVertCount = faceManager.GetUserFaceVertexCount(userId);
if (iVertCount > 0)
{
faceVertices = new Vector3[iVertCount];
}
}
if (faceVertices != null)
{
if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
{
if (faceVertices != null && faceVertices[(int)facePoint] != Vector3.zero)
{
Vector3 facePointPos = faceVertices [(int)facePoint];
if (foregroundCamera)
{
facePointPos = GetOverlayPosition(facePointPos);
}
if (facePointTransform)
{
facePointTransform.position = facePointPos;
}
if (faceInfoText)
{
string sStatus = string.Format("{0}: {1}", facePoint, facePointPos);
faceInfoText.text = sStatus;
}
}
}
}
}
}
// Update is called once per frame
void Update()
{
Matrix4x4 kinectToWorld = Matrix4x4.zero;
//Quaternion quatTiltAngle = Quaternion.Euler(-manager.sensorAngle, 0.0f, 0.0f);
//kinectToWorld.SetTRS(new Vector3(0.0f, manager.sensorHeight, 0.0f), quatTiltAngle, Vector3.one);
if (!manager)
{
manager = KinectManager.Instance;
}
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
// get the face points
if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
long userId = manager.GetUserIdByIndex(playerIndex);
if (faceVertices == null)
{
//int iVertCount = faceManager.GetUserFaceVertexCount(userId);
int iVertCount = 1347;
if (iVertCount > 0)
{
faceVertices = new Vector3[iVertCount];
}
check_f = false;
}
if (faceVertices != null)
{
if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
{
//-------------- get 35 special point-------------------
//Matrix4x4 kinectToWorld = manager.GetKinectToWorldMatrix();
HighDetailFacePoints[] facePoints = (HighDetailFacePoints[])System.Enum.GetValues(typeof(HighDetailFacePoints));
for (int i = 0; i < facePoints.Length; i++)
{
HighDetailFacePoints point = facePoints[i];
//dictFacePoints[point] = kinectToWorld.MultiplyPoint3x4(faceVertices[(int)point]);
dictFacePoints[point] = faceVertices[(int)point];
}
check_f = true;
}
}
}
}
void Update()
{
// reference to face manager
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
if (kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
// check for tracked user
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId != 0 && kinectManager.IsUserTracked(userId))
{
if (faceManager.GetUserAnimUnits(userId, ref dictAnimUnits))
{
// process the blend shapes -> anim units
for (int i = 0; i < blendShapeCount; i++)
{
if (blendShape2AnimUnit.ContainsKey(blendShapeNames[i]))
{
KinectInterop.FaceShapeAnimations faceAnim = blendShape2AnimUnit[blendShapeNames[i]];
float animValue = dictAnimUnits[faceAnim];
// check for multiplier
float mul = 1f;
if (blendShape2Multiplier.ContainsKey(blendShapeNames[i]))
{
mul = blendShape2Multiplier[blendShapeNames[i]];
}
if (animValue * mul < 0f)
{
animValue = 0f;
}
// lerp to the new value
blendShapeValues[i] = Mathf.Lerp(blendShapeValues [i], animValue * mul * 100f, smoothFactor * Time.deltaTime);
skinnedMeshRenderer.SetBlendShapeWeight(i, blendShapeValues[i]);
}
}
}
}
}
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
//unsafe for editor but can optimise if commented
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
if (userId == 0)
{
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
if (!colorTex)
{
colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
}
try
{
RenderTexture foregroundTex = (RenderTexture)backManager.GetForegroundTex();
KinectInterop.RenderTex2Tex2D(foregroundTex, ref colorTex);
}
catch (System.Exception)
{
colorTex = (Texture2D)backManager.GetForegroundTex();
}
}
else
{
// use color camera image
colorTex = kinectManager.GetUsersClrTex();
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
{
return;
}
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
if (targetRenderer && targetRenderer.material)
{
targetRenderer.material.mainTexture = faceTex;
}
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId == 0)
{
if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
{
targetObject.material.mainTexture = null;
}
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (!bSkipUpdatingForegroundTex && backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
Texture bmForeTex = backManager.GetForegroundTex();
if (bmForeTex is RenderTexture)
{
foregroundTex = (RenderTexture)bmForeTex;
bSkipUpdatingForegroundTex = (foregroundTex != null);
}
else
{
if (foregroundTex == null)
{
foregroundTex = new RenderTexture(bmForeTex.width, bmForeTex.height, 0);
}
Graphics.Blit(bmForeTex, foregroundTex);
bSkipUpdatingForegroundTex = false;
}
// if (!colorTex)
// {
// colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
// }
//
// try
// {
// foregroundTex = (RenderTexture)backManager.GetForegroundTex ();
// //KinectInterop.RenderTex2Tex2D (foregroundTex, ref colorTex);
// }
// catch (System.Exception)
// {
// colorTex = (Texture2D)backManager.GetForegroundTex ();
// }
}
else
{
// use color camera image
if (!colorTex)
{
colorTex = kinectManager.GetUsersClrTex2D();
}
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (faceRect.width > 0 && faceRect.height > 0)
{
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if (foregroundTex)
{
if ((faceX + faceW) > foregroundTex.width)
{
faceW = foregroundTex.width - faceX;
}
if ((faceY + faceH) > foregroundTex.height)
{
faceH = foregroundTex.height - faceY;
}
}
else if (colorTex)
{
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
if (foregroundTex)
{
KinectInterop.RenderTex2Tex2D(foregroundTex, faceX, foregroundTex.height - faceY - faceH, faceW, faceH, ref faceTex);
}
else if (colorTex)
{
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
}
if (targetObject && !targetObject.gameObject.activeSelf)
{
targetObject.gameObject.SetActive(true);
}
if (targetObject && targetObject.material)
{
targetObject.material.mainTexture = faceTex;
}
// don't rotate the transform - mesh follows the head rotation
if (targetObject.transform.rotation != Quaternion.identity)
{
targetObject.transform.rotation = Quaternion.identity;
}
}
else
{
if (targetObject && targetObject.gameObject.activeSelf)
{
targetObject.gameObject.SetActive(false);
}
if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
{
targetObject.material.mainTexture = null;
}
}
}
void Update()
{
int recHostId;
int connectionId;
int recChannelId;
int dataSize;
bool connListUpdated = false;
if (backgroundImage && backgroundImage.texture == null)
{
backgroundImage.texture = manager ? manager.GetUsersLblTex() : null;
}
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (gestureManager == null)
{
gestureManager = VisualGestureManager.Instance;
}
if (speechManager == null)
{
speechManager = SpeechManager.Instance;
}
try
{
byte error = 0;
NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out recChannelId, recBuffer, bufferSize, out dataSize, out error);
switch (recData)
{
case NetworkEventType.Nothing: //1
break;
case NetworkEventType.ConnectEvent: //2
if (recHostId == serverHostId && recChannelId == serverChannelId &&
!dictConnection.ContainsKey(connectionId))
{
HostConnection conn = new HostConnection();
conn.hostId = recHostId;
conn.connectionId = connectionId;
conn.channelId = recChannelId;
conn.keepAlive = true;
conn.reqDataType = "ka,kb,km,kh";
//conn.matrixSent = false;
dictConnection[connectionId] = conn;
connListUpdated = true;
//LogToConsole(connectionId + "-conn: " + conn.reqDataType);
}
// // reset chunked face messages
// sendFvMsg = string.Empty;
// sendFvNextOfs = 0;
//
// sendFtMsg = string.Empty;
// sendFtNextOfs = 0;
break;
case NetworkEventType.DataEvent: //3
if (recHostId == serverHostId && recChannelId == serverChannelId &&
dictConnection.ContainsKey(connectionId))
{
HostConnection conn = dictConnection[connectionId];
int decompSize = 0;
if (decompressor != null && (recBuffer[0] > 127 || recBuffer[0] < 32))
{
decompSize = decompressor.Decompress(recBuffer, 0, compressBuffer, 0, dataSize);
}
else
{
System.Buffer.BlockCopy(recBuffer, 0, compressBuffer, 0, dataSize);
decompSize = dataSize;
}
string sRecvMessage = System.Text.Encoding.UTF8.GetString(compressBuffer, 0, decompSize);
if (sRecvMessage.StartsWith("ka"))
{
if (sRecvMessage == "ka") // vr-examples v1.0 keep-alive message
{
sRecvMessage = "ka,kb,km,kh";
}
conn.keepAlive = true;
conn.reqDataType = sRecvMessage;
dictConnection[connectionId] = conn;
//LogToConsole(connectionId + "-recv: " + conn.reqDataType);
// check for SR phrase-reset
int iIndexSR = sRecvMessage.IndexOf(",sr");
if (iIndexSR >= 0 && speechManager)
{
speechManager.ClearPhraseRecognized();
//LogToConsole("phrase cleared");
}
}
}
break;
case NetworkEventType.DisconnectEvent: //4
if (dictConnection.ContainsKey(connectionId))
{
dictConnection.Remove(connectionId);
connListUpdated = true;
}
break;
}
if (connListUpdated)
{
// get all connection IDs
alConnectionId.Clear();
alConnectionId.AddRange(dictConnection.Keys);
// display the number of connections
StringBuilder sbConnStatus = new StringBuilder();
sbConnStatus.AppendFormat("Server running: {0} connection(s)", dictConnection.Count);
foreach (int connId in dictConnection.Keys)
{
HostConnection conn = dictConnection[connId];
int iPort = 0; string sAddress = string.Empty; NetworkID network; NodeID destNode;
NetworkTransport.GetConnectionInfo(conn.hostId, conn.connectionId, out sAddress, out iPort, out network, out destNode, out error);
if (error == (int)NetworkError.Ok)
{
sbConnStatus.AppendLine().Append(" ").Append(sAddress).Append(":").Append(iPort);
}
}
LogToConsole(sbConnStatus.ToString());
if (connStatusText)
{
connStatusText.text = sbConnStatus.ToString();
}
}
// send body frame to available connections
const char delimiter = ',';
string sBodyFrame = manager ? manager.GetBodyFrameData(ref liRelTime, ref fCurrentTime, delimiter) : string.Empty;
if (sBodyFrame.Length > 0 && dictConnection.Count > 0)
{
StringBuilder sbSendMessage = new StringBuilder();
bool bFaceParamsRequested = IsFaceParamsRequested();
sbSendMessage.Append(manager.GetWorldMatrixData(delimiter)).Append('|');
sbSendMessage.Append(sBodyFrame).Append('|');
sbSendMessage.Append(manager.GetBodyHandData(ref liRelTime, delimiter)).Append('|');
if (bFaceParamsRequested && faceManager && faceManager.IsFaceTrackingInitialized())
{
sbSendMessage.Append(faceManager.GetFaceParamsAsCsv(delimiter)).Append('|');
}
if (gestureManager && gestureManager.IsVisualGestureInitialized())
{
sbSendMessage.Append(gestureManager.GetGestureDataAsCsv(delimiter)).Append('|');
}
if (speechManager && speechManager.IsSapiInitialized())
{
sbSendMessage.Append(speechManager.GetSpeechDataAsCsv(delimiter)).Append('|');
}
if (sbSendMessage.Length > 0 && sbSendMessage[sbSendMessage.Length - 1] == '|')
{
sbSendMessage.Remove(sbSendMessage.Length - 1, 1);
}
byte[] btSendMessage = System.Text.Encoding.UTF8.GetBytes(sbSendMessage.ToString());
//Debug.Log("Message " + sbSendMessage.Length + " chars: " + sbSendMessage.ToString());
//Debug.Log("Encoded into " + btSendMessage.Length + " bytes: " + ByteArrayToString(btSendMessage, btSendMessage.Length));
int compSize = 0;
if (compressor != null && btSendMessage.Length > 100 && !websocketHost)
{
compSize = compressor.Compress(btSendMessage, 0, btSendMessage.Length, compressBuffer, 0);
}
else
{
System.Buffer.BlockCopy(btSendMessage, 0, compressBuffer, 0, btSendMessage.Length);
compSize = btSendMessage.Length;
}
//Debug.Log("Compressed into " + compSize + " bytes: " + ByteArrayToString(compressBuffer, compSize));
// // check face-tracking requests
// bool bFaceParams = false, bFaceVertices = false, bFaceUvs = false, bFaceTriangles = false;
// if(faceManager && faceManager.IsFaceTrackingInitialized())
// CheckFacetrackRequests(out bFaceParams, out bFaceVertices, out bFaceUvs, out bFaceTriangles);
//
// byte[] btFaceParams = null;
// if(bFaceParams)
// {
// string sFaceParams = faceManager.GetFaceParamsAsCsv();
// if(!string.IsNullOrEmpty(sFaceParams))
// btFaceParams = System.Text.Encoding.UTF8.GetBytes(sFaceParams);
// }
//
// // next chunk of data for face vertices
// byte[] btFaceVertices = null;
// string sFvMsgHead = string.Empty;
// GetNextFaceVertsChunk(bFaceVertices, bFaceUvs, ref btFaceVertices, out sFvMsgHead);
//
// // next chunk of data for face triangles
// byte[] btFaceTriangles = null;
// string sFtMsgHead = string.Empty;
// GetNextFaceTrisChunk(bFaceTriangles, ref btFaceTriangles, out sFtMsgHead);
foreach (int connId in alConnectionId)
{
HostConnection conn = dictConnection[connId];
if (conn.keepAlive)
{
conn.keepAlive = false;
dictConnection[connId] = conn;
if (conn.reqDataType != null && conn.reqDataType.Contains("kb,"))
{
//LogToConsole(conn.connectionId + "-sendkb: " + conn.reqDataType);
error = 0;
//if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btSendMessage, btSendMessage.Length, out error))
if (!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, compressBuffer, compSize, out error))
{
string sMessage = "Error sending body data via conn " + conn.connectionId + ": " + (NetworkError)error;
LogErrorToConsole(sMessage);
if (serverStatusText)
{
serverStatusText.text = sMessage;
}
}
}
// if(bFaceParams && btFaceParams != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("fp,"))
// {
// //Debug.Log(conn.connectionId + "-sendfp: " + conn.reqDataType);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceParams, btFaceParams.Length, out error))
// {
// string sMessage = "Error sending face params via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
//
// if(bFaceVertices && btFaceVertices != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("fv,"))
// {
// //Debug.Log(conn.connectionId + "-sendfv: " + conn.reqDataType + " - " + sFvMsgHead);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceVertices, btFaceVertices.Length, out error))
// {
// string sMessage = "Error sending face verts via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
//
// if(bFaceTriangles && btFaceTriangles != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("ft,"))
// {
// //Debug.Log(conn.connectionId + "-sendft: " + conn.reqDataType + " - " + sFtMsgHead);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceTriangles, btFaceTriangles.Length, out error))
// {
// string sMessage = "Error sending face tris via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
}
}
}
}
catch (System.Exception ex)
{
LogErrorToConsole(ex.Message + "\n" + ex.StackTrace);
if (serverStatusText)
{
serverStatusText.text = ex.Message;
}
}
}
void Update()
{
if (!manager)
{
manager = KinectManager.Instance;
}
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
// // get reference to the Kinect2Interface
// if(k2interface == null)
// {
// manager = KinectManager.Instance;
//
// if(manager && manager.IsInitialized())
// {
// KinectInterop.SensorData sensorData = manager.GetSensorData();
//
// if(sensorData != null && sensorData.sensorInterface != null)
// {
// k2interface = (Kinect2Interface)sensorData.sensorInterface;
// }
// }
// }
// get the face points
if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
long userId = manager.GetUserIdByIndex(playerIndex);
if (faceVertices == null)
{
int iVertCount = faceManager.GetUserFaceVertexCount(userId);
if (iVertCount > 0)
{
faceVertices = new Vector3[iVertCount];
}
}
if (faceVertices != null)
{
if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
{
Matrix4x4 kinectToWorld = manager.GetKinectToWorldMatrix();
HighDetailFacePoints[] facePoints = (HighDetailFacePoints[])System.Enum.GetValues(typeof(HighDetailFacePoints));
for (int i = 0; i < facePoints.Length; i++)
{
HighDetailFacePoints point = facePoints[i];
dictFacePoints[point] = kinectToWorld.MultiplyPoint3x4(faceVertices[(int)point]);
}
}
}
}
if (faceVertices != null && faceVertices[(int)facePoint] != Vector3.zero)
{
Vector3 facePointPos = faceVertices [(int)facePoint];
if (facePointTransform)
{
facePointTransform.position = facePointPos;
}
if (faceInfoText)
{
string sStatus = string.Format("{0}: {1}", facePoint, facePointPos);
faceInfoText.text = sStatus;
}
}
}
void Update()
{
blendShape2Multiplier ["Jaw_Left"] = -1f * jawLeftMult;
blendShape2Multiplier ["BrowsUp_Right"] = -1f * browUpRightMult;
blendShape2Multiplier ["BrowsUp_Left"] = -1f * browUpLeftMult;
blendShape2Multiplier ["Jaw_Down"] = jawDownMult;
blendShape2Multiplier ["BrowsDown_Right"] = browDownRightMult;
blendShape2Multiplier ["CheekPuff_Left"] = cheekLeftMult;
blendShape2Multiplier ["CheekPuff_Right"] = cheekRightMult;
blendShape2Multiplier ["Smile_Right"] = smileRightMult;
blendShape2Multiplier ["Smile_Left"] = smileLeftMult;
blendShape2Multiplier ["Jaw_Right"] = jawRightMult;
blendShape2Multiplier ["Blink_Right"] = blinkRightMult;
blendShape2Multiplier ["Blink_Left"] = blinkLeftMult;
blendShape2Multiplier ["Midmouth_Left"] = midmouthMult;
blendShape2Multiplier ["Midmouth_Right"] = midmouthMult;
blendShape2Multiplier ["LowerLipDown_Right"] = depressorRightMult;
blendShape2Multiplier ["LowerLipDown_Left"] = depressorLeftMult;
blendShape2Multiplier ["Lips_Stretch_L"] = stretcherLeftMult;
blendShape2Multiplier ["Lips_Stretch_R"] = stretcherRightMult;
blendShape2Multiplier ["Frown_Right"] = frownRightMult;
blendShape2Multiplier ["Frown_Left"] = frownLeftMult;
// reference to face manager
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
if (kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
// check for tracked user
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId != 0 && kinectManager.IsUserTracked(userId))
{
if (faceManager.GetUserAnimUnits(userId, ref dictAnimUnits))
{
// process the blend shapes -> anim units
for (int i = 0; i < blendShapeCount; i++)
{
if (blendShape2AnimUnit.ContainsKey(blendShapeNames [i]))
{
KinectInterop.FaceShapeAnimations faceAnim = blendShape2AnimUnit [blendShapeNames [i]];
float animValue = dictAnimUnits [faceAnim];
// check for multiplier
float mul = 1f;
if (blendShape2Multiplier.ContainsKey(blendShapeNames [i]))
{
mul = blendShape2Multiplier [blendShapeNames [i]];
}
if (animValue * mul < 0f)
{
animValue = 0f;
}
// lerp to the new value
blendShapeValues [i] = Mathf.Lerp(blendShapeValues [i], animValue * mul * 100f, smoothFactor * Time.deltaTime);
skinnedMeshRenderer.SetBlendShapeWeight(i, blendShapeValues [i]);
//rotation
Vector3 angles = kinectManager.GetJointOrientation(userId, (int)KinectInterop.JointType.Head, true).eulerAngles;
transform.parent.parent.rotation = Quaternion.Euler(new Vector3(-angles.x, -angles.y, angles.z));
}
}
}
}
}
}
void Update()
{
if (skinnedMeshRenderer == null)
{
return;
}
if (faceAnimUnits.Count != faceBlendShapes.Count)
{
return;
}
if (blendShapeValues == null || blendShapeValues.Length != faceAnimUnits.Count)
{
blendShapeValues = new float[faceAnimUnits.Count];
}
// reference to face manager
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
if (kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
// check for tracked user
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId != 0 && kinectManager.IsUserTracked(userId))
{
// move the model transform according to user's head
if (headTransform != null)
{
MoveHeadTransform(userId);
}
// animate blend shapes
if (faceManager.GetUserAnimUnits(userId, ref dictAnimUnits))
{
for (int i = 0; i < faceAnimUnits.Count; i++)
{
KinectInterop.FaceShapeAnimations faceAnim = faceAnimUnits[i];
if (dictAnimUnits.ContainsKey(faceAnim))
{
float animValue = dictAnimUnits[faceAnim];
string blendShapeName = faceBlendShapes[i];
int blendShapeIndex = !string.IsNullOrEmpty(blendShapeName) ? skinnedMesh.GetBlendShapeIndex(blendShapeName) : -1;
if (blendShapeIndex >= 0)
{
if (animValue < 0f)
{
//Debug.Log(faceAnim + ": " + animValue);
animValue = -animValue;
}
// lerp to the new value
blendShapeValues[i] = Mathf.Lerp(blendShapeValues[i], animValue * valueMultiplier, blendSmoothFactor * Time.deltaTime);
skinnedMeshRenderer.SetBlendShapeWeight(blendShapeIndex, blendShapeValues[i]);
}
else
{
Debug.LogError("Blend shape '" + blendShapeName + "' not found!");
}
}
}
}
}
}
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
if (userId == 0)
{
return;
}
colorTex = kinectManager.GetUsersClrTex();
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
{
return;
}
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
if (targetRenderer && targetRenderer.material)
{
targetRenderer.material.mainTexture = faceTex;
}
}
