void StartFacetracker()
{
try
{
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please, wait...";
}
// initialize Kinect sensor as needed
int rc = FacetrackingWrapper.InitKinectSensor((int)FacetrackingWrapper.Constants.ColorImageResolution, (int)FacetrackingWrapper.Constants.DepthImageResolution, FacetrackingWrapper.Constants.IsNearMode);
if (rc != 0)
{
throw new Exception("Initialization of Kinect sensor failed");
}
// Initialize the kinect speech wrapper
rc = FacetrackingWrapper.InitFaceTracking();
if (rc < 0)
{
throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc));
}
if (ComputeColorMap)
{
// Initialize color map related stuff
usersClrTex = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight(), TextureFormat.ARGB32, false);
usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height);
colorImage = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()];
videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4];
}
instance = this;
facetrackingInitialized = true;
DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
// Make sure to kill the Kinect on quitting.
void OnApplicationQuit()
{
// Shutdown Speech Recognizer and Kinect
FacetrackingWrapper.FinishFaceTracking();
FacetrackingWrapper.ShutdownKinectSensor();
facetrackingInitialized = false;
instance = null;
}
///// end of added
//----------------------------------- end of public functions --------------------------------------//
void Awake()
{
//debugText = GameObject.Find("DebugText");
// ensure the needed dlls are in place
if (FacetrackingWrapper.EnsureKinectWrapperPresence())
{
// reload the same level
Application.LoadLevel(Application.loadedLevel);
}
}
void Update()
{
// start Kinect face tracker as needed
if (!facetrackingInitialized)
{
StartFacetracker();
if (!facetrackingInitialized)
{
Application.Quit();
return;
}
}
if (facetrackingInitialized)
{
// update the face tracker
int rc = FacetrackingWrapper.UpdateFaceTracking();
if (rc >= 0)
{
// poll the video frame as needed
if (ComputeColorMap)
{
if (FacetrackingWrapper.PollVideo(ref videoBuffer, ref colorImage))
{
usersClrTex.SetPixels32(colorImage);
usersClrTex.Apply();
}
}
// estimate the tracking state
isTracking = FacetrackingWrapper.IsFaceTracked();
faceTrackingID = FacetrackingWrapper.GetFaceTrackingID();
// get the facetracking parameters
if (isTracking)
{
//float fTimeNow = Time.realtimeSinceStartup;
// get face rectangle
bool bGotFaceRect = FacetrackingWrapper.GetFaceRect(ref faceRect);
// get head position and rotation
Vector4 vHeadPos = Vector4.zero, vHeadRot = Vector4.zero;
if (FacetrackingWrapper.GetHeadPosition(ref vHeadPos))
{
headPos = (Vector3)vHeadPos;
if (!MirroredHeadMovement)
{
headPos.z = -headPos.z;
}
}
if (FacetrackingWrapper.GetHeadRotation(ref vHeadRot))
{
if (MirroredHeadMovement)
{
vHeadRot.x = -vHeadRot.x;
vHeadRot.z = -vHeadRot.z;
}
else
{
vHeadRot.x = -vHeadRot.x;
vHeadRot.y = -vHeadRot.y;
}
headRot = Quaternion.Euler((Vector3)vHeadRot);
}
// get the animation units
int iNumAU = FacetrackingWrapper.GetAnimUnitsCount();
bGotAU = false;
if (iNumAU > 0)
{
if (afAU == null)
{
afAU = new float[iNumAU];
}
bGotAU = FacetrackingWrapper.GetAnimUnits(afAU, ref iNumAU);
}
// get the shape units
isConverged = FacetrackingWrapper.IsShapeConverged();
int iNumSU = FacetrackingWrapper.GetShapeUnitsCount();
bGotSU = false;
if (iNumSU > 0)
{
if (afSU == null)
{
afSU = new float[iNumSU];
}
bGotSU = FacetrackingWrapper.GetShapeUnits(afSU, ref iNumSU);
}
// get the shape points
int iNumPoints = FacetrackingWrapper.GetShapePointsCount();
bGotPoints = false;
if (iNumPoints > 0)
{
int iNumPointsXY = iNumPoints << 1;
if (avPointsXY == null)
{
avPointsXY = new float[iNumPointsXY];
avPoints = new Vector2[iNumPoints];
}
bGotPoints = FacetrackingWrapper.GetShapePoints(avPointsXY, ref iNumPointsXY);
if (bGotPoints)
{
for (int i = 0; i < iNumPoints; i++)
{
int iXY = i << 1;
avPoints[i].x = avPointsXY[iXY];
avPoints[i].y = avPointsXY[iXY + 1];
}
}
}
///// added by Takefumi
// get the 3D shape points
int i3DNumPoints = FacetrackingWrapper.Get3DShapePointsCount();
bGot3DPoints = false;
if (i3DNumPoints > 0)
{
int iNumPointsXYZ = i3DNumPoints * 3;
if (avPointsXYZ == null)
{
avPointsXYZ = new float[iNumPointsXYZ];
av3DPoints = new Vector3[i3DNumPoints];
}
bGot3DPoints = FacetrackingWrapper.Get3DShapePoints(avPointsXYZ, ref iNumPointsXYZ);
if (bGot3DPoints)
{
for (int i = 0; i < i3DNumPoints; i++)
{
int iXYZ = i * 3;
av3DPoints[i].x = avPointsXYZ[iXYZ];
av3DPoints[i].y = avPointsXYZ[iXYZ + 1];
av3DPoints[i].z = avPointsXYZ[iXYZ + 2];
}
}
}
///// end of added
if (ComputeColorMap)
{
if (DisplayFaceRect && bGotFaceRect)
{
DrawFacetrackerRect(usersClrTex, !VisualizeFacetracker);
}
if (VisualizeFacetracker)
{
DrawFacetrackerLines(usersClrTex, avPoints, true);
}
}
}
}
}
}
// returns the user ID at the given index
// the index must be between 0 and (usersCount - 1)
public uint GetUserIdAt(int index)
{
return(FacetrackingWrapper.GetSkeletonTrackingID((uint)index));
}
// returns the number of Kinect users
public int GetUsersCount()
{
return(FacetrackingWrapper.GetInteractorsCount());
}
// sets new user ID (primary skeleton ID) to be used by the native wrapper
public void SetUserId(uint userId)
{
FacetrackingWrapper.SetSkeletonTrackingID(userId);
}
// returns the user ID (primary skeleton ID), or 0 if no user is currently tracked
public uint GetUserID()
{
return(FacetrackingWrapper.GetSkeletonTrackingID());
}
void Update()
{
// start Kinect face tracker as needed
if (!facetrackingInitialized)
{
StartFacetracker();
if (!facetrackingInitialized)
{
Application.Quit();
return;
}
}
if (facetrackingInitialized)
{
// update the face tracker
int rc = FacetrackingWrapper.UpdateFaceTracking();
if (rc >= 0)
{
// poll the video frame as needed
if (computeColorMap)
{
if (FacetrackingWrapper.PollVideo(ref videoBuffer, ref colorImage))
{
usersClrTex.SetPixels32(colorImage);
usersClrTex.Apply();
}
}
// estimate the tracking state
isTrackingFace = FacetrackingWrapper.IsFaceTracked();
faceTrackingID = FacetrackingWrapper.GetFaceTrackingID();
Debug.Log(isTrackingFace);
// get the facetracking parameters
if (isTrackingFace)
{
lastFaceTrackedTime = Time.realtimeSinceStartup;
// get face rectangle
bool bGotFaceRect = FacetrackingWrapper.GetFaceRect(ref faceRect);
// get head position and rotation
Vector4 vHeadPos = Vector4.zero, vHeadRot = Vector4.zero;
if (FacetrackingWrapper.GetHeadPosition(ref vHeadPos))
{
headPos = (Vector3)vHeadPos;
if (!mirroredHeadMovement)
{
headPos.z = -headPos.z;
}
}
if (FacetrackingWrapper.GetHeadRotation(ref vHeadRot))
{
if (mirroredHeadMovement)
{
vHeadRot.x = -vHeadRot.x;
vHeadRot.z = -vHeadRot.z;
}
else
{
vHeadRot.x = -vHeadRot.x;
vHeadRot.y = -vHeadRot.y;
}
headRot = Quaternion.Euler((Vector3)vHeadRot);
}
// get the animation units
bGotAU = false;
int iNumAU = FacetrackingWrapper.GetAnimUnitsCount();
if (iNumAU > 0)
{
if (afAU == null || afAU.Length == 0)
{
afAU = new float[iNumAU];
}
if (afAU != null && afAU.Length > 0)
{
bGotAU = FacetrackingWrapper.GetAnimUnits(ref afAU);
}
}
// get the shape units
//isConverged = FacetrackingWrapper.IsShapeConverged();
bGotSU = false;
int iNumSU = FacetrackingWrapper.GetShapeUnitsCount();
if (iNumSU > 0)
{
if (afSU == null || afSU.Length == 0)
{
afSU = new float[iNumSU];
}
if (afSU != null && afSU.Length > 0)
{
bGotSU = FacetrackingWrapper.GetShapeUnits(ref afSU);
}
}
// get the shape points
bGotShapePoints = false;
int iNumPoints = FacetrackingWrapper.GetShapePointsCount();
if (iNumPoints > 0)
{
if (avShapePoints == null || avShapePoints.Length == 0)
{
avShapePoints = new Vector2[iNumPoints];
}
if (avShapePoints != null || avShapePoints.Length > 0)
{
bGotShapePoints = FacetrackingWrapper.GetShapePoints(ref avShapePoints);
}
}
// get the 3D model points
bGotFaceModelPoints = false;
int iFaceModelVertexCount = FacetrackingWrapper.GetFaceModelVerticesCount();
if (iFaceModelVertexCount > 0)
{
if (avFaceModelTriangles == null || avFaceModelTriangles.Length == 0)
{
int iFaceModelTrianglesCount = FacetrackingWrapper.GetFaceModelTrianglesCount();
avFaceModelTriangles = new int[iFaceModelTrianglesCount];
FacetrackingWrapper.GetFaceModelTriangles(mirroredHeadMovement, ref avFaceModelTriangles);
}
if (avFaceModelPoints == null || avFaceModelPoints.Length == 0)
{
avFaceModelPoints = new Vector3[iFaceModelVertexCount];
}
if (avFaceModelPoints != null && avFaceModelPoints.Length > 0)
{
bGotFaceModelPoints = FacetrackingWrapper.GetFaceModelVertices(ref avFaceModelPoints);
}
}
if (computeColorMap)
{
if (displayFaceRect && bGotFaceRect)
{
DrawFacetrackerRect(usersClrTex, !visualizeFacetracker);
}
if (visualizeFacetracker && bGotShapePoints)
{
DrawFacetrackerLines(usersClrTex, avShapePoints, true);
}
}
}
else if ((Time.realtimeSinceStartup - lastFaceTrackedTime) <= faceTrackingTolerance)
{
// allow tolerance in tracking
isTrackingFace = true;
}
}
}
}
void Update()
{
// start Kinect face tracker as needed
if (!facetrackingInitialized)
{
StartFacetracker();
if (!facetrackingInitialized)
{
Application.Quit();
return;
}
}
if (facetrackingInitialized)
{
// update the face tracker
int rc = FacetrackingWrapper.UpdateFaceTracking();
if (rc >= 0)
{
// poll the video frame as needed
if (ComputeColorMap)
{
//float fTimeNow2 = Time.realtimeSinceStartup;
if (FacetrackingWrapper.PollVideo(ref videoBuffer, ref colorImage))
{
usersClrTex.SetPixels32(colorImage);
usersClrTex.Apply();
}
//fTimeNow2 = Time.realtimeSinceStartup - fTimeNow2;
//Debug.Log("PollVideo() took " + fTimeNow2 + " s.");
}
// estimate the tracking state
isTracking = FacetrackingWrapper.IsFaceTracked();
// get the facetracking parameters
if (isTracking)
{
//float fTimeNow = Time.realtimeSinceStartup;
// get head position and rotation
Vector4 vHeadPos = Vector4.zero, vHeadRot = Vector4.zero;
if (FacetrackingWrapper.GetHeadPosition(ref vHeadPos))
{
headPos = (Vector3)vHeadPos;
}
if (FacetrackingWrapper.GetHeadRotation(ref vHeadRot))
{
vHeadRot.x = -vHeadRot.x;
vHeadRot.z = -vHeadRot.z;
headRot = Quaternion.Euler((Vector3)vHeadRot);
}
// get the animation units
int iNumAU = FacetrackingWrapper.GetAnimUnitsCount();
bGotAU = false;
if (iNumAU > 0)
{
if (afAU == null)
{
afAU = new float[iNumAU];
}
bGotAU = FacetrackingWrapper.GetAnimUnits(afAU, ref iNumAU);
}
// // debug anim. units
// sAuDebug = "AU: ";
// for(int i = 0; i < iNumAU; i++)
// {
// sAuDebug += String.Format("{0}:{1:F2} ", i, afAU[i]);
// }
// get the shape units
isConverged = FacetrackingWrapper.IsShapeConverged();
int iNumSU = FacetrackingWrapper.GetShapeUnitsCount();
bGotSU = false;
if (iNumSU > 0)
{
if (afSU == null)
{
afSU = new float[iNumSU];
}
bGotSU = FacetrackingWrapper.GetShapeUnits(afSU, ref iNumSU);
}
// get the shape points
int iNumPoints = FacetrackingWrapper.GetShapePointsCount();
bGotPoints = false;
if (iNumPoints > 0)
{
int iNumPointsXY = iNumPoints << 1;
if (avPointsXY == null)
{
avPointsXY = new float[iNumPointsXY];
avPoints = new Vector2[iNumPoints];
}
bGotPoints = FacetrackingWrapper.GetShapePoints(avPointsXY, ref iNumPointsXY);
if (bGotPoints)
{
for (int i = 0; i < iNumPoints; i++)
{
int iXY = i << 1;
avPoints[i].x = avPointsXY[iXY];
avPoints[i].y = avPointsXY[iXY + 1];
}
}
}
//fTimeNow = Time.realtimeSinceStartup - fTimeNow;
//Debug.Log("GetAU/SU/Points() took " + fTimeNow + " s.");
if (ComputeColorMap && VisualizeFacetracker)
{
//fTimeNow = Time.realtimeSinceStartup;
DrawFacetrackerLines(usersClrTex, avPoints);
//fTimeNow = Time.realtimeSinceStartup - fTimeNow;
//Debug.Log("DrawFacetrackerLines() took " + fTimeNow + " s.");
}
}
}
}
}