public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = 6;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
sensorData.colorImageWidth = 1920;
sensorData.colorImageHeight = 1080;
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
sensorData.depthImageWidth = 512;
sensorData.depthImageHeight = 424;
// look for face-tracking manager
faceManager = GameObject.FindObjectOfType <FacetrackingManager>();
bFaceManagerAvailable = faceManager != null;
bFaceTrackingInited = false;
Debug.Log("DummyK2-sensor opened");
return(sensorData);
}
void Update()
{
// get the face-tracking manager instance
if (faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
// get user-id by user-index
long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0;
if (kinectManager && kinectManager.IsInitialized() && userId != 0 &&
faceManager && faceManager.IsTrackingFace(userId))
{
if (faceManager.GetUserFaceProperties(userId, ref faceProps))
{
string sFaceProps = "";
foreach (string propName in faceProps.Keys)
{
string propValue = faceProps[propName];
sFaceProps += propName + "=" + propValue + "\n";
}
if (infoText && sFaceProps.Length > 0)
{
//Debug.Log("FaceProps: " + sFaceProps);
infoText.text = sFaceProps;
}
}
}
}
// checks if FacetrackingManager is initialized or not
private IEnumerator CheckFacetrackingManager()
{
// wait for 2 seconds
yield return(new WaitForSeconds(2f));
string sStatusMsg = string.Empty;
FacetrackingManager facetrackingManager = FacetrackingManager.Instance;
if (!facetrackingManager)
{
sStatusMsg = "FacetrackingManager is missing!";
}
else if (!facetrackingManager.IsFaceTrackingInitialized())
{
sStatusMsg = "FacetrackingManager not initialized! Check the log-file for details.";
}
else
{
LogToConsole("FacetrackingManager is ready.");
}
if (sStatusMsg.Length > 0)
{
LogErrorToConsole(sStatusMsg);
}
}
void Update()
{
if (facetrackingManager == null)
{
facetrackingManager = FacetrackingManager.Instance;
}
if (facetrackingManager && facetrackingManager.IsTrackingFace())
{
// set head position & rotation
if (head != null)
{
//Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition();
//HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, 3 * Time.deltaTime);
//Quaternion newRotation = HeadInitialRotation * facetrackingManager.GetHeadRotation();
//head.localRotation = Quaternion.Slerp(head.localRotation, newRotation, 3 * Time.deltaTime);
//head.localRotation = HeadInitialRotation * facetrackingManager.GetHeadRotation();
Vector3 targetOrientation = Vector3.zero;
targetOrientation.x = facetrackingManager.GetHeadRotation().eulerAngles.x;
targetOrientation.y = facetrackingManager.GetHeadRotation().eulerAngles.y;
head.localRotation = Quaternion.Euler(targetOrientation);
}
}
}
void StartFacetracker()
{
try
{
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please, wait...";
}
// initialize Kinect sensor as needed
int rc = FacetrackingWrapper.InitKinectSensor((int)FacetrackingWrapper.Constants.ColorImageResolution, (int)FacetrackingWrapper.Constants.DepthImageResolution, FacetrackingWrapper.Constants.IsNearMode);
if (rc != 0)
{
throw new Exception("Initialization of Kinect sensor failed");
}
// Initialize the kinect speech wrapper
rc = FacetrackingWrapper.InitFaceTracking();
if (rc < 0)
{
throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc));
}
if (ComputeColorMap)
{
// Initialize color map related stuff
usersClrTex = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight(), TextureFormat.ARGB32, false);
usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height);
colorImage = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()];
videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4];
}
instance = this;
facetrackingInitialized = true;
DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
private Quaternion RotateBone(OverlayObject oo, KinectManager kinectManager, FacetrackingManager facetrackingManager, long key)
{
int iJointIndex = (int)oo.TrackedJointType;
Quaternion qRotObject = Quaternion.identity;
if (iJointIndex == 3) // if it's the head
{
if (facetrackingManager) // && facetrackingManager.IsFaceTrackingInitialized())
{
qRotObject *= facetrackingManager.GetHeadRotation(key, bMirroredMovement: !RearProjection);
if (oo.BoneGameObject.FindChild("Jaw") != null)
{
Transform Jaw;
if (facetrackingManager.bGotAU && oo.BoneGameObject.FindChild("Jaw") != null)
{
Jaw = oo.BoneGameObject.FindChild("Jaw");
// print("Jaw angle : " + facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER);
Jaw.localRotation = Quaternion.AngleAxis(facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER, Vector3.right);
}
}
}
}
else // all joints not the head
{
qRotObject = kinectManager.GetJointOrientation(key, iJointIndex, flip: RearProjection);
Vector3 rotAngles = qRotObject.eulerAngles - oo.InitialRotation.eulerAngles;
qRotObject = Quaternion.Euler(rotAngles);
}
return(Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime));
}
public static float GetNeckAngle(FacetrackingManager facetrackingManager, JointName joint)
{
float angle = 0;
if (facetrackingManager == null)
{
facetrackingManager = FacetrackingManager.Instance;
}
if (facetrackingManager && facetrackingManager.IsTrackingFace())
{
switch (joint)
{
case JointName.pan:
angle = facetrackingManager.GetHeadRotation().eulerAngles.y;
if (angle > 180f)
{
angle = angle - 360f; //180도를 넘을리는 없지만 각의 범위를 -180~180로 쓰겠다는 의미
}
break;
case JointName.tilt:
angle = facetrackingManager.GetHeadRotation().eulerAngles.x;
if (angle > 180f)
{
angle = angle - 360f;
}
break;
default: break;
}
}
return(angle);
}
void Update()
{
// get the face-tracking manager instance
if(faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if(faceManager && faceManager.IsTrackingFace())
{
// head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(true);
if(smoothFactor != 0f)
transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
else
transform.rotation = newRotation;
// head position
Vector3 newPosition = faceManager.GetHeadPosition(true);
if(verticalOffset != 0f)
{
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = transform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// if(smoothFactor != 0f)
// transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
// else
transform.position = newPosition;
}
}
void Update()
{
if (!manager)
{
manager = KinectManager.Instance;
}
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
// get the face points
if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
long userId = manager.GetUserIdByIndex(playerIndex);
if (userId == 0)
{
return;
}
if (faceVertices == null)
{
int iVertCount = faceManager.GetUserFaceVertexCount(userId);
if (iVertCount > 0)
{
faceVertices = new Vector3[iVertCount];
}
}
if (faceVertices != null)
{
if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
{
if (faceVertices != null && faceVertices[(int)facePoint] != Vector3.zero)
{
Vector3 facePointPos = faceVertices [(int)facePoint];
if (foregroundCamera)
{
facePointPos = GetOverlayPosition(facePointPos);
}
if (facePointTransform)
{
facePointTransform.position = facePointPos;
}
if (faceInfoText)
{
string sStatus = string.Format("{0}: {1}", facePoint, facePointPos);
faceInfoText.text = sStatus;
}
}
}
}
}
}
void Update()
{
// get the face-tracking manager instance
if (faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
// get user-id by user-index
long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0;
if (kinectManager && kinectManager.IsInitialized() && userId != 0 &&
faceManager && faceManager.IsTrackingFace(userId))
{
if (faceManager.GetUserFaceProperties(userId, ref faceProps))
{
//NuitrackInterface.FaceLandmarks faceLM = new NuitrackInterface.FaceLandmarks();
//if(faceProps.ContainsKey("landmarks"))
//{
// faceLM = JsonUtility.FromJson<NuitrackInterface.FaceLandmarks>(faceProps["landmarks"]);
// // do something with the face landmarks
//}
//Vector2 leftEyePos = Vector2.zero;
//if (faceProps.ContainsKey("lefteye"))
//{
// leftEyePos = JsonUtility.FromJson<Vector2>(faceProps["lefteye"]);
// // do something with the left-eye position
//}
//Vector2 rightEyePos = Vector2.zero;
//if (faceProps.ContainsKey("righteye"))
//{
// rightEyePos = JsonUtility.FromJson<Vector2>(faceProps["righteye"]);
// // do something with the right-eye position
//}
string sFaceProps = "";
foreach (string propName in faceProps.Keys)
{
string propValue = faceProps[propName];
if (propValue != null && propValue.Length > maxPropLength)
{
propValue = propValue.Substring(0, maxPropLength) + "...";
}
sFaceProps += propName + "=" + propValue + "\n";
}
if (infoText && sFaceProps.Length > 0)
{
//Debug.Log("FaceProps: " + sFaceProps);
infoText.text = sFaceProps;
}
}
}
}
void Update ()
{
// get the face-tracking manager instance
if(faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
if (kinectManager && faceManager && faceManager.IsTrackingFace ()) {
// get user-id by user-index
long userId = kinectManager.GetUserIdByIndex (playerIndex);
if (userId == 0)
return;
// head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation (userId, true);
if (smoothFactor != 0f)
transform.rotation = Quaternion.Slerp (transform.rotation, newRotation, smoothFactor * Time.deltaTime);
else
transform.rotation = newRotation;
// head position
Vector3 newPosition = faceManager.GetHeadPosition (userId, true);
//Debug.Log(newPosition.ToString());
if (verticalOffset != 0f) {
Vector3 dirHead = new Vector3 (0, verticalOffset, 0);
dirHead = transform.InverseTransformDirection (dirHead);
newPosition += dirHead;
}
// if(smoothFactor != 0f)
// transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
// else
//<----------------------------------------------correct modal position --------------------------------------------->
//Debug.Log ("new Positionx " + newPosition.x.ToString());
if(newPosition.x > 0f)
newPosition.x = newPosition.x * 0.8f - deltaX;
else if(newPosition.x < 0f)
newPosition.x = newPosition.x * 0.8f + deltaX;
//<------------------------------------------------------------------------------------------------------------------>
transform.position = newPosition;
//Debug.Log ("transform.position " + transform.position.x.ToString());
//Debug.Log(newPosition.ToString());
} else if (kinectManager && faceManager && !faceManager.IsTrackingFace ()) {
transform.position = -Vector3.one;
}
}
// Make sure to kill the Kinect on quitting.
void OnApplicationQuit()
{
// Shutdown Speech Recognizer and Kinect
FacetrackingWrapper.FinishFaceTracking();
FacetrackingWrapper.ShutdownKinectSensor();
facetrackingInitialized = false;
instance = null;
}
// Update is called once per frame
void Update()
{
Matrix4x4 kinectToWorld = Matrix4x4.zero;
//Quaternion quatTiltAngle = Quaternion.Euler(-manager.sensorAngle, 0.0f, 0.0f);
//kinectToWorld.SetTRS(new Vector3(0.0f, manager.sensorHeight, 0.0f), quatTiltAngle, Vector3.one);
if (!manager)
{
manager = KinectManager.Instance;
}
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
// get the face points
if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
long userId = manager.GetUserIdByIndex(playerIndex);
if (faceVertices == null)
{
//int iVertCount = faceManager.GetUserFaceVertexCount(userId);
int iVertCount = 1347;
if (iVertCount > 0)
{
faceVertices = new Vector3[iVertCount];
}
check_f = false;
}
if (faceVertices != null)
{
if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
{
//-------------- get 35 special point-------------------
//Matrix4x4 kinectToWorld = manager.GetKinectToWorldMatrix();
HighDetailFacePoints[] facePoints = (HighDetailFacePoints[])System.Enum.GetValues(typeof(HighDetailFacePoints));
for (int i = 0; i < facePoints.Length; i++)
{
HighDetailFacePoints point = facePoints[i];
//dictFacePoints[point] = kinectToWorld.MultiplyPoint3x4(faceVertices[(int)point]);
dictFacePoints[point] = faceVertices[(int)point];
}
check_f = true;
}
}
}
}
// Update is called once per frame
void FixedUpdate()
{
Debug.Log("inside fixedupdate of engagement meter");
/*
* updateCounter++;
* float timeRequired=30f;
* float iterationsRequired=timeRequired*50f;
*
* combinedPlayerEngagement/=50f;
* leftPlayerEngagement/=50f;
* rightPlayerEngagement/=50f;
*
* //Debug.Log("update counter"+updateCounter);
* if(updateCounter==iterationsRequired)
* {
*
* //Debug.Log("twoengagement "+combinedPlayerEngagement);
* //Debug.Log("leftPlayerEngagement "+leftPlayerEngagement);
* //Debug.Log("rightPlayerEngagement "+rightPlayerEngagement);
* sendToCsv();
* pauseGame();
* }
*
*
* // delay -= Time.deltaTime;
* // if(delay < 0){
* // //Debug.Log(faceManagerP1.GetHeadRotation(false));
* //
* // delay = 1f;
* // }
*
* //sendToCsv();
*
* //Debug.Log(_KinectManager.GetJointPosition(_KinectManager.GetUserIdByIndex(0),4));
*/
//lockArrow();
//leadArrow();
fixedUpdateCounter++;
Debug.Log("in fixed update");
if (faceManager == null)
{
//Debug.Log("In FixedUpdate, faceManager was null");
faceManager = FacetrackingManager.Instance;
}
if (fixedUpdateCounter == 24)
{
measureEngagement();
fixedUpdateCounter = 0;
}
//measureEngagement();
//Resources.UnloadUnusedAssets();
}
// Use this for initialization
void Start()
{
faceManager = FacetrackingManager.Instance;
//Debug.Log("in Start: facemanager instance"+ faceManager);
leftPlayerEngagement = 0;
rightPlayerEngagement = 0;
combinedPlayerEngagement = 0;
leftPlayerEngagementsecondHalf = 0;
rightPlayerEngagementsecondHalf = 0;
combinedPlayerEngagementsecondHalf = 0;
// updateCounter=0;
}
private Vector3 TransformBones(KinectManager kinectManager, FacetrackingManager facetrackingManager)
{
Vector3 jp = Vector3.zero;
for (int i = 0; i < userIDs.Count; i++) // do this for every USer ID in
{
List <OverlayObject> bonesList;
long userID = userIDs[i];
if (bagOfBonesDictionary.ContainsKey(userID))
{
bonesList = bagOfBonesDictionary[userID];
foreach (OverlayObject oo in bonesList)
{
int iJointIndex = (int)oo.TrackedJointType;
if (kinectManager.IsJointTracked(userID, iJointIndex))
{
var TryPos = TranslateBone(oo, userID, kinectManager);
if (TryPos != Vector3.zero)
{
oo.BoneGameObject.position = TryPos;
}
oo.BoneGameObject.rotation = RotateBone(oo, kinectManager, facetrackingManager, userID); //Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime);
if (oo.ReferenceGameObjectTransform != null)
{
oo.BoneGameObject.localScale = ScaleBone(kinectManager, userID, oo, iJointIndex);
}
else
{
oo.BoneGameObject.localScale = new Vector3(1, 1, 1);
}
}
else // joint not tracked
{
oo.BoneGameObject.position = oo.BoneGameObject.position;
oo.BoneGameObject.localRotation = oo.BoneGameObject.localRotation;
oo.BoneGameObject.localScale = oo.BoneGameObject.localScale;
}
}
}
else
{
print("Bag of Bones is Missing userID");
}
}
return(jp);
}
void Update()
{ // float translation = Time.deltaTime * 10;
// get the face-tracking manager instance
if (manager == null)
{
manager = FacetrackingManager.Instance;
}
if (manager && manager.IsTrackingFace())
{
// set head position & rotation
// FaceFrameReference faceRef = e.FrameReference;
}
}
void OnDestroy()
{
if (isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null)
{
// finish face tracking
sensorData.sensorInterface.FinishFaceTracking();
}
// // clean up
// Resources.UnloadUnusedAssets();
// GC.Collect();
isFacetrackingInitialized = false;
instance = null;
}
void Update ()
{
if(faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if(!kinectManager || !kinectManager.IsInitialized())
return;
if(!faceManager || !faceManager.IsFaceTrackingInitialized())
return;
long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
if(userId == 0)
return;
colorTex = kinectManager.GetUsersClrTex();
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if(!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
return;
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if(faceX < 0) faceX = 0;
if(faceY < 0) faceY = 0;
if((faceX + faceW) > colorTex.width) faceW = colorTex.width - faceX;
if((faceY + faceH) > colorTex.height) faceH = colorTex.height - faceY;
if(faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
if(targetRenderer && targetRenderer.material)
{
targetRenderer.material.mainTexture = faceTex;
}
}
void Start()
{
//pec = new GameObject[arraySize];
//Associate the variables with Components in the MainCamera
grabScript = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <GrabDropScript>();
interactionManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponents <InteractionManager>();
result = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <Timer>();
scriptLog = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <log>();
engagementMeter = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <EngagementMeter>();
faceTrackingManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <FacetrackingManager>();
scoreKeepBody = GameObject.Find("BodyPlaceHolders").GetComponent <ScoreKeep>();
source = GetComponent <AudioSource>(); //Gets AudioSource from the attached Object
}
void Update()
{
// get the face-tracking manager instance
if (faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
if (kinectManager && faceManager && faceManager.IsTrackingFace())
{
// get user-id by user-index
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId == 0)
{
return;
}
// head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);
if (smoothFactor != 0f)
{
transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
}
else
{
transform.rotation = newRotation;
}
// head position
Vector3 newPosition = faceManager.GetHeadPosition(userId, true);
if (verticalOffset != 0f)
{
Vector3 dirHead = new Vector3(horizontalOffset, verticalOffset, zOffset);
dirHead = transform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// if(smoothFactor != 0f)
// transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
// else
transform.position = newPosition;
}
}
void Update()
{
// reference to face manager
if (!faceManager)
{
faceManager = FacetrackingManager.Instance;
}
if (kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
{
// check for tracked user
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId != 0 && kinectManager.IsUserTracked(userId))
{
if (faceManager.GetUserAnimUnits(userId, ref dictAnimUnits))
{
// process the blend shapes -> anim units
for (int i = 0; i < blendShapeCount; i++)
{
if (blendShape2AnimUnit.ContainsKey(blendShapeNames[i]))
{
KinectInterop.FaceShapeAnimations faceAnim = blendShape2AnimUnit[blendShapeNames[i]];
float animValue = dictAnimUnits[faceAnim];
// check for multiplier
float mul = 1f;
if (blendShape2Multiplier.ContainsKey(blendShapeNames[i]))
{
mul = blendShape2Multiplier[blendShapeNames[i]];
}
if (animValue * mul < 0f)
{
animValue = 0f;
}
// lerp to the new value
blendShapeValues[i] = Mathf.Lerp(blendShapeValues [i], animValue * mul * 100f, smoothFactor * Time.deltaTime);
skinnedMeshRenderer.SetBlendShapeWeight(i, blendShapeValues[i]);
}
}
}
}
}
}
// enables or disables the face-tracking component
public void EnableFaceTracking(bool bEnable)
{
FacetrackingManager facetrackingManager = gameObject.GetComponent <FacetrackingManager>();
if (facetrackingManager)
{
facetrackingManager.enabled = bEnable;
LogToConsole("Face tracking is " + (bEnable ? "enabled" : "disabled"));
if (bEnable)
{
StartCoroutine(CheckFacetrackingManager());
}
}
else
{
LogErrorToConsole("FacetrackingManager-component not found.");
}
}
void Update()
{
// get the face-tracking manager instance
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (faceManager && faceManager.IsTrackingFace())
{
// head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(true);
if (smoothFactor != 0f)
{
transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
}
else
{
transform.rotation = newRotation;
}
// head position
Vector3 newPosition = faceManager.GetHeadPosition(true);
if (verticalOffset != 0f)
{
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = transform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// if(smoothFactor != 0f)
// transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
// else
transform.position = newPosition;
}
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
//unsafe for editor but can optimise if commented
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
if (userId == 0)
{
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
if (!colorTex)
{
colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
}
try
{
RenderTexture foregroundTex = (RenderTexture)backManager.GetForegroundTex();
KinectInterop.RenderTex2Tex2D(foregroundTex, ref colorTex);
}
catch (System.Exception)
{
colorTex = (Texture2D)backManager.GetForegroundTex();
}
}
else
{
// use color camera image
colorTex = kinectManager.GetUsersClrTex();
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
{
return;
}
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
if (targetRenderer && targetRenderer.material)
{
targetRenderer.material.mainTexture = faceTex;
}
}
void Awake()
{
manager = Camera.main.GetComponent<FacetrackingManager>();
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if(kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if(sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if(debugText != null)
{
debugText.GetComponent<GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if(sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if(bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
instance = this;
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if(debugText != null)
{
debugText.GetComponent<GUIText>().text = "Ready.";
}
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
void OnDestroy()
{
if(isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null)
{
// finish face tracking
sensorData.sensorInterface.FinishFaceTracking();
}
// // clean up
// Resources.UnloadUnusedAssets();
// GC.Collect();
isFacetrackingInitialized = false;
instance = null;
}
void StartFacetracker()
{
try
{
if(debugText != null)
debugText.guiText.text = "Please, wait...";
// initialize Kinect sensor as needed
int rc = FacetrackingWrapper.InitKinectSensor();
if(rc != 0)
{
throw new Exception("Initialization of Kinect sensor failed");
}
// Initialize the kinect speech wrapper
rc = FacetrackingWrapper.InitFaceTracking();
if (rc < 0)
{
throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc));
}
if(ComputeColorMap)
{
// Initialize color map related stuff
usersClrTex = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight());
usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height);
colorImage = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()];
videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4];
}
instance = this;
facetrackingInitialized = true;
DontDestroyOnLoad(gameObject);
if(debugText != null)
debugText.guiText.text = "Ready.";
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.guiText.text = "Please check the Kinect and FT-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.guiText.text = ex.Message;
}
}
void Update()
{
// get the face-tracking manager instance
if(faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
if(kinectManager && kinectManager.IsInitialized() &&
faceManager && faceManager.IsTrackingFace() && foregroundCamera)
{
// get user-id by user-index
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if(userId == 0)
return;
// get head position
Vector3 newPosition = faceManager.GetHeadPosition(userId, true);
// get head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);
// rotational fix, provided by Richard Borys:
// The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;
addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;
newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
// end of rotational fix
if(smoothFactorRotation != 0f)
transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactorRotation * Time.deltaTime);
else
transform.rotation = newRotation;
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if(portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
// model position
newPosition = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
if(newPosition == Vector3.zero)
{
// hide the model behind the camera
newPosition.z = -10f;
}
if(verticalOffset != 0f)
{
// add the vertical offset
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = transform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// go to the new position
if(smoothFactorMovement != 0f && transform.position.z >= 0f)
transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactorMovement * Time.deltaTime);
else
transform.position = newPosition;
// scale the model if needed
if(transform.localScale.x != modelScaleFactor)
{
transform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
}
}
else
{
// hide the model behind the camera
if(transform.position.z >= 0f)
{
transform.position = new Vector3(0f, 0f, -10f);
}
}
}
void Update()
{
// get the face-tracking manager instance
if(manager == null)
{
manager = FacetrackingManager.Instance;
}
if(manager && manager.IsTrackingFace())
{
// set head position & rotation
if(HeadTransform != null)
{
// head position
Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement);
if(smoothFactor != 0f)
HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime);
else
HeadTransform.localPosition = newPosition;
// head rotation
Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);
if(smoothFactor != 0f)
HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime);
else
HeadTransform.localRotation = newRotation;
}
// apply animation units
// AU0 - Upper Lip Raiser
// 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);
// AU1 - Jaw Lowerer
// 0=closed; 1=fully open; -1= closed, like 0
float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);
// AU2 – Lip Stretcher
// 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);
float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);
// AU3 – Brow Lowerer
// 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);
float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);
// AU4 – Lip Corner Depressor
// 0=neutral; -1=very happy smile; +1=very sad frown
float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);
float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);
// AU6, AU7 – Eyelid closed
// 0=neutral; -1=raised; +1=fully lowered
float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);
float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised);
}
}
void Update()
{
// get the face-tracking manager instance
if(manager == null)
{
manager = FacetrackingManager.Instance;
}
if(manager && manager.IsTrackingFace())
{
// set head position & rotation
if(HeadTransform != null)
{
// head position
Vector3 newPosition = manager.GetHeadPosition(mirroredHeadMovement);
// head rotation
Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);
// rotational fix, provided by Richard Borys:
// The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;
addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;
newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
// end of rotational fix
if(smoothFactor != 0f)
HeadTransform.rotation = Quaternion.Slerp(HeadTransform.rotation, newRotation, smoothFactor * Time.deltaTime);
else
HeadTransform.rotation = newRotation;
// check for head pos overlay
if(foregroundCamera)
{
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if(portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
KinectManager kinectManager = KinectManager.Instance;
if(kinectManager)
{
long userId = kinectManager.GetUserIdByIndex(manager.playerIndex);
Vector3 posColorOverlay = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
if(posColorOverlay != Vector3.zero)
{
newPosition = posColorOverlay;
// if(overlayObj)
// {
// overlayObj.position = newPosition;
// }
}
}
}
else
{
// move around the initial position
newPosition += HeadInitialPosition;
}
// vertical offet
if(verticalOffset != 0f)
{
// add the vertical offset
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = HeadTransform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// set the position
if(smoothFactor != 0f)
HeadTransform.position = Vector3.Lerp(HeadTransform.position, newPosition, smoothFactor * Time.deltaTime);
else
HeadTransform.position = newPosition;
// scale factor
if(HeadTransform.localScale.x != modelScaleFactor)
{
HeadTransform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
}
}
// apply animation units
// AU0 - Upper Lip Raiser
// 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);
// AU1 - Jaw Lowerer
// 0=closed; 1=fully open; -1= closed, like 0
float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);
// AU2 – Lip Stretcher
// 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);
float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);
// AU3 – Brow Lowerer
// 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);
float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);
// AU4 – Lip Corner Depressor
// 0=neutral; -1=very happy smile; +1=very sad frown
float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);
float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);
// AU6, AU7 – Eyelid closed
// 0=neutral; -1=raised; +1=fully lowered
float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);
float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised);
}
else
{
// hide the model behind the camera
if(HeadTransform && HeadTransform.position.z >= 0f)
{
HeadTransform.position = new Vector3(0f, 0f, -10f);
}
}
}
void Awake()
{
manager = Camera.main.GetComponent <FacetrackingManager>();
}
// Make sure to kill the Kinect on quitting.
void OnApplicationQuit()
{
// Shutdown Speech Recognizer and Kinect
FacetrackingWrapper.FinishFaceTracking();
FacetrackingWrapper.ShutdownKinectSensor();
facetrackingInitialized = false;
instance = null;
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(playerIndex);
if (userId == 0)
{
if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
{
targetObject.material.mainTexture = null;
}
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (!bSkipUpdatingForegroundTex && backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
Texture bmForeTex = backManager.GetForegroundTex();
if (bmForeTex is RenderTexture)
{
foregroundTex = (RenderTexture)bmForeTex;
bSkipUpdatingForegroundTex = (foregroundTex != null);
}
else
{
if (foregroundTex == null)
{
foregroundTex = new RenderTexture(bmForeTex.width, bmForeTex.height, 0);
}
Graphics.Blit(bmForeTex, foregroundTex);
bSkipUpdatingForegroundTex = false;
}
// if (!colorTex)
// {
// colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
// }
//
// try
// {
// foregroundTex = (RenderTexture)backManager.GetForegroundTex ();
// //KinectInterop.RenderTex2Tex2D (foregroundTex, ref colorTex);
// }
// catch (System.Exception)
// {
// colorTex = (Texture2D)backManager.GetForegroundTex ();
// }
}
else
{
// use color camera image
if (!colorTex)
{
colorTex = kinectManager.GetUsersClrTex2D();
}
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (faceRect.width > 0 && faceRect.height > 0)
{
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if (foregroundTex)
{
if ((faceX + faceW) > foregroundTex.width)
{
faceW = foregroundTex.width - faceX;
}
if ((faceY + faceH) > foregroundTex.height)
{
faceH = foregroundTex.height - faceY;
}
}
else if (colorTex)
{
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
if (foregroundTex)
{
KinectInterop.RenderTex2Tex2D(foregroundTex, faceX, foregroundTex.height - faceY - faceH, faceW, faceH, ref faceTex);
}
else if (colorTex)
{
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
}
if (targetObject && !targetObject.gameObject.activeSelf)
{
targetObject.gameObject.SetActive(true);
}
if (targetObject && targetObject.material)
{
targetObject.material.mainTexture = faceTex;
}
// don't rotate the transform - mesh follows the head rotation
if (targetObject.transform.rotation != Quaternion.identity)
{
targetObject.transform.rotation = Quaternion.identity;
}
}
else
{
if (targetObject && targetObject.gameObject.activeSelf)
{
targetObject.gameObject.SetActive(false);
}
if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
{
targetObject.material.mainTexture = null;
}
}
}
void Update()
{
int recHostId;
int connectionId;
int recChannelId;
int dataSize;
bool connListUpdated = false;
if (backgroundImage && backgroundImage.texture == null)
{
backgroundImage.texture = manager ? manager.GetUsersLblTex() : null;
}
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (gestureManager == null)
{
gestureManager = VisualGestureManager.Instance;
}
if (speechManager == null)
{
speechManager = SpeechManager.Instance;
}
try
{
byte error = 0;
NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out recChannelId, recBuffer, bufferSize, out dataSize, out error);
switch (recData)
{
case NetworkEventType.Nothing: //1
break;
case NetworkEventType.ConnectEvent: //2
if (recHostId == serverHostId && recChannelId == serverChannelId &&
!dictConnection.ContainsKey(connectionId))
{
HostConnection conn = new HostConnection();
conn.hostId = recHostId;
conn.connectionId = connectionId;
conn.channelId = recChannelId;
conn.keepAlive = true;
conn.reqDataType = "ka,kb,km,kh";
//conn.matrixSent = false;
dictConnection[connectionId] = conn;
connListUpdated = true;
//LogToConsole(connectionId + "-conn: " + conn.reqDataType);
}
// // reset chunked face messages
// sendFvMsg = string.Empty;
// sendFvNextOfs = 0;
//
// sendFtMsg = string.Empty;
// sendFtNextOfs = 0;
break;
case NetworkEventType.DataEvent: //3
if (recHostId == serverHostId && recChannelId == serverChannelId &&
dictConnection.ContainsKey(connectionId))
{
HostConnection conn = dictConnection[connectionId];
int decompSize = 0;
if (decompressor != null && (recBuffer[0] > 127 || recBuffer[0] < 32))
{
decompSize = decompressor.Decompress(recBuffer, 0, compressBuffer, 0, dataSize);
}
else
{
System.Buffer.BlockCopy(recBuffer, 0, compressBuffer, 0, dataSize);
decompSize = dataSize;
}
string sRecvMessage = System.Text.Encoding.UTF8.GetString(compressBuffer, 0, decompSize);
if (sRecvMessage.StartsWith("ka"))
{
if (sRecvMessage == "ka") // vr-examples v1.0 keep-alive message
{
sRecvMessage = "ka,kb,km,kh";
}
conn.keepAlive = true;
conn.reqDataType = sRecvMessage;
dictConnection[connectionId] = conn;
//LogToConsole(connectionId + "-recv: " + conn.reqDataType);
// check for SR phrase-reset
int iIndexSR = sRecvMessage.IndexOf(",sr");
if (iIndexSR >= 0 && speechManager)
{
speechManager.ClearPhraseRecognized();
//LogToConsole("phrase cleared");
}
}
}
break;
case NetworkEventType.DisconnectEvent: //4
if (dictConnection.ContainsKey(connectionId))
{
dictConnection.Remove(connectionId);
connListUpdated = true;
}
break;
}
if (connListUpdated)
{
// get all connection IDs
alConnectionId.Clear();
alConnectionId.AddRange(dictConnection.Keys);
// display the number of connections
StringBuilder sbConnStatus = new StringBuilder();
sbConnStatus.AppendFormat("Server running: {0} connection(s)", dictConnection.Count);
foreach (int connId in dictConnection.Keys)
{
HostConnection conn = dictConnection[connId];
int iPort = 0; string sAddress = string.Empty; NetworkID network; NodeID destNode;
NetworkTransport.GetConnectionInfo(conn.hostId, conn.connectionId, out sAddress, out iPort, out network, out destNode, out error);
if (error == (int)NetworkError.Ok)
{
sbConnStatus.AppendLine().Append(" ").Append(sAddress).Append(":").Append(iPort);
}
}
LogToConsole(sbConnStatus.ToString());
if (connStatusText)
{
connStatusText.text = sbConnStatus.ToString();
}
}
// send body frame to available connections
const char delimiter = ',';
string sBodyFrame = manager ? manager.GetBodyFrameData(ref liRelTime, ref fCurrentTime, delimiter) : string.Empty;
if (sBodyFrame.Length > 0 && dictConnection.Count > 0)
{
StringBuilder sbSendMessage = new StringBuilder();
bool bFaceParamsRequested = IsFaceParamsRequested();
sbSendMessage.Append(manager.GetWorldMatrixData(delimiter)).Append('|');
sbSendMessage.Append(sBodyFrame).Append('|');
sbSendMessage.Append(manager.GetBodyHandData(ref liRelTime, delimiter)).Append('|');
if (bFaceParamsRequested && faceManager && faceManager.IsFaceTrackingInitialized())
{
sbSendMessage.Append(faceManager.GetFaceParamsAsCsv(delimiter)).Append('|');
}
if (gestureManager && gestureManager.IsVisualGestureInitialized())
{
sbSendMessage.Append(gestureManager.GetGestureDataAsCsv(delimiter)).Append('|');
}
if (speechManager && speechManager.IsSapiInitialized())
{
sbSendMessage.Append(speechManager.GetSpeechDataAsCsv(delimiter)).Append('|');
}
if (sbSendMessage.Length > 0 && sbSendMessage[sbSendMessage.Length - 1] == '|')
{
sbSendMessage.Remove(sbSendMessage.Length - 1, 1);
}
byte[] btSendMessage = System.Text.Encoding.UTF8.GetBytes(sbSendMessage.ToString());
//Debug.Log("Message " + sbSendMessage.Length + " chars: " + sbSendMessage.ToString());
//Debug.Log("Encoded into " + btSendMessage.Length + " bytes: " + ByteArrayToString(btSendMessage, btSendMessage.Length));
int compSize = 0;
if (compressor != null && btSendMessage.Length > 100 && !websocketHost)
{
compSize = compressor.Compress(btSendMessage, 0, btSendMessage.Length, compressBuffer, 0);
}
else
{
System.Buffer.BlockCopy(btSendMessage, 0, compressBuffer, 0, btSendMessage.Length);
compSize = btSendMessage.Length;
}
//Debug.Log("Compressed into " + compSize + " bytes: " + ByteArrayToString(compressBuffer, compSize));
// // check face-tracking requests
// bool bFaceParams = false, bFaceVertices = false, bFaceUvs = false, bFaceTriangles = false;
// if(faceManager && faceManager.IsFaceTrackingInitialized())
// CheckFacetrackRequests(out bFaceParams, out bFaceVertices, out bFaceUvs, out bFaceTriangles);
//
// byte[] btFaceParams = null;
// if(bFaceParams)
// {
// string sFaceParams = faceManager.GetFaceParamsAsCsv();
// if(!string.IsNullOrEmpty(sFaceParams))
// btFaceParams = System.Text.Encoding.UTF8.GetBytes(sFaceParams);
// }
//
// // next chunk of data for face vertices
// byte[] btFaceVertices = null;
// string sFvMsgHead = string.Empty;
// GetNextFaceVertsChunk(bFaceVertices, bFaceUvs, ref btFaceVertices, out sFvMsgHead);
//
// // next chunk of data for face triangles
// byte[] btFaceTriangles = null;
// string sFtMsgHead = string.Empty;
// GetNextFaceTrisChunk(bFaceTriangles, ref btFaceTriangles, out sFtMsgHead);
foreach (int connId in alConnectionId)
{
HostConnection conn = dictConnection[connId];
if (conn.keepAlive)
{
conn.keepAlive = false;
dictConnection[connId] = conn;
if (conn.reqDataType != null && conn.reqDataType.Contains("kb,"))
{
//LogToConsole(conn.connectionId + "-sendkb: " + conn.reqDataType);
error = 0;
//if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btSendMessage, btSendMessage.Length, out error))
if (!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, compressBuffer, compSize, out error))
{
string sMessage = "Error sending body data via conn " + conn.connectionId + ": " + (NetworkError)error;
LogErrorToConsole(sMessage);
if (serverStatusText)
{
serverStatusText.text = sMessage;
}
}
}
// if(bFaceParams && btFaceParams != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("fp,"))
// {
// //Debug.Log(conn.connectionId + "-sendfp: " + conn.reqDataType);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceParams, btFaceParams.Length, out error))
// {
// string sMessage = "Error sending face params via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
//
// if(bFaceVertices && btFaceVertices != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("fv,"))
// {
// //Debug.Log(conn.connectionId + "-sendfv: " + conn.reqDataType + " - " + sFvMsgHead);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceVertices, btFaceVertices.Length, out error))
// {
// string sMessage = "Error sending face verts via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
//
// if(bFaceTriangles && btFaceTriangles != null &&
// conn.reqDataType != null && conn.reqDataType.Contains("ft,"))
// {
// //Debug.Log(conn.connectionId + "-sendft: " + conn.reqDataType + " - " + sFtMsgHead);
//
// error = 0;
// if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceTriangles, btFaceTriangles.Length, out error))
// {
// string sMessage = "Error sending face tris via conn " + conn.connectionId + ": " + (NetworkError)error;
// Debug.LogError(sMessage);
//
// if(serverStatusText)
// {
// serverStatusText.text = sMessage;
// }
// }
// }
}
}
}
}
catch (System.Exception ex)
{
LogErrorToConsole(ex.Message + "\n" + ex.StackTrace);
if (serverStatusText)
{
serverStatusText.text = ex.Message;
}
}
}
void Update()
{
// get the face-tracking manager instance
if (faceManager == null)
{
kinectManager = KinectManager.Instance;
faceManager = FacetrackingManager.Instance;
}
// get user-id by user-index
long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0;
if (kinectManager && kinectManager.IsInitialized() && userId != 0 &&
faceManager && faceManager.IsTrackingFace(userId) && foregroundCamera)
{
// get head position
Vector3 newPosition = faceManager.GetHeadPosition(userId, true);
// get head rotation
Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);
// rotational fix, provided by Richard Borys:
// The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;
addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;
newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
// end of rotational fix
if (smoothFactorRotation != 0f)
{
transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactorRotation * Time.deltaTime);
}
else
{
transform.rotation = newRotation;
}
// get the background rectangle (use the portrait background, if available)
Rect backgroundRect = foregroundCamera.pixelRect;
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.enabled)
{
backgroundRect = portraitBack.GetBackgroundRect();
}
// model position
newPosition = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
if (newPosition == Vector3.zero)
{
// hide the model behind the camera
newPosition.z = -10f;
}
if (verticalOffset != 0f)
{
// add the vertical offset
Vector3 dirHead = new Vector3(0, verticalOffset, 0);
dirHead = transform.InverseTransformDirection(dirHead);
newPosition += dirHead;
}
// go to the new position
if (smoothFactorMovement != 0f && transform.position.z >= 0f)
{
transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactorMovement * Time.deltaTime);
}
else
{
transform.position = newPosition;
}
// scale the model if needed
if (transform.localScale.x != modelScaleFactor)
{
transform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
}
}
else
{
// hide the model behind the camera
if (transform.position.z >= 0f)
{
transform.position = new Vector3(0f, 0f, -10f);
}
}
}
//----------------------------------- end of public functions --------------------------------------//
void Awake()
{
instance = this;
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
instance = this;
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = ex.Message;
}
}
}
void Update()
{
// get the face-tracking manager instance
if (manager == null)
{
manager = FacetrackingManager.Instance;
}
if (manager && manager.IsTrackingFace())
{
// set head position & rotation
if (HeadTransform != null)
{
// head position
Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement);
if (smoothFactor != 0f)
{
HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime);
}
else
{
HeadTransform.localPosition = newPosition;
}
// head rotation
Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);
if (smoothFactor != 0f)
{
HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime);
}
else
{
HeadTransform.localRotation = newRotation;
}
}
// apply animation units
// AU0 - Upper Lip Raiser
// 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);
// AU1 - Jaw Lowerer
// 0=closed; 1=fully open; -1= closed, like 0
float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);
// AU2 – Lip Stretcher
// 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);
float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);
// AU3 – Brow Lowerer
// 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);
float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);
// AU4 – Lip Corner Depressor
// 0=neutral; -1=very happy smile; +1=very sad frown
float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);
float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);
// AU6, AU7 – Eyelid closed
// 0=neutral; -1=raised; +1=fully lowered
float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);
float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, LowerEyelidRightRaised);
}
}
void OnDestroy()
{
if(isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null)
{
// finish face tracking
if(sensorData != null && sensorData.sensorInterface != null)
{
sensorData.sensorInterface.FinishFaceTracking();
}
isFacetrackingInitialized = false;
instance = null;
}
}
private void CheckFaceTrackingAvailability(long userId)
{
faceTracker = FacetrackingManager.Instance;
faceTrackingAvailable [userId] = faceTracker != null && faceTracker.IsTrackingFace (userId);
}
public bool IsFaceTrackingAvailable(long userId)
{
faceTracker = FacetrackingManager.Instance;
return (userId == firstUserId || userId == secondUserId) && faceTracker.IsTrackingFace (userId);
}
