public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); sensorData.bodyCount = 6; sensorData.jointCount = 25; sensorData.depthCameraFOV = 60f; sensorData.colorCameraFOV = 53.8f; sensorData.depthCameraOffset = 0f; sensorData.faceOverlayOffset = 0f; sensorData.colorImageWidth = 1920; sensorData.colorImageHeight = 1080; // flip color image vertically sensorData.colorImageScale = new Vector3(1f, -1f, 1f); sensorData.depthImageWidth = 512; sensorData.depthImageHeight = 424; // look for face-tracking manager faceManager = GameObject.FindObjectOfType <FacetrackingManager>(); bFaceManagerAvailable = faceManager != null; bFaceTrackingInited = false; Debug.Log("DummyK2-sensor opened"); return(sensorData); }
void Update() { // get the face-tracking manager instance if (faceManager == null) { kinectManager = KinectManager.Instance; faceManager = FacetrackingManager.Instance; } // get user-id by user-index long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0; if (kinectManager && kinectManager.IsInitialized() && userId != 0 && faceManager && faceManager.IsTrackingFace(userId)) { if (faceManager.GetUserFaceProperties(userId, ref faceProps)) { string sFaceProps = ""; foreach (string propName in faceProps.Keys) { string propValue = faceProps[propName]; sFaceProps += propName + "=" + propValue + "\n"; } if (infoText && sFaceProps.Length > 0) { //Debug.Log("FaceProps: " + sFaceProps); infoText.text = sFaceProps; } } } }
// checks if FacetrackingManager is initialized or not private IEnumerator CheckFacetrackingManager() { // wait for 2 seconds yield return(new WaitForSeconds(2f)); string sStatusMsg = string.Empty; FacetrackingManager facetrackingManager = FacetrackingManager.Instance; if (!facetrackingManager) { sStatusMsg = "FacetrackingManager is missing!"; } else if (!facetrackingManager.IsFaceTrackingInitialized()) { sStatusMsg = "FacetrackingManager not initialized! Check the log-file for details."; } else { LogToConsole("FacetrackingManager is ready."); } if (sStatusMsg.Length > 0) { LogErrorToConsole(sStatusMsg); } }
void Update() { if (facetrackingManager == null) { facetrackingManager = FacetrackingManager.Instance; } if (facetrackingManager && facetrackingManager.IsTrackingFace()) { // set head position & rotation if (head != null) { //Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(); //HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, 3 * Time.deltaTime); //Quaternion newRotation = HeadInitialRotation * facetrackingManager.GetHeadRotation(); //head.localRotation = Quaternion.Slerp(head.localRotation, newRotation, 3 * Time.deltaTime); //head.localRotation = HeadInitialRotation * facetrackingManager.GetHeadRotation(); Vector3 targetOrientation = Vector3.zero; targetOrientation.x = facetrackingManager.GetHeadRotation().eulerAngles.x; targetOrientation.y = facetrackingManager.GetHeadRotation().eulerAngles.y; head.localRotation = Quaternion.Euler(targetOrientation); } } }
void StartFacetracker() { try { if (debugText != null) { debugText.GetComponent <GUIText>().text = "Please, wait..."; } // initialize Kinect sensor as needed int rc = FacetrackingWrapper.InitKinectSensor((int)FacetrackingWrapper.Constants.ColorImageResolution, (int)FacetrackingWrapper.Constants.DepthImageResolution, FacetrackingWrapper.Constants.IsNearMode); if (rc != 0) { throw new Exception("Initialization of Kinect sensor failed"); } // Initialize the kinect speech wrapper rc = FacetrackingWrapper.InitFaceTracking(); if (rc < 0) { throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc)); } if (ComputeColorMap) { // Initialize color map related stuff usersClrTex = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight(), TextureFormat.ARGB32, false); usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height); colorImage = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()]; videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4]; } instance = this; facetrackingInitialized = true; DontDestroyOnLoad(gameObject); if (debugText != null) { debugText.GetComponent <GUIText>().text = "Ready."; } } catch (DllNotFoundException ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations."; } } catch (Exception ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.GetComponent <GUIText>().text = ex.Message; } } }
private Quaternion RotateBone(OverlayObject oo, KinectManager kinectManager, FacetrackingManager facetrackingManager, long key) { int iJointIndex = (int)oo.TrackedJointType; Quaternion qRotObject = Quaternion.identity; if (iJointIndex == 3) // if it's the head { if (facetrackingManager) // && facetrackingManager.IsFaceTrackingInitialized()) { qRotObject *= facetrackingManager.GetHeadRotation(key, bMirroredMovement: !RearProjection); if (oo.BoneGameObject.FindChild("Jaw") != null) { Transform Jaw; if (facetrackingManager.bGotAU && oo.BoneGameObject.FindChild("Jaw") != null) { Jaw = oo.BoneGameObject.FindChild("Jaw"); // print("Jaw angle : " + facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER); Jaw.localRotation = Quaternion.AngleAxis(facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER, Vector3.right); } } } } else // all joints not the head { qRotObject = kinectManager.GetJointOrientation(key, iJointIndex, flip: RearProjection); Vector3 rotAngles = qRotObject.eulerAngles - oo.InitialRotation.eulerAngles; qRotObject = Quaternion.Euler(rotAngles); } return(Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime)); }
public static float GetNeckAngle(FacetrackingManager facetrackingManager, JointName joint) { float angle = 0; if (facetrackingManager == null) { facetrackingManager = FacetrackingManager.Instance; } if (facetrackingManager && facetrackingManager.IsTrackingFace()) { switch (joint) { case JointName.pan: angle = facetrackingManager.GetHeadRotation().eulerAngles.y; if (angle > 180f) { angle = angle - 360f; //180도를 넘을리는 없지만 각의 범위를 -180~180로 쓰겠다는 의미 } break; case JointName.tilt: angle = facetrackingManager.GetHeadRotation().eulerAngles.x; if (angle > 180f) { angle = angle - 360f; } break; default: break; } } return(angle); }
void Update() { // get the face-tracking manager instance if(faceManager == null) { faceManager = FacetrackingManager.Instance; } if(faceManager && faceManager.IsTrackingFace()) { // head rotation Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(true); if(smoothFactor != 0f) transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime); else transform.rotation = newRotation; // head position Vector3 newPosition = faceManager.GetHeadPosition(true); if(verticalOffset != 0f) { Vector3 dirHead = new Vector3(0, verticalOffset, 0); dirHead = transform.InverseTransformDirection(dirHead); newPosition += dirHead; } // if(smoothFactor != 0f) // transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime); // else transform.position = newPosition; } }
void Update() { if (!manager) { manager = KinectManager.Instance; } if (!faceManager) { faceManager = FacetrackingManager.Instance; } // get the face points if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized()) { long userId = manager.GetUserIdByIndex(playerIndex); if (userId == 0) { return; } if (faceVertices == null) { int iVertCount = faceManager.GetUserFaceVertexCount(userId); if (iVertCount > 0) { faceVertices = new Vector3[iVertCount]; } } if (faceVertices != null) { if (faceManager.GetUserFaceVertices(userId, ref faceVertices)) { if (faceVertices != null && faceVertices[(int)facePoint] != Vector3.zero) { Vector3 facePointPos = faceVertices [(int)facePoint]; if (foregroundCamera) { facePointPos = GetOverlayPosition(facePointPos); } if (facePointTransform) { facePointTransform.position = facePointPos; } if (faceInfoText) { string sStatus = string.Format("{0}: {1}", facePoint, facePointPos); faceInfoText.text = sStatus; } } } } } }
void Update() { // get the face-tracking manager instance if (faceManager == null) { kinectManager = KinectManager.Instance; faceManager = FacetrackingManager.Instance; } // get user-id by user-index long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0; if (kinectManager && kinectManager.IsInitialized() && userId != 0 && faceManager && faceManager.IsTrackingFace(userId)) { if (faceManager.GetUserFaceProperties(userId, ref faceProps)) { //NuitrackInterface.FaceLandmarks faceLM = new NuitrackInterface.FaceLandmarks(); //if(faceProps.ContainsKey("landmarks")) //{ // faceLM = JsonUtility.FromJson<NuitrackInterface.FaceLandmarks>(faceProps["landmarks"]); // // do something with the face landmarks //} //Vector2 leftEyePos = Vector2.zero; //if (faceProps.ContainsKey("lefteye")) //{ // leftEyePos = JsonUtility.FromJson<Vector2>(faceProps["lefteye"]); // // do something with the left-eye position //} //Vector2 rightEyePos = Vector2.zero; //if (faceProps.ContainsKey("righteye")) //{ // rightEyePos = JsonUtility.FromJson<Vector2>(faceProps["righteye"]); // // do something with the right-eye position //} string sFaceProps = ""; foreach (string propName in faceProps.Keys) { string propValue = faceProps[propName]; if (propValue != null && propValue.Length > maxPropLength) { propValue = propValue.Substring(0, maxPropLength) + "..."; } sFaceProps += propName + "=" + propValue + "\n"; } if (infoText && sFaceProps.Length > 0) { //Debug.Log("FaceProps: " + sFaceProps); infoText.text = sFaceProps; } } } }
void Update () { // get the face-tracking manager instance if(faceManager == null) { kinectManager = KinectManager.Instance; faceManager = FacetrackingManager.Instance; } if (kinectManager && faceManager && faceManager.IsTrackingFace ()) { // get user-id by user-index long userId = kinectManager.GetUserIdByIndex (playerIndex); if (userId == 0) return; // head rotation Quaternion newRotation = initialRotation * faceManager.GetHeadRotation (userId, true); if (smoothFactor != 0f) transform.rotation = Quaternion.Slerp (transform.rotation, newRotation, smoothFactor * Time.deltaTime); else transform.rotation = newRotation; // head position Vector3 newPosition = faceManager.GetHeadPosition (userId, true); //Debug.Log(newPosition.ToString()); if (verticalOffset != 0f) { Vector3 dirHead = new Vector3 (0, verticalOffset, 0); dirHead = transform.InverseTransformDirection (dirHead); newPosition += dirHead; } // if(smoothFactor != 0f) // transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime); // else //<----------------------------------------------correct modal position ---------------------------------------------> //Debug.Log ("new Positionx " + newPosition.x.ToString()); if(newPosition.x > 0f) newPosition.x = newPosition.x * 0.8f - deltaX; else if(newPosition.x < 0f) newPosition.x = newPosition.x * 0.8f + deltaX; //<------------------------------------------------------------------------------------------------------------------> transform.position = newPosition; //Debug.Log ("transform.position " + transform.position.x.ToString()); //Debug.Log(newPosition.ToString()); } else if (kinectManager && faceManager && !faceManager.IsTrackingFace ()) { transform.position = -Vector3.one; } }
// Make sure to kill the Kinect on quitting. void OnApplicationQuit() { // Shutdown Speech Recognizer and Kinect FacetrackingWrapper.FinishFaceTracking(); FacetrackingWrapper.ShutdownKinectSensor(); facetrackingInitialized = false; instance = null; }
// Update is called once per frame void Update() { Matrix4x4 kinectToWorld = Matrix4x4.zero; //Quaternion quatTiltAngle = Quaternion.Euler(-manager.sensorAngle, 0.0f, 0.0f); //kinectToWorld.SetTRS(new Vector3(0.0f, manager.sensorHeight, 0.0f), quatTiltAngle, Vector3.one); if (!manager) { manager = KinectManager.Instance; } if (!faceManager) { faceManager = FacetrackingManager.Instance; } // get the face points if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized()) { long userId = manager.GetUserIdByIndex(playerIndex); if (faceVertices == null) { //int iVertCount = faceManager.GetUserFaceVertexCount(userId); int iVertCount = 1347; if (iVertCount > 0) { faceVertices = new Vector3[iVertCount]; } check_f = false; } if (faceVertices != null) { if (faceManager.GetUserFaceVertices(userId, ref faceVertices)) { //-------------- get 35 special point------------------- //Matrix4x4 kinectToWorld = manager.GetKinectToWorldMatrix(); HighDetailFacePoints[] facePoints = (HighDetailFacePoints[])System.Enum.GetValues(typeof(HighDetailFacePoints)); for (int i = 0; i < facePoints.Length; i++) { HighDetailFacePoints point = facePoints[i]; //dictFacePoints[point] = kinectToWorld.MultiplyPoint3x4(faceVertices[(int)point]); dictFacePoints[point] = faceVertices[(int)point]; } check_f = true; } } } }
// Update is called once per frame void FixedUpdate() { Debug.Log("inside fixedupdate of engagement meter"); /* * updateCounter++; * float timeRequired=30f; * float iterationsRequired=timeRequired*50f; * * combinedPlayerEngagement/=50f; * leftPlayerEngagement/=50f; * rightPlayerEngagement/=50f; * * //Debug.Log("update counter"+updateCounter); * if(updateCounter==iterationsRequired) * { * * //Debug.Log("twoengagement "+combinedPlayerEngagement); * //Debug.Log("leftPlayerEngagement "+leftPlayerEngagement); * //Debug.Log("rightPlayerEngagement "+rightPlayerEngagement); * sendToCsv(); * pauseGame(); * } * * * // delay -= Time.deltaTime; * // if(delay < 0){ * // //Debug.Log(faceManagerP1.GetHeadRotation(false)); * // * // delay = 1f; * // } * * //sendToCsv(); * * //Debug.Log(_KinectManager.GetJointPosition(_KinectManager.GetUserIdByIndex(0),4)); */ //lockArrow(); //leadArrow(); fixedUpdateCounter++; Debug.Log("in fixed update"); if (faceManager == null) { //Debug.Log("In FixedUpdate, faceManager was null"); faceManager = FacetrackingManager.Instance; } if (fixedUpdateCounter == 24) { measureEngagement(); fixedUpdateCounter = 0; } //measureEngagement(); //Resources.UnloadUnusedAssets(); }
// Use this for initialization void Start() { faceManager = FacetrackingManager.Instance; //Debug.Log("in Start: facemanager instance"+ faceManager); leftPlayerEngagement = 0; rightPlayerEngagement = 0; combinedPlayerEngagement = 0; leftPlayerEngagementsecondHalf = 0; rightPlayerEngagementsecondHalf = 0; combinedPlayerEngagementsecondHalf = 0; // updateCounter=0; }
private Vector3 TransformBones(KinectManager kinectManager, FacetrackingManager facetrackingManager) { Vector3 jp = Vector3.zero; for (int i = 0; i < userIDs.Count; i++) // do this for every USer ID in { List <OverlayObject> bonesList; long userID = userIDs[i]; if (bagOfBonesDictionary.ContainsKey(userID)) { bonesList = bagOfBonesDictionary[userID]; foreach (OverlayObject oo in bonesList) { int iJointIndex = (int)oo.TrackedJointType; if (kinectManager.IsJointTracked(userID, iJointIndex)) { var TryPos = TranslateBone(oo, userID, kinectManager); if (TryPos != Vector3.zero) { oo.BoneGameObject.position = TryPos; } oo.BoneGameObject.rotation = RotateBone(oo, kinectManager, facetrackingManager, userID); //Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime); if (oo.ReferenceGameObjectTransform != null) { oo.BoneGameObject.localScale = ScaleBone(kinectManager, userID, oo, iJointIndex); } else { oo.BoneGameObject.localScale = new Vector3(1, 1, 1); } } else // joint not tracked { oo.BoneGameObject.position = oo.BoneGameObject.position; oo.BoneGameObject.localRotation = oo.BoneGameObject.localRotation; oo.BoneGameObject.localScale = oo.BoneGameObject.localScale; } } } else { print("Bag of Bones is Missing userID"); } } return(jp); }
void Update() { // float translation = Time.deltaTime * 10; // get the face-tracking manager instance if (manager == null) { manager = FacetrackingManager.Instance; } if (manager && manager.IsTrackingFace()) { // set head position & rotation // FaceFrameReference faceRef = e.FrameReference; } }
void OnDestroy() { if (isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null) { // finish face tracking sensorData.sensorInterface.FinishFaceTracking(); } // // clean up // Resources.UnloadUnusedAssets(); // GC.Collect(); isFacetrackingInitialized = false; instance = null; }
void Update () { if(faceManager == null) { faceManager = FacetrackingManager.Instance; } if(!kinectManager || !kinectManager.IsInitialized()) return; if(!faceManager || !faceManager.IsFaceTrackingInitialized()) return; long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex); if(userId == 0) return; colorTex = kinectManager.GetUsersClrTex(); //faceRect = faceManager.GetFaceColorRect(userId); faceRect = GetHeadJointFaceRect(userId); if(!colorTex || faceRect.width <= 0 || faceRect.height <= 0) return; int faceX = (int)faceRect.x; int faceY = (int)faceRect.y; int faceW = (int)faceRect.width; int faceH = (int)faceRect.height; if(faceX < 0) faceX = 0; if(faceY < 0) faceY = 0; if((faceX + faceW) > colorTex.width) faceW = colorTex.width - faceX; if((faceY + faceH) > colorTex.height) faceH = colorTex.height - faceY; if(faceTex.width != faceW || faceTex.height != faceH) { faceTex.Resize(faceW, faceH); } Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0); faceTex.SetPixels(colorPixels); faceTex.Apply(); if(targetRenderer && targetRenderer.material) { targetRenderer.material.mainTexture = faceTex; } }
void Start() { //pec = new GameObject[arraySize]; //Associate the variables with Components in the MainCamera grabScript = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <GrabDropScript>(); interactionManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponents <InteractionManager>(); result = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <Timer>(); scriptLog = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <log>(); engagementMeter = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <EngagementMeter>(); faceTrackingManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <FacetrackingManager>(); scoreKeepBody = GameObject.Find("BodyPlaceHolders").GetComponent <ScoreKeep>(); source = GetComponent <AudioSource>(); //Gets AudioSource from the attached Object }
void Update() { // get the face-tracking manager instance if (faceManager == null) { kinectManager = KinectManager.Instance; faceManager = FacetrackingManager.Instance; } if (kinectManager && faceManager && faceManager.IsTrackingFace()) { // get user-id by user-index long userId = kinectManager.GetUserIdByIndex(playerIndex); if (userId == 0) { return; } // head rotation Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true); if (smoothFactor != 0f) { transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime); } else { transform.rotation = newRotation; } // head position Vector3 newPosition = faceManager.GetHeadPosition(userId, true); if (verticalOffset != 0f) { Vector3 dirHead = new Vector3(horizontalOffset, verticalOffset, zOffset); dirHead = transform.InverseTransformDirection(dirHead); newPosition += dirHead; } // if(smoothFactor != 0f) // transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime); // else transform.position = newPosition; } }
void Update() { // reference to face manager if (!faceManager) { faceManager = FacetrackingManager.Instance; } if (kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized()) { // check for tracked user long userId = kinectManager.GetUserIdByIndex(playerIndex); if (userId != 0 && kinectManager.IsUserTracked(userId)) { if (faceManager.GetUserAnimUnits(userId, ref dictAnimUnits)) { // process the blend shapes -> anim units for (int i = 0; i < blendShapeCount; i++) { if (blendShape2AnimUnit.ContainsKey(blendShapeNames[i])) { KinectInterop.FaceShapeAnimations faceAnim = blendShape2AnimUnit[blendShapeNames[i]]; float animValue = dictAnimUnits[faceAnim]; // check for multiplier float mul = 1f; if (blendShape2Multiplier.ContainsKey(blendShapeNames[i])) { mul = blendShape2Multiplier[blendShapeNames[i]]; } if (animValue * mul < 0f) { animValue = 0f; } // lerp to the new value blendShapeValues[i] = Mathf.Lerp(blendShapeValues [i], animValue * mul * 100f, smoothFactor * Time.deltaTime); skinnedMeshRenderer.SetBlendShapeWeight(i, blendShapeValues[i]); } } } } } }
// enables or disables the face-tracking component public void EnableFaceTracking(bool bEnable) { FacetrackingManager facetrackingManager = gameObject.GetComponent <FacetrackingManager>(); if (facetrackingManager) { facetrackingManager.enabled = bEnable; LogToConsole("Face tracking is " + (bEnable ? "enabled" : "disabled")); if (bEnable) { StartCoroutine(CheckFacetrackingManager()); } } else { LogErrorToConsole("FacetrackingManager-component not found."); } }
void Update() { // get the face-tracking manager instance if (faceManager == null) { faceManager = FacetrackingManager.Instance; } if (faceManager && faceManager.IsTrackingFace()) { // head rotation Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(true); if (smoothFactor != 0f) { transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime); } else { transform.rotation = newRotation; } // head position Vector3 newPosition = faceManager.GetHeadPosition(true); if (verticalOffset != 0f) { Vector3 dirHead = new Vector3(0, verticalOffset, 0); dirHead = transform.InverseTransformDirection(dirHead); newPosition += dirHead; } // if(smoothFactor != 0f) // transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime); // else transform.position = newPosition; } }
void Update() { if (faceManager == null) { faceManager = FacetrackingManager.Instance; } //unsafe for editor but can optimise if commented if (!kinectManager || !kinectManager.IsInitialized()) { return; } if (!faceManager || !faceManager.IsFaceTrackingInitialized()) { return; } long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex); if (userId == 0) { return; } if (!backManager) { backManager = BackgroundRemovalManager.Instance; if (backManager) { // re-initialize the texture colorTex = null; } } if (backManager && backManager.IsBackgroundRemovalInitialized()) { // use foreground image if (!colorTex) { colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false); } try { RenderTexture foregroundTex = (RenderTexture)backManager.GetForegroundTex(); KinectInterop.RenderTex2Tex2D(foregroundTex, ref colorTex); } catch (System.Exception) { colorTex = (Texture2D)backManager.GetForegroundTex(); } } else { // use color camera image colorTex = kinectManager.GetUsersClrTex(); } //faceRect = faceManager.GetFaceColorRect(userId); faceRect = GetHeadJointFaceRect(userId); if (!colorTex || faceRect.width <= 0 || faceRect.height <= 0) { return; } int faceX = (int)faceRect.x; int faceY = (int)faceRect.y; int faceW = (int)faceRect.width; int faceH = (int)faceRect.height; if (faceX < 0) { faceX = 0; } if (faceY < 0) { faceY = 0; } if ((faceX + faceW) > colorTex.width) { faceW = colorTex.width - faceX; } if ((faceY + faceH) > colorTex.height) { faceH = colorTex.height - faceY; } if (faceTex.width != faceW || faceTex.height != faceH) { faceTex.Resize(faceW, faceH); } Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0); faceTex.SetPixels(colorPixels); faceTex.Apply(); if (targetRenderer && targetRenderer.material) { targetRenderer.material.mainTexture = faceTex; } }
void Awake() { manager = Camera.main.GetComponent<FacetrackingManager>(); }
//----------------------------------- end of public functions --------------------------------------// void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if(kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if(sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized."); } if(debugText != null) { debugText.GetComponent<GUIText>().text = "Please, wait..."; } // ensure the needed dlls are in place and face tracking is available for this interface bool bNeedRestart = false; if(sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart)) { if(bNeedRestart) { KinectInterop.RestartLevel(gameObject, "FM"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Face tracking is not supported!"); } // Initialize the face tracker if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect)) { throw new Exception("Face tracking could not be initialized."); } instance = this; isFacetrackingInitialized = true; //DontDestroyOnLoad(gameObject); if(debugText != null) { debugText.GetComponent<GUIText>().text = "Ready."; } } catch(DllNotFoundException ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations."; } catch (Exception ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = ex.Message; } }
void OnDestroy() { if(isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null) { // finish face tracking sensorData.sensorInterface.FinishFaceTracking(); } // // clean up // Resources.UnloadUnusedAssets(); // GC.Collect(); isFacetrackingInitialized = false; instance = null; }
void StartFacetracker() { try { if(debugText != null) debugText.guiText.text = "Please, wait..."; // initialize Kinect sensor as needed int rc = FacetrackingWrapper.InitKinectSensor(); if(rc != 0) { throw new Exception("Initialization of Kinect sensor failed"); } // Initialize the kinect speech wrapper rc = FacetrackingWrapper.InitFaceTracking(); if (rc < 0) { throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc)); } if(ComputeColorMap) { // Initialize color map related stuff usersClrTex = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight()); usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height); colorImage = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()]; videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4]; } instance = this; facetrackingInitialized = true; DontDestroyOnLoad(gameObject); if(debugText != null) debugText.guiText.text = "Ready."; } catch(DllNotFoundException ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.guiText.text = "Please check the Kinect and FT-Library installations."; } catch (Exception ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.guiText.text = ex.Message; } }
void Update() { // get the face-tracking manager instance if(faceManager == null) { kinectManager = KinectManager.Instance; faceManager = FacetrackingManager.Instance; } if(kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsTrackingFace() && foregroundCamera) { // get user-id by user-index long userId = kinectManager.GetUserIdByIndex(playerIndex); if(userId == 0) return; // get head position Vector3 newPosition = faceManager.GetHeadPosition(userId, true); // get head rotation Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true); // rotational fix, provided by Richard Borys: // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z), Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero; addedRotation.x = newRotation.eulerAngles.x + addedRotation.x; addedRotation.y = newRotation.eulerAngles.y + addedRotation.y; addedRotation.z = newRotation.eulerAngles.z + addedRotation.z; newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z); // end of rotational fix if(smoothFactorRotation != 0f) transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactorRotation * Time.deltaTime); else transform.rotation = newRotation; // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if(portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } // model position newPosition = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect); if(newPosition == Vector3.zero) { // hide the model behind the camera newPosition.z = -10f; } if(verticalOffset != 0f) { // add the vertical offset Vector3 dirHead = new Vector3(0, verticalOffset, 0); dirHead = transform.InverseTransformDirection(dirHead); newPosition += dirHead; } // go to the new position if(smoothFactorMovement != 0f && transform.position.z >= 0f) transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactorMovement * Time.deltaTime); else transform.position = newPosition; // scale the model if needed if(transform.localScale.x != modelScaleFactor) { transform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor); } } else { // hide the model behind the camera if(transform.position.z >= 0f) { transform.position = new Vector3(0f, 0f, -10f); } } }
void Update() { // get the face-tracking manager instance if(manager == null) { manager = FacetrackingManager.Instance; } if(manager && manager.IsTrackingFace()) { // set head position & rotation if(HeadTransform != null) { // head position Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement); if(smoothFactor != 0f) HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime); else HeadTransform.localPosition = newPosition; // head rotation Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); if(smoothFactor != 0f) HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime); else HeadTransform.localRotation = newRotation; } // apply animation units // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker); SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp); SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen); SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft); fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left; SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched); float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight); fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right; SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer); fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left; SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered); float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer); fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right; SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered); // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft); fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left; SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed); float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight); fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right; SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed); // AU6, AU7 – Eyelid closed // 0=neutral; -1=raised; +1=fully lowered float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed); fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left; SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered); SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised); float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed); fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right; SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered); SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised); } }
void Update() { // get the face-tracking manager instance if(manager == null) { manager = FacetrackingManager.Instance; } if(manager && manager.IsTrackingFace()) { // set head position & rotation if(HeadTransform != null) { // head position Vector3 newPosition = manager.GetHeadPosition(mirroredHeadMovement); // head rotation Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); // rotational fix, provided by Richard Borys: // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z), Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero; addedRotation.x = newRotation.eulerAngles.x + addedRotation.x; addedRotation.y = newRotation.eulerAngles.y + addedRotation.y; addedRotation.z = newRotation.eulerAngles.z + addedRotation.z; newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z); // end of rotational fix if(smoothFactor != 0f) HeadTransform.rotation = Quaternion.Slerp(HeadTransform.rotation, newRotation, smoothFactor * Time.deltaTime); else HeadTransform.rotation = newRotation; // check for head pos overlay if(foregroundCamera) { // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if(portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } KinectManager kinectManager = KinectManager.Instance; if(kinectManager) { long userId = kinectManager.GetUserIdByIndex(manager.playerIndex); Vector3 posColorOverlay = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect); if(posColorOverlay != Vector3.zero) { newPosition = posColorOverlay; // if(overlayObj) // { // overlayObj.position = newPosition; // } } } } else { // move around the initial position newPosition += HeadInitialPosition; } // vertical offet if(verticalOffset != 0f) { // add the vertical offset Vector3 dirHead = new Vector3(0, verticalOffset, 0); dirHead = HeadTransform.InverseTransformDirection(dirHead); newPosition += dirHead; } // set the position if(smoothFactor != 0f) HeadTransform.position = Vector3.Lerp(HeadTransform.position, newPosition, smoothFactor * Time.deltaTime); else HeadTransform.position = newPosition; // scale factor if(HeadTransform.localScale.x != modelScaleFactor) { HeadTransform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor); } } // apply animation units // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker); SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp); SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen); SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft); fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left; SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched); float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight); fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right; SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer); fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left; SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered); float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer); fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right; SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered); // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft); fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left; SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed); float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight); fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right; SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed); // AU6, AU7 – Eyelid closed // 0=neutral; -1=raised; +1=fully lowered float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed); fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left; SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered); SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised); float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed); fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right; SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered); SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised); } else { // hide the model behind the camera if(HeadTransform && HeadTransform.position.z >= 0f) { HeadTransform.position = new Vector3(0f, 0f, -10f); } } }
void Awake() { manager = Camera.main.GetComponent <FacetrackingManager>(); }
void Update() { if (faceManager == null) { faceManager = FacetrackingManager.Instance; } if (!kinectManager || !kinectManager.IsInitialized()) { return; } if (!faceManager || !faceManager.IsFaceTrackingInitialized()) { return; } long userId = kinectManager.GetUserIdByIndex(playerIndex); if (userId == 0) { if (targetObject && targetObject.material && targetObject.material.mainTexture != null) { targetObject.material.mainTexture = null; } return; } if (!backManager) { backManager = BackgroundRemovalManager.Instance; if (backManager) { // re-initialize the texture colorTex = null; } } if (!bSkipUpdatingForegroundTex && backManager && backManager.IsBackgroundRemovalInitialized()) { // use foreground image Texture bmForeTex = backManager.GetForegroundTex(); if (bmForeTex is RenderTexture) { foregroundTex = (RenderTexture)bmForeTex; bSkipUpdatingForegroundTex = (foregroundTex != null); } else { if (foregroundTex == null) { foregroundTex = new RenderTexture(bmForeTex.width, bmForeTex.height, 0); } Graphics.Blit(bmForeTex, foregroundTex); bSkipUpdatingForegroundTex = false; } // if (!colorTex) // { // colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false); // } // // try // { // foregroundTex = (RenderTexture)backManager.GetForegroundTex (); // //KinectInterop.RenderTex2Tex2D (foregroundTex, ref colorTex); // } // catch (System.Exception) // { // colorTex = (Texture2D)backManager.GetForegroundTex (); // } } else { // use color camera image if (!colorTex) { colorTex = kinectManager.GetUsersClrTex2D(); } } //faceRect = faceManager.GetFaceColorRect(userId); faceRect = GetHeadJointFaceRect(userId); if (faceRect.width > 0 && faceRect.height > 0) { int faceX = (int)faceRect.x; int faceY = (int)faceRect.y; int faceW = (int)faceRect.width; int faceH = (int)faceRect.height; if (faceX < 0) { faceX = 0; } if (faceY < 0) { faceY = 0; } if (foregroundTex) { if ((faceX + faceW) > foregroundTex.width) { faceW = foregroundTex.width - faceX; } if ((faceY + faceH) > foregroundTex.height) { faceH = foregroundTex.height - faceY; } } else if (colorTex) { if ((faceX + faceW) > colorTex.width) { faceW = colorTex.width - faceX; } if ((faceY + faceH) > colorTex.height) { faceH = colorTex.height - faceY; } } if (faceTex.width != faceW || faceTex.height != faceH) { faceTex.Resize(faceW, faceH); } if (foregroundTex) { KinectInterop.RenderTex2Tex2D(foregroundTex, faceX, foregroundTex.height - faceY - faceH, faceW, faceH, ref faceTex); } else if (colorTex) { Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0); faceTex.SetPixels(colorPixels); faceTex.Apply(); } if (targetObject && !targetObject.gameObject.activeSelf) { targetObject.gameObject.SetActive(true); } if (targetObject && targetObject.material) { targetObject.material.mainTexture = faceTex; } // don't rotate the transform - mesh follows the head rotation if (targetObject.transform.rotation != Quaternion.identity) { targetObject.transform.rotation = Quaternion.identity; } } else { if (targetObject && targetObject.gameObject.activeSelf) { targetObject.gameObject.SetActive(false); } if (targetObject && targetObject.material && targetObject.material.mainTexture != null) { targetObject.material.mainTexture = null; } } }
void Update() { int recHostId; int connectionId; int recChannelId; int dataSize; bool connListUpdated = false; if (backgroundImage && backgroundImage.texture == null) { backgroundImage.texture = manager ? manager.GetUsersLblTex() : null; } if (faceManager == null) { faceManager = FacetrackingManager.Instance; } if (gestureManager == null) { gestureManager = VisualGestureManager.Instance; } if (speechManager == null) { speechManager = SpeechManager.Instance; } try { byte error = 0; NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out recChannelId, recBuffer, bufferSize, out dataSize, out error); switch (recData) { case NetworkEventType.Nothing: //1 break; case NetworkEventType.ConnectEvent: //2 if (recHostId == serverHostId && recChannelId == serverChannelId && !dictConnection.ContainsKey(connectionId)) { HostConnection conn = new HostConnection(); conn.hostId = recHostId; conn.connectionId = connectionId; conn.channelId = recChannelId; conn.keepAlive = true; conn.reqDataType = "ka,kb,km,kh"; //conn.matrixSent = false; dictConnection[connectionId] = conn; connListUpdated = true; //LogToConsole(connectionId + "-conn: " + conn.reqDataType); } // // reset chunked face messages // sendFvMsg = string.Empty; // sendFvNextOfs = 0; // // sendFtMsg = string.Empty; // sendFtNextOfs = 0; break; case NetworkEventType.DataEvent: //3 if (recHostId == serverHostId && recChannelId == serverChannelId && dictConnection.ContainsKey(connectionId)) { HostConnection conn = dictConnection[connectionId]; int decompSize = 0; if (decompressor != null && (recBuffer[0] > 127 || recBuffer[0] < 32)) { decompSize = decompressor.Decompress(recBuffer, 0, compressBuffer, 0, dataSize); } else { System.Buffer.BlockCopy(recBuffer, 0, compressBuffer, 0, dataSize); decompSize = dataSize; } string sRecvMessage = System.Text.Encoding.UTF8.GetString(compressBuffer, 0, decompSize); if (sRecvMessage.StartsWith("ka")) { if (sRecvMessage == "ka") // vr-examples v1.0 keep-alive message { sRecvMessage = "ka,kb,km,kh"; } conn.keepAlive = true; conn.reqDataType = sRecvMessage; dictConnection[connectionId] = conn; //LogToConsole(connectionId + "-recv: " + conn.reqDataType); // check for SR phrase-reset int iIndexSR = sRecvMessage.IndexOf(",sr"); if (iIndexSR >= 0 && speechManager) { speechManager.ClearPhraseRecognized(); //LogToConsole("phrase cleared"); } } } break; case NetworkEventType.DisconnectEvent: //4 if (dictConnection.ContainsKey(connectionId)) { dictConnection.Remove(connectionId); connListUpdated = true; } break; } if (connListUpdated) { // get all connection IDs alConnectionId.Clear(); alConnectionId.AddRange(dictConnection.Keys); // display the number of connections StringBuilder sbConnStatus = new StringBuilder(); sbConnStatus.AppendFormat("Server running: {0} connection(s)", dictConnection.Count); foreach (int connId in dictConnection.Keys) { HostConnection conn = dictConnection[connId]; int iPort = 0; string sAddress = string.Empty; NetworkID network; NodeID destNode; NetworkTransport.GetConnectionInfo(conn.hostId, conn.connectionId, out sAddress, out iPort, out network, out destNode, out error); if (error == (int)NetworkError.Ok) { sbConnStatus.AppendLine().Append(" ").Append(sAddress).Append(":").Append(iPort); } } LogToConsole(sbConnStatus.ToString()); if (connStatusText) { connStatusText.text = sbConnStatus.ToString(); } } // send body frame to available connections const char delimiter = ','; string sBodyFrame = manager ? manager.GetBodyFrameData(ref liRelTime, ref fCurrentTime, delimiter) : string.Empty; if (sBodyFrame.Length > 0 && dictConnection.Count > 0) { StringBuilder sbSendMessage = new StringBuilder(); bool bFaceParamsRequested = IsFaceParamsRequested(); sbSendMessage.Append(manager.GetWorldMatrixData(delimiter)).Append('|'); sbSendMessage.Append(sBodyFrame).Append('|'); sbSendMessage.Append(manager.GetBodyHandData(ref liRelTime, delimiter)).Append('|'); if (bFaceParamsRequested && faceManager && faceManager.IsFaceTrackingInitialized()) { sbSendMessage.Append(faceManager.GetFaceParamsAsCsv(delimiter)).Append('|'); } if (gestureManager && gestureManager.IsVisualGestureInitialized()) { sbSendMessage.Append(gestureManager.GetGestureDataAsCsv(delimiter)).Append('|'); } if (speechManager && speechManager.IsSapiInitialized()) { sbSendMessage.Append(speechManager.GetSpeechDataAsCsv(delimiter)).Append('|'); } if (sbSendMessage.Length > 0 && sbSendMessage[sbSendMessage.Length - 1] == '|') { sbSendMessage.Remove(sbSendMessage.Length - 1, 1); } byte[] btSendMessage = System.Text.Encoding.UTF8.GetBytes(sbSendMessage.ToString()); //Debug.Log("Message " + sbSendMessage.Length + " chars: " + sbSendMessage.ToString()); //Debug.Log("Encoded into " + btSendMessage.Length + " bytes: " + ByteArrayToString(btSendMessage, btSendMessage.Length)); int compSize = 0; if (compressor != null && btSendMessage.Length > 100 && !websocketHost) { compSize = compressor.Compress(btSendMessage, 0, btSendMessage.Length, compressBuffer, 0); } else { System.Buffer.BlockCopy(btSendMessage, 0, compressBuffer, 0, btSendMessage.Length); compSize = btSendMessage.Length; } //Debug.Log("Compressed into " + compSize + " bytes: " + ByteArrayToString(compressBuffer, compSize)); // // check face-tracking requests // bool bFaceParams = false, bFaceVertices = false, bFaceUvs = false, bFaceTriangles = false; // if(faceManager && faceManager.IsFaceTrackingInitialized()) // CheckFacetrackRequests(out bFaceParams, out bFaceVertices, out bFaceUvs, out bFaceTriangles); // // byte[] btFaceParams = null; // if(bFaceParams) // { // string sFaceParams = faceManager.GetFaceParamsAsCsv(); // if(!string.IsNullOrEmpty(sFaceParams)) // btFaceParams = System.Text.Encoding.UTF8.GetBytes(sFaceParams); // } // // // next chunk of data for face vertices // byte[] btFaceVertices = null; // string sFvMsgHead = string.Empty; // GetNextFaceVertsChunk(bFaceVertices, bFaceUvs, ref btFaceVertices, out sFvMsgHead); // // // next chunk of data for face triangles // byte[] btFaceTriangles = null; // string sFtMsgHead = string.Empty; // GetNextFaceTrisChunk(bFaceTriangles, ref btFaceTriangles, out sFtMsgHead); foreach (int connId in alConnectionId) { HostConnection conn = dictConnection[connId]; if (conn.keepAlive) { conn.keepAlive = false; dictConnection[connId] = conn; if (conn.reqDataType != null && conn.reqDataType.Contains("kb,")) { //LogToConsole(conn.connectionId + "-sendkb: " + conn.reqDataType); error = 0; //if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btSendMessage, btSendMessage.Length, out error)) if (!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, compressBuffer, compSize, out error)) { string sMessage = "Error sending body data via conn " + conn.connectionId + ": " + (NetworkError)error; LogErrorToConsole(sMessage); if (serverStatusText) { serverStatusText.text = sMessage; } } } // if(bFaceParams && btFaceParams != null && // conn.reqDataType != null && conn.reqDataType.Contains("fp,")) // { // //Debug.Log(conn.connectionId + "-sendfp: " + conn.reqDataType); // // error = 0; // if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceParams, btFaceParams.Length, out error)) // { // string sMessage = "Error sending face params via conn " + conn.connectionId + ": " + (NetworkError)error; // Debug.LogError(sMessage); // // if(serverStatusText) // { // serverStatusText.text = sMessage; // } // } // } // // if(bFaceVertices && btFaceVertices != null && // conn.reqDataType != null && conn.reqDataType.Contains("fv,")) // { // //Debug.Log(conn.connectionId + "-sendfv: " + conn.reqDataType + " - " + sFvMsgHead); // // error = 0; // if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceVertices, btFaceVertices.Length, out error)) // { // string sMessage = "Error sending face verts via conn " + conn.connectionId + ": " + (NetworkError)error; // Debug.LogError(sMessage); // // if(serverStatusText) // { // serverStatusText.text = sMessage; // } // } // } // // if(bFaceTriangles && btFaceTriangles != null && // conn.reqDataType != null && conn.reqDataType.Contains("ft,")) // { // //Debug.Log(conn.connectionId + "-sendft: " + conn.reqDataType + " - " + sFtMsgHead); // // error = 0; // if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceTriangles, btFaceTriangles.Length, out error)) // { // string sMessage = "Error sending face tris via conn " + conn.connectionId + ": " + (NetworkError)error; // Debug.LogError(sMessage); // // if(serverStatusText) // { // serverStatusText.text = sMessage; // } // } // } } } } } catch (System.Exception ex) { LogErrorToConsole(ex.Message + "\n" + ex.StackTrace); if (serverStatusText) { serverStatusText.text = ex.Message; } } }
void Update() { // get the face-tracking manager instance if (faceManager == null) { kinectManager = KinectManager.Instance; faceManager = FacetrackingManager.Instance; } // get user-id by user-index long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0; if (kinectManager && kinectManager.IsInitialized() && userId != 0 && faceManager && faceManager.IsTrackingFace(userId) && foregroundCamera) { // get head position Vector3 newPosition = faceManager.GetHeadPosition(userId, true); // get head rotation Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true); // rotational fix, provided by Richard Borys: // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z), Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero; addedRotation.x = newRotation.eulerAngles.x + addedRotation.x; addedRotation.y = newRotation.eulerAngles.y + addedRotation.y; addedRotation.z = newRotation.eulerAngles.z + addedRotation.z; newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z); // end of rotational fix if (smoothFactorRotation != 0f) { transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactorRotation * Time.deltaTime); } else { transform.rotation = newRotation; } // get the background rectangle (use the portrait background, if available) Rect backgroundRect = foregroundCamera.pixelRect; PortraitBackground portraitBack = PortraitBackground.Instance; if (portraitBack && portraitBack.enabled) { backgroundRect = portraitBack.GetBackgroundRect(); } // model position newPosition = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect); if (newPosition == Vector3.zero) { // hide the model behind the camera newPosition.z = -10f; } if (verticalOffset != 0f) { // add the vertical offset Vector3 dirHead = new Vector3(0, verticalOffset, 0); dirHead = transform.InverseTransformDirection(dirHead); newPosition += dirHead; } // go to the new position if (smoothFactorMovement != 0f && transform.position.z >= 0f) { transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactorMovement * Time.deltaTime); } else { transform.position = newPosition; } // scale the model if needed if (transform.localScale.x != modelScaleFactor) { transform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor); } } else { // hide the model behind the camera if (transform.position.z >= 0f) { transform.position = new Vector3(0f, 0f, -10f); } } }
//----------------------------------- end of public functions --------------------------------------// void Awake() { instance = this; }
//----------------------------------- end of public functions --------------------------------------// void Start() { instance = this; try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if (kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if (sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized."); } if (debugText != null) { debugText.text = "Please, wait..."; } // ensure the needed dlls are in place and face tracking is available for this interface bool bNeedRestart = false; if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart)) { if (bNeedRestart) { KinectInterop.RestartLevel(gameObject, "FM"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Face tracking is not supported!"); } // Initialize the face tracker if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect)) { throw new Exception("Face tracking could not be initialized."); } isFacetrackingInitialized = true; //DontDestroyOnLoad(gameObject); if (debugText != null) { debugText.text = "Ready."; } } catch (DllNotFoundException ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.text = "Please check the Kinect and FT-Library installations."; } } catch (Exception ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.text = ex.Message; } } }
void Update() { // get the face-tracking manager instance if (manager == null) { manager = FacetrackingManager.Instance; } if (manager && manager.IsTrackingFace()) { // set head position & rotation if (HeadTransform != null) { // head position Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement); if (smoothFactor != 0f) { HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime); } else { HeadTransform.localPosition = newPosition; } // head rotation Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement); if (smoothFactor != 0f) { HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime); } else { HeadTransform.localRotation = newRotation; } } // apply animation units // AU0 - Upper Lip Raiser // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker); SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp); SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp); // AU1 - Jaw Lowerer // 0=closed; 1=fully open; -1= closed, like 0 float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen); SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown); // AU2 – Lip Stretcher // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth) float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft); fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left; SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched); float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight); fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right; SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched); // AU3 – Brow Lowerer // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes) float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer); fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left; SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered); float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer); fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right; SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered); // AU4 – Lip Corner Depressor // 0=neutral; -1=very happy smile; +1=very sad frown float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft); fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left; SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed); float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight); fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right; SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed); // AU6, AU7 – Eyelid closed // 0=neutral; -1=raised; +1=fully lowered float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed); fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left; SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered); SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised); float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed); fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right; SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered); SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, LowerEyelidRightRaised); } }
void OnDestroy() { if(isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null) { // finish face tracking if(sensorData != null && sensorData.sensorInterface != null) { sensorData.sensorInterface.FinishFaceTracking(); } isFacetrackingInitialized = false; instance = null; } }
private void CheckFaceTrackingAvailability(long userId) { faceTracker = FacetrackingManager.Instance; faceTrackingAvailable [userId] = faceTracker != null && faceTracker.IsTrackingFace (userId); }
public bool IsFaceTrackingAvailable(long userId) { faceTracker = FacetrackingManager.Instance; return (userId == firstUserId || userId == secondUserId) && faceTracker.IsTrackingFace (userId); }