Exemplo n.º 1
0
    public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
    {
        KinectInterop.SensorData sensorData = new KinectInterop.SensorData();

        sensorData.bodyCount  = 6;
        sensorData.jointCount = 25;

        sensorData.depthCameraFOV    = 60f;
        sensorData.colorCameraFOV    = 53.8f;
        sensorData.depthCameraOffset = 0f;
        sensorData.faceOverlayOffset = 0f;

        sensorData.colorImageWidth  = 1920;
        sensorData.colorImageHeight = 1080;

        // flip color image vertically
        sensorData.colorImageScale = new Vector3(1f, -1f, 1f);

        sensorData.depthImageWidth  = 512;
        sensorData.depthImageHeight = 424;

        // look for face-tracking manager
        faceManager           = GameObject.FindObjectOfType <FacetrackingManager>();
        bFaceManagerAvailable = faceManager != null;
        bFaceTrackingInited   = false;

        Debug.Log("DummyK2-sensor opened");

        return(sensorData);
    }
    void Update()
    {
        // get the face-tracking manager instance
        if (faceManager == null)
        {
            kinectManager = KinectManager.Instance;
            faceManager   = FacetrackingManager.Instance;
        }

        // get user-id by user-index
        long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0;

        if (kinectManager && kinectManager.IsInitialized() && userId != 0 &&
            faceManager && faceManager.IsTrackingFace(userId))
        {
            if (faceManager.GetUserFaceProperties(userId, ref faceProps))
            {
                string sFaceProps = "";
                foreach (string propName in faceProps.Keys)
                {
                    string propValue = faceProps[propName];
                    sFaceProps += propName + "=" + propValue + "\n";
                }

                if (infoText && sFaceProps.Length > 0)
                {
                    //Debug.Log("FaceProps: " + sFaceProps);
                    infoText.text = sFaceProps;
                }
            }
        }
    }
Exemplo n.º 3
0
    // checks if FacetrackingManager is initialized or not
    private IEnumerator CheckFacetrackingManager()
    {
        // wait for 2 seconds
        yield return(new WaitForSeconds(2f));

        string sStatusMsg = string.Empty;
        FacetrackingManager facetrackingManager = FacetrackingManager.Instance;

        if (!facetrackingManager)
        {
            sStatusMsg = "FacetrackingManager is missing!";
        }
        else if (!facetrackingManager.IsFaceTrackingInitialized())
        {
            sStatusMsg = "FacetrackingManager not initialized! Check the log-file for details.";
        }
        else
        {
            LogToConsole("FacetrackingManager is ready.");
        }

        if (sStatusMsg.Length > 0)
        {
            LogErrorToConsole(sStatusMsg);
        }
    }
Exemplo n.º 4
0
    void Update()
    {
        if (facetrackingManager == null)
        {
            facetrackingManager = FacetrackingManager.Instance;
        }

        if (facetrackingManager && facetrackingManager.IsTrackingFace())
        {
            // set head position & rotation
            if (head != null)
            {
                //Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition();
                //HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, 3 * Time.deltaTime);

                //Quaternion newRotation = HeadInitialRotation * facetrackingManager.GetHeadRotation();
                //head.localRotation = Quaternion.Slerp(head.localRotation, newRotation, 3 * Time.deltaTime);
                //head.localRotation = HeadInitialRotation * facetrackingManager.GetHeadRotation();
                Vector3 targetOrientation = Vector3.zero;
                targetOrientation.x = facetrackingManager.GetHeadRotation().eulerAngles.x;
                targetOrientation.y = facetrackingManager.GetHeadRotation().eulerAngles.y;

                head.localRotation = Quaternion.Euler(targetOrientation);
            }
        }
    }
Exemplo n.º 5
0
    void StartFacetracker()
    {
        try
        {
            if (debugText != null)
            {
                debugText.GetComponent <GUIText>().text = "Please, wait...";
            }

            // initialize Kinect sensor as needed
            int rc = FacetrackingWrapper.InitKinectSensor((int)FacetrackingWrapper.Constants.ColorImageResolution, (int)FacetrackingWrapper.Constants.DepthImageResolution, FacetrackingWrapper.Constants.IsNearMode);
            if (rc != 0)
            {
                throw new Exception("Initialization of Kinect sensor failed");
            }

            // Initialize the kinect speech wrapper
            rc = FacetrackingWrapper.InitFaceTracking();
            if (rc < 0)
            {
                throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc));
            }

            if (ComputeColorMap)
            {
                // Initialize color map related stuff
                usersClrTex  = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight(), TextureFormat.ARGB32, false);
                usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height);

                colorImage  = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()];
                videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4];
            }

            instance = this;
            facetrackingInitialized = true;

            DontDestroyOnLoad(gameObject);

            if (debugText != null)
            {
                debugText.GetComponent <GUIText>().text = "Ready.";
            }
        }
        catch (DllNotFoundException ex)
        {
            Debug.LogError(ex.ToString());
            if (debugText != null)
            {
                debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
            }
        }
        catch (Exception ex)
        {
            Debug.LogError(ex.ToString());
            if (debugText != null)
            {
                debugText.GetComponent <GUIText>().text = ex.Message;
            }
        }
    }
Exemplo n.º 6
0
        private Quaternion RotateBone(OverlayObject oo, KinectManager kinectManager, FacetrackingManager facetrackingManager, long key)
        {
            int        iJointIndex = (int)oo.TrackedJointType;
            Quaternion qRotObject  = Quaternion.identity;

            if (iJointIndex == 3)        // if it's the head
            {
                if (facetrackingManager) // && facetrackingManager.IsFaceTrackingInitialized())
                {
                    qRotObject *= facetrackingManager.GetHeadRotation(key, bMirroredMovement: !RearProjection);

                    if (oo.BoneGameObject.FindChild("Jaw") != null)
                    {
                        Transform Jaw;
                        if (facetrackingManager.bGotAU && oo.BoneGameObject.FindChild("Jaw") != null)
                        {
                            Jaw = oo.BoneGameObject.FindChild("Jaw");
                            //  print("Jaw angle : " + facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER);
                            Jaw.localRotation = Quaternion.AngleAxis(facetrackingManager.dictAU[0] * JAWROTATEMULIPLIER, Vector3.right);
                        }
                    }
                }
            }
            else // all joints not the head
            {
                qRotObject = kinectManager.GetJointOrientation(key, iJointIndex, flip: RearProjection);
                Vector3 rotAngles = qRotObject.eulerAngles - oo.InitialRotation.eulerAngles;
                qRotObject = Quaternion.Euler(rotAngles);
            }
            return(Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime));
        }
Exemplo n.º 7
0
    public static float GetNeckAngle(FacetrackingManager facetrackingManager, JointName joint)
    {
        float angle = 0;

        if (facetrackingManager == null)
        {
            facetrackingManager = FacetrackingManager.Instance;
        }

        if (facetrackingManager && facetrackingManager.IsTrackingFace())
        {
            switch (joint)
            {
            case JointName.pan:
                angle = facetrackingManager.GetHeadRotation().eulerAngles.y;
                if (angle > 180f)
                {
                    angle = angle - 360f;                   //180도를 넘을리는 없지만 각의 범위를 -180~180로 쓰겠다는 의미
                }
                break;

            case JointName.tilt:
                angle = facetrackingManager.GetHeadRotation().eulerAngles.x;
                if (angle > 180f)
                {
                    angle = angle - 360f;
                }
                break;

            default: break;
            }
        }

        return(angle);
    }
Exemplo n.º 8
0
    void Update()
    {
        // get the face-tracking manager instance
        if(faceManager == null)
        {
            faceManager = FacetrackingManager.Instance;
        }

        if(faceManager && faceManager.IsTrackingFace())
        {
            // head rotation
            Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(true);

            if(smoothFactor != 0f)
                transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
            else
                transform.rotation = newRotation;

            // head position
            Vector3 newPosition = faceManager.GetHeadPosition(true);

            if(verticalOffset != 0f)
            {
                Vector3 dirHead = new Vector3(0, verticalOffset, 0);
                dirHead = transform.InverseTransformDirection(dirHead);
                newPosition += dirHead;
            }

        //			if(smoothFactor != 0f)
        //				transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
        //			else
                transform.position = newPosition;
        }
    }
Exemplo n.º 9
0
    void Update()
    {
        if (!manager)
        {
            manager = KinectManager.Instance;
        }

        if (!faceManager)
        {
            faceManager = FacetrackingManager.Instance;
        }

        // get the face points
        if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
        {
            long userId = manager.GetUserIdByIndex(playerIndex);
            if (userId == 0)
            {
                return;
            }

            if (faceVertices == null)
            {
                int iVertCount = faceManager.GetUserFaceVertexCount(userId);

                if (iVertCount > 0)
                {
                    faceVertices = new Vector3[iVertCount];
                }
            }

            if (faceVertices != null)
            {
                if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
                {
                    if (faceVertices != null && faceVertices[(int)facePoint] != Vector3.zero)
                    {
                        Vector3 facePointPos = faceVertices [(int)facePoint];
                        if (foregroundCamera)
                        {
                            facePointPos = GetOverlayPosition(facePointPos);
                        }

                        if (facePointTransform)
                        {
                            facePointTransform.position = facePointPos;
                        }

                        if (faceInfoText)
                        {
                            string sStatus = string.Format("{0}: {1}", facePoint, facePointPos);
                            faceInfoText.text = sStatus;
                        }
                    }
                }
            }
        }
    }
Exemplo n.º 10
0
    void Update()
    {
        // get the face-tracking manager instance
        if (faceManager == null)
        {
            kinectManager = KinectManager.Instance;
            faceManager   = FacetrackingManager.Instance;
        }

        // get user-id by user-index
        long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0;

        if (kinectManager && kinectManager.IsInitialized() && userId != 0 &&
            faceManager && faceManager.IsTrackingFace(userId))
        {
            if (faceManager.GetUserFaceProperties(userId, ref faceProps))
            {
                //NuitrackInterface.FaceLandmarks faceLM = new NuitrackInterface.FaceLandmarks();
                //if(faceProps.ContainsKey("landmarks"))
                //{
                //    faceLM = JsonUtility.FromJson<NuitrackInterface.FaceLandmarks>(faceProps["landmarks"]);
                //    // do something with the face landmarks
                //}

                //Vector2 leftEyePos = Vector2.zero;
                //if (faceProps.ContainsKey("lefteye"))
                //{
                //    leftEyePos = JsonUtility.FromJson<Vector2>(faceProps["lefteye"]);
                //    // do something with the left-eye position
                //}

                //Vector2 rightEyePos = Vector2.zero;
                //if (faceProps.ContainsKey("righteye"))
                //{
                //    rightEyePos = JsonUtility.FromJson<Vector2>(faceProps["righteye"]);
                //    // do something with the right-eye position
                //}

                string sFaceProps = "";
                foreach (string propName in faceProps.Keys)
                {
                    string propValue = faceProps[propName];
                    if (propValue != null && propValue.Length > maxPropLength)
                    {
                        propValue = propValue.Substring(0, maxPropLength) + "...";
                    }

                    sFaceProps += propName + "=" + propValue + "\n";
                }

                if (infoText && sFaceProps.Length > 0)
                {
                    //Debug.Log("FaceProps: " + sFaceProps);
                    infoText.text = sFaceProps;
                }
            }
        }
    }
Exemplo n.º 11
0
	void Update () 
	{
		// get the face-tracking manager instance
		if(faceManager == null)
		{
			kinectManager = KinectManager.Instance;
			faceManager = FacetrackingManager.Instance;
		}
		
		if (kinectManager && faceManager && faceManager.IsTrackingFace ()) {
			// get user-id by user-index
			long userId = kinectManager.GetUserIdByIndex (playerIndex);
			if (userId == 0)
				return;

			// head rotation
			Quaternion newRotation = initialRotation * faceManager.GetHeadRotation (userId, true);
			
			if (smoothFactor != 0f)
				transform.rotation = Quaternion.Slerp (transform.rotation, newRotation, smoothFactor * Time.deltaTime);
			else
				transform.rotation = newRotation;

			// head position
			Vector3 newPosition = faceManager.GetHeadPosition (userId, true);
			//Debug.Log(newPosition.ToString());

			if (verticalOffset != 0f) {
				Vector3 dirHead = new Vector3 (0, verticalOffset, 0);
				dirHead = transform.InverseTransformDirection (dirHead);
				newPosition += dirHead;
			}
			
//			if(smoothFactor != 0f)
//				transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
//			else

			//<----------------------------------------------correct modal position --------------------------------------------->
			//Debug.Log ("new Positionx   "  + newPosition.x.ToString());

			if(newPosition.x > 0f)
				newPosition.x = newPosition.x * 0.8f - deltaX; 
			else if(newPosition.x < 0f)
				newPosition.x = newPosition.x * 0.8f + deltaX;
			//<------------------------------------------------------------------------------------------------------------------>


			transform.position = newPosition;

			//Debug.Log ("transform.position   " + transform.position.x.ToString());
			//Debug.Log(newPosition.ToString());
		} else if (kinectManager && faceManager && !faceManager.IsTrackingFace ()) {
		
			transform.position = -Vector3.one;
		}

	}
Exemplo n.º 12
0
    // Make sure to kill the Kinect on quitting.
    void OnApplicationQuit()
    {
        // Shutdown Speech Recognizer and Kinect
        FacetrackingWrapper.FinishFaceTracking();
        FacetrackingWrapper.ShutdownKinectSensor();

        facetrackingInitialized = false;
        instance = null;
    }
    // Update is called once per frame
    void Update()
    {
        Matrix4x4 kinectToWorld = Matrix4x4.zero;

        //Quaternion quatTiltAngle = Quaternion.Euler(-manager.sensorAngle, 0.0f, 0.0f);
        //kinectToWorld.SetTRS(new Vector3(0.0f, manager.sensorHeight, 0.0f), quatTiltAngle, Vector3.one);

        if (!manager)
        {
            manager = KinectManager.Instance;
        }

        if (!faceManager)
        {
            faceManager = FacetrackingManager.Instance;
        }

        // get the face points
        if (manager != null && manager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
        {
            long userId = manager.GetUserIdByIndex(playerIndex);

            if (faceVertices == null)
            {
                //int iVertCount = faceManager.GetUserFaceVertexCount(userId);
                int iVertCount = 1347;

                if (iVertCount > 0)
                {
                    faceVertices = new Vector3[iVertCount];
                }

                check_f = false;
            }

            if (faceVertices != null)
            {
                if (faceManager.GetUserFaceVertices(userId, ref faceVertices))
                {
                    //-------------- get 35 special point-------------------
                    //Matrix4x4 kinectToWorld = manager.GetKinectToWorldMatrix();
                    HighDetailFacePoints[] facePoints = (HighDetailFacePoints[])System.Enum.GetValues(typeof(HighDetailFacePoints));

                    for (int i = 0; i < facePoints.Length; i++)
                    {
                        HighDetailFacePoints point = facePoints[i];
                        //dictFacePoints[point] = kinectToWorld.MultiplyPoint3x4(faceVertices[(int)point]);

                        dictFacePoints[point] = faceVertices[(int)point];
                    }

                    check_f = true;
                }
            }
        }
    }
Exemplo n.º 14
0
    // Update is called once per frame
    void FixedUpdate()
    {
        Debug.Log("inside fixedupdate of engagement meter");

        /*
         * updateCounter++;
         * float timeRequired=30f;
         * float iterationsRequired=timeRequired*50f;
         *
         * combinedPlayerEngagement/=50f;
         * leftPlayerEngagement/=50f;
         * rightPlayerEngagement/=50f;
         *
         * //Debug.Log("update counter"+updateCounter);
         * if(updateCounter==iterationsRequired)
         * {
         *
         *      //Debug.Log("twoengagement "+combinedPlayerEngagement);
         *      //Debug.Log("leftPlayerEngagement "+leftPlayerEngagement);
         *      //Debug.Log("rightPlayerEngagement "+rightPlayerEngagement);
         *      sendToCsv();
         *      pauseGame();
         * }
         *
         *
         * //		delay -= Time.deltaTime;
         * //		if(delay < 0){
         * //			//Debug.Log(faceManagerP1.GetHeadRotation(false));
         * //
         * //			delay = 1f;
         * //		}
         *
         * //sendToCsv();
         *
         * //Debug.Log(_KinectManager.GetJointPosition(_KinectManager.GetUserIdByIndex(0),4));
         */
        //lockArrow();
        //leadArrow();

        fixedUpdateCounter++;
        Debug.Log("in fixed update");
        if (faceManager == null)
        {
            //Debug.Log("In FixedUpdate, faceManager was null");
            faceManager = FacetrackingManager.Instance;
        }
        if (fixedUpdateCounter == 24)
        {
            measureEngagement();
            fixedUpdateCounter = 0;
        }
        //measureEngagement();
        //Resources.UnloadUnusedAssets();
    }
Exemplo n.º 15
0
    // Use this for initialization
    void Start()
    {
        faceManager = FacetrackingManager.Instance;
        //Debug.Log("in Start: facemanager instance"+ faceManager);

        leftPlayerEngagement               = 0;
        rightPlayerEngagement              = 0;
        combinedPlayerEngagement           = 0;
        leftPlayerEngagementsecondHalf     = 0;
        rightPlayerEngagementsecondHalf    = 0;
        combinedPlayerEngagementsecondHalf = 0;
        // updateCounter=0;
    }
Exemplo n.º 16
0
        private Vector3 TransformBones(KinectManager kinectManager, FacetrackingManager facetrackingManager)
        {
            Vector3 jp = Vector3.zero;

            for (int i = 0; i < userIDs.Count; i++)    // do this for every USer ID in
            {
                List <OverlayObject> bonesList;
                long userID = userIDs[i];
                if (bagOfBonesDictionary.ContainsKey(userID))
                {
                    bonesList = bagOfBonesDictionary[userID];
                    foreach (OverlayObject oo in bonesList)
                    {
                        int iJointIndex = (int)oo.TrackedJointType;


                        if (kinectManager.IsJointTracked(userID, iJointIndex))
                        {
                            var TryPos = TranslateBone(oo, userID, kinectManager);
                            if (TryPos != Vector3.zero)
                            {
                                oo.BoneGameObject.position = TryPos;
                            }

                            oo.BoneGameObject.rotation = RotateBone(oo, kinectManager, facetrackingManager, userID); //Quaternion.Slerp(oo.BoneGameObject.transform.rotation, qRotObject, smoothFactor * Time.deltaTime);

                            if (oo.ReferenceGameObjectTransform != null)
                            {
                                oo.BoneGameObject.localScale = ScaleBone(kinectManager, userID, oo, iJointIndex);
                            }
                            else
                            {
                                oo.BoneGameObject.localScale = new Vector3(1, 1, 1);
                            }
                        }

                        else // joint not tracked
                        {
                            oo.BoneGameObject.position      = oo.BoneGameObject.position;
                            oo.BoneGameObject.localRotation = oo.BoneGameObject.localRotation;
                            oo.BoneGameObject.localScale    = oo.BoneGameObject.localScale;
                        }
                    }
                }
                else
                {
                    print("Bag of Bones is Missing userID");
                }
            }
            return(jp);
        }
Exemplo n.º 17
0
    void Update()

    {      // float translation = Time.deltaTime * 10;
        // get the face-tracking manager instance
        if (manager == null)
        {
            manager = FacetrackingManager.Instance;
        }

        if (manager && manager.IsTrackingFace())
        {
            // set head position & rotation
            //	FaceFrameReference faceRef = e.FrameReference;
        }
    }
Exemplo n.º 18
0
    void OnDestroy()
    {
        if (isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null)
        {
            // finish face tracking
            sensorData.sensorInterface.FinishFaceTracking();
        }

//		// clean up
//		Resources.UnloadUnusedAssets();
//		GC.Collect();

        isFacetrackingInitialized = false;
        instance = null;
    }
Exemplo n.º 19
0
	void Update () 
	{
		if(faceManager == null)
		{
			faceManager = FacetrackingManager.Instance;
		}

		if(!kinectManager || !kinectManager.IsInitialized())
			return;
		if(!faceManager || !faceManager.IsFaceTrackingInitialized())
			return;

		long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
		if(userId == 0)
			return;

		colorTex = kinectManager.GetUsersClrTex();
		//faceRect = faceManager.GetFaceColorRect(userId);
		faceRect = GetHeadJointFaceRect(userId);

		if(!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
			return;

		int faceX = (int)faceRect.x;
		int faceY = (int)faceRect.y;
		int faceW = (int)faceRect.width;
		int faceH = (int)faceRect.height;

		if(faceX < 0) faceX = 0;
		if(faceY < 0) faceY = 0;
		if((faceX + faceW) > colorTex.width) faceW = colorTex.width - faceX;
		if((faceY + faceH) > colorTex.height) faceH = colorTex.height - faceY;

		if(faceTex.width != faceW || faceTex.height != faceH)
		{
			faceTex.Resize(faceW, faceH);
		}

		Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
		faceTex.SetPixels(colorPixels);
		faceTex.Apply();

		if(targetRenderer && targetRenderer.material)
		{
			targetRenderer.material.mainTexture = faceTex;
		}
	}
Exemplo n.º 20
0
    void Start()
    {
        //pec = new GameObject[arraySize];

        //Associate the variables with Components in the MainCamera
        grabScript          = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <GrabDropScript>();
        interactionManager  = GameObject.FindGameObjectWithTag("MainCamera").GetComponents <InteractionManager>();
        result              = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <Timer>();
        scriptLog           = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <log>();
        engagementMeter     = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <EngagementMeter>();
        faceTrackingManager = GameObject.FindGameObjectWithTag("MainCamera").GetComponent <FacetrackingManager>();
        scoreKeepBody       = GameObject.Find("BodyPlaceHolders").GetComponent <ScoreKeep>();



        source = GetComponent <AudioSource>();        //Gets AudioSource from the attached Object
    }
Exemplo n.º 21
0
    void Update()
    {
        // get the face-tracking manager instance
        if (faceManager == null)
        {
            kinectManager = KinectManager.Instance;
            faceManager   = FacetrackingManager.Instance;
        }

        if (kinectManager && faceManager && faceManager.IsTrackingFace())
        {
            // get user-id by user-index
            long userId = kinectManager.GetUserIdByIndex(playerIndex);
            if (userId == 0)
            {
                return;
            }

            // head rotation
            Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);

            if (smoothFactor != 0f)
            {
                transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
            }
            else
            {
                transform.rotation = newRotation;
            }

            // head position
            Vector3 newPosition = faceManager.GetHeadPosition(userId, true);

            if (verticalOffset != 0f)
            {
                Vector3 dirHead = new Vector3(horizontalOffset, verticalOffset, zOffset);
                dirHead      = transform.InverseTransformDirection(dirHead);
                newPosition += dirHead;
            }

//			if(smoothFactor != 0f)
//				transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
//			else
            transform.position = newPosition;
        }
    }
Exemplo n.º 22
0
    void Update()
    {
        // reference to face manager
        if (!faceManager)
        {
            faceManager = FacetrackingManager.Instance;
        }

        if (kinectManager && kinectManager.IsInitialized() && faceManager && faceManager.IsFaceTrackingInitialized())
        {
            // check for tracked user
            long userId = kinectManager.GetUserIdByIndex(playerIndex);

            if (userId != 0 && kinectManager.IsUserTracked(userId))
            {
                if (faceManager.GetUserAnimUnits(userId, ref dictAnimUnits))
                {
                    // process the blend shapes -> anim units
                    for (int i = 0; i < blendShapeCount; i++)
                    {
                        if (blendShape2AnimUnit.ContainsKey(blendShapeNames[i]))
                        {
                            KinectInterop.FaceShapeAnimations faceAnim = blendShape2AnimUnit[blendShapeNames[i]];
                            float animValue = dictAnimUnits[faceAnim];

                            // check for multiplier
                            float mul = 1f;
                            if (blendShape2Multiplier.ContainsKey(blendShapeNames[i]))
                            {
                                mul = blendShape2Multiplier[blendShapeNames[i]];
                            }

                            if (animValue * mul < 0f)
                            {
                                animValue = 0f;
                            }

                            // lerp to the new value
                            blendShapeValues[i] = Mathf.Lerp(blendShapeValues [i], animValue * mul * 100f, smoothFactor * Time.deltaTime);
                            skinnedMeshRenderer.SetBlendShapeWeight(i, blendShapeValues[i]);
                        }
                    }
                }
            }
        }
    }
Exemplo n.º 23
0
    // enables or disables the face-tracking component
    public void EnableFaceTracking(bool bEnable)
    {
        FacetrackingManager facetrackingManager = gameObject.GetComponent <FacetrackingManager>();

        if (facetrackingManager)
        {
            facetrackingManager.enabled = bEnable;
            LogToConsole("Face tracking is " + (bEnable ? "enabled" : "disabled"));

            if (bEnable)
            {
                StartCoroutine(CheckFacetrackingManager());
            }
        }
        else
        {
            LogErrorToConsole("FacetrackingManager-component not found.");
        }
    }
Exemplo n.º 24
0
    void Update()
    {
        // get the face-tracking manager instance
        if (faceManager == null)
        {
            faceManager = FacetrackingManager.Instance;
        }

        if (faceManager && faceManager.IsTrackingFace())
        {
            // head rotation
            Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(true);

            if (smoothFactor != 0f)
            {
                transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactor * Time.deltaTime);
            }
            else
            {
                transform.rotation = newRotation;
            }

            // head position
            Vector3 newPosition = faceManager.GetHeadPosition(true);

            if (verticalOffset != 0f)
            {
                Vector3 dirHead = new Vector3(0, verticalOffset, 0);
                dirHead      = transform.InverseTransformDirection(dirHead);
                newPosition += dirHead;
            }

//			if(smoothFactor != 0f)
//				transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactor * Time.deltaTime);
//			else
            transform.position = newPosition;
        }
    }
Exemplo n.º 25
0
    void Update()
    {
        if (faceManager == null)
        {
            faceManager = FacetrackingManager.Instance;
        }
        //unsafe for editor but can optimise if commented
        if (!kinectManager || !kinectManager.IsInitialized())
        {
            return;
        }
        if (!faceManager || !faceManager.IsFaceTrackingInitialized())
        {
            return;
        }


        long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);

        if (userId == 0)
        {
            return;
        }

        if (!backManager)
        {
            backManager = BackgroundRemovalManager.Instance;

            if (backManager)
            {
                // re-initialize the texture
                colorTex = null;
            }
        }

        if (backManager && backManager.IsBackgroundRemovalInitialized())
        {
            // use foreground image
            if (!colorTex)
            {
                colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
            }

            try
            {
                RenderTexture foregroundTex = (RenderTexture)backManager.GetForegroundTex();
                KinectInterop.RenderTex2Tex2D(foregroundTex, ref colorTex);
            }
            catch (System.Exception)
            {
                colorTex = (Texture2D)backManager.GetForegroundTex();
            }
        }
        else
        {
            // use color camera image
            colorTex = kinectManager.GetUsersClrTex();
        }

        //faceRect = faceManager.GetFaceColorRect(userId);
        faceRect = GetHeadJointFaceRect(userId);

        if (!colorTex || faceRect.width <= 0 || faceRect.height <= 0)
        {
            return;
        }

        int faceX = (int)faceRect.x;
        int faceY = (int)faceRect.y;
        int faceW = (int)faceRect.width;
        int faceH = (int)faceRect.height;

        if (faceX < 0)
        {
            faceX = 0;
        }
        if (faceY < 0)
        {
            faceY = 0;
        }
        if ((faceX + faceW) > colorTex.width)
        {
            faceW = colorTex.width - faceX;
        }
        if ((faceY + faceH) > colorTex.height)
        {
            faceH = colorTex.height - faceY;
        }

        if (faceTex.width != faceW || faceTex.height != faceH)
        {
            faceTex.Resize(faceW, faceH);
        }

        Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
        faceTex.SetPixels(colorPixels);
        faceTex.Apply();

        if (targetRenderer && targetRenderer.material)
        {
            targetRenderer.material.mainTexture = faceTex;
        }
    }
 void Awake()
 {
     manager = Camera.main.GetComponent<FacetrackingManager>();
 }
Exemplo n.º 27
0
    //----------------------------------- end of public functions --------------------------------------//
    void Start()
    {
        try
        {
            // get sensor data
            KinectManager kinectManager = KinectManager.Instance;
            if(kinectManager && kinectManager.IsInitialized())
            {
                sensorData = kinectManager.GetSensorData();
            }

            if(sensorData == null || sensorData.sensorInterface == null)
            {
                throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
            }

            if(debugText != null)
            {
                debugText.GetComponent<GUIText>().text = "Please, wait...";
            }

            // ensure the needed dlls are in place and face tracking is available for this interface
            bool bNeedRestart = false;
            if(sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
            {
                if(bNeedRestart)
                {
                    KinectInterop.RestartLevel(gameObject, "FM");
                    return;
                }
            }
            else
            {
                string sInterfaceName = sensorData.sensorInterface.GetType().Name;
                throw new Exception(sInterfaceName + ": Face tracking is not supported!");
            }

            // Initialize the face tracker
            if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
            {
                throw new Exception("Face tracking could not be initialized.");
            }

            instance = this;
            isFacetrackingInitialized = true;

            //DontDestroyOnLoad(gameObject);

            if(debugText != null)
            {
                debugText.GetComponent<GUIText>().text = "Ready.";
            }
        }
        catch(DllNotFoundException ex)
        {
            Debug.LogError(ex.ToString());
            if(debugText != null)
                debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations.";
        }
        catch (Exception ex)
        {
            Debug.LogError(ex.ToString());
            if(debugText != null)
                debugText.GetComponent<GUIText>().text = ex.Message;
        }
    }
Exemplo n.º 28
0
    void OnDestroy()
    {
        if(isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null)
        {
            // finish face tracking
            sensorData.sensorInterface.FinishFaceTracking();
        }

        //		// clean up
        //		Resources.UnloadUnusedAssets();
        //		GC.Collect();

        isFacetrackingInitialized = false;
        instance = null;
    }
    void StartFacetracker()
    {
        try
        {
            if(debugText != null)
                debugText.guiText.text = "Please, wait...";

            // initialize Kinect sensor as needed
            int rc = FacetrackingWrapper.InitKinectSensor();
            if(rc != 0)
            {
                throw new Exception("Initialization of Kinect sensor failed");
            }

            // Initialize the kinect speech wrapper
            rc = FacetrackingWrapper.InitFaceTracking();
            if (rc < 0)
            {
                throw new Exception(String.Format("Error initializing Kinect/FT: hr=0x{0:X}", rc));
            }

            if(ComputeColorMap)
            {
                // Initialize color map related stuff
                usersClrTex = new Texture2D(FacetrackingWrapper.GetImageWidth(), FacetrackingWrapper.GetImageHeight());
                usersClrRect = new Rect(Screen.width, Screen.height - usersClrTex.height, -usersClrTex.width, usersClrTex.height);

                colorImage = new Color32[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight()];
                videoBuffer = new byte[FacetrackingWrapper.GetImageWidth() * FacetrackingWrapper.GetImageHeight() * 4];
            }

            instance = this;
            facetrackingInitialized = true;

            DontDestroyOnLoad(gameObject);

            if(debugText != null)
                debugText.guiText.text = "Ready.";
        }
        catch(DllNotFoundException ex)
        {
            Debug.LogError(ex.ToString());
            if(debugText != null)
                debugText.guiText.text = "Please check the Kinect and FT-Library installations.";
        }
        catch (Exception ex)
        {
            Debug.LogError(ex.ToString());
            if(debugText != null)
                debugText.guiText.text = ex.Message;
        }
    }
    void Update()
    {
        // get the face-tracking manager instance
        if(faceManager == null)
        {
            kinectManager = KinectManager.Instance;
            faceManager = FacetrackingManager.Instance;
        }

        if(kinectManager && kinectManager.IsInitialized() &&
           faceManager && faceManager.IsTrackingFace() && foregroundCamera)
        {
            // get user-id by user-index
            long userId = kinectManager.GetUserIdByIndex(playerIndex);
            if(userId == 0)
                return;

            // get head position
            Vector3 newPosition = faceManager.GetHeadPosition(userId, true);

            // get head rotation
            Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);

            // rotational fix, provided by Richard Borys:
            // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
            Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
                Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;

            addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
            addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
            addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;

            newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
            // end of rotational fix

            if(smoothFactorRotation != 0f)
                transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactorRotation * Time.deltaTime);
            else
                transform.rotation = newRotation;

            // get the background rectangle (use the portrait background, if available)
            Rect backgroundRect = foregroundCamera.pixelRect;
            PortraitBackground portraitBack = PortraitBackground.Instance;

            if(portraitBack && portraitBack.enabled)
            {
                backgroundRect = portraitBack.GetBackgroundRect();
            }

            // model position
            newPosition = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
            if(newPosition == Vector3.zero)
            {
                // hide the model behind the camera
                newPosition.z = -10f;
            }

            if(verticalOffset != 0f)
            {
                // add the vertical offset
                Vector3 dirHead = new Vector3(0, verticalOffset, 0);
                dirHead = transform.InverseTransformDirection(dirHead);
                newPosition += dirHead;
            }

            // go to the new position
            if(smoothFactorMovement != 0f && transform.position.z >= 0f)
                transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactorMovement * Time.deltaTime);
            else
                transform.position = newPosition;

            // scale the model if needed
            if(transform.localScale.x != modelScaleFactor)
            {
                transform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
            }
        }
        else
        {
            // hide the model behind the camera
            if(transform.position.z >= 0f)
            {
                transform.position = new Vector3(0f, 0f, -10f);
            }
        }
    }
Exemplo n.º 31
0
    void Update()
    {
        // get the face-tracking manager instance
        if(manager == null)
        {
            manager = FacetrackingManager.Instance;
        }

        if(manager && manager.IsTrackingFace())
        {
            // set head position & rotation
            if(HeadTransform != null)
            {
                // head position
                Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement);

                if(smoothFactor != 0f)
                    HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime);
                else
                    HeadTransform.localPosition = newPosition;

                // head rotation
                Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);

                if(smoothFactor != 0f)
                    HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime);
                else
                    HeadTransform.localRotation = newRotation;
            }

            // apply animation units

            // AU0 - Upper Lip Raiser
            // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
            float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
            SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
            SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);

            // AU1 - Jaw Lowerer
            // 0=closed; 1=fully open; -1= closed, like 0
            float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
            SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);

            // AU2 – Lip Stretcher
            // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
            float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
            fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
            SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);

            float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
            fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
            SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);

            // AU3 – Brow Lowerer
            // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
            float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
            fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
            SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);

            float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
            fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
            SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);

            // AU4 – Lip Corner Depressor
            // 0=neutral; -1=very happy smile; +1=very sad frown
            float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
            fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
            SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);

            float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
            fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
            SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);

            // AU6, AU7 – Eyelid closed
            // 0=neutral; -1=raised; +1=fully lowered
            float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
            fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
            SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
            SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);

            float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
            fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
            SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
            SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised);
        }
    }
    void Update()
    {
        // get the face-tracking manager instance
        if(manager == null)
        {
            manager = FacetrackingManager.Instance;
        }

        if(manager && manager.IsTrackingFace())
        {
            // set head position & rotation
            if(HeadTransform != null)
            {
                // head position
                Vector3 newPosition = manager.GetHeadPosition(mirroredHeadMovement);

                // head rotation
                Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);

                // rotational fix, provided by Richard Borys:
                // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
                Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
                                                                          Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;

                addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
                addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
                addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;

                newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
                // end of rotational fix

                if(smoothFactor != 0f)
                    HeadTransform.rotation = Quaternion.Slerp(HeadTransform.rotation, newRotation, smoothFactor * Time.deltaTime);
                else
                    HeadTransform.rotation = newRotation;

                // check for head pos overlay
                if(foregroundCamera)
                {
                    // get the background rectangle (use the portrait background, if available)
                    Rect backgroundRect = foregroundCamera.pixelRect;
                    PortraitBackground portraitBack = PortraitBackground.Instance;

                    if(portraitBack && portraitBack.enabled)
                    {
                        backgroundRect = portraitBack.GetBackgroundRect();
                    }

                    KinectManager kinectManager = KinectManager.Instance;

                    if(kinectManager)
                    {
                        long userId = kinectManager.GetUserIdByIndex(manager.playerIndex);
                        Vector3 posColorOverlay = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);

                        if(posColorOverlay != Vector3.zero)
                        {
                            newPosition = posColorOverlay;

        //							if(overlayObj)
        //							{
        //								overlayObj.position = newPosition;
        //							}
                        }
                    }
                }
                else
                {
                    // move around the initial position
                    newPosition += HeadInitialPosition;
                }

                // vertical offet
                if(verticalOffset != 0f)
                {
                    // add the vertical offset
                    Vector3 dirHead = new Vector3(0, verticalOffset, 0);
                    dirHead = HeadTransform.InverseTransformDirection(dirHead);
                    newPosition += dirHead;
                }

                // set the position
                if(smoothFactor != 0f)
                    HeadTransform.position = Vector3.Lerp(HeadTransform.position, newPosition, smoothFactor * Time.deltaTime);
                else
                    HeadTransform.position = newPosition;

                // scale factor
                if(HeadTransform.localScale.x != modelScaleFactor)
                {
                    HeadTransform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
                }
            }

            // apply animation units

            // AU0 - Upper Lip Raiser
            // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
            float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
            SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
            SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);

            // AU1 - Jaw Lowerer
            // 0=closed; 1=fully open; -1= closed, like 0
            float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
            SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);

            // AU2 – Lip Stretcher
            // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
            float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
            fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
            SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);

            float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
            fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
            SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);

            // AU3 – Brow Lowerer
            // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
            float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
            fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
            SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);

            float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
            fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
            SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);

            // AU4 – Lip Corner Depressor
            // 0=neutral; -1=very happy smile; +1=very sad frown
            float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
            fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
            SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);

            float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
            fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
            SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);

            // AU6, AU7 – Eyelid closed
            // 0=neutral; -1=raised; +1=fully lowered
            float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
            fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
            SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
            SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);

            float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
            fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
            SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
            SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, LowerEyelidRightNeutral, LowerEyelidRightRaised);
        }
        else
        {
            // hide the model behind the camera
            if(HeadTransform && HeadTransform.position.z >= 0f)
            {
                HeadTransform.position = new Vector3(0f, 0f, -10f);
            }
        }
    }
Exemplo n.º 33
0
 void Awake()
 {
     manager = Camera.main.GetComponent <FacetrackingManager>();
 }
    // Make sure to kill the Kinect on quitting.
    void OnApplicationQuit()
    {
        // Shutdown Speech Recognizer and Kinect
        FacetrackingWrapper.FinishFaceTracking();
        FacetrackingWrapper.ShutdownKinectSensor();

        facetrackingInitialized = false;
        instance = null;
    }
Exemplo n.º 35
0
    void Update()
    {
        if (faceManager == null)
        {
            faceManager = FacetrackingManager.Instance;
        }

        if (!kinectManager || !kinectManager.IsInitialized())
        {
            return;
        }
        if (!faceManager || !faceManager.IsFaceTrackingInitialized())
        {
            return;
        }

        long userId = kinectManager.GetUserIdByIndex(playerIndex);

        if (userId == 0)
        {
            if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
            {
                targetObject.material.mainTexture = null;
            }

            return;
        }

        if (!backManager)
        {
            backManager = BackgroundRemovalManager.Instance;

            if (backManager)
            {
                // re-initialize the texture
                colorTex = null;
            }
        }

        if (!bSkipUpdatingForegroundTex && backManager && backManager.IsBackgroundRemovalInitialized())
        {
            // use foreground image
            Texture bmForeTex = backManager.GetForegroundTex();

            if (bmForeTex is RenderTexture)
            {
                foregroundTex = (RenderTexture)bmForeTex;
                bSkipUpdatingForegroundTex = (foregroundTex != null);
            }
            else
            {
                if (foregroundTex == null)
                {
                    foregroundTex = new RenderTexture(bmForeTex.width, bmForeTex.height, 0);
                }

                Graphics.Blit(bmForeTex, foregroundTex);
                bSkipUpdatingForegroundTex = false;
            }

//			if (!colorTex)
//			{
//				colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
//			}
//
//			try
//			{
//				foregroundTex = (RenderTexture)backManager.GetForegroundTex ();
//				//KinectInterop.RenderTex2Tex2D (foregroundTex, ref colorTex);
//			}
//			catch (System.Exception)
//			{
//				colorTex = (Texture2D)backManager.GetForegroundTex ();
//			}
        }
        else
        {
            // use color camera image
            if (!colorTex)
            {
                colorTex = kinectManager.GetUsersClrTex2D();
            }
        }

        //faceRect = faceManager.GetFaceColorRect(userId);
        faceRect = GetHeadJointFaceRect(userId);

        if (faceRect.width > 0 && faceRect.height > 0)
        {
            int faceX = (int)faceRect.x;
            int faceY = (int)faceRect.y;
            int faceW = (int)faceRect.width;
            int faceH = (int)faceRect.height;

            if (faceX < 0)
            {
                faceX = 0;
            }
            if (faceY < 0)
            {
                faceY = 0;
            }

            if (foregroundTex)
            {
                if ((faceX + faceW) > foregroundTex.width)
                {
                    faceW = foregroundTex.width - faceX;
                }
                if ((faceY + faceH) > foregroundTex.height)
                {
                    faceH = foregroundTex.height - faceY;
                }
            }
            else if (colorTex)
            {
                if ((faceX + faceW) > colorTex.width)
                {
                    faceW = colorTex.width - faceX;
                }
                if ((faceY + faceH) > colorTex.height)
                {
                    faceH = colorTex.height - faceY;
                }
            }

            if (faceTex.width != faceW || faceTex.height != faceH)
            {
                faceTex.Resize(faceW, faceH);
            }

            if (foregroundTex)
            {
                KinectInterop.RenderTex2Tex2D(foregroundTex, faceX, foregroundTex.height - faceY - faceH, faceW, faceH, ref faceTex);
            }
            else if (colorTex)
            {
                Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
                faceTex.SetPixels(colorPixels);
                faceTex.Apply();
            }

            if (targetObject && !targetObject.gameObject.activeSelf)
            {
                targetObject.gameObject.SetActive(true);
            }

            if (targetObject && targetObject.material)
            {
                targetObject.material.mainTexture = faceTex;
            }

            // don't rotate the transform - mesh follows the head rotation
            if (targetObject.transform.rotation != Quaternion.identity)
            {
                targetObject.transform.rotation = Quaternion.identity;
            }
        }
        else
        {
            if (targetObject && targetObject.gameObject.activeSelf)
            {
                targetObject.gameObject.SetActive(false);
            }

            if (targetObject && targetObject.material && targetObject.material.mainTexture != null)
            {
                targetObject.material.mainTexture = null;
            }
        }
    }
Exemplo n.º 36
0
    void Update()
    {
        int recHostId;
        int connectionId;
        int recChannelId;
        int dataSize;

        bool connListUpdated = false;

        if (backgroundImage && backgroundImage.texture == null)
        {
            backgroundImage.texture = manager ? manager.GetUsersLblTex() : null;
        }

        if (faceManager == null)
        {
            faceManager = FacetrackingManager.Instance;
        }

        if (gestureManager == null)
        {
            gestureManager = VisualGestureManager.Instance;
        }

        if (speechManager == null)
        {
            speechManager = SpeechManager.Instance;
        }

        try
        {
            byte             error   = 0;
            NetworkEventType recData = NetworkTransport.Receive(out recHostId, out connectionId, out recChannelId, recBuffer, bufferSize, out dataSize, out error);

            switch (recData)
            {
            case NetworkEventType.Nothing:                     //1
                break;

            case NetworkEventType.ConnectEvent:                //2
                if (recHostId == serverHostId && recChannelId == serverChannelId &&
                    !dictConnection.ContainsKey(connectionId))
                {
                    HostConnection conn = new HostConnection();
                    conn.hostId       = recHostId;
                    conn.connectionId = connectionId;
                    conn.channelId    = recChannelId;
                    conn.keepAlive    = true;
                    conn.reqDataType  = "ka,kb,km,kh";
                    //conn.matrixSent = false;

                    dictConnection[connectionId] = conn;
                    connListUpdated = true;

                    //LogToConsole(connectionId + "-conn: " + conn.reqDataType);
                }

//				// reset chunked face messages
//				sendFvMsg = string.Empty;
//				sendFvNextOfs = 0;
//
//				sendFtMsg = string.Empty;
//				sendFtNextOfs = 0;
                break;

            case NetworkEventType.DataEvent:                   //3
                if (recHostId == serverHostId && recChannelId == serverChannelId &&
                    dictConnection.ContainsKey(connectionId))
                {
                    HostConnection conn = dictConnection[connectionId];

                    int decompSize = 0;
                    if (decompressor != null && (recBuffer[0] > 127 || recBuffer[0] < 32))
                    {
                        decompSize = decompressor.Decompress(recBuffer, 0, compressBuffer, 0, dataSize);
                    }
                    else
                    {
                        System.Buffer.BlockCopy(recBuffer, 0, compressBuffer, 0, dataSize);
                        decompSize = dataSize;
                    }

                    string sRecvMessage = System.Text.Encoding.UTF8.GetString(compressBuffer, 0, decompSize);

                    if (sRecvMessage.StartsWith("ka"))
                    {
                        if (sRecvMessage == "ka")                         // vr-examples v1.0 keep-alive message
                        {
                            sRecvMessage = "ka,kb,km,kh";
                        }

                        conn.keepAlive               = true;
                        conn.reqDataType             = sRecvMessage;
                        dictConnection[connectionId] = conn;

                        //LogToConsole(connectionId + "-recv: " + conn.reqDataType);

                        // check for SR phrase-reset
                        int iIndexSR = sRecvMessage.IndexOf(",sr");
                        if (iIndexSR >= 0 && speechManager)
                        {
                            speechManager.ClearPhraseRecognized();
                            //LogToConsole("phrase cleared");
                        }
                    }
                }
                break;

            case NetworkEventType.DisconnectEvent:             //4
                if (dictConnection.ContainsKey(connectionId))
                {
                    dictConnection.Remove(connectionId);
                    connListUpdated = true;
                }
                break;
            }

            if (connListUpdated)
            {
                // get all connection IDs
                alConnectionId.Clear();
                alConnectionId.AddRange(dictConnection.Keys);

                // display the number of connections
                StringBuilder sbConnStatus = new StringBuilder();
                sbConnStatus.AppendFormat("Server running: {0} connection(s)", dictConnection.Count);

                foreach (int connId in dictConnection.Keys)
                {
                    HostConnection conn = dictConnection[connId];
                    int            iPort = 0; string sAddress = string.Empty; NetworkID network; NodeID destNode;

                    NetworkTransport.GetConnectionInfo(conn.hostId, conn.connectionId, out sAddress, out iPort, out network, out destNode, out error);
                    if (error == (int)NetworkError.Ok)
                    {
                        sbConnStatus.AppendLine().Append("    ").Append(sAddress).Append(":").Append(iPort);
                    }
                }

                LogToConsole(sbConnStatus.ToString());

                if (connStatusText)
                {
                    connStatusText.text = sbConnStatus.ToString();
                }
            }

            // send body frame to available connections
            const char delimiter  = ',';
            string     sBodyFrame = manager ? manager.GetBodyFrameData(ref liRelTime, ref fCurrentTime, delimiter) : string.Empty;

            if (sBodyFrame.Length > 0 && dictConnection.Count > 0)
            {
                StringBuilder sbSendMessage        = new StringBuilder();
                bool          bFaceParamsRequested = IsFaceParamsRequested();

                sbSendMessage.Append(manager.GetWorldMatrixData(delimiter)).Append('|');
                sbSendMessage.Append(sBodyFrame).Append('|');
                sbSendMessage.Append(manager.GetBodyHandData(ref liRelTime, delimiter)).Append('|');

                if (bFaceParamsRequested && faceManager && faceManager.IsFaceTrackingInitialized())
                {
                    sbSendMessage.Append(faceManager.GetFaceParamsAsCsv(delimiter)).Append('|');
                }

                if (gestureManager && gestureManager.IsVisualGestureInitialized())
                {
                    sbSendMessage.Append(gestureManager.GetGestureDataAsCsv(delimiter)).Append('|');
                }

                if (speechManager && speechManager.IsSapiInitialized())
                {
                    sbSendMessage.Append(speechManager.GetSpeechDataAsCsv(delimiter)).Append('|');
                }

                if (sbSendMessage.Length > 0 && sbSendMessage[sbSendMessage.Length - 1] == '|')
                {
                    sbSendMessage.Remove(sbSendMessage.Length - 1, 1);
                }

                byte[] btSendMessage = System.Text.Encoding.UTF8.GetBytes(sbSendMessage.ToString());

                //Debug.Log("Message " + sbSendMessage.Length + " chars: " + sbSendMessage.ToString());
                //Debug.Log("Encoded into " + btSendMessage.Length + " bytes: " + ByteArrayToString(btSendMessage, btSendMessage.Length));

                int compSize = 0;
                if (compressor != null && btSendMessage.Length > 100 && !websocketHost)
                {
                    compSize = compressor.Compress(btSendMessage, 0, btSendMessage.Length, compressBuffer, 0);
                }
                else
                {
                    System.Buffer.BlockCopy(btSendMessage, 0, compressBuffer, 0, btSendMessage.Length);
                    compSize = btSendMessage.Length;
                }

                //Debug.Log("Compressed into " + compSize + " bytes: " + ByteArrayToString(compressBuffer, compSize));

//				// check face-tracking requests
//				bool bFaceParams = false, bFaceVertices = false, bFaceUvs = false, bFaceTriangles = false;
//				if(faceManager && faceManager.IsFaceTrackingInitialized())
//					CheckFacetrackRequests(out bFaceParams, out bFaceVertices, out bFaceUvs, out bFaceTriangles);
//
//				byte[] btFaceParams = null;
//				if(bFaceParams)
//				{
//					string sFaceParams = faceManager.GetFaceParamsAsCsv();
//					if(!string.IsNullOrEmpty(sFaceParams))
//						btFaceParams = System.Text.Encoding.UTF8.GetBytes(sFaceParams);
//				}
//
//				// next chunk of data for face vertices
//				byte[] btFaceVertices = null;
//				string sFvMsgHead = string.Empty;
//				GetNextFaceVertsChunk(bFaceVertices, bFaceUvs, ref btFaceVertices, out sFvMsgHead);
//
//				// next chunk of data for face triangles
//				byte[] btFaceTriangles = null;
//				string sFtMsgHead = string.Empty;
//				GetNextFaceTrisChunk(bFaceTriangles, ref btFaceTriangles, out sFtMsgHead);

                foreach (int connId in alConnectionId)
                {
                    HostConnection conn = dictConnection[connId];

                    if (conn.keepAlive)
                    {
                        conn.keepAlive         = false;
                        dictConnection[connId] = conn;

                        if (conn.reqDataType != null && conn.reqDataType.Contains("kb,"))
                        {
                            //LogToConsole(conn.connectionId + "-sendkb: " + conn.reqDataType);

                            error = 0;
                            //if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btSendMessage, btSendMessage.Length, out error))
                            if (!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, compressBuffer, compSize, out error))
                            {
                                string sMessage = "Error sending body data via conn " + conn.connectionId + ": " + (NetworkError)error;
                                LogErrorToConsole(sMessage);

                                if (serverStatusText)
                                {
                                    serverStatusText.text = sMessage;
                                }
                            }
                        }

//						if(bFaceParams && btFaceParams != null &&
//							conn.reqDataType != null && conn.reqDataType.Contains("fp,"))
//						{
//							//Debug.Log(conn.connectionId + "-sendfp: " + conn.reqDataType);
//
//							error = 0;
//							if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceParams, btFaceParams.Length, out error))
//							{
//								string sMessage = "Error sending face params via conn " + conn.connectionId + ": " + (NetworkError)error;
//								Debug.LogError(sMessage);
//
//								if(serverStatusText)
//								{
//									serverStatusText.text = sMessage;
//								}
//							}
//						}
//
//						if(bFaceVertices && btFaceVertices != null &&
//							conn.reqDataType != null && conn.reqDataType.Contains("fv,"))
//						{
//							//Debug.Log(conn.connectionId + "-sendfv: " + conn.reqDataType + " - " + sFvMsgHead);
//
//							error = 0;
//							if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceVertices, btFaceVertices.Length, out error))
//							{
//								string sMessage = "Error sending face verts via conn " + conn.connectionId + ": " + (NetworkError)error;
//								Debug.LogError(sMessage);
//
//								if(serverStatusText)
//								{
//									serverStatusText.text = sMessage;
//								}
//							}
//						}
//
//						if(bFaceTriangles && btFaceTriangles != null &&
//							conn.reqDataType != null && conn.reqDataType.Contains("ft,"))
//						{
//							//Debug.Log(conn.connectionId + "-sendft: " + conn.reqDataType + " - " + sFtMsgHead);
//
//							error = 0;
//							if(!NetworkTransport.Send(conn.hostId, conn.connectionId, conn.channelId, btFaceTriangles, btFaceTriangles.Length, out error))
//							{
//								string sMessage = "Error sending face tris via conn " + conn.connectionId + ": " + (NetworkError)error;
//								Debug.LogError(sMessage);
//
//								if(serverStatusText)
//								{
//									serverStatusText.text = sMessage;
//								}
//							}
//						}
                    }
                }
            }
        }
        catch (System.Exception ex)
        {
            LogErrorToConsole(ex.Message + "\n" + ex.StackTrace);

            if (serverStatusText)
            {
                serverStatusText.text = ex.Message;
            }
        }
    }
Exemplo n.º 37
0
    void Update()
    {
        // get the face-tracking manager instance
        if (faceManager == null)
        {
            kinectManager = KinectManager.Instance;
            faceManager   = FacetrackingManager.Instance;
        }

        // get user-id by user-index
        long userId = kinectManager ? kinectManager.GetUserIdByIndex(playerIndex) : 0;

        if (kinectManager && kinectManager.IsInitialized() && userId != 0 &&
            faceManager && faceManager.IsTrackingFace(userId) && foregroundCamera)
        {
            // get head position
            Vector3 newPosition = faceManager.GetHeadPosition(userId, true);

            // get head rotation
            Quaternion newRotation = initialRotation * faceManager.GetHeadRotation(userId, true);

            // rotational fix, provided by Richard Borys:
            // The added rotation fixes rotational error that occurs when person is not centered in the middle of the kinect
            Vector3 addedRotation = newPosition.z != 0f ? new Vector3(Mathf.Rad2Deg * (Mathf.Tan(newPosition.y) / newPosition.z),
                                                                      Mathf.Rad2Deg * (Mathf.Tan(newPosition.x) / newPosition.z), 0) : Vector3.zero;

            addedRotation.x = newRotation.eulerAngles.x + addedRotation.x;
            addedRotation.y = newRotation.eulerAngles.y + addedRotation.y;
            addedRotation.z = newRotation.eulerAngles.z + addedRotation.z;

            newRotation = Quaternion.Euler(addedRotation.x, addedRotation.y, addedRotation.z);
            // end of rotational fix

            if (smoothFactorRotation != 0f)
            {
                transform.rotation = Quaternion.Slerp(transform.rotation, newRotation, smoothFactorRotation * Time.deltaTime);
            }
            else
            {
                transform.rotation = newRotation;
            }

            // get the background rectangle (use the portrait background, if available)
            Rect backgroundRect             = foregroundCamera.pixelRect;
            PortraitBackground portraitBack = PortraitBackground.Instance;

            if (portraitBack && portraitBack.enabled)
            {
                backgroundRect = portraitBack.GetBackgroundRect();
            }

            // model position
            newPosition = kinectManager.GetJointPosColorOverlay(userId, (int)KinectInterop.JointType.Head, foregroundCamera, backgroundRect);
            if (newPosition == Vector3.zero)
            {
                // hide the model behind the camera
                newPosition.z = -10f;
            }

            if (verticalOffset != 0f)
            {
                // add the vertical offset
                Vector3 dirHead = new Vector3(0, verticalOffset, 0);
                dirHead      = transform.InverseTransformDirection(dirHead);
                newPosition += dirHead;
            }

            // go to the new position
            if (smoothFactorMovement != 0f && transform.position.z >= 0f)
            {
                transform.position = Vector3.Lerp(transform.position, newPosition, smoothFactorMovement * Time.deltaTime);
            }
            else
            {
                transform.position = newPosition;
            }

            // scale the model if needed
            if (transform.localScale.x != modelScaleFactor)
            {
                transform.localScale = new Vector3(modelScaleFactor, modelScaleFactor, modelScaleFactor);
            }
        }
        else
        {
            // hide the model behind the camera
            if (transform.position.z >= 0f)
            {
                transform.position = new Vector3(0f, 0f, -10f);
            }
        }
    }
Exemplo n.º 38
0
    //----------------------------------- end of public functions --------------------------------------//

    void Awake()
    {
        instance = this;
    }
Exemplo n.º 39
0
    //----------------------------------- end of public functions --------------------------------------//


    void Start()
    {
        instance = this;

        try
        {
            // get sensor data
            KinectManager kinectManager = KinectManager.Instance;
            if (kinectManager && kinectManager.IsInitialized())
            {
                sensorData = kinectManager.GetSensorData();
            }

            if (sensorData == null || sensorData.sensorInterface == null)
            {
                throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
            }

            if (debugText != null)
            {
                debugText.text = "Please, wait...";
            }

            // ensure the needed dlls are in place and face tracking is available for this interface
            bool bNeedRestart = false;
            if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
            {
                if (bNeedRestart)
                {
                    KinectInterop.RestartLevel(gameObject, "FM");
                    return;
                }
            }
            else
            {
                string sInterfaceName = sensorData.sensorInterface.GetType().Name;
                throw new Exception(sInterfaceName + ": Face tracking is not supported!");
            }

            // Initialize the face tracker
            if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
            {
                throw new Exception("Face tracking could not be initialized.");
            }

            isFacetrackingInitialized = true;

            //DontDestroyOnLoad(gameObject);

            if (debugText != null)
            {
                debugText.text = "Ready.";
            }
        }
        catch (DllNotFoundException ex)
        {
            Debug.LogError(ex.ToString());
            if (debugText != null)
            {
                debugText.text = "Please check the Kinect and FT-Library installations.";
            }
        }
        catch (Exception ex)
        {
            Debug.LogError(ex.ToString());
            if (debugText != null)
            {
                debugText.text = ex.Message;
            }
        }
    }
Exemplo n.º 40
0
    void Update()
    {
        // get the face-tracking manager instance
        if (manager == null)
        {
            manager = FacetrackingManager.Instance;
        }

        if (manager && manager.IsTrackingFace())
        {
            // set head position & rotation
            if (HeadTransform != null)
            {
                // head position
                Vector3 newPosition = HeadInitialPosition + manager.GetHeadPosition(mirroredHeadMovement);

                if (smoothFactor != 0f)
                {
                    HeadTransform.localPosition = Vector3.Lerp(HeadTransform.localPosition, newPosition, smoothFactor * Time.deltaTime);
                }
                else
                {
                    HeadTransform.localPosition = newPosition;
                }

                // head rotation
                Quaternion newRotation = HeadInitialRotation * manager.GetHeadRotation(mirroredHeadMovement);

                if (smoothFactor != 0f)
                {
                    HeadTransform.localRotation = Quaternion.Slerp(HeadTransform.localRotation, newRotation, smoothFactor * Time.deltaTime);
                }
                else
                {
                    HeadTransform.localRotation = newRotation;
                }
            }

            // apply animation units

            // AU0 - Upper Lip Raiser
            // 0=neutral, covering teeth; 1=showing teeth fully; -1=maximal possible pushed down lip
            float fAU0 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipPucker);
            SetJointRotation(UpperLipLeft, UpperLipLeftAxis, fAU0, UpperLipLeftNeutral, UpperLipLeftUp);
            SetJointRotation(UpperLipRight, UpperLipRightAxis, fAU0, UpperLipRightNeutral, UpperLipRightUp);

            // AU1 - Jaw Lowerer
            // 0=closed; 1=fully open; -1= closed, like 0
            float fAU1 = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.JawOpen);
            SetJointRotation(Jaw, JawAxis, fAU1, JawNeutral, JawDown);

            // AU2 – Lip Stretcher
            // 0=neutral; 1=fully stretched (joker’s smile); -1=fully rounded (kissing mouth)
            float fAU2_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherLeft);
            fAU2_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_left * 2 - 1) : fAU2_left;
            SetJointRotation(LipLeft, LipLeftAxis, fAU2_left, LipLeftNeutral, LipLeftStretched);

            float fAU2_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipStretcherRight);
            fAU2_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU2_right * 2 - 1) : fAU2_right;
            SetJointRotation(LipRight, LipRightAxis, fAU2_right, LipRightNeutral, LipRightStretched);

            // AU3 – Brow Lowerer
            // 0=neutral; -1=raised almost all the way; +1=fully lowered (to the limit of the eyes)
            float fAU3_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyebrowLowerer);
            fAU3_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_left * 2 - 1) : fAU3_left;
            SetJointRotation(EyebrowLeft, EyebrowLeftAxis, fAU3_left, EyebrowLeftNeutral, EyebrowLeftLowered);

            float fAU3_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyebrowLowerer);
            fAU3_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU3_right * 2 - 1) : fAU3_right;
            SetJointRotation(EyebrowRight, EyebrowRightAxis, fAU3_right, EyebrowRightNeutral, EyebrowRightLowered);

            // AU4 – Lip Corner Depressor
            // 0=neutral; -1=very happy smile; +1=very sad frown
            float fAU4_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorLeft);
            fAU4_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_left * 2) : fAU4_left;
            SetJointRotation(LipCornerLeft, LipCornerLeftAxis, fAU4_left, LipCornerLeftNeutral, LipCornerLeftDepressed);

            float fAU4_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LipCornerDepressorRight);
            fAU4_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU4_right * 2) : fAU4_right;
            SetJointRotation(LipCornerRight, LipCornerRightAxis, fAU4_right, LipCornerRightNeutral, LipCornerRightDepressed);

            // AU6, AU7 – Eyelid closed
            // 0=neutral; -1=raised; +1=fully lowered
            float fAU6_left = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.LefteyeClosed);
            fAU6_left = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_left * 2 - 1) : fAU6_left;
            SetJointRotation(UpperEyelidLeft, UpperEyelidLeftAxis, fAU6_left, UpperEyelidLeftNeutral, UpperEyelidLeftLowered);
            SetJointRotation(LowerEyelidLeft, LowerEyelidLeftAxis, fAU6_left, LowerEyelidLeftNeutral, LowerEyelidLeftRaised);

            float fAU6_right = manager.GetAnimUnit(KinectInterop.FaceShapeAnimations.RighteyeClosed);
            fAU6_right = (platform == KinectInterop.DepthSensorPlatform.KinectSDKv2) ? (fAU6_right * 2 - 1) : fAU6_right;
            SetJointRotation(UpperEyelidRight, UpperEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, UpperEyelidRightLowered);
            SetJointRotation(LowerEyelidRight, LowerEyelidRightAxis, fAU6_right, UpperEyelidRightNeutral, LowerEyelidRightRaised);
        }
    }
Exemplo n.º 41
0
    void OnDestroy()
    {
        if(isFacetrackingInitialized && sensorData != null && sensorData.sensorInterface != null)
        {
            // finish face tracking
            if(sensorData != null && sensorData.sensorInterface != null)
            {
                sensorData.sensorInterface.FinishFaceTracking();
            }

            isFacetrackingInitialized = false;
            instance = null;
        }
    }
Exemplo n.º 42
0
 private void CheckFaceTrackingAvailability(long userId)
 {
     faceTracker = FacetrackingManager.Instance;
     faceTrackingAvailable [userId] = faceTracker != null && faceTracker.IsTrackingFace (userId);
 }
Exemplo n.º 43
0
 public bool IsFaceTrackingAvailable(long userId)
 {
     faceTracker = FacetrackingManager.Instance;
     return (userId == firstUserId || userId == secondUserId) && faceTracker.IsTrackingFace (userId);
 }