Ejemplo n.º 1
0
        public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
        {
            sourceFlags = dwFlags;

            NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
                NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;

            if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
            {
                nuiFlags |= NuiInitializeFlags.UsesSkeleton;
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
            {
                nuiFlags |= NuiInitializeFlags.UsesColor;
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
            {
                nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
            {
                nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
            {
                nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeAudio) != 0)
            {
                nuiFlags |= NuiInitializeFlags.UsesAudio;
            }

            FacetrackingManager[] faceManagers = GameObject.FindObjectsOfType(typeof(FacetrackingManager)) as FacetrackingManager[];
            if (faceManagers != null && faceManagers.Length > 0)
            {
                for (int i = 0; i < faceManagers.Length; i++)
                {
                    if (faceManagers[i].enabled)
                    {
                        //Debug.Log("Found FacetrackingManager => UsesColor");
                        nuiFlags |= NuiInitializeFlags.UsesColor;
                        break;
                    }
                }
            }

            SpeechManager[] speechManagers = GameObject.FindObjectsOfType(typeof(SpeechManager)) as SpeechManager[];
            if (speechManagers != null && speechManagers.Length > 0)
            {
                for (int i = 0; i < speechManagers.Length; i++)
                {
                    if (speechManagers[i].enabled)
                    {
                        //Debug.Log("Found SpeechManager => UsesAudio");
                        nuiFlags |= NuiInitializeFlags.UsesAudio;
                        break;
                    }
                }
            }

            int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);

            if (hr == 0)
            {
                // set sensor angle
                SetKinectElevationAngle((int)sensorAngle);

                // initialize Kinect interaction
                hr = InitKinectInteraction();
                if (hr != 0)
                {
                    Debug.LogError(string.Format("Error initializing KinectInteraction: hr=0x{0:X}", hr));
                }

                KinectInterop.SensorData sensorData = new KinectInterop.SensorData();

                sensorData.bodyCount = Constants.NuiSkeletonCount;
                sensorData.jointCount = Constants.JointCount;

                sensorData.depthCameraFOV = 46.6f;
                sensorData.colorCameraFOV = 48.6f;
                sensorData.depthCameraOffset = 0.01f;
                sensorData.faceOverlayOffset = 0.01f;

                NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
                //			sensorData.colorImageWidth = Constants.ColorImageWidth;
                //			sensorData.colorImageHeight = Constants.ColorImageHeight;

                if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
                {
                    //colorStreamHandle =  GetColorStreamHandle();
                    sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
                }

                NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
                //			sensorData.depthImageWidth = Constants.DepthImageWidth;
                //			sensorData.depthImageHeight = Constants.DepthImageHeight;

                if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
                {
                    //depthStreamHandle = GetDepthStreamHandle();
                    sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
                }

                if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
                {
                    sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
                }

                if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
                {
                    sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
                }

                if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
                {
                    skeletonFrame = new NuiSkeletonFrame()
                    {
                        SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
                    };

                    // default values used to pass to smoothing function
                    smoothParameters = new NuiTransformSmoothParameters();

                    smoothParameters.fSmoothing = 0.5f;
                    smoothParameters.fCorrection = 0.5f;
                    smoothParameters.fPrediction = 0.5f;
                    smoothParameters.fJitterRadius = 0.05f;
                    smoothParameters.fMaxDeviationRadius = 0.04f;
                }

                return sensorData;
            }
            else
            {
                Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
            }

            return null;
        }
Ejemplo n.º 2
0
        public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
        {
            KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
            sensorFlags = dwFlags;

            kinectSensor = KinectSensor.GetDefault();
            if (kinectSensor == null)
                return null;

            coordMapper = kinectSensor.CoordinateMapper;

            this.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
            sensorData.bodyCount = this.bodyCount;
            sensorData.jointCount = 25;

            sensorData.depthCameraFOV = 60f;
            sensorData.colorCameraFOV = 53.8f;
            sensorData.depthCameraOffset = -0.05f;
            sensorData.faceOverlayOffset = -0.04f;

            if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
            {
                if (!bUseMultiSource)
                    bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();

                bodyData = new Body[sensorData.bodyCount];
            }

            var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
            sensorData.colorImageWidth = frameDesc.Width;
            sensorData.colorImageHeight = frameDesc.Height;

            if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
            {
                if (!bUseMultiSource)
                    colorFrameReader = kinectSensor.ColorFrameSource.OpenReader();

                sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels];
            }

            sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width;
            sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height;

            if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
            {
                if (!bUseMultiSource)
                    depthFrameReader = kinectSensor.DepthFrameSource.OpenReader();

                sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels];
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
            {
                if (!bUseMultiSource)
                    bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader();

                sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
            }

            if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
            {
                if (!bUseMultiSource)
                    infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader();

                sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels];
            }

            //if(!kinectSensor.IsOpen)
            {
                //Debug.Log("Opening sensor, available: " + kinectSensor.IsAvailable);
                kinectSensor.Open();
            }

            float fWaitTime = Time.realtimeSinceStartup + 3f;
            while (!kinectSensor.IsAvailable && Time.realtimeSinceStartup < fWaitTime)
            {
                // wait for sensor to open
            }

            Debug.Log("K2-sensor " + (kinectSensor.IsOpen ? "opened" : "closed") +
                      ", available: " + kinectSensor.IsAvailable);

            if (bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
            {
                multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)((int)dwFlags & 0x3F));
            }

            return sensorData;
        }
Ejemplo n.º 3
0
        //----------------------------------- end of public functions --------------------------------------//
        void Start()
        {
            try
            {
                // get sensor data
                KinectManager kinectManager = KinectManager.Instance;
                if (kinectManager && kinectManager.IsInitialized())
                {
                    sensorData = kinectManager.GetSensorData();
                }

                if (sensorData == null || sensorData.sensorInterface == null)
                {
                    throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized.");
                }

                if (debugText != null)
                {
                    debugText.GetComponent<GUIText>().text = "Please, wait...";
                }

                // ensure the needed dlls are in place and speech recognition is available for this interface
                bool bNeedRestart = false;
                if (sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart))
                {
                    if (bNeedRestart)
                    {
                        KinectInterop.RestartLevel(gameObject, "SM");
                        return;
                    }
                }
                else
                {
                    string sInterfaceName = sensorData.sensorInterface.GetType().Name;
                    throw new Exception(sInterfaceName + ": Speech recognition is not supported!");
                }

                // Initialize the speech recognizer
                string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode);
                int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false);
                if (rc < 0)
                {
                    string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
                    throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage));
                }

                if (requiredConfidence > 0)
                {
                    sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence);
                }

                if (grammarFileName != string.Empty)
                {
                    // copy the grammar file from Resources, if available
                    //if(!File.Exists(grammarFileName))
                    {
                        TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset;

                        if (textRes != null)
                        {
                            string sResText = textRes.text;
                            File.WriteAllText(grammarFileName, sResText);
                        }
                    }

                    // load the grammar file
                    rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode, dynamicGrammar);
                    if (rc < 0)
                    {
                        string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
                        throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage);
                    }

                    //				// test dynamic grammar phrases
                    //				AddGrammarPhrase("addressBook", string.Empty, "Nancy Anderson", true, false);
                    //				AddGrammarPhrase("addressBook", string.Empty, "Cindy White", false, false);
                    //				AddGrammarPhrase("addressBook", string.Empty, "Oliver Lee", false, false);
                    //				AddGrammarPhrase("addressBook", string.Empty, "Alan Brewer", false, false);
                    //				AddGrammarPhrase("addressBook", string.Empty, "April Reagan", false, true);
                }

                instance = this;
                sapiInitialized = true;

                //DontDestroyOnLoad(gameObject);

                if (debugText != null)
                {
                    debugText.GetComponent<GUIText>().text = "Ready.";
                }
            }
            catch (DllNotFoundException ex)
            {
                Debug.LogError(ex.ToString());
                if (debugText != null)
                    debugText.GetComponent<GUIText>().text = "Please check the Kinect and SAPI installations.";
            }
            catch (Exception ex)
            {
                Debug.LogError(ex.ToString());
                if (debugText != null)
                    debugText.GetComponent<GUIText>().text = ex.Message;
            }
        }
        //----------------------------------- end of public functions --------------------------------------//
        void Start()
        {
            try
            {
                // get sensor data
                KinectManager kinectManager = KinectManager.Instance;
                if (kinectManager && kinectManager.IsInitialized())
                {
                    sensorData = kinectManager.GetSensorData();
                }

                if (sensorData == null || sensorData.sensorInterface == null)
                {
                    throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized.");
                }

                // ensure the needed dlls are in place and speech recognition is available for this interface
                bool bNeedRestart = false;
                bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart);

                if (bSuccess)
                {
                    if (bNeedRestart)
                    {
                        KinectInterop.RestartLevel(gameObject, "BR");
                        return;
                    }
                }
                else
                {
                    string sInterfaceName = sensorData.sensorInterface.GetType().Name;
                    throw new Exception(sInterfaceName + ": Background removal is not supported!");
                }

                // Initialize the background removal
                bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution);

                if (!bSuccess)
                {
                    throw new Exception("Background removal could not be initialized.");
                }

                // create the foreground image and alpha-image
                int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution);
                foregroundImage = new byte[imageLength];

                // get the needed rectangle
                Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution);

                // create the foreground texture
                foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false);

                // calculate the foreground rectangle
                if (foregroundCamera != null)
                {
                    Rect cameraRect = foregroundCamera.pixelRect;
                    float rectHeight = cameraRect.height;
                    float rectWidth = cameraRect.width;

                    if (rectWidth > rectHeight)
                        rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height);
                    else
                        rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width);

                    foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
                }

                instance = this;
                isBrInited = true;

                //DontDestroyOnLoad(gameObject);
            }
            catch (DllNotFoundException ex)
            {
                Debug.LogError(ex.ToString());
                if (debugText != null)
                    debugText.GetComponent<GUIText>().text = "Please check the Kinect and BR-Library installations.";
            }
            catch (Exception ex)
            {
                Debug.LogError(ex.ToString());
                if (debugText != null)
                    debugText.GetComponent<GUIText>().text = ex.Message;
            }
        }
Ejemplo n.º 5
0
        //----------------------------------- end of public functions --------------------------------------//
        void Start()
        {
            try
            {
                // get sensor data
                KinectManager kinectManager = KinectManager.Instance;
                if (kinectManager && kinectManager.IsInitialized())
                {
                    sensorData = kinectManager.GetSensorData();
                }

                if (sensorData == null || sensorData.sensorInterface == null)
                {
                    throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
                }

                if (debugText != null)
                {
                    debugText.GetComponent<GUIText>().text = "Please, wait...";
                }

                // ensure the needed dlls are in place and face tracking is available for this interface
                bool bNeedRestart = false;
                if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
                {
                    if (bNeedRestart)
                    {
                        KinectInterop.RestartLevel(gameObject, "FM");
                        return;
                    }
                }
                else
                {
                    string sInterfaceName = sensorData.sensorInterface.GetType().Name;
                    throw new Exception(sInterfaceName + ": Face tracking is not supported!");
                }

                // Initialize the face tracker
                if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
                {
                    throw new Exception("Face tracking could not be initialized.");
                }

                instance = this;
                isFacetrackingInitialized = true;

                //DontDestroyOnLoad(gameObject);

                if (debugText != null)
                {
                    debugText.GetComponent<GUIText>().text = "Ready.";
                }
            }
            catch (DllNotFoundException ex)
            {
                Debug.LogError(ex.ToString());
                if (debugText != null)
                    debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations.";
            }
            catch (Exception ex)
            {
                Debug.LogError(ex.ToString());
                if (debugText != null)
                    debugText.GetComponent<GUIText>().text = ex.Message;
            }
        }
Ejemplo n.º 6
0
        public void StartKinect()
        {
            try
            {
                // try to initialize the default Kinect2 sensor
                KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody;

                if (computeUserMap)
                    dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex;
                if (computeColorMap)
                    dwFlags |= KinectInterop.FrameSource.TypeColor;
                if (computeInfraredMap)
                    dwFlags |= KinectInterop.FrameSource.TypeInfrared;
                //			if(useAudioSource)
                //				dwFlags |= KinectInterop.FrameSource.TypeAudio;

                // open the default sensor
                sensorData = KinectInterop.OpenDefaultSensor(sensorInterfaces, dwFlags, sensorAngle, useMultiSourceReader);
                if (sensorData == null)
                {
                    if (sensorInterfaces == null || sensorInterfaces.Count == 0)
                    {
                        transform.parent.gameObject.SetActive(false);
                        //throw new Exception("No sensor found. Make sure you have installed the SDK and the sensor is connected.");
                        return;
                    }
                    else
                        throw new Exception("OpenDefaultSensor failed.");
                }

                // enable or disable getting height and angle info
                sensorData.hintHeightAngle = (autoHeightAngle != AutoHeightAngle.DontUse);

                //create the transform matrix - kinect to world
                Quaternion quatTiltAngle = Quaternion.Euler(-sensorAngle, 0.0f, 0.0f);
                kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), quatTiltAngle, Vector3.one);
            }
            catch (DllNotFoundException ex)
            {
                string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation.";

                //Debug.LogError(message);
                Debug.LogException(ex);

                if (calibrationText != null)
                {
                    calibrationText.GetComponent<GUIText>().text = message;
                }

                return;
            }
            catch (Exception ex)
            {
                string message = ex.Message;

                //Debug.LogError(message);
                Debug.LogException(ex);

                if (calibrationText != null)
                {
                    calibrationText.GetComponent<GUIText>().text = message;
                }

                return;
            }

            // set the singleton instance
            instance = this;

            // init skeleton structures
            bodyFrame = new KinectInterop.BodyFrameData(sensorData.bodyCount, KinectInterop.Constants.JointCount); // sensorData.jointCount
            bodyFrame.bTurnAnalisys = allowTurnArounds;

            KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters();

            switch (smoothing)
            {
                case Smoothing.Default:
                    smoothParameters.smoothing = 0.5f;
                    smoothParameters.correction = 0.5f;
                    smoothParameters.prediction = 0.5f;
                    smoothParameters.jitterRadius = 0.05f;
                    smoothParameters.maxDeviationRadius = 0.04f;
                    break;
                case Smoothing.Medium:
                    smoothParameters.smoothing = 0.5f;
                    smoothParameters.correction = 0.1f;
                    smoothParameters.prediction = 0.5f;
                    smoothParameters.jitterRadius = 0.1f;
                    smoothParameters.maxDeviationRadius = 0.1f;
                    break;
                case Smoothing.Aggressive:
                    smoothParameters.smoothing = 0.7f;
                    smoothParameters.correction = 0.3f;
                    smoothParameters.prediction = 1.0f;
                    smoothParameters.jitterRadius = 1.0f;
                    smoothParameters.maxDeviationRadius = 1.0f;
                    break;
            }

            // init data filters
            jointPositionFilter = new JointPositionsFilter();
            jointPositionFilter.Init(smoothParameters);

            // init the bone orientation constraints
            if (useBoneOrientationConstraints)
            {
                boneConstraintsFilter = new BoneOrientationsConstraint();
                boneConstraintsFilter.AddDefaultConstraints();
                boneConstraintsFilter.SetDebugText(calibrationText);
            }

            if (computeUserMap)
            {
                // Initialize depth & label map related stuff
                usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight, TextureFormat.ARGB32, false);

                usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
                usersHistogramImage = new Color32[usersMapSize];
                usersPrevState = new ushort[usersMapSize];
                usersHistogramMap = new float[5001];
            }

            if (computeColorMap)
            {
                // Initialize color map related stuff
                //usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false);
                usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight;
            }

            // try to automatically use the available avatar controllers in the scene
            if (avatarControllers.Count == 0)
            {
                MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];

                foreach (MonoBehaviour monoScript in monoScripts)
                {
                    if (typeof(AvatarController).IsAssignableFrom(monoScript.GetType()) &&
                       monoScript.enabled)
                    {
                        AvatarController avatar = (AvatarController)monoScript;
                        avatarControllers.Add(avatar);
                    }
                }
            }

            // try to automatically use the available gesture listeners in the scene
            if (gestureListeners.Count == 0)
            {
                MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];

                foreach (MonoBehaviour monoScript in monoScripts)
                {
                    if (typeof(KinectGestures.GestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
                       monoScript.enabled)
                    {
                        //KinectGestures.GestureListenerInterface gl = (KinectGestures.GestureListenerInterface)monoScript;
                        gestureListeners.Add(monoScript);
                    }
                }
            }

            // Initialize user list to contain all users.
            //alUserIds = new List<Int64>();
            //dictUserIdToIndex = new Dictionary<Int64, int>();

            kinectInitialized = true;

#if USE_SINGLE_KM_IN_MULTIPLE_SCENES
            //DontDestroyOnLoad(gameObject);
#endif

            // GUI Text.
            if (calibrationText != null)
            {
                calibrationText.GetComponent<GUIText>().text = "WAITING FOR USERS";
            }

            Debug.Log("Waiting for users.");
        }