public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); sensorFlags = dwFlags; kinectSensor = KinectSensor.GetDefault(); if(kinectSensor == null) return null; coordMapper = kinectSensor.CoordinateMapper; this.bodyCount = kinectSensor.BodyFrameSource.BodyCount; sensorData.bodyCount = this.bodyCount; sensorData.jointCount = 25; sensorData.depthCameraFOV = 60f; sensorData.colorCameraFOV = 53.8f; sensorData.depthCameraOffset = -0.05f; sensorData.faceOverlayOffset = -0.04f; if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { if(!bUseMultiSource) bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader(); bodyData = new Body[sensorData.bodyCount]; } var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba); sensorData.colorImageWidth = frameDesc.Width; sensorData.colorImageHeight = frameDesc.Height; if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { if(!bUseMultiSource) colorFrameReader = kinectSensor.ColorFrameSource.OpenReader(); sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels]; } sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width; sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height; if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { if(!bUseMultiSource) depthFrameReader = kinectSensor.DepthFrameSource.OpenReader(); sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels]; } if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { if(!bUseMultiSource) bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader(); sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels]; } if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { if(!bUseMultiSource) infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader(); sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels]; } //if(!kinectSensor.IsOpen) { //Debug.Log("Opening sensor, available: " + kinectSensor.IsAvailable); kinectSensor.Open(); } float fWaitTime = Time.realtimeSinceStartup + 3f; while(!kinectSensor.IsAvailable && Time.realtimeSinceStartup < fWaitTime) { // wait for sensor to open } Debug.Log("K2-sensor " + (kinectSensor.IsOpen ? "opened" : "closed") + ", available: " + kinectSensor.IsAvailable); if(bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen) { multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)((int)dwFlags & 0x3F)); } return sensorData; }
void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null; if (sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Visual gesture tracking cannot be started, because the KinectManager is missing or not initialized."); } if (sensorData.sensorInterface.GetSensorPlatform() != KinectInterop.DepthSensorPlatform.KinectSDKv2) { throw new Exception("Visual gesture tracking is only supported by Kinect SDK v2"); } // ensure the needed dlls are in place and face tracking is available for this interface bool bNeedRestart = false; if (IsVisualGesturesAvailable(ref bNeedRestart)) { if (bNeedRestart) { KinectInterop.RestartLevel(gameObject, "VG"); return; } } else { throw new Exception("Visual gesture tracking is not supported!"); } // initialize visual gesture tracker if (!InitVisualGestures()) { throw new Exception("Visual gesture tracking could not be initialized."); } // try to automatically detect the available gesture listeners in the scene if (visualGestureListeners.Count == 0) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach (MonoBehaviour monoScript in monoScripts) { // if(typeof(VisualGestureListenerInterface).IsAssignableFrom(monoScript.GetType()) && // monoScript.enabled) if ((monoScript is VisualGestureListenerInterface) && monoScript.enabled) { visualGestureListeners.Add(monoScript); } } } isVisualGestureInitialized = true; } catch (DllNotFoundException ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.text = "Please check the Kinect and VGB-Library installations."; } } catch (Exception ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.text = ex.Message; } } }
void Start() { manager = KinectManager.Instance; if (manager != null) { sensorData = manager.GetSensorData(); minDepth = Mathf.RoundToInt(minDistance * 1000f); maxDepth = Mathf.RoundToInt(maxDistance * 1000f); colorWidth = manager.GetColorImageWidth(); colorHeight = manager.GetColorImageHeight(); depthWidth = manager.GetDepthImageWidth(); depthHeight = manager.GetDepthImageHeight(); sampledWidth = depthWidth / sampleSize; sampledHeight = depthHeight / sampleSize; kinectToWorld = manager.GetKinectToWorldMatrix(); if(sensorData.depth2SpaceCoords == null) { sensorData.depth2SpaceCoords = new Vector3[depthWidth * depthHeight]; } sceneMeshPos = transform.position; if(!mirroredScene) { sceneMeshPos.x = -sceneMeshPos.x; } vertexType = new byte[sampledWidth * sampledHeight]; vertexIndex = new int[sampledWidth * sampledHeight]; CreateMesh(sampledWidth, sampledHeight); } }
void Start() { if (foregroundCamera == null) { // by default use the main camera foregroundCamera = Camera.main; } manager = KinectManager.Instance; if (manager && manager.IsInitialized()) { KinectInterop.SensorData sensorData = manager.GetSensorData(); if (sensorData != null && sensorData.sensorInterface != null && foregroundCamera != null) { // sensor platform sensorPlatform = sensorData.sensorIntPlatform; // get depth image size depthImageWidth = sensorData.depthImageWidth; depthImageHeight = sensorData.depthImageHeight; // calculate the foreground rectangles Rect cameraRect = foregroundCamera.pixelRect; float rectHeight = cameraRect.height; float rectWidth = cameraRect.width; if (rectWidth > rectHeight) { rectWidth = rectHeight * depthImageWidth / depthImageHeight; } else { rectHeight = rectWidth * depthImageHeight / depthImageWidth; } float foregroundOfsX = (cameraRect.width - rectWidth) / 2; float foregroundOfsY = (cameraRect.height - rectHeight) / 2; foregroundImgRect = new Rect(foregroundOfsX, foregroundOfsY, rectWidth, rectHeight); // create joint colliders numColliders = sensorData.jointCount; jointColliders = new GameObject[numColliders]; for (int i = 0; i < numColliders; i++) { string sColObjectName = ((KinectInterop.JointType)i).ToString() + "Collider"; jointColliders[i] = new GameObject(sColObjectName); jointColliders[i].transform.parent = transform; if (i == 0) { // circle collider for body center CircleCollider2D collider = jointColliders[i].AddComponent <CircleCollider2D>(); collider.radius = colliderWidth; } else { // box colliders for bones BoxCollider2D collider = jointColliders[i].AddComponent <BoxCollider2D>(); collider.size = new Vector2(colliderWidth, colliderWidth); } } } } }
public bool PollDepthFrame(KinectInterop.SensorData sensorData) { bool bNewFrame = false; if (bMultiSource && !bMultiFramesReady) { return(bNewFrame); } // poll for depth frame if (depthFrame != null && (long)depthFrame.Timestamp != lastDepthFrameTimestamp) { lastDepthFrameTimestamp = (long)depthFrame.Timestamp; if (dontHFlipDepthFrame) { // copy depth data, no flip int copyLength = sensorData.depthImage.Length * sizeof(ushort); Buffer.BlockCopy(depthFrame.Data, 0, sensorData.depthImage, 0, copyLength); } else { // copy depth data, h-flip int lenDepthFrame = sensorData.depthImage.Length; int lenDepthRow = sensorData.depthImageWidth; for (int i = 0; i < lenDepthFrame; i += lenDepthRow) { int iNext = i + lenDepthRow; for (int s = i, d = iNext - 1; s < iNext; s++, d--) { sensorData.depthImage[d] = depthFrame[s]; } } } sensorData.lastDepthFrameTime = (long)depthFrame.Timestamp; bNewFrame = true; if (bMultiSource) { bMultiFrameDepth = false; } } // poll for body-index frame if (userFrame != null && (long)userFrame.Timestamp != lastUserFrameTimestamp) { lastUserFrameTimestamp = (long)userFrame.Timestamp; int lenBodyIndexFrame = sensorData.bodyIndexImage.Length; int lenBodyIndexRow = sensorData.depthImageWidth; if (dontHFlipDepthFrame) { // copy body-index data, no flip for (int i = 0; i < lenBodyIndexFrame; i += lenBodyIndexRow) { int iNext = i + lenBodyIndexRow; for (int s = i, d = i; s < iNext; s++, d++) { ushort u = userFrame[s]; byte b = 255; if (u != 0 && bodyIdToIndex.ContainsKey(u)) { b = bodyIdToIndex[u]; } sensorData.bodyIndexImage[d] = b; } } } else { // copy body-index data, h-flip for (int i = 0; i < lenBodyIndexFrame; i += lenBodyIndexRow) { int iNext = i + lenBodyIndexRow; for (int s = i, d = iNext - 1; s < iNext; s++, d--) { ushort u = userFrame[s]; byte b = 255; if (u != 0 && bodyIdToIndex.ContainsKey(u)) { b = bodyIdToIndex[u]; } sensorData.bodyIndexImage[d] = b; } } } sensorData.lastBodyIndexFrameTime = (long)userFrame.Timestamp; bNewFrame = true; if (bMultiSource) { bMultiFrameBodyIndex = false; } // move floor estimation here if (sensorData.hintHeightAngle) { Vector3 floorNormal = new Vector3(userFrame.FloorNormal.X, userFrame.FloorNormal.Y, userFrame.FloorNormal.Z); if (floorNormal != Vector3.zero) { sensorData.sensorRotDetected = Quaternion.FromToRotation(floorNormal, Vector3.up); sensorData.sensorHgtDetected = Mathf.Abs(userFrame.Floor.Y / 1000f); } } } return(bNewFrame); }
public bool GetMultiSourceFrame(KinectInterop.SensorData sensorData) { return(false); }
public void FixJointOrientations(KinectInterop.SensorData sensorData, ref KinectInterop.BodyData bodyData) { }
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); sensorData.bodyCount = 6; sensorData.jointCount = 25; sensorData.depthCameraFOV = 60f; sensorData.colorCameraFOV = 53.8f; sensorData.depthCameraOffset = 0f; sensorData.faceOverlayOffset = 0f; sensorData.colorImageWidth = 1920; sensorData.colorImageHeight = 1080; sensorData.depthImageWidth = 512; sensorData.depthImageHeight = 424; Debug.Log("DummyK2-sensor opened"); return sensorData; }
public bool MapColorFrameToDepthCoords(KinectInterop.SensorData sensorData, ref Vector2[] vDepthCoords) { return(false); }
public bool PollInfraredFrame(KinectInterop.SensorData sensorData) { return(false); }
public bool MapDepthFrameToSpaceCoords(KinectInterop.SensorData sensorData, ref Vector3[] vSpaceCoords) { return(false); }
public bool PollBodyFrame(KinectInterop.SensorData sensorData, ref KinectInterop.BodyFrameData bodyFrame, ref Matrix4x4 kinectToWorld, bool bIgnoreJointZ) { bool bNewFrame = false; // look for frame int hr = PollBodyFrame(); if (hr != 0) { Debug.Log("PollBodyFrame error: " + hr); } if (hr == 0) { int bodyFrameIndex = GetBodyFrameIndex(); if (bodyFrameIndex != lastBodyFrameIndex) { lastBodyFrameIndex = bodyFrameIndex; long timeNowTicks = DateTime.Now.Ticks; // get body index frame var pBodyIndexData = GCHandle.Alloc(sensorData.bodyIndexImage, GCHandleType.Pinned); hr = GetBodyIndexData(pBodyIndexData.AddrOfPinnedObject()); pBodyIndexData.Free(); if (hr != 0) { Debug.Log("GetBodyIndexData() error: " + hr); } sensorData.lastBodyIndexFrameTime = (hr == 0) ? timeNowTicks : sensorData.lastBodyIndexFrameTime; bodyFrame.liPreviousTime = bodyFrame.liRelativeTime; bodyFrame.liRelativeTime = timeNowTicks; if (sensorData.hintHeightAngle) { Vector4 floorInfo = Vector4.zero; if (GetFloorInfo(ref floorInfo) != 0) { Vector3 floorPlane = (Vector3)floorInfo; sensorData.sensorRotDetected = Quaternion.FromToRotation(floorPlane, Vector3.up); sensorData.sensorHgtDetected = Mathf.Abs(floorInfo.w / 1000f); } } // int bodySize1 = Marshal.SizeOf(typeof(ObtBody)); // int bodySize2 = GetBodyDataSize(); int bodyCount = GetBodyCount(); if (lastBodyCount != bodyCount) { sbDebugBodies.Append(bodyCount).Append(" bodies - "); } for (int i = 0; i < sensorData.bodyCount; i++) { // compare to real body count if (i >= bodyCount || !obtBodyInited) { bodyFrame.bodyData[i].bIsTracked = 0; continue; } // get body and joints data hr = GetBodyData(i, ref obtBody); if (lastBodyCount != bodyCount) { sbDebugBodies.Append(obtBody.id).Append(":").Append(obtBody.status).Append(":").Append(hr).Append(" "); } // if there is error, consider body as not-tracked if (hr != 0) { bodyFrame.bodyData[i].bIsTracked = 0; continue; } // set body tracking state bodyFrame.bodyData[i].bIsTracked = (short)(obtBody.status != (byte)ObtBodyStatus.BODY_STATUS_NOT_TRACKING ? 1 : 0); if (obtBody.status != (byte)ObtBodyStatus.BODY_STATUS_NOT_TRACKING) { // transfer body and joints data bodyFrame.bodyData[i].liTrackingID = (long)obtBody.id; for (int j = 0; j < sensorData.jointCount; j++) { KinectInterop.JointData jointData = bodyFrame.bodyData[i].joint[j]; int obtJI = BodyJoint2AstraJoint[j]; if (obtJI >= 0) { jointData.trackingState = (KinectInterop.TrackingState)obtBody.joints[obtJI].status; } else { jointData.trackingState = KinectInterop.TrackingState.NotTracked; } if (jointData.trackingState != KinectInterop.TrackingState.NotTracked) { Vector3 jointPos = obtBody.joints[obtJI].worldPosition / 1000f; float jPosZ = (bIgnoreJointZ && j > 0) ? bodyFrame.bodyData[i].joint[0].kinectPos.z : jointPos.z; jointData.kinectPos = jointPos; jointData.position = kinectToWorld.MultiplyPoint3x4(new Vector3(jointPos.x, jointPos.y, jPosZ)); } jointData.orientation = Quaternion.identity; if (j == 0) { bodyFrame.bodyData[i].position = jointData.position; bodyFrame.bodyData[i].orientation = jointData.orientation; } bodyFrame.bodyData[i].joint[j] = jointData; } // processes some special joint cases ProcessBodyFrameSpecialCases(i, ref bodyFrame); // // hand states - is this available on orbbec? // bodyFrame.bodyData[i].leftHandState = (KinectInterop.HandState)body.HandLeftState; // bodyFrame.bodyData[i].leftHandConfidence = (KinectInterop.TrackingConfidence)body.HandLeftConfidence; // // bodyFrame.bodyData[i].rightHandState = (KinectInterop.HandState)body.HandRightState; // bodyFrame.bodyData[i].rightHandConfidence = (KinectInterop.TrackingConfidence)body.HandRightConfidence; } } if (sbDebugBodies.Length > 0) { Debug.Log(sbDebugBodies.ToString()); sbDebugBodies.Remove(0, sbDebugBodies.Length); } lastBodyCount = bodyCount; bNewFrame = true; } // release the frame ReleaseBodyFrame(); } return(bNewFrame); }
public void FreeMultiSourceFrame(KinectInterop.SensorData sensorData) { }
// private Vector3 sizeObjectHandLeft = Vector3.zero; // private Vector3 sizeObjectHandRight = Vector3.zero; void Start() { manager = KinectManager.Instance; sensorData = manager ? manager.GetSensorData() : null; }
// public bool IsBodyTurned(ref KinectInterop.BodyData bodyData) // { // return false; // } public Vector2 MapSpacePointToDepthCoords(KinectInterop.SensorData sensorData, Vector3 spacePos) { Vector2 depthPos = coordMapper != null?coordMapper.MapSpacePointToDepthCoords(spacePos) : Vector2.zero; return(depthPos); }
//----------------------------------- end of public functions --------------------------------------// void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if(kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if(sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized."); } if(debugText != null) { debugText.GetComponent<GUIText>().text = "Please, wait..."; } // ensure the needed dlls are in place and speech recognition is available for this interface bool bNeedRestart = false; if(sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart)) { if(bNeedRestart) { KinectInterop.RestartLevel(gameObject, "SM"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Speech recognition is not supported!"); } // Initialize the speech recognizer string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode); int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false); if (rc < 0) { string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc); throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage)); } if(requiredConfidence > 0) { sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence); } if(grammarFileName != string.Empty) { // copy the grammar file from Resources, if available if(!File.Exists(grammarFileName)) { TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset; if(textRes != null) { string sResText = textRes.text; File.WriteAllText(grammarFileName, sResText); } } // load the grammar file rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode); if (rc < 0) { string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc); throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage); } } instance = this; sapiInitialized = true; //DontDestroyOnLoad(gameObject); if(debugText != null) { debugText.GetComponent<GUIText>().text = "Ready."; } } catch(DllNotFoundException ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = "Please check the Kinect and SAPI installations."; } catch (Exception ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = ex.Message; } }
public Vector3 MapDepthPointToSpaceCoords(KinectInterop.SensorData sensorData, Vector2 depthPos, ushort depthVal) { Vector3 spacePos = coordMapper != null?coordMapper.MapDepthPointToSpaceCoords(depthPos, depthVal) : Vector3.zero; return(spacePos); }
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); //sensorFlags = dwFlags; kinectSensor = KinectSensor.GetDefault(); if(kinectSensor == null) return null; coordMapper = kinectSensor.CoordinateMapper; this.bodyCount = kinectSensor.BodyFrameSource.BodyCount; sensorData.bodyCount = this.bodyCount; sensorData.jointCount = 25; sensorData.depthCameraFOV = 60f; sensorData.colorCameraFOV = 53.8f; sensorData.depthCameraOffset = -0.03f; if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { if(!bUseMultiSource) bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader(); bodyData = new Body[sensorData.bodyCount]; } var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba); sensorData.colorImageWidth = frameDesc.Width; sensorData.colorImageHeight = frameDesc.Height; if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { if(!bUseMultiSource) colorFrameReader = kinectSensor.ColorFrameSource.OpenReader(); sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels]; } sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width; sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height; if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { if(!bUseMultiSource) depthFrameReader = kinectSensor.DepthFrameSource.OpenReader(); sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels]; } if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { if(!bUseMultiSource) bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader(); sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels]; } if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { if(!bUseMultiSource) infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader(); sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels]; } if(!kinectSensor.IsOpen) { kinectSensor.Open(); } if(bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen) { multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)dwFlags); } return sensorData; }
public bool MapDepthFrameToSpaceCoords(KinectInterop.SensorData sensorData, ref Vector3[] vSpaceCoords) { bool bSuccess = coordMapper != null?coordMapper.MapDepthFrameToSpaceCoords(sensorData, ref vSpaceCoords) : false; return(bSuccess); }
//----------------------------------- end of public functions --------------------------------------// void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if(kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if(sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized."); } // ensure the needed dlls are in place and speech recognition is available for this interface bool bNeedRestart = false; bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart); if(bSuccess) { if(bNeedRestart) { KinectInterop.RestartLevel(gameObject, "BR"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Background removal is not supported!"); } // Initialize the background removal bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution); if (!bSuccess) { throw new Exception("Background removal could not be initialized."); } // create the foreground image and alpha-image int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution); foregroundImage = new byte[imageLength]; // get the needed rectangle Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution); // create the foreground texture foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false); // calculate the foreground rectangle if(foregroundCamera != null) { Rect cameraRect = foregroundCamera.pixelRect; float rectHeight = cameraRect.height; float rectWidth = cameraRect.width; if(rectWidth > rectHeight) rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height); else rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width); foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight); } instance = this; isBrInited = true; //DontDestroyOnLoad(gameObject); } catch(DllNotFoundException ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = "Please check the Kinect and BR-Library installations."; } catch (Exception ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = ex.Message; } }
public Vector2 MapDepthPointToColorCoords(KinectInterop.SensorData sensorData, Vector2 depthPos, ushort depthVal) { Vector2 colorPos = coordMapper != null?coordMapper.MapDepthPointToColorCoords(depthPos, depthVal) : Vector2.zero; return(colorPos); }
public bool PollColorFrame(KinectInterop.SensorData sensorData) { bool bNewFrame = false; if (bMultiSource && !bMultiFramesReady) { return(bNewFrame); } // check if color camera is webcam if (bWebColorStream && colorWebcamData != null && colorWebcamTimestamp != lastColorFrameTimestamp) { lastColorFrameTimestamp = colorWebcamTimestamp; if (sensorData.colorImageTexture2D && sensorData.colorImageTexture2D.width == colorWebCam.width && sensorData.colorImageTexture2D.height == colorWebCam.height) { sensorData.colorImageTexture2D.SetPixels32(colorWebcamData); } sensorData.lastColorFrameTime = colorWebcamTimestamp; bNewFrame = true; } // poll for color frame else if (colorFrame != null && (long)colorFrame.Timestamp != lastColorFrameTimestamp) { lastColorFrameTimestamp = (long)colorFrame.Timestamp; //sensorData.colorImage = colorFrame.Data; // copy color data h-flip, BGR to RGB int lenColorFrame = sensorData.colorImage.Length; int lenColorRow = sensorData.colorImageWidth * 3; if (dontHFlipColorFrame) { // copy depth data, no flip for (int i = 0; i < lenColorFrame; i += lenColorRow) { int iNext = i + lenColorRow; for (int s = i, d = i; s < iNext; s += 3, d += 3) { sensorData.colorImage[d] = colorFrame.Data[s + 2]; sensorData.colorImage[d + 1] = colorFrame.Data[s + 1]; sensorData.colorImage[d + 2] = colorFrame.Data[s]; } } } else { // copy color data, h-flip for (int i = 0; i < lenColorFrame; i += lenColorRow) { int iNext = i + lenColorRow; for (int s = i, d = iNext - 3; s < iNext; s += 3, d -= 3) { sensorData.colorImage[d] = colorFrame.Data[s + 2]; sensorData.colorImage[d + 1] = colorFrame.Data[s + 1]; sensorData.colorImage[d + 2] = colorFrame.Data[s]; } } } sensorData.lastColorFrameTime = (long)colorFrame.Timestamp; bNewFrame = true; } if (bNewFrame && bMultiSource) { bMultiFrameColor = false; } return(bNewFrame); }
public bool InitBackgroundRemoval(KinectInterop.SensorData sensorData, bool isHiResPrefered) { return(KinectInterop.InitBackgroundRemoval(sensorData, isHiResPrefered)); }
void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if (kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if (sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized."); } if (debugText != null) { debugText.text = "Please, wait..."; } // ensure the needed dlls are in place and speech recognition is available for this interface bool bNeedRestart = false; if (sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart)) { if (bNeedRestart) { KinectInterop.RestartLevel(gameObject, "SM"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Speech recognition is not supported!"); } // Initialize the speech recognizer string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode); int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false); if (rc < 0) { string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc); throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage)); } if (requiredConfidence > 0) { sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence); } if (grammarFileName != string.Empty) { // copy the grammar file from Resources, if available //if(!File.Exists(grammarFileName)) { TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset; if (textRes != null) { string sResText = textRes.text; File.WriteAllText(grammarFileName, sResText); } else { throw new Exception("Couldn't find grammar resource: " + grammarFileName + ".txt"); } } // load the grammar file rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode, dynamicGrammar); if (rc < 0) { string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc); throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage); } // // test dynamic grammar phrases // AddGrammarPhrase("addressBook", string.Empty, "Nancy Anderson", true, false); // AddGrammarPhrase("addressBook", string.Empty, "Cindy White", false, false); // AddGrammarPhrase("addressBook", string.Empty, "Oliver Lee", false, false); // AddGrammarPhrase("addressBook", string.Empty, "Alan Brewer", false, false); // AddGrammarPhrase("addressBook", string.Empty, "April Reagan", false, true); } sapiInitialized = true; //DontDestroyOnLoad(gameObject); if (debugText != null) { debugText.text = "Ready."; } // try to automatically detect the available speech recognition listeners in the scene if (speechRecognitionListeners.Count == 0) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach (MonoBehaviour monoScript in monoScripts) { if ((monoScript is SpeechRecognitionInterface) && monoScript.enabled) { speechRecognitionListeners.Add(monoScript); } } } } catch (DllNotFoundException ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.text = "Please check the Kinect and SAPI installations."; } } catch (Exception ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.text = ex.Message; } } }
public void FinishBackgroundRemoval(KinectInterop.SensorData sensorData) { KinectInterop.FinishBackgroundRemoval(sensorData); bBackgroundRemovalInited = false; }
//----------------------------------- end of public functions --------------------------------------// void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if (kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if (sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized."); } if (debugText != null) { debugText.GetComponent <GUIText>().text = "Please, wait..."; } // ensure the needed dlls are in place and face tracking is available for this interface bool bNeedRestart = false; if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart)) { if (bNeedRestart) { KinectInterop.RestartLevel(gameObject, "FM"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Face tracking is not supported!"); } // Initialize the face tracker if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect)) { throw new Exception("Face tracking could not be initialized."); } instance = this; isFacetrackingInitialized = true; //DontDestroyOnLoad(gameObject); if (debugText != null) { debugText.GetComponent <GUIText>().text = "Ready."; } } catch (DllNotFoundException ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations."; } } catch (Exception ex) { Debug.LogError(ex.ToString()); if (debugText != null) { debugText.GetComponent <GUIText>().text = ex.Message; } } }
public bool UpdateBackgroundRemoval(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, bool bAlphaTexOnly) { return(KinectInterop.UpdateBackgroundRemoval(sensorData, isHiResPrefered, defaultColor, bAlphaTexOnly)); }
void Start() { // int hr = 0; try { // if(startKinectServer) // { // // start the Kinect-server app // hr = StartKinectServer(); // if (hr != 0) // { // throw new Exception("Kinect2Server not started"); // } // } // try to initialize the default Kinect2 sensor KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody; if(computeUserMap) dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex; if(computeColorMap) dwFlags |= KinectInterop.FrameSource.TypeColor; if(computeInfraredMap) dwFlags |= KinectInterop.FrameSource.TypeInfrared; // hr = KinectInterop.InitDefaultKinectSensor(dwFlags, KinectInterop.Constants.ColorImageWidth, KinectInterop.Constants.ColorImageHeight); // if (hr != 0) // { // throw new Exception("InitDefaultKinectSensor failed"); // } // open the default kinect sensor sensorData = KinectInterop.OpenDefaultKinectSensor(dwFlags); if (sensorData == null) { throw new Exception("OpenDefaultKinectSensor failed"); } // transform matrix - kinect to world kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), Quaternion.identity, Vector3.one); } catch(DllNotFoundException ex) { string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation."; Debug.LogError(message); Debug.LogException(ex); if(calibrationText != null) { calibrationText.guiText.text = message; } return; } catch(Exception ex) { string message = ex.Message; Debug.LogError(message); Debug.LogException(ex); if(calibrationText != null) { calibrationText.guiText.text = message; } return; } // init skeleton structures bodyFrame = new KinectInterop.BodyFrameData(true); KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters(); switch(smoothing) { case Smoothing.Default: smoothParameters.smoothing = 0.5f; smoothParameters.correction = 0.5f; smoothParameters.prediction = 0.5f; smoothParameters.jitterRadius = 0.05f; smoothParameters.maxDeviationRadius = 0.04f; break; case Smoothing.Medium: smoothParameters.smoothing = 0.5f; smoothParameters.correction = 0.1f; smoothParameters.prediction = 0.5f; smoothParameters.jitterRadius = 0.1f; smoothParameters.maxDeviationRadius = 0.1f; break; case Smoothing.Aggressive: smoothParameters.smoothing = 0.7f; smoothParameters.correction = 0.3f; smoothParameters.prediction = 1.0f; smoothParameters.jitterRadius = 1.0f; smoothParameters.maxDeviationRadius = 1.0f; break; } // init data filters jointPositionFilter = new JointPositionsFilter(); jointPositionFilter.Init(smoothParameters); // init the bone orientation constraints if(useBoneOrientationConstraints) { boneConstraintsFilter = new BoneOrientationsConstraint(); boneConstraintsFilter.AddDefaultConstraints(); boneConstraintsFilter.SetDebugText(calibrationText); } // get the main camera rectangle Rect cameraRect = Camera.main.pixelRect; // calculate map width and height in percent, if needed if(MapsPercentWidth == 0f) MapsPercentWidth = (sensorData.depthImageWidth / 2) / cameraRect.width; if(MapsPercentHeight == 0f) MapsPercentHeight = (sensorData.depthImageHeight / 2) / cameraRect.height; if(computeUserMap) { // init user-depth structures //depthImage = new KinectInterop.DepthBuffer(true); //bodyIndexImage = new KinectInterop.BodyIndexBuffer(true); // Initialize depth & label map related stuff usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight); usersMapRect = new Rect(cameraRect.width - cameraRect.width * MapsPercentWidth, cameraRect.height, cameraRect.width * MapsPercentWidth, -cameraRect.height * MapsPercentHeight); usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight; usersHistogramImage = new Color32[usersMapSize]; usersPrevState = new ushort[usersMapSize]; usersHistogramMap = new float[5001]; } if(computeColorMap) { // init color image structures //colorImage = new KinectInterop.ColorBuffer(true); // Initialize color map related stuff usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false); usersClrRect = new Rect(cameraRect.width - 320, cameraRect.height, 320, -180); usersCaliBox = new Rect(cameraRect.width - 160 - 320 / 14, cameraRect.height - 90, 320 / 7, 90); usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight; if(computeUserMap && displayColorMap) { usersMapRect.x -= cameraRect.width * MapsPercentWidth; //usersClrTex.width / 2; } } // try to automatically find the available avatar controllers at the scene if(avatarControllers.Count == 0) { AvatarController[] avatars = FindObjectsOfType(typeof(AvatarController)) as AvatarController[]; foreach(AvatarController avatar in avatars) { avatarControllers.Add(avatar); } } // Initialize user list to contain all users. alUserIds = new List<Int64>(); dictUserIdToIndex = new Dictionary<Int64, int>(); kinectInitialized = true; instance = this; //DontDestroyOnLoad(gameObject); // GUI Text. if(calibrationText != null) { calibrationText.guiText.text = "WAITING FOR USERS"; } Debug.Log("Waiting for users."); }
public Rect GetForegroundFrameRect(KinectInterop.SensorData sensorData, bool isHiResPrefered) { return(KinectInterop.GetForegroundFrameRect(sensorData, isHiResPrefered)); }
private bool InitVisualGestures() { KinectManager kinectManager = KinectManager.Instance; KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null; Kinect2Interface kinectInterface = sensorData.sensorInterface as Kinect2Interface; KinectSensor kinectSensor = kinectInterface != null ? kinectInterface.kinectSensor : null; if (kinectSensor == null) { return(false); } Debug.Log("GestureDB: " + gestureDatabase); if (gestureDatabase == string.Empty) { Debug.LogError("Please specify gesture database file!"); return(false); } // copy the gesture database file from Resources, if available if (!File.Exists(gestureDatabase)) { TextAsset textRes = Resources.Load(gestureDatabase, typeof(TextAsset)) as TextAsset; if (textRes != null && textRes.bytes.Length != 0) { File.WriteAllBytes(gestureDatabase, textRes.bytes); } } // create the vgb source vgbFrameSource = VisualGestureBuilderFrameSource.Create(kinectSensor, 0); // open the reader vgbFrameReader = vgbFrameSource != null?vgbFrameSource.OpenReader() : null; if (vgbFrameReader != null) { vgbFrameReader.IsPaused = true; } using (VisualGestureBuilderDatabase database = VisualGestureBuilderDatabase.Create(gestureDatabase)) { if (database == null) { Debug.LogError("Gesture database not found: " + gestureDatabase); return(false); } // check if we need to load all gestures bool bAllGestures = (gestureNames.Count == 0); foreach (Gesture gesture in database.AvailableGestures) { bool bAddGesture = bAllGestures || gestureNames.Contains(gesture.Name); if (bAddGesture) { string sGestureName = gesture.Name; vgbFrameSource.AddGesture(gesture); if (!gestureNames.Contains(sGestureName)) { gestureNames.Add(sGestureName); Debug.Log("Detect gesture: " + sGestureName); } if (!gestureData.ContainsKey(sGestureName)) { VisualGestureData data = new VisualGestureData(); data.gestureName = sGestureName; data.timestamp = Time.realtimeSinceStartup; data.isDiscrete = (gesture.GestureType == GestureType.Discrete); data.isContinuous = (gesture.GestureType == GestureType.Continuous); gestureData.Add(sGestureName, data); } } } } return(true); }
public int GetForegroundFrameLength(KinectInterop.SensorData sensorData, bool isHiResPrefered) { return(KinectInterop.GetForegroundFrameLength(sensorData, isHiResPrefered)); }
public bool UpdateSensorData(KinectInterop.SensorData sensorData) { return(true); }
public bool PollForegroundFrame(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, bool bLimitedUsers, ICollection <int> alTrackedIndexes, ref byte[] foregroundImage) { return(KinectInterop.PollForegroundFrame(sensorData, isHiResPrefered, defaultColor, bLimitedUsers, alTrackedIndexes, ref foregroundImage)); }
void StartKinect() { try { // try to initialize the default Kinect2 sensor KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody; if(computeUserMap) dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex; if(computeColorMap) dwFlags |= KinectInterop.FrameSource.TypeColor; if(computeInfraredMap) dwFlags |= KinectInterop.FrameSource.TypeInfrared; // if(useAudioSource) // dwFlags |= KinectInterop.FrameSource.TypeAudio; // open the default sensor sensorData = KinectInterop.OpenDefaultSensor(sensorInterfaces, dwFlags, sensorAngle, useMultiSourceReader); if (sensorData == null) { throw new Exception("OpenDefaultSensor failed"); } //create the transform matrix - kinect to world Quaternion quatTiltAngle = new Quaternion(); quatTiltAngle.eulerAngles = new Vector3(-sensorAngle, 0.0f, 0.0f); kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), quatTiltAngle, Vector3.one); } catch(DllNotFoundException ex) { string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation."; Debug.LogError(message); Debug.LogException(ex); if(calibrationText != null) { calibrationText.guiText.text = message; } return; } catch(Exception ex) { string message = ex.Message; Debug.LogError(message); Debug.LogException(ex); if(calibrationText != null) { calibrationText.guiText.text = message; } return; } // set the singleton instance instance = this; // init skeleton structures bodyFrame = new KinectInterop.BodyFrameData(sensorData.bodyCount, KinectInterop.Constants.JointCount); // sensorData.jointCount KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters(); switch(smoothing) { case Smoothing.Default: smoothParameters.smoothing = 0.5f; smoothParameters.correction = 0.5f; smoothParameters.prediction = 0.5f; smoothParameters.jitterRadius = 0.05f; smoothParameters.maxDeviationRadius = 0.04f; break; case Smoothing.Medium: smoothParameters.smoothing = 0.5f; smoothParameters.correction = 0.1f; smoothParameters.prediction = 0.5f; smoothParameters.jitterRadius = 0.1f; smoothParameters.maxDeviationRadius = 0.1f; break; case Smoothing.Aggressive: smoothParameters.smoothing = 0.7f; smoothParameters.correction = 0.3f; smoothParameters.prediction = 1.0f; smoothParameters.jitterRadius = 1.0f; smoothParameters.maxDeviationRadius = 1.0f; break; } // init data filters jointPositionFilter = new JointPositionsFilter(); jointPositionFilter.Init(smoothParameters); // init the bone orientation constraints if(useBoneOrientationConstraints) { boneConstraintsFilter = new BoneOrientationsConstraint(); boneConstraintsFilter.AddDefaultConstraints(); //boneConstraintsFilter.SetDebugText(calibrationText); } // get the main camera rectangle Rect cameraRect = Camera.main.pixelRect; // calculate map width and height in percent, if needed if(DisplayMapsWidthPercent == 0f) { DisplayMapsWidthPercent = (sensorData.depthImageWidth / 2) * 100 / cameraRect.width; } if(computeUserMap) { float displayMapsWidthPercent = DisplayMapsWidthPercent / 100f; float displayMapsHeightPercent = displayMapsWidthPercent * sensorData.depthImageHeight / sensorData.depthImageWidth; float displayWidth = cameraRect.width * displayMapsWidthPercent; float displayHeight = cameraRect.width * displayMapsHeightPercent; // Initialize depth & label map related stuff usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight); usersMapRect = new Rect(cameraRect.width - displayWidth, cameraRect.height, displayWidth, -displayHeight); usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight; usersHistogramImage = new Color32[usersMapSize]; usersPrevState = new ushort[usersMapSize]; usersHistogramMap = new float[5001]; } if(computeColorMap) { float displayMapsWidthPercent = DisplayMapsWidthPercent / 100f; float displayMapsHeightPercent = displayMapsWidthPercent * sensorData.colorImageHeight / sensorData.colorImageWidth; float displayWidth = cameraRect.width * displayMapsWidthPercent; float displayHeight = cameraRect.width * displayMapsHeightPercent; // Initialize color map related stuff usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false); usersClrRect = new Rect(cameraRect.width - displayWidth, cameraRect.height, displayWidth, -displayHeight); usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight; // if(computeUserMap && displayColorMap) // { // usersMapRect.x -= cameraRect.width * displayMapsWidthPercent; // } } // try to automatically use the available avatar controllers in the scene if(avatarControllers.Count == 0) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach(MonoBehaviour monoScript in monoScripts) { if(typeof(AvatarController).IsAssignableFrom(monoScript.GetType())) { AvatarController avatar = (AvatarController)monoScript; avatarControllers.Add(avatar); } } } // try to automatically use the available gesture listeners in the scene if(gestureListeners.Count == 0) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach(MonoBehaviour monoScript in monoScripts) { if(typeof(KinectGestures.GestureListenerInterface).IsAssignableFrom(monoScript.GetType())) { print ("Found Listener"); //KinectGestures.GestureListenerInterface gl = (KinectGestures.GestureListenerInterface)monoScript; gestureListeners.Add(monoScript); } } } // Initialize user list to contain all users. alUserIds = new List<Int64>(); dictUserIdToIndex = new Dictionary<Int64, int>(); kinectInitialized = true; DontDestroyOnLoad(gameObject); // GUI Text. if(calibrationText != null) { calibrationText.guiText.text = "WAITING FOR USERS"; } Debug.Log("Waiting for users."); }
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { // init interface if (!bNuitrackInited) { bNuitrackInited = NuitrackInit(); if (!bNuitrackInited) { return(null); } } sensorFlags = dwFlags; bMultiSource = bUseMultiSource; KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { for (int i = 0; i < WebCamTexture.devices.Length; i++) { if (WebCamTexture.devices[i].name.IndexOf("astra", StringComparison.CurrentCultureIgnoreCase) >= 0) { Debug.Log(" " + WebCamTexture.devices[i].name + "- AstraPro detected."); colorWebCam = new WebCamTexture(WebCamTexture.devices[i].name, 640, 480, 30); break; } } if (!colorWebCam) { colorSensor = nuitrack.ColorSensor.Create(); colorSensor.OnUpdateEvent += HandleOnColorSensorUpdateEvent; } else { bWebColorStream = true; colorWebCam.Play(); sensorData.colorImageTexture = colorWebCam; } } if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { depthSensor = nuitrack.DepthSensor.Create(); // depthSensor.SetMirror (true); depthSensor.OnUpdateEvent += HandleOnDepthSensorUpdateEvent; } if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { userTracker = nuitrack.UserTracker.Create(); userTracker.OnUpdateEvent += HandleOnUserTrackerUpdateEvent; } if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { skeletonTracker = nuitrack.SkeletonTracker.Create(); skeletonTracker.OnSkeletonUpdateEvent += HandleOnSkeletonUpdateEvent; handTracker = nuitrack.HandTracker.Create(); handTracker.OnUpdateEvent += HandleOnHandsUpdateEvent; gestureRecognizer = nuitrack.GestureRecognizer.Create(); gestureRecognizer.OnNewGesturesEvent += OnNewGestures; } // if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) // { // } nuitrack.Nuitrack.onIssueUpdateEvent += OnIssuesUpdate; nuitrack.Nuitrack.Run(); sensorData.bodyCount = Constants.SkeletonCount; sensorData.jointCount = Constants.JointCount; sensorData.depthCameraOffset = 0f; sensorData.faceOverlayOffset = 0f; if (!bWebColorStream) { // // wait for color frame // if (colorSensor != null) // { // colorFrame = colorSensor.GetColorFrame(); // float waitTillTime = Time.realtimeSinceStartup + 2.5f; // // while (colorFrame == null && Time.realtimeSinceStartup <= waitTillTime) // { // nuitrack.Nuitrack.Update(); // System.Threading.Thread.Sleep(50); // colorFrame = colorSensor.GetColorFrame(); // } // } sensorData.colorImageWidth = colorSensor != null?colorSensor.GetOutputMode().XRes : 640; sensorData.colorImageHeight = colorSensor != null?colorSensor.GetOutputMode().YRes : 480; // flip color image vertically sensorData.colorImageScale = new Vector3(1f, -1f, 1f); } else { sensorData.colorImageWidth = colorWebCam.width; sensorData.colorImageHeight = colorWebCam.height; // flip color image horizontally sensorData.colorImageScale = new Vector3(-1f, 1f, 1f); } Debug.Log(" Color sensor: " + (colorSensor != null ? colorSensor.ToString() : "-") + ", width: " + sensorData.colorImageWidth + ", height: " + sensorData.colorImageHeight); // // wait for depth frame // if (depthSensor != null) // { // depthFrame = depthSensor.GetDepthFrame(); // float waitTillTime = Time.realtimeSinceStartup + 2.5f; // // while (depthFrame == null && Time.realtimeSinceStartup <= waitTillTime) // { // nuitrack.Nuitrack.Update(); // System.Threading.Thread.Sleep(50); // depthFrame = depthSensor.GetDepthFrame(); // } // } sensorData.depthImageWidth = depthSensor != null?depthSensor.GetOutputMode().XRes : 640; sensorData.depthImageHeight = depthSensor != null?depthSensor.GetOutputMode().YRes : 480; Debug.Log(" Depth sensor: " + (depthSensor != null ? depthSensor.ToString() : "-") + ", width: " + sensorData.depthImageWidth + ", height: " + sensorData.depthImageHeight); // color & depth FOV float hfovr = colorSensor != null?colorSensor.GetOutputMode().HFOV : 0f; float vfovr = 2f * Mathf.Atan(Mathf.Tan(hfovr / 2f) * sensorData.depthImageHeight / sensorData.depthImageWidth); sensorData.colorCameraFOV = vfovr * Mathf.Rad2Deg; hfovr = depthSensor != null?depthSensor.GetOutputMode().HFOV : 0f; vfovr = 2f * Mathf.Atan(Mathf.Tan(hfovr / 2f) * sensorData.depthImageHeight / sensorData.depthImageWidth); sensorData.depthCameraFOV = vfovr * Mathf.Rad2Deg; if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { int colorImageSize = !colorWebCam ? (sensorData.colorImageWidth * sensorData.colorImageHeight * 3) : 0; sensorData.colorImage = new byte[colorImageSize]; if (!colorWebCam) { sensorData.colorImageTexture2D = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGB24, false); } } if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { int depthImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight; sensorData.depthImage = new ushort[depthImageSize]; } if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { int bodyIndexImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight; sensorData.bodyIndexImage = new byte[bodyIndexImageSize]; } if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { int depthImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight; sensorData.infraredImage = new ushort[depthImageSize]; } // setup coordinate mapper coordMapper = new OrbbecAstraMapper(); coordMapper.SetupSpaceMapping(sensorData.depthImageWidth, sensorData.depthImageHeight, hfovr, vfovr); // d2c-calibration data is valid for Orbbec-Astra only (sensor id & calib.data not provided by Nuitrack) coordMapper.SetupCalibrationData(bWebColorStream); // set lost-user time tolerance equal to KM if (KinectManager.Instance != null) { waitTimeBeforeRemove = KinectManager.Instance.waitTimeBeforeRemove; } Debug.Log("Nuitrack sensor opened"); return(sensorData); }
public KinectInterop.SensorData OpenDefaultSensor (KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { sourceFlags = dwFlags; NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone; NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex; if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { nuiFlags |= NuiInitializeFlags.UsesSkeleton; } if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { nuiFlags |= NuiInitializeFlags.UsesColor; } if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex; } if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex; } if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000); } int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode); if(hr == 0) { // set sensor angle SetKinectElevationAngle((int)sensorAngle); // initialize Kinect interaction hr = InitKinectInteraction(); if(hr != 0) { Debug.LogError("Initialization of KinectInteraction failed."); } KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); sensorData.bodyCount = Constants.NuiSkeletonCount; sensorData.jointCount = 20; NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight); // sensorData.colorImageWidth = Constants.ColorImageWidth; // sensorData.colorImageHeight = Constants.ColorImageHeight; if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { //colorStreamHandle = GetColorStreamHandle(); sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4]; } NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight); // sensorData.depthImageWidth = Constants.DepthImageWidth; // sensorData.depthImageHeight = Constants.DepthImageHeight; if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { //depthStreamHandle = GetDepthStreamHandle(); sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight]; } if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight]; } if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight]; } if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { skeletonFrame = new NuiSkeletonFrame() { SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount] }; // default values used to pass to smoothing function smoothParameters = new NuiTransformSmoothParameters(); smoothParameters.fSmoothing = 0.5f; smoothParameters.fCorrection = 0.5f; smoothParameters.fPrediction = 0.5f; smoothParameters.fJitterRadius = 0.05f; smoothParameters.fMaxDeviationRadius = 0.04f; } return sensorData; } else { Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr)); } return null; }
public void CloseSensor(KinectInterop.SensorData sensorData) { if (colorWebCam) { colorWebCam.Stop(); colorWebCam = null; } if (coordMapper != null) { coordMapper.CleanUp(); coordMapper = null; } if (colorSensor != null) { colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent; colorSensor = null; colorFrame = null; } if (depthSensor != null) { depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent; depthSensor = null; depthFrame = null; } if (userTracker != null) { userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent; userTracker = null; userFrame = null; } if (skeletonTracker != null) { skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent; skeletonTracker = null; skeletonData = null; } if (handTracker != null) { handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent; handTracker = null; handTrackerData = null; } if (gestureRecognizer != null) { gestureRecognizer.OnNewGesturesEvent -= OnNewGestures; gestureRecognizer = null; } if (bNuitrackInited) { bNuitrackInited = false; nuitrack.Nuitrack.onIssueUpdateEvent -= OnIssuesUpdate; NuitrackTerm(); } Debug.Log("Nuitrack sensor closed"); }
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource) { sourceFlags = dwFlags; NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone; NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex; if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { nuiFlags |= NuiInitializeFlags.UsesSkeleton; } if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { nuiFlags |= NuiInitializeFlags.UsesColor; } if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex; } if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex; } if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000); } if((dwFlags & KinectInterop.FrameSource.TypeAudio) != 0) { nuiFlags |= NuiInitializeFlags.UsesAudio; } FacetrackingManager[] faceManagers = GameObject.FindObjectsOfType(typeof(FacetrackingManager)) as FacetrackingManager[]; if(faceManagers != null && faceManagers.Length > 0) { for(int i = 0; i < faceManagers.Length; i++) { if(faceManagers[i].enabled) { //Debug.Log("Found FacetrackingManager => UsesColor"); nuiFlags |= NuiInitializeFlags.UsesColor; break; } } } SpeechManager[] speechManagers = GameObject.FindObjectsOfType(typeof(SpeechManager)) as SpeechManager[]; if(speechManagers != null && speechManagers.Length > 0) { for(int i = 0; i < speechManagers.Length; i++) { if(speechManagers[i].enabled) { //Debug.Log("Found SpeechManager => UsesAudio"); nuiFlags |= NuiInitializeFlags.UsesAudio; break; } } } int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode); if(hr == 0) { // set sensor angle SetKinectElevationAngle((int)sensorAngle); // initialize Kinect interaction hr = InitKinectInteraction(); if(hr != 0) { Debug.LogError(string.Format("Error initializing KinectInteraction: hr=0x{0:X}", hr)); } KinectInterop.SensorData sensorData = new KinectInterop.SensorData(); sensorData.bodyCount = Constants.NuiSkeletonCount; sensorData.jointCount = 20; sensorData.depthCameraFOV = 46.6f; sensorData.colorCameraFOV = 48.6f; sensorData.depthCameraOffset = 0.01f; NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight); // sensorData.colorImageWidth = Constants.ColorImageWidth; // sensorData.colorImageHeight = Constants.ColorImageHeight; if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0) { //colorStreamHandle = GetColorStreamHandle(); sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4]; } NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight); // sensorData.depthImageWidth = Constants.DepthImageWidth; // sensorData.depthImageHeight = Constants.DepthImageHeight; if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0) { //depthStreamHandle = GetDepthStreamHandle(); sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight]; } if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0) { sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight]; } if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0) { sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight]; } if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0) { skeletonFrame = new NuiSkeletonFrame() { SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount] }; // default values used to pass to smoothing function smoothParameters = new NuiTransformSmoothParameters(); smoothParameters.fSmoothing = 0.5f; smoothParameters.fCorrection = 0.5f; smoothParameters.fPrediction = 0.5f; smoothParameters.fJitterRadius = 0.05f; smoothParameters.fMaxDeviationRadius = 0.04f; } return sensorData; } else { Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr)); } return null; }
public void FreeMultiSourceFrame(KinectInterop.SensorData sensorData) { bMultiFramesReady = false; }
void StartKinect() { try { // try to initialize the default Kinect2 sensor KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody; if(computeUserMap != UserMapType.None) dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex; if(computeColorMap) dwFlags |= KinectInterop.FrameSource.TypeColor; if(computeInfraredMap) dwFlags |= KinectInterop.FrameSource.TypeInfrared; // if(useAudioSource) // dwFlags |= KinectInterop.FrameSource.TypeAudio; // open the default sensor BackgroundRemovalManager brManager = gameObject.GetComponentInChildren<BackgroundRemovalManager>(); sensorData = KinectInterop.OpenDefaultSensor(sensorInterfaces, dwFlags, sensorAngle, useMultiSourceReader, computeUserMap, brManager); if (sensorData == null) { if(sensorInterfaces == null || sensorInterfaces.Count == 0) throw new Exception("No sensor found. Make sure you have installed the SDK and the sensor is connected."); else throw new Exception("OpenDefaultSensor failed."); } // enable or disable getting height and angle info sensorData.hintHeightAngle = (autoHeightAngle != AutoHeightAngle.DontUse); //create the transform matrix - kinect to world Quaternion quatTiltAngle = Quaternion.Euler(-sensorAngle, 0.0f, 0.0f); kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), quatTiltAngle, Vector3.one); } catch(DllNotFoundException ex) { string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation."; Debug.LogError(message); Debug.LogException(ex); if(calibrationText != null) { calibrationText.text = message; } return; } catch(Exception ex) { string message = ex.Message; Debug.LogError(message); Debug.LogException(ex); if(calibrationText != null) { calibrationText.text = message; } return; } // set the singleton instance instance = this; // init skeleton structures bodyFrame = new KinectInterop.BodyFrameData(sensorData.bodyCount, KinectInterop.Constants.MaxJointCount); // sensorData.jointCount bodyFrame.bTurnAnalisys = allowTurnArounds; KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters(); switch(smoothing) { case Smoothing.Default: smoothParameters.smoothing = 0.5f; smoothParameters.correction = 0.5f; smoothParameters.prediction = 0.5f; smoothParameters.jitterRadius = 0.05f; smoothParameters.maxDeviationRadius = 0.04f; break; case Smoothing.Medium: smoothParameters.smoothing = 0.5f; smoothParameters.correction = 0.1f; smoothParameters.prediction = 0.5f; smoothParameters.jitterRadius = 0.1f; smoothParameters.maxDeviationRadius = 0.1f; break; case Smoothing.Aggressive: smoothParameters.smoothing = 0.7f; smoothParameters.correction = 0.3f; smoothParameters.prediction = 1.0f; smoothParameters.jitterRadius = 1.0f; smoothParameters.maxDeviationRadius = 1.0f; break; } // init data filters jointPositionFilter = new JointPositionsFilter(); jointPositionFilter.Init(smoothParameters); // init the bone orientation constraints if(useBoneOrientationConstraints) { boneConstraintsFilter = new BoneOrientationsConstraint(); boneConstraintsFilter.AddDefaultConstraints(); boneConstraintsFilter.SetDebugText(calibrationText); } if(computeUserMap != UserMapType.None && computeUserMap != UserMapType.RawUserDepth) { // Initialize depth & label map related stuff usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight, TextureFormat.ARGB32, false); usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight; usersHistogramImage = new Color32[usersMapSize]; usersPrevState = new ushort[usersMapSize]; usersHistogramMap = new float[5001]; } if(computeColorMap) { // Initialize color map related stuff //usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false); usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight; } // try to automatically use the available avatar controllers in the scene if(avatarControllers.Count == 0) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach(MonoBehaviour monoScript in monoScripts) { if(typeof(AvatarController).IsAssignableFrom(monoScript.GetType()) && monoScript.enabled) { AvatarController avatar = (AvatarController)monoScript; avatarControllers.Add(avatar); } } } // set up the gesture manager, if not already set if(gestureManager == null) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach(MonoBehaviour monoScript in monoScripts) { if(typeof(KinectGestures).IsAssignableFrom(monoScript.GetType()) && monoScript.enabled) { gestureManager = (KinectGestures)monoScript; break; } } } // try to automatically use the available gesture listeners in the scene if(gestureListeners.Count == 0) { MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[]; foreach(MonoBehaviour monoScript in monoScripts) { if(typeof(KinectGestures.GestureListenerInterface).IsAssignableFrom(monoScript.GetType()) && monoScript.enabled) { //KinectGestures.GestureListenerInterface gl = (KinectGestures.GestureListenerInterface)monoScript; gestureListeners.Add(monoScript); } } } // Initialize user list to contain all users. //alUserIds = new List<Int64>(); //dictUserIdToIndex = new Dictionary<Int64, int>(); // // start the background reader // kinectReaderThread = new System.Threading.Thread(UpdateKinectStreamsThread); // kinectReaderThread.Name = "KinectReaderThread"; // kinectReaderThread.IsBackground = true; // kinectReaderThread.Start(); // kinectReaderRunning = true; kinectInitialized = true; #if USE_SINGLE_KM_IN_MULTIPLE_SCENES DontDestroyOnLoad(gameObject); #endif // GUI Text. if(calibrationText != null) { calibrationText.text = "WAITING FOR USERS"; } Debug.Log("Waiting for users."); }
public bool PollBodyFrame(KinectInterop.SensorData sensorData, ref KinectInterop.BodyFrameData bodyFrame, ref Matrix4x4 kinectToWorld, bool bIgnoreJointZ) { bool bNewFrame = false; // look for skeleton frame if (skeletonData != null && skeletonDataTimestamp != lastSkeletonFrameTimestamp) { lastSkeletonFrameTimestamp = skeletonDataTimestamp; //long timeNowTicks = DateTime.Now.Ticks; bodyFrame.liPreviousTime = bodyFrame.liRelativeTime; bodyFrame.liRelativeTime = skeletonDataTimestamp; int bodyCount = skeletonData.Skeletons != null ? skeletonData.Skeletons.Length : 0; if (lastBodyCount != bodyCount) { sbDebugBodies.Append(bodyCount).Append(" bodies - "); } // clear id2index //bodyIdToIndex.Clear(); // create bodyIndexUsed-array on the 1st use if (bodyIndexUsed == null) { bodyIndexUsed = new bool[sensorData.bodyCount]; } // clear the tracked flags and find empty index int eIndex = -1; for (int i = 0; i < sensorData.bodyCount; i++) { bodyFrame.bodyData[i].bIsTracked = 0; if (eIndex < 0 && !bodyIndexUsed[i]) { eIndex = i; } } for (int i = 0; i < sensorData.bodyCount; i++) { // compare to real body count if (i >= bodyCount) { //bodyFrame.bodyData[i].bIsTracked = 0; continue; } // get body and joints data nuitrack.Skeleton nuiBody = skeletonData.Skeletons[i]; if (lastBodyCount != bodyCount) { sbDebugBodies.Append(nuiBody.ID).Append(":"); nuitrack.Joint jUser = nuiBody.Joints[(int)nuitrack.JointType.Waist]; Vector3 vUserPos = new Vector3(dontHFlipBodyFrame ? jUser.Real.X : -jUser.Real.X, jUser.Real.Y, jUser.Real.Z); sbDebugBodies.Append(vUserPos); sbDebugBodies.Append(" "); } // create the body index if needed ushort uBodyId = (ushort)nuiBody.ID; if (!bodyIdToIndex.ContainsKey(uBodyId)) { Debug.Log(" New body ID:" + uBodyId + ", index: " + eIndex); bodyIdToIndex[uBodyId] = (byte)eIndex; bodyIndexUsed[eIndex] = true; } // get existing body index int bi = bodyIdToIndex[uBodyId]; bodyIdToTime[uBodyId] = Time.time; // set body tracking state bodyFrame.bodyData[bi].bIsTracked = 1; // transfer body and joints data bodyFrame.bodyData[bi].liTrackingID = (long)nuiBody.ID; //bodyIdToIndex[(ushort)nuiBody.ID] = (byte)i; // z-position of the waist float waistPosZ = nuiBody.Joints[(int)nuitrack.JointType.Waist].Real.Z / 1000f; for (int j = 0; j < sensorData.jointCount; j++) { KinectInterop.JointData jointData = bodyFrame.bodyData[bi].joint[j]; int nuiJI = !dontHFlipBodyFrame ? BodyJoint2NormalNuitrackJoint[j] : BodyJoint2MirroredNuitrackJoint[j]; if (nuiJI >= 0) { nuitrack.Joint nuiJoint = nuiBody.Joints[nuiJI]; if (nuiJoint.Confidence >= 0.5f) { jointData.trackingState = KinectInterop.TrackingState.Tracked; } else if (nuiJoint.Confidence >= 0.1f) { jointData.trackingState = KinectInterop.TrackingState.Inferred; } else { jointData.trackingState = KinectInterop.TrackingState.NotTracked; } Vector3 jointPos = new Vector3((dontHFlipBodyFrame ? nuiJoint.Real.X : -nuiJoint.Real.X) / 1000f, nuiJoint.Real.Y / 1000f, nuiJoint.Real.Z / 1000f); float jPosZ = (bIgnoreJointZ && j > 0) ? waistPosZ : jointPos.z; jointData.kinectPos = jointPos; jointData.position = kinectToWorld.MultiplyPoint3x4(new Vector3(jointPos.x, jointPos.y, jPosZ)); jointData.orientation = Quaternion.identity; } else { jointData.trackingState = KinectInterop.TrackingState.NotTracked; } if (j == 0) { bodyFrame.bodyData[bi].position = jointData.position; bodyFrame.bodyData[bi].orientation = jointData.orientation; } bodyFrame.bodyData[bi].joint[j] = jointData; } if (handTrackerData != null) { nuitrack.UserHands hands = handTrackerData.GetUserHandsByID(nuiBody.ID); if (hands != null) { nuitrack.HandContent?leftHand = !dontHFlipBodyFrame ? hands.LeftHand : hands.RightHand; bodyFrame.bodyData[bi].leftHandState = leftHand.HasValue && leftHand.Value.Click ? KinectInterop.HandState.Closed : KinectInterop.HandState.Open; bodyFrame.bodyData[bi].leftHandConfidence = leftHand.HasValue ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low; nuitrack.HandContent?rightHand = !dontHFlipBodyFrame ? hands.RightHand : hands.LeftHand; bodyFrame.bodyData[bi].rightHandState = rightHand.HasValue && rightHand.Value.Click ? KinectInterop.HandState.Closed : KinectInterop.HandState.Open; bodyFrame.bodyData[bi].rightHandConfidence = rightHand.HasValue ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low; } } // processes some special joint cases ProcessBodyFrameSpecialCases(bi, ref bodyFrame); } // check for lost users List <ushort> lostUsers = new List <ushort>(); foreach (ushort uBodyId in bodyIdToTime.Keys) { // prevent user removal upon sporadical tracking failures if ((Time.time - bodyIdToTime[uBodyId]) > waitTimeBeforeRemove) { lostUsers.Add(uBodyId); } } // remove the lost users if (lostUsers.Count > 0) { foreach (ushort uBodyId in lostUsers) { Debug.Log(" Lost body ID:" + uBodyId + ", index: " + bodyIdToIndex[uBodyId]); int bi = bodyIdToIndex[uBodyId]; bodyIndexUsed[bi] = false; bodyIdToIndex.Remove(uBodyId); bodyIdToTime.Remove(uBodyId); } // clean up lostUsers.Clear(); } // write bodies-debug info, if needed if (sbDebugBodies.Length > 0) { sbDebugBodies.Append("Time: ").Append(Time.realtimeSinceStartup); Debug.Log(sbDebugBodies.ToString()); sbDebugBodies.Remove(0, sbDebugBodies.Length); } lastBodyCount = bodyCount; bNewFrame = true; } return(bNewFrame); }
//----------------------------------- end of public functions --------------------------------------// void Start() { try { // get sensor data KinectManager kinectManager = KinectManager.Instance; if(kinectManager && kinectManager.IsInitialized()) { sensorData = kinectManager.GetSensorData(); } if(sensorData == null || sensorData.sensorInterface == null) { throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized."); } if(debugText != null) { debugText.GetComponent<GUIText>().text = "Please, wait..."; } // ensure the needed dlls are in place and face tracking is available for this interface bool bNeedRestart = false; if(sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart)) { if(bNeedRestart) { KinectInterop.RestartLevel(gameObject, "FM"); return; } } else { string sInterfaceName = sensorData.sensorInterface.GetType().Name; throw new Exception(sInterfaceName + ": Face tracking is not supported!"); } // Initialize the face tracker if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect)) { throw new Exception("Face tracking could not be initialized."); } instance = this; isFacetrackingInitialized = true; //DontDestroyOnLoad(gameObject); if(debugText != null) { debugText.GetComponent<GUIText>().text = "Ready."; } } catch(DllNotFoundException ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations."; } catch (Exception ex) { Debug.LogError(ex.ToString()); if(debugText != null) debugText.GetComponent<GUIText>().text = ex.Message; } }
void Start() { manager = KinectManager.Instance; if (manager != null) { sensorData = manager.GetSensorData(); depthWidth = manager.GetDepthImageWidth(); depthHeight = manager.GetDepthImageHeight(); sampledWidth = depthWidth / sampleSize; sampledHeight = depthHeight / sampleSize; kinectToWorld = manager.GetKinectToWorldMatrix(); //spaceCoords = new Vector3[depthWidth * depthHeight]; if(sensorData.depth2SpaceCoords == null) { sensorData.depth2SpaceCoords = new Vector3[depthWidth * depthHeight]; } vertexType = new byte[sampledWidth * sampledHeight]; vertexIndex = new int[sampledWidth * sampledHeight]; CreateMesh(sampledWidth, sampledHeight); } }