public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorFlags = dwFlags;
kinectSensor = KinectSensor.GetDefault();
if(kinectSensor == null)
return null;
coordMapper = kinectSensor.CoordinateMapper;
this.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
sensorData.bodyCount = this.bodyCount;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = -0.05f;
sensorData.faceOverlayOffset = -0.04f;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
if(!bUseMultiSource)
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
bodyData = new Body[sensorData.bodyCount];
}
var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
sensorData.colorImageWidth = frameDesc.Width;
sensorData.colorImageHeight = frameDesc.Height;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
if(!bUseMultiSource)
colorFrameReader = kinectSensor.ColorFrameSource.OpenReader();
sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels];
}
sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width;
sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
if(!bUseMultiSource)
depthFrameReader = kinectSensor.DepthFrameSource.OpenReader();
sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
if(!bUseMultiSource)
bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader();
sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if(!bUseMultiSource)
infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader();
sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels];
}
//if(!kinectSensor.IsOpen)
{
//Debug.Log("Opening sensor, available: " + kinectSensor.IsAvailable);
kinectSensor.Open();
}
float fWaitTime = Time.realtimeSinceStartup + 3f;
while(!kinectSensor.IsAvailable && Time.realtimeSinceStartup < fWaitTime)
{
// wait for sensor to open
}
Debug.Log("K2-sensor " + (kinectSensor.IsOpen ? "opened" : "closed") +
", available: " + kinectSensor.IsAvailable);
if(bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
{
multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)((int)dwFlags & 0x3F));
}
return sensorData;
}
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null;
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Visual gesture tracking cannot be started, because the KinectManager is missing or not initialized.");
}
if (sensorData.sensorInterface.GetSensorPlatform() != KinectInterop.DepthSensorPlatform.KinectSDKv2)
{
throw new Exception("Visual gesture tracking is only supported by Kinect SDK v2");
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (IsVisualGesturesAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "VG");
return;
}
}
else
{
throw new Exception("Visual gesture tracking is not supported!");
}
// initialize visual gesture tracker
if (!InitVisualGestures())
{
throw new Exception("Visual gesture tracking could not be initialized.");
}
// try to automatically detect the available gesture listeners in the scene
if (visualGestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
// if(typeof(VisualGestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
// monoScript.enabled)
if ((monoScript is VisualGestureListenerInterface) && monoScript.enabled)
{
visualGestureListeners.Add(monoScript);
}
}
}
isVisualGestureInitialized = true;
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = "Please check the Kinect and VGB-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = ex.Message;
}
}
}
void Start()
{
manager = KinectManager.Instance;
if (manager != null)
{
sensorData = manager.GetSensorData();
minDepth = Mathf.RoundToInt(minDistance * 1000f);
maxDepth = Mathf.RoundToInt(maxDistance * 1000f);
colorWidth = manager.GetColorImageWidth();
colorHeight = manager.GetColorImageHeight();
depthWidth = manager.GetDepthImageWidth();
depthHeight = manager.GetDepthImageHeight();
sampledWidth = depthWidth / sampleSize;
sampledHeight = depthHeight / sampleSize;
kinectToWorld = manager.GetKinectToWorldMatrix();
if(sensorData.depth2SpaceCoords == null)
{
sensorData.depth2SpaceCoords = new Vector3[depthWidth * depthHeight];
}
sceneMeshPos = transform.position;
if(!mirroredScene)
{
sceneMeshPos.x = -sceneMeshPos.x;
}
vertexType = new byte[sampledWidth * sampledHeight];
vertexIndex = new int[sampledWidth * sampledHeight];
CreateMesh(sampledWidth, sampledHeight);
}
}
void Start()
{
if (foregroundCamera == null)
{
// by default use the main camera
foregroundCamera = Camera.main;
}
manager = KinectManager.Instance;
if (manager && manager.IsInitialized())
{
KinectInterop.SensorData sensorData = manager.GetSensorData();
if (sensorData != null && sensorData.sensorInterface != null && foregroundCamera != null)
{
// sensor platform
sensorPlatform = sensorData.sensorIntPlatform;
// get depth image size
depthImageWidth = sensorData.depthImageWidth;
depthImageHeight = sensorData.depthImageHeight;
// calculate the foreground rectangles
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if (rectWidth > rectHeight)
{
rectWidth = rectHeight * depthImageWidth / depthImageHeight;
}
else
{
rectHeight = rectWidth * depthImageHeight / depthImageWidth;
}
float foregroundOfsX = (cameraRect.width - rectWidth) / 2;
float foregroundOfsY = (cameraRect.height - rectHeight) / 2;
foregroundImgRect = new Rect(foregroundOfsX, foregroundOfsY, rectWidth, rectHeight);
// create joint colliders
numColliders = sensorData.jointCount;
jointColliders = new GameObject[numColliders];
for (int i = 0; i < numColliders; i++)
{
string sColObjectName = ((KinectInterop.JointType)i).ToString() + "Collider";
jointColliders[i] = new GameObject(sColObjectName);
jointColliders[i].transform.parent = transform;
if (i == 0)
{
// circle collider for body center
CircleCollider2D collider = jointColliders[i].AddComponent <CircleCollider2D>();
collider.radius = colliderWidth;
}
else
{
// box colliders for bones
BoxCollider2D collider = jointColliders[i].AddComponent <BoxCollider2D>();
collider.size = new Vector2(colliderWidth, colliderWidth);
}
}
}
}
}
public bool PollDepthFrame(KinectInterop.SensorData sensorData)
{
bool bNewFrame = false;
if (bMultiSource && !bMultiFramesReady)
{
return(bNewFrame);
}
// poll for depth frame
if (depthFrame != null && (long)depthFrame.Timestamp != lastDepthFrameTimestamp)
{
lastDepthFrameTimestamp = (long)depthFrame.Timestamp;
if (dontHFlipDepthFrame)
{
// copy depth data, no flip
int copyLength = sensorData.depthImage.Length * sizeof(ushort);
Buffer.BlockCopy(depthFrame.Data, 0, sensorData.depthImage, 0, copyLength);
}
else
{
// copy depth data, h-flip
int lenDepthFrame = sensorData.depthImage.Length;
int lenDepthRow = sensorData.depthImageWidth;
for (int i = 0; i < lenDepthFrame; i += lenDepthRow)
{
int iNext = i + lenDepthRow;
for (int s = i, d = iNext - 1; s < iNext; s++, d--)
{
sensorData.depthImage[d] = depthFrame[s];
}
}
}
sensorData.lastDepthFrameTime = (long)depthFrame.Timestamp;
bNewFrame = true;
if (bMultiSource)
{
bMultiFrameDepth = false;
}
}
// poll for body-index frame
if (userFrame != null && (long)userFrame.Timestamp != lastUserFrameTimestamp)
{
lastUserFrameTimestamp = (long)userFrame.Timestamp;
int lenBodyIndexFrame = sensorData.bodyIndexImage.Length;
int lenBodyIndexRow = sensorData.depthImageWidth;
if (dontHFlipDepthFrame)
{
// copy body-index data, no flip
for (int i = 0; i < lenBodyIndexFrame; i += lenBodyIndexRow)
{
int iNext = i + lenBodyIndexRow;
for (int s = i, d = i; s < iNext; s++, d++)
{
ushort u = userFrame[s];
byte b = 255;
if (u != 0 && bodyIdToIndex.ContainsKey(u))
{
b = bodyIdToIndex[u];
}
sensorData.bodyIndexImage[d] = b;
}
}
}
else
{
// copy body-index data, h-flip
for (int i = 0; i < lenBodyIndexFrame; i += lenBodyIndexRow)
{
int iNext = i + lenBodyIndexRow;
for (int s = i, d = iNext - 1; s < iNext; s++, d--)
{
ushort u = userFrame[s];
byte b = 255;
if (u != 0 && bodyIdToIndex.ContainsKey(u))
{
b = bodyIdToIndex[u];
}
sensorData.bodyIndexImage[d] = b;
}
}
}
sensorData.lastBodyIndexFrameTime = (long)userFrame.Timestamp;
bNewFrame = true;
if (bMultiSource)
{
bMultiFrameBodyIndex = false;
}
// move floor estimation here
if (sensorData.hintHeightAngle)
{
Vector3 floorNormal = new Vector3(userFrame.FloorNormal.X, userFrame.FloorNormal.Y, userFrame.FloorNormal.Z);
if (floorNormal != Vector3.zero)
{
sensorData.sensorRotDetected = Quaternion.FromToRotation(floorNormal, Vector3.up);
sensorData.sensorHgtDetected = Mathf.Abs(userFrame.Floor.Y / 1000f);
}
}
}
return(bNewFrame);
}
public bool GetMultiSourceFrame(KinectInterop.SensorData sensorData)
{
return(false);
}
public void FixJointOrientations(KinectInterop.SensorData sensorData, ref KinectInterop.BodyData bodyData)
{
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = 6;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
sensorData.colorImageWidth = 1920;
sensorData.colorImageHeight = 1080;
sensorData.depthImageWidth = 512;
sensorData.depthImageHeight = 424;
Debug.Log("DummyK2-sensor opened");
return sensorData;
}
public bool MapColorFrameToDepthCoords(KinectInterop.SensorData sensorData, ref Vector2[] vDepthCoords)
{
return(false);
}
public bool PollInfraredFrame(KinectInterop.SensorData sensorData)
{
return(false);
}
public bool MapDepthFrameToSpaceCoords(KinectInterop.SensorData sensorData, ref Vector3[] vSpaceCoords)
{
return(false);
}
public bool PollBodyFrame(KinectInterop.SensorData sensorData, ref KinectInterop.BodyFrameData bodyFrame,
ref Matrix4x4 kinectToWorld, bool bIgnoreJointZ)
{
bool bNewFrame = false;
// look for frame
int hr = PollBodyFrame();
if (hr != 0)
{
Debug.Log("PollBodyFrame error: " + hr);
}
if (hr == 0)
{
int bodyFrameIndex = GetBodyFrameIndex();
if (bodyFrameIndex != lastBodyFrameIndex)
{
lastBodyFrameIndex = bodyFrameIndex;
long timeNowTicks = DateTime.Now.Ticks;
// get body index frame
var pBodyIndexData = GCHandle.Alloc(sensorData.bodyIndexImage, GCHandleType.Pinned);
hr = GetBodyIndexData(pBodyIndexData.AddrOfPinnedObject());
pBodyIndexData.Free();
if (hr != 0)
{
Debug.Log("GetBodyIndexData() error: " + hr);
}
sensorData.lastBodyIndexFrameTime = (hr == 0) ? timeNowTicks : sensorData.lastBodyIndexFrameTime;
bodyFrame.liPreviousTime = bodyFrame.liRelativeTime;
bodyFrame.liRelativeTime = timeNowTicks;
if (sensorData.hintHeightAngle)
{
Vector4 floorInfo = Vector4.zero;
if (GetFloorInfo(ref floorInfo) != 0)
{
Vector3 floorPlane = (Vector3)floorInfo;
sensorData.sensorRotDetected = Quaternion.FromToRotation(floorPlane, Vector3.up);
sensorData.sensorHgtDetected = Mathf.Abs(floorInfo.w / 1000f);
}
}
// int bodySize1 = Marshal.SizeOf(typeof(ObtBody));
// int bodySize2 = GetBodyDataSize();
int bodyCount = GetBodyCount();
if (lastBodyCount != bodyCount)
{
sbDebugBodies.Append(bodyCount).Append(" bodies - ");
}
for (int i = 0; i < sensorData.bodyCount; i++)
{
// compare to real body count
if (i >= bodyCount || !obtBodyInited)
{
bodyFrame.bodyData[i].bIsTracked = 0;
continue;
}
// get body and joints data
hr = GetBodyData(i, ref obtBody);
if (lastBodyCount != bodyCount)
{
sbDebugBodies.Append(obtBody.id).Append(":").Append(obtBody.status).Append(":").Append(hr).Append(" ");
}
// if there is error, consider body as not-tracked
if (hr != 0)
{
bodyFrame.bodyData[i].bIsTracked = 0;
continue;
}
// set body tracking state
bodyFrame.bodyData[i].bIsTracked = (short)(obtBody.status != (byte)ObtBodyStatus.BODY_STATUS_NOT_TRACKING ? 1 : 0);
if (obtBody.status != (byte)ObtBodyStatus.BODY_STATUS_NOT_TRACKING)
{
// transfer body and joints data
bodyFrame.bodyData[i].liTrackingID = (long)obtBody.id;
for (int j = 0; j < sensorData.jointCount; j++)
{
KinectInterop.JointData jointData = bodyFrame.bodyData[i].joint[j];
int obtJI = BodyJoint2AstraJoint[j];
if (obtJI >= 0)
{
jointData.trackingState = (KinectInterop.TrackingState)obtBody.joints[obtJI].status;
}
else
{
jointData.trackingState = KinectInterop.TrackingState.NotTracked;
}
if (jointData.trackingState != KinectInterop.TrackingState.NotTracked)
{
Vector3 jointPos = obtBody.joints[obtJI].worldPosition / 1000f;
float jPosZ = (bIgnoreJointZ && j > 0) ? bodyFrame.bodyData[i].joint[0].kinectPos.z : jointPos.z;
jointData.kinectPos = jointPos;
jointData.position = kinectToWorld.MultiplyPoint3x4(new Vector3(jointPos.x, jointPos.y, jPosZ));
}
jointData.orientation = Quaternion.identity;
if (j == 0)
{
bodyFrame.bodyData[i].position = jointData.position;
bodyFrame.bodyData[i].orientation = jointData.orientation;
}
bodyFrame.bodyData[i].joint[j] = jointData;
}
// processes some special joint cases
ProcessBodyFrameSpecialCases(i, ref bodyFrame);
// // hand states - is this available on orbbec?
// bodyFrame.bodyData[i].leftHandState = (KinectInterop.HandState)body.HandLeftState;
// bodyFrame.bodyData[i].leftHandConfidence = (KinectInterop.TrackingConfidence)body.HandLeftConfidence;
//
// bodyFrame.bodyData[i].rightHandState = (KinectInterop.HandState)body.HandRightState;
// bodyFrame.bodyData[i].rightHandConfidence = (KinectInterop.TrackingConfidence)body.HandRightConfidence;
}
}
if (sbDebugBodies.Length > 0)
{
Debug.Log(sbDebugBodies.ToString());
sbDebugBodies.Remove(0, sbDebugBodies.Length);
}
lastBodyCount = bodyCount;
bNewFrame = true;
}
// release the frame
ReleaseBodyFrame();
}
return(bNewFrame);
}
public void FreeMultiSourceFrame(KinectInterop.SensorData sensorData)
{
}
// private Vector3 sizeObjectHandLeft = Vector3.zero;
// private Vector3 sizeObjectHandRight = Vector3.zero;
void Start()
{
manager = KinectManager.Instance;
sensorData = manager ? manager.GetSensorData() : null;
}
// public bool IsBodyTurned(ref KinectInterop.BodyData bodyData)
// {
// return false;
// }
public Vector2 MapSpacePointToDepthCoords(KinectInterop.SensorData sensorData, Vector3 spacePos)
{
Vector2 depthPos = coordMapper != null?coordMapper.MapSpacePointToDepthCoords(spacePos) : Vector2.zero;
return(depthPos);
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if(kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if(sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized.");
}
if(debugText != null)
{
debugText.GetComponent<GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
if(sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart))
{
if(bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "SM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Speech recognition is not supported!");
}
// Initialize the speech recognizer
string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode);
int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage));
}
if(requiredConfidence > 0)
{
sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence);
}
if(grammarFileName != string.Empty)
{
// copy the grammar file from Resources, if available
if(!File.Exists(grammarFileName))
{
TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset;
if(textRes != null)
{
string sResText = textRes.text;
File.WriteAllText(grammarFileName, sResText);
}
}
// load the grammar file
rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage);
}
}
instance = this;
sapiInitialized = true;
//DontDestroyOnLoad(gameObject);
if(debugText != null)
{
debugText.GetComponent<GUIText>().text = "Ready.";
}
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and SAPI installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
public Vector3 MapDepthPointToSpaceCoords(KinectInterop.SensorData sensorData, Vector2 depthPos, ushort depthVal)
{
Vector3 spacePos = coordMapper != null?coordMapper.MapDepthPointToSpaceCoords(depthPos, depthVal) : Vector3.zero;
return(spacePos);
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
//sensorFlags = dwFlags;
kinectSensor = KinectSensor.GetDefault();
if(kinectSensor == null)
return null;
coordMapper = kinectSensor.CoordinateMapper;
this.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
sensorData.bodyCount = this.bodyCount;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = -0.03f;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
if(!bUseMultiSource)
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
bodyData = new Body[sensorData.bodyCount];
}
var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
sensorData.colorImageWidth = frameDesc.Width;
sensorData.colorImageHeight = frameDesc.Height;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
if(!bUseMultiSource)
colorFrameReader = kinectSensor.ColorFrameSource.OpenReader();
sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels];
}
sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width;
sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
if(!bUseMultiSource)
depthFrameReader = kinectSensor.DepthFrameSource.OpenReader();
sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
if(!bUseMultiSource)
bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader();
sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if(!bUseMultiSource)
infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader();
sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels];
}
if(!kinectSensor.IsOpen)
{
kinectSensor.Open();
}
if(bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
{
multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)dwFlags);
}
return sensorData;
}
public bool MapDepthFrameToSpaceCoords(KinectInterop.SensorData sensorData, ref Vector3[] vSpaceCoords)
{
bool bSuccess = coordMapper != null?coordMapper.MapDepthFrameToSpaceCoords(sensorData, ref vSpaceCoords) : false;
return(bSuccess);
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if(kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if(sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized.");
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart);
if(bSuccess)
{
if(bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "BR");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Background removal is not supported!");
}
// Initialize the background removal
bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution);
if (!bSuccess)
{
throw new Exception("Background removal could not be initialized.");
}
// create the foreground image and alpha-image
int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution);
foregroundImage = new byte[imageLength];
// get the needed rectangle
Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution);
// create the foreground texture
foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false);
// calculate the foreground rectangle
if(foregroundCamera != null)
{
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if(rectWidth > rectHeight)
rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height);
else
rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width);
foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
}
instance = this;
isBrInited = true;
//DontDestroyOnLoad(gameObject);
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and BR-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
public Vector2 MapDepthPointToColorCoords(KinectInterop.SensorData sensorData, Vector2 depthPos, ushort depthVal)
{
Vector2 colorPos = coordMapper != null?coordMapper.MapDepthPointToColorCoords(depthPos, depthVal) : Vector2.zero;
return(colorPos);
}
public bool PollColorFrame(KinectInterop.SensorData sensorData)
{
bool bNewFrame = false;
if (bMultiSource && !bMultiFramesReady)
{
return(bNewFrame);
}
// check if color camera is webcam
if (bWebColorStream && colorWebcamData != null && colorWebcamTimestamp != lastColorFrameTimestamp)
{
lastColorFrameTimestamp = colorWebcamTimestamp;
if (sensorData.colorImageTexture2D && sensorData.colorImageTexture2D.width == colorWebCam.width && sensorData.colorImageTexture2D.height == colorWebCam.height)
{
sensorData.colorImageTexture2D.SetPixels32(colorWebcamData);
}
sensorData.lastColorFrameTime = colorWebcamTimestamp;
bNewFrame = true;
}
// poll for color frame
else if (colorFrame != null && (long)colorFrame.Timestamp != lastColorFrameTimestamp)
{
lastColorFrameTimestamp = (long)colorFrame.Timestamp;
//sensorData.colorImage = colorFrame.Data;
// copy color data h-flip, BGR to RGB
int lenColorFrame = sensorData.colorImage.Length;
int lenColorRow = sensorData.colorImageWidth * 3;
if (dontHFlipColorFrame)
{
// copy depth data, no flip
for (int i = 0; i < lenColorFrame; i += lenColorRow)
{
int iNext = i + lenColorRow;
for (int s = i, d = i; s < iNext; s += 3, d += 3)
{
sensorData.colorImage[d] = colorFrame.Data[s + 2];
sensorData.colorImage[d + 1] = colorFrame.Data[s + 1];
sensorData.colorImage[d + 2] = colorFrame.Data[s];
}
}
}
else
{
// copy color data, h-flip
for (int i = 0; i < lenColorFrame; i += lenColorRow)
{
int iNext = i + lenColorRow;
for (int s = i, d = iNext - 3; s < iNext; s += 3, d -= 3)
{
sensorData.colorImage[d] = colorFrame.Data[s + 2];
sensorData.colorImage[d + 1] = colorFrame.Data[s + 1];
sensorData.colorImage[d + 2] = colorFrame.Data[s];
}
}
}
sensorData.lastColorFrameTime = (long)colorFrame.Timestamp;
bNewFrame = true;
}
if (bNewFrame && bMultiSource)
{
bMultiFrameColor = false;
}
return(bNewFrame);
}
public bool InitBackgroundRemoval(KinectInterop.SensorData sensorData, bool isHiResPrefered)
{
return(KinectInterop.InitBackgroundRemoval(sensorData, isHiResPrefered));
}
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.text = "Please, wait...";
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "SM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Speech recognition is not supported!");
}
// Initialize the speech recognizer
string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode);
int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage));
}
if (requiredConfidence > 0)
{
sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence);
}
if (grammarFileName != string.Empty)
{
// copy the grammar file from Resources, if available
//if(!File.Exists(grammarFileName))
{
TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset;
if (textRes != null)
{
string sResText = textRes.text;
File.WriteAllText(grammarFileName, sResText);
}
else
{
throw new Exception("Couldn't find grammar resource: " + grammarFileName + ".txt");
}
}
// load the grammar file
rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode, dynamicGrammar);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage);
}
// // test dynamic grammar phrases
// AddGrammarPhrase("addressBook", string.Empty, "Nancy Anderson", true, false);
// AddGrammarPhrase("addressBook", string.Empty, "Cindy White", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "Oliver Lee", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "Alan Brewer", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "April Reagan", false, true);
}
sapiInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.text = "Ready.";
}
// try to automatically detect the available speech recognition listeners in the scene
if (speechRecognitionListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
if ((monoScript is SpeechRecognitionInterface) && monoScript.enabled)
{
speechRecognitionListeners.Add(monoScript);
}
}
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = "Please check the Kinect and SAPI installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = ex.Message;
}
}
}
public void FinishBackgroundRemoval(KinectInterop.SensorData sensorData)
{
KinectInterop.FinishBackgroundRemoval(sensorData);
bBackgroundRemovalInited = false;
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
instance = this;
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
public bool UpdateBackgroundRemoval(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, bool bAlphaTexOnly)
{
return(KinectInterop.UpdateBackgroundRemoval(sensorData, isHiResPrefered, defaultColor, bAlphaTexOnly));
}
void Start()
{
// int hr = 0;
try
{
// if(startKinectServer)
// {
// // start the Kinect-server app
// hr = StartKinectServer();
// if (hr != 0)
// {
// throw new Exception("Kinect2Server not started");
// }
// }
// try to initialize the default Kinect2 sensor
KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody;
if(computeUserMap)
dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex;
if(computeColorMap)
dwFlags |= KinectInterop.FrameSource.TypeColor;
if(computeInfraredMap)
dwFlags |= KinectInterop.FrameSource.TypeInfrared;
// hr = KinectInterop.InitDefaultKinectSensor(dwFlags, KinectInterop.Constants.ColorImageWidth, KinectInterop.Constants.ColorImageHeight);
// if (hr != 0)
// {
// throw new Exception("InitDefaultKinectSensor failed");
// }
// open the default kinect sensor
sensorData = KinectInterop.OpenDefaultKinectSensor(dwFlags);
if (sensorData == null)
{
throw new Exception("OpenDefaultKinectSensor failed");
}
// transform matrix - kinect to world
kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), Quaternion.identity, Vector3.one);
}
catch(DllNotFoundException ex)
{
string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation.";
Debug.LogError(message);
Debug.LogException(ex);
if(calibrationText != null)
{
calibrationText.guiText.text = message;
}
return;
}
catch(Exception ex)
{
string message = ex.Message;
Debug.LogError(message);
Debug.LogException(ex);
if(calibrationText != null)
{
calibrationText.guiText.text = message;
}
return;
}
// init skeleton structures
bodyFrame = new KinectInterop.BodyFrameData(true);
KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters();
switch(smoothing)
{
case Smoothing.Default:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.5f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.05f;
smoothParameters.maxDeviationRadius = 0.04f;
break;
case Smoothing.Medium:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.1f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.1f;
smoothParameters.maxDeviationRadius = 0.1f;
break;
case Smoothing.Aggressive:
smoothParameters.smoothing = 0.7f;
smoothParameters.correction = 0.3f;
smoothParameters.prediction = 1.0f;
smoothParameters.jitterRadius = 1.0f;
smoothParameters.maxDeviationRadius = 1.0f;
break;
}
// init data filters
jointPositionFilter = new JointPositionsFilter();
jointPositionFilter.Init(smoothParameters);
// init the bone orientation constraints
if(useBoneOrientationConstraints)
{
boneConstraintsFilter = new BoneOrientationsConstraint();
boneConstraintsFilter.AddDefaultConstraints();
boneConstraintsFilter.SetDebugText(calibrationText);
}
// get the main camera rectangle
Rect cameraRect = Camera.main.pixelRect;
// calculate map width and height in percent, if needed
if(MapsPercentWidth == 0f)
MapsPercentWidth = (sensorData.depthImageWidth / 2) / cameraRect.width;
if(MapsPercentHeight == 0f)
MapsPercentHeight = (sensorData.depthImageHeight / 2) / cameraRect.height;
if(computeUserMap)
{
// init user-depth structures
//depthImage = new KinectInterop.DepthBuffer(true);
//bodyIndexImage = new KinectInterop.BodyIndexBuffer(true);
// Initialize depth & label map related stuff
usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight);
usersMapRect = new Rect(cameraRect.width - cameraRect.width * MapsPercentWidth, cameraRect.height, cameraRect.width * MapsPercentWidth, -cameraRect.height * MapsPercentHeight);
usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
usersHistogramImage = new Color32[usersMapSize];
usersPrevState = new ushort[usersMapSize];
usersHistogramMap = new float[5001];
}
if(computeColorMap)
{
// init color image structures
//colorImage = new KinectInterop.ColorBuffer(true);
// Initialize color map related stuff
usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false);
usersClrRect = new Rect(cameraRect.width - 320, cameraRect.height, 320, -180);
usersCaliBox = new Rect(cameraRect.width - 160 - 320 / 14, cameraRect.height - 90, 320 / 7, 90);
usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight;
if(computeUserMap && displayColorMap)
{
usersMapRect.x -= cameraRect.width * MapsPercentWidth; //usersClrTex.width / 2;
}
}
// try to automatically find the available avatar controllers at the scene
if(avatarControllers.Count == 0)
{
AvatarController[] avatars = FindObjectsOfType(typeof(AvatarController)) as AvatarController[];
foreach(AvatarController avatar in avatars)
{
avatarControllers.Add(avatar);
}
}
// Initialize user list to contain all users.
alUserIds = new List<Int64>();
dictUserIdToIndex = new Dictionary<Int64, int>();
kinectInitialized = true;
instance = this;
//DontDestroyOnLoad(gameObject);
// GUI Text.
if(calibrationText != null)
{
calibrationText.guiText.text = "WAITING FOR USERS";
}
Debug.Log("Waiting for users.");
}
public Rect GetForegroundFrameRect(KinectInterop.SensorData sensorData, bool isHiResPrefered)
{
return(KinectInterop.GetForegroundFrameRect(sensorData, isHiResPrefered));
}
private bool InitVisualGestures()
{
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null;
Kinect2Interface kinectInterface = sensorData.sensorInterface as Kinect2Interface;
KinectSensor kinectSensor = kinectInterface != null ? kinectInterface.kinectSensor : null;
if (kinectSensor == null)
{
return(false);
}
Debug.Log("GestureDB: " + gestureDatabase);
if (gestureDatabase == string.Empty)
{
Debug.LogError("Please specify gesture database file!");
return(false);
}
// copy the gesture database file from Resources, if available
if (!File.Exists(gestureDatabase))
{
TextAsset textRes = Resources.Load(gestureDatabase, typeof(TextAsset)) as TextAsset;
if (textRes != null && textRes.bytes.Length != 0)
{
File.WriteAllBytes(gestureDatabase, textRes.bytes);
}
}
// create the vgb source
vgbFrameSource = VisualGestureBuilderFrameSource.Create(kinectSensor, 0);
// open the reader
vgbFrameReader = vgbFrameSource != null?vgbFrameSource.OpenReader() : null;
if (vgbFrameReader != null)
{
vgbFrameReader.IsPaused = true;
}
using (VisualGestureBuilderDatabase database = VisualGestureBuilderDatabase.Create(gestureDatabase))
{
if (database == null)
{
Debug.LogError("Gesture database not found: " + gestureDatabase);
return(false);
}
// check if we need to load all gestures
bool bAllGestures = (gestureNames.Count == 0);
foreach (Gesture gesture in database.AvailableGestures)
{
bool bAddGesture = bAllGestures || gestureNames.Contains(gesture.Name);
if (bAddGesture)
{
string sGestureName = gesture.Name;
vgbFrameSource.AddGesture(gesture);
if (!gestureNames.Contains(sGestureName))
{
gestureNames.Add(sGestureName);
Debug.Log("Detect gesture: " + sGestureName);
}
if (!gestureData.ContainsKey(sGestureName))
{
VisualGestureData data = new VisualGestureData();
data.gestureName = sGestureName;
data.timestamp = Time.realtimeSinceStartup;
data.isDiscrete = (gesture.GestureType == GestureType.Discrete);
data.isContinuous = (gesture.GestureType == GestureType.Continuous);
gestureData.Add(sGestureName, data);
}
}
}
}
return(true);
}
public int GetForegroundFrameLength(KinectInterop.SensorData sensorData, bool isHiResPrefered)
{
return(KinectInterop.GetForegroundFrameLength(sensorData, isHiResPrefered));
}
public bool UpdateSensorData(KinectInterop.SensorData sensorData)
{
return(true);
}
public bool PollForegroundFrame(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, bool bLimitedUsers, ICollection <int> alTrackedIndexes, ref byte[] foregroundImage)
{
return(KinectInterop.PollForegroundFrame(sensorData, isHiResPrefered, defaultColor, bLimitedUsers, alTrackedIndexes, ref foregroundImage));
}
void StartKinect()
{
try
{
// try to initialize the default Kinect2 sensor
KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody;
if(computeUserMap)
dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex;
if(computeColorMap)
dwFlags |= KinectInterop.FrameSource.TypeColor;
if(computeInfraredMap)
dwFlags |= KinectInterop.FrameSource.TypeInfrared;
// if(useAudioSource)
// dwFlags |= KinectInterop.FrameSource.TypeAudio;
// open the default sensor
sensorData = KinectInterop.OpenDefaultSensor(sensorInterfaces, dwFlags, sensorAngle, useMultiSourceReader);
if (sensorData == null)
{
throw new Exception("OpenDefaultSensor failed");
}
//create the transform matrix - kinect to world
Quaternion quatTiltAngle = new Quaternion();
quatTiltAngle.eulerAngles = new Vector3(-sensorAngle, 0.0f, 0.0f);
kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), quatTiltAngle, Vector3.one);
}
catch(DllNotFoundException ex)
{
string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation.";
Debug.LogError(message);
Debug.LogException(ex);
if(calibrationText != null)
{
calibrationText.guiText.text = message;
}
return;
}
catch(Exception ex)
{
string message = ex.Message;
Debug.LogError(message);
Debug.LogException(ex);
if(calibrationText != null)
{
calibrationText.guiText.text = message;
}
return;
}
// set the singleton instance
instance = this;
// init skeleton structures
bodyFrame = new KinectInterop.BodyFrameData(sensorData.bodyCount, KinectInterop.Constants.JointCount); // sensorData.jointCount
KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters();
switch(smoothing)
{
case Smoothing.Default:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.5f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.05f;
smoothParameters.maxDeviationRadius = 0.04f;
break;
case Smoothing.Medium:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.1f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.1f;
smoothParameters.maxDeviationRadius = 0.1f;
break;
case Smoothing.Aggressive:
smoothParameters.smoothing = 0.7f;
smoothParameters.correction = 0.3f;
smoothParameters.prediction = 1.0f;
smoothParameters.jitterRadius = 1.0f;
smoothParameters.maxDeviationRadius = 1.0f;
break;
}
// init data filters
jointPositionFilter = new JointPositionsFilter();
jointPositionFilter.Init(smoothParameters);
// init the bone orientation constraints
if(useBoneOrientationConstraints)
{
boneConstraintsFilter = new BoneOrientationsConstraint();
boneConstraintsFilter.AddDefaultConstraints();
//boneConstraintsFilter.SetDebugText(calibrationText);
}
// get the main camera rectangle
Rect cameraRect = Camera.main.pixelRect;
// calculate map width and height in percent, if needed
if(DisplayMapsWidthPercent == 0f)
{
DisplayMapsWidthPercent = (sensorData.depthImageWidth / 2) * 100 / cameraRect.width;
}
if(computeUserMap)
{
float displayMapsWidthPercent = DisplayMapsWidthPercent / 100f;
float displayMapsHeightPercent = displayMapsWidthPercent * sensorData.depthImageHeight / sensorData.depthImageWidth;
float displayWidth = cameraRect.width * displayMapsWidthPercent;
float displayHeight = cameraRect.width * displayMapsHeightPercent;
// Initialize depth & label map related stuff
usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight);
usersMapRect = new Rect(cameraRect.width - displayWidth, cameraRect.height, displayWidth, -displayHeight);
usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
usersHistogramImage = new Color32[usersMapSize];
usersPrevState = new ushort[usersMapSize];
usersHistogramMap = new float[5001];
}
if(computeColorMap)
{
float displayMapsWidthPercent = DisplayMapsWidthPercent / 100f;
float displayMapsHeightPercent = displayMapsWidthPercent * sensorData.colorImageHeight / sensorData.colorImageWidth;
float displayWidth = cameraRect.width * displayMapsWidthPercent;
float displayHeight = cameraRect.width * displayMapsHeightPercent;
// Initialize color map related stuff
usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false);
usersClrRect = new Rect(cameraRect.width - displayWidth, cameraRect.height, displayWidth, -displayHeight);
usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight;
// if(computeUserMap && displayColorMap)
// {
// usersMapRect.x -= cameraRect.width * displayMapsWidthPercent;
// }
}
// try to automatically use the available avatar controllers in the scene
if(avatarControllers.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach(MonoBehaviour monoScript in monoScripts)
{
if(typeof(AvatarController).IsAssignableFrom(monoScript.GetType()))
{
AvatarController avatar = (AvatarController)monoScript;
avatarControllers.Add(avatar);
}
}
}
// try to automatically use the available gesture listeners in the scene
if(gestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach(MonoBehaviour monoScript in monoScripts)
{
if(typeof(KinectGestures.GestureListenerInterface).IsAssignableFrom(monoScript.GetType()))
{
print ("Found Listener");
//KinectGestures.GestureListenerInterface gl = (KinectGestures.GestureListenerInterface)monoScript;
gestureListeners.Add(monoScript);
}
}
}
// Initialize user list to contain all users.
alUserIds = new List<Int64>();
dictUserIdToIndex = new Dictionary<Int64, int>();
kinectInitialized = true;
DontDestroyOnLoad(gameObject);
// GUI Text.
if(calibrationText != null)
{
calibrationText.guiText.text = "WAITING FOR USERS";
}
Debug.Log("Waiting for users.");
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
// init interface
if (!bNuitrackInited)
{
bNuitrackInited = NuitrackInit();
if (!bNuitrackInited)
{
return(null);
}
}
sensorFlags = dwFlags;
bMultiSource = bUseMultiSource;
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
for (int i = 0; i < WebCamTexture.devices.Length; i++)
{
if (WebCamTexture.devices[i].name.IndexOf("astra", StringComparison.CurrentCultureIgnoreCase) >= 0)
{
Debug.Log(" " + WebCamTexture.devices[i].name + "- AstraPro detected.");
colorWebCam = new WebCamTexture(WebCamTexture.devices[i].name, 640, 480, 30);
break;
}
}
if (!colorWebCam)
{
colorSensor = nuitrack.ColorSensor.Create();
colorSensor.OnUpdateEvent += HandleOnColorSensorUpdateEvent;
}
else
{
bWebColorStream = true;
colorWebCam.Play();
sensorData.colorImageTexture = colorWebCam;
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
depthSensor = nuitrack.DepthSensor.Create();
// depthSensor.SetMirror (true);
depthSensor.OnUpdateEvent += HandleOnDepthSensorUpdateEvent;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
userTracker = nuitrack.UserTracker.Create();
userTracker.OnUpdateEvent += HandleOnUserTrackerUpdateEvent;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonTracker = nuitrack.SkeletonTracker.Create();
skeletonTracker.OnSkeletonUpdateEvent += HandleOnSkeletonUpdateEvent;
handTracker = nuitrack.HandTracker.Create();
handTracker.OnUpdateEvent += HandleOnHandsUpdateEvent;
gestureRecognizer = nuitrack.GestureRecognizer.Create();
gestureRecognizer.OnNewGesturesEvent += OnNewGestures;
}
// if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
// {
// }
nuitrack.Nuitrack.onIssueUpdateEvent += OnIssuesUpdate;
nuitrack.Nuitrack.Run();
sensorData.bodyCount = Constants.SkeletonCount;
sensorData.jointCount = Constants.JointCount;
sensorData.depthCameraOffset = 0f;
sensorData.faceOverlayOffset = 0f;
if (!bWebColorStream)
{
// // wait for color frame
// if (colorSensor != null)
// {
// colorFrame = colorSensor.GetColorFrame();
// float waitTillTime = Time.realtimeSinceStartup + 2.5f;
//
// while (colorFrame == null && Time.realtimeSinceStartup <= waitTillTime)
// {
// nuitrack.Nuitrack.Update();
// System.Threading.Thread.Sleep(50);
// colorFrame = colorSensor.GetColorFrame();
// }
// }
sensorData.colorImageWidth = colorSensor != null?colorSensor.GetOutputMode().XRes : 640;
sensorData.colorImageHeight = colorSensor != null?colorSensor.GetOutputMode().YRes : 480;
// flip color image vertically
sensorData.colorImageScale = new Vector3(1f, -1f, 1f);
}
else
{
sensorData.colorImageWidth = colorWebCam.width;
sensorData.colorImageHeight = colorWebCam.height;
// flip color image horizontally
sensorData.colorImageScale = new Vector3(-1f, 1f, 1f);
}
Debug.Log(" Color sensor: " + (colorSensor != null ? colorSensor.ToString() : "-") +
", width: " + sensorData.colorImageWidth + ", height: " + sensorData.colorImageHeight);
// // wait for depth frame
// if (depthSensor != null)
// {
// depthFrame = depthSensor.GetDepthFrame();
// float waitTillTime = Time.realtimeSinceStartup + 2.5f;
//
// while (depthFrame == null && Time.realtimeSinceStartup <= waitTillTime)
// {
// nuitrack.Nuitrack.Update();
// System.Threading.Thread.Sleep(50);
// depthFrame = depthSensor.GetDepthFrame();
// }
// }
sensorData.depthImageWidth = depthSensor != null?depthSensor.GetOutputMode().XRes : 640;
sensorData.depthImageHeight = depthSensor != null?depthSensor.GetOutputMode().YRes : 480;
Debug.Log(" Depth sensor: " + (depthSensor != null ? depthSensor.ToString() : "-") +
", width: " + sensorData.depthImageWidth + ", height: " + sensorData.depthImageHeight);
// color & depth FOV
float hfovr = colorSensor != null?colorSensor.GetOutputMode().HFOV : 0f;
float vfovr = 2f * Mathf.Atan(Mathf.Tan(hfovr / 2f) * sensorData.depthImageHeight / sensorData.depthImageWidth);
sensorData.colorCameraFOV = vfovr * Mathf.Rad2Deg;
hfovr = depthSensor != null?depthSensor.GetOutputMode().HFOV : 0f;
vfovr = 2f * Mathf.Atan(Mathf.Tan(hfovr / 2f) * sensorData.depthImageHeight / sensorData.depthImageWidth);
sensorData.depthCameraFOV = vfovr * Mathf.Rad2Deg;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
int colorImageSize = !colorWebCam ? (sensorData.colorImageWidth * sensorData.colorImageHeight * 3) : 0;
sensorData.colorImage = new byte[colorImageSize];
if (!colorWebCam)
{
sensorData.colorImageTexture2D = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGB24, false);
}
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
int depthImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
sensorData.depthImage = new ushort[depthImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
int bodyIndexImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
sensorData.bodyIndexImage = new byte[bodyIndexImageSize];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
int depthImageSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
sensorData.infraredImage = new ushort[depthImageSize];
}
// setup coordinate mapper
coordMapper = new OrbbecAstraMapper();
coordMapper.SetupSpaceMapping(sensorData.depthImageWidth, sensorData.depthImageHeight, hfovr, vfovr);
// d2c-calibration data is valid for Orbbec-Astra only (sensor id & calib.data not provided by Nuitrack)
coordMapper.SetupCalibrationData(bWebColorStream);
// set lost-user time tolerance equal to KM
if (KinectManager.Instance != null)
{
waitTimeBeforeRemove = KinectManager.Instance.waitTimeBeforeRemove;
}
Debug.Log("Nuitrack sensor opened");
return(sensorData);
}
public KinectInterop.SensorData OpenDefaultSensor (KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if(hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if(hr != 0)
{
Debug.LogError("Initialization of KinectInteraction failed.");
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public void CloseSensor(KinectInterop.SensorData sensorData)
{
if (colorWebCam)
{
colorWebCam.Stop();
colorWebCam = null;
}
if (coordMapper != null)
{
coordMapper.CleanUp();
coordMapper = null;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
colorSensor = null;
colorFrame = null;
}
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
depthSensor = null;
depthFrame = null;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
userTracker = null;
userFrame = null;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
skeletonTracker = null;
skeletonData = null;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
handTracker = null;
handTrackerData = null;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
gestureRecognizer = null;
}
if (bNuitrackInited)
{
bNuitrackInited = false;
nuitrack.Nuitrack.onIssueUpdateEvent -= OnIssuesUpdate;
NuitrackTerm();
}
Debug.Log("Nuitrack sensor closed");
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
if((dwFlags & KinectInterop.FrameSource.TypeAudio) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesAudio;
}
FacetrackingManager[] faceManagers = GameObject.FindObjectsOfType(typeof(FacetrackingManager)) as FacetrackingManager[];
if(faceManagers != null && faceManagers.Length > 0)
{
for(int i = 0; i < faceManagers.Length; i++)
{
if(faceManagers[i].enabled)
{
//Debug.Log("Found FacetrackingManager => UsesColor");
nuiFlags |= NuiInitializeFlags.UsesColor;
break;
}
}
}
SpeechManager[] speechManagers = GameObject.FindObjectsOfType(typeof(SpeechManager)) as SpeechManager[];
if(speechManagers != null && speechManagers.Length > 0)
{
for(int i = 0; i < speechManagers.Length; i++)
{
if(speechManagers[i].enabled)
{
//Debug.Log("Found SpeechManager => UsesAudio");
nuiFlags |= NuiInitializeFlags.UsesAudio;
break;
}
}
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if(hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if(hr != 0)
{
Debug.LogError(string.Format("Error initializing KinectInteraction: hr=0x{0:X}", hr));
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = 20;
sensorData.depthCameraFOV = 46.6f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0.01f;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public void FreeMultiSourceFrame(KinectInterop.SensorData sensorData)
{
bMultiFramesReady = false;
}
void StartKinect()
{
try
{
// try to initialize the default Kinect2 sensor
KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody;
if(computeUserMap != UserMapType.None)
dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex;
if(computeColorMap)
dwFlags |= KinectInterop.FrameSource.TypeColor;
if(computeInfraredMap)
dwFlags |= KinectInterop.FrameSource.TypeInfrared;
// if(useAudioSource)
// dwFlags |= KinectInterop.FrameSource.TypeAudio;
// open the default sensor
BackgroundRemovalManager brManager = gameObject.GetComponentInChildren<BackgroundRemovalManager>();
sensorData = KinectInterop.OpenDefaultSensor(sensorInterfaces, dwFlags, sensorAngle, useMultiSourceReader, computeUserMap, brManager);
if (sensorData == null)
{
if(sensorInterfaces == null || sensorInterfaces.Count == 0)
throw new Exception("No sensor found. Make sure you have installed the SDK and the sensor is connected.");
else
throw new Exception("OpenDefaultSensor failed.");
}
// enable or disable getting height and angle info
sensorData.hintHeightAngle = (autoHeightAngle != AutoHeightAngle.DontUse);
//create the transform matrix - kinect to world
Quaternion quatTiltAngle = Quaternion.Euler(-sensorAngle, 0.0f, 0.0f);
kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), quatTiltAngle, Vector3.one);
}
catch(DllNotFoundException ex)
{
string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation.";
Debug.LogError(message);
Debug.LogException(ex);
if(calibrationText != null)
{
calibrationText.text = message;
}
return;
}
catch(Exception ex)
{
string message = ex.Message;
Debug.LogError(message);
Debug.LogException(ex);
if(calibrationText != null)
{
calibrationText.text = message;
}
return;
}
// set the singleton instance
instance = this;
// init skeleton structures
bodyFrame = new KinectInterop.BodyFrameData(sensorData.bodyCount, KinectInterop.Constants.MaxJointCount); // sensorData.jointCount
bodyFrame.bTurnAnalisys = allowTurnArounds;
KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters();
switch(smoothing)
{
case Smoothing.Default:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.5f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.05f;
smoothParameters.maxDeviationRadius = 0.04f;
break;
case Smoothing.Medium:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.1f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.1f;
smoothParameters.maxDeviationRadius = 0.1f;
break;
case Smoothing.Aggressive:
smoothParameters.smoothing = 0.7f;
smoothParameters.correction = 0.3f;
smoothParameters.prediction = 1.0f;
smoothParameters.jitterRadius = 1.0f;
smoothParameters.maxDeviationRadius = 1.0f;
break;
}
// init data filters
jointPositionFilter = new JointPositionsFilter();
jointPositionFilter.Init(smoothParameters);
// init the bone orientation constraints
if(useBoneOrientationConstraints)
{
boneConstraintsFilter = new BoneOrientationsConstraint();
boneConstraintsFilter.AddDefaultConstraints();
boneConstraintsFilter.SetDebugText(calibrationText);
}
if(computeUserMap != UserMapType.None && computeUserMap != UserMapType.RawUserDepth)
{
// Initialize depth & label map related stuff
usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight, TextureFormat.ARGB32, false);
usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
usersHistogramImage = new Color32[usersMapSize];
usersPrevState = new ushort[usersMapSize];
usersHistogramMap = new float[5001];
}
if(computeColorMap)
{
// Initialize color map related stuff
//usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false);
usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight;
}
// try to automatically use the available avatar controllers in the scene
if(avatarControllers.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach(MonoBehaviour monoScript in monoScripts)
{
if(typeof(AvatarController).IsAssignableFrom(monoScript.GetType()) && monoScript.enabled)
{
AvatarController avatar = (AvatarController)monoScript;
avatarControllers.Add(avatar);
}
}
}
// set up the gesture manager, if not already set
if(gestureManager == null)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach(MonoBehaviour monoScript in monoScripts)
{
if(typeof(KinectGestures).IsAssignableFrom(monoScript.GetType()) && monoScript.enabled)
{
gestureManager = (KinectGestures)monoScript;
break;
}
}
}
// try to automatically use the available gesture listeners in the scene
if(gestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach(MonoBehaviour monoScript in monoScripts)
{
if(typeof(KinectGestures.GestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
monoScript.enabled)
{
//KinectGestures.GestureListenerInterface gl = (KinectGestures.GestureListenerInterface)monoScript;
gestureListeners.Add(monoScript);
}
}
}
// Initialize user list to contain all users.
//alUserIds = new List<Int64>();
//dictUserIdToIndex = new Dictionary<Int64, int>();
// // start the background reader
// kinectReaderThread = new System.Threading.Thread(UpdateKinectStreamsThread);
// kinectReaderThread.Name = "KinectReaderThread";
// kinectReaderThread.IsBackground = true;
// kinectReaderThread.Start();
// kinectReaderRunning = true;
kinectInitialized = true;
#if USE_SINGLE_KM_IN_MULTIPLE_SCENES
DontDestroyOnLoad(gameObject);
#endif
// GUI Text.
if(calibrationText != null)
{
calibrationText.text = "WAITING FOR USERS";
}
Debug.Log("Waiting for users.");
}
public bool PollBodyFrame(KinectInterop.SensorData sensorData, ref KinectInterop.BodyFrameData bodyFrame,
ref Matrix4x4 kinectToWorld, bool bIgnoreJointZ)
{
bool bNewFrame = false;
// look for skeleton frame
if (skeletonData != null && skeletonDataTimestamp != lastSkeletonFrameTimestamp)
{
lastSkeletonFrameTimestamp = skeletonDataTimestamp;
//long timeNowTicks = DateTime.Now.Ticks;
bodyFrame.liPreviousTime = bodyFrame.liRelativeTime;
bodyFrame.liRelativeTime = skeletonDataTimestamp;
int bodyCount = skeletonData.Skeletons != null ? skeletonData.Skeletons.Length : 0;
if (lastBodyCount != bodyCount)
{
sbDebugBodies.Append(bodyCount).Append(" bodies - ");
}
// clear id2index
//bodyIdToIndex.Clear();
// create bodyIndexUsed-array on the 1st use
if (bodyIndexUsed == null)
{
bodyIndexUsed = new bool[sensorData.bodyCount];
}
// clear the tracked flags and find empty index
int eIndex = -1;
for (int i = 0; i < sensorData.bodyCount; i++)
{
bodyFrame.bodyData[i].bIsTracked = 0;
if (eIndex < 0 && !bodyIndexUsed[i])
{
eIndex = i;
}
}
for (int i = 0; i < sensorData.bodyCount; i++)
{
// compare to real body count
if (i >= bodyCount)
{
//bodyFrame.bodyData[i].bIsTracked = 0;
continue;
}
// get body and joints data
nuitrack.Skeleton nuiBody = skeletonData.Skeletons[i];
if (lastBodyCount != bodyCount)
{
sbDebugBodies.Append(nuiBody.ID).Append(":");
nuitrack.Joint jUser = nuiBody.Joints[(int)nuitrack.JointType.Waist];
Vector3 vUserPos = new Vector3(dontHFlipBodyFrame ? jUser.Real.X : -jUser.Real.X, jUser.Real.Y, jUser.Real.Z);
sbDebugBodies.Append(vUserPos);
sbDebugBodies.Append(" ");
}
// create the body index if needed
ushort uBodyId = (ushort)nuiBody.ID;
if (!bodyIdToIndex.ContainsKey(uBodyId))
{
Debug.Log(" New body ID:" + uBodyId + ", index: " + eIndex);
bodyIdToIndex[uBodyId] = (byte)eIndex;
bodyIndexUsed[eIndex] = true;
}
// get existing body index
int bi = bodyIdToIndex[uBodyId];
bodyIdToTime[uBodyId] = Time.time;
// set body tracking state
bodyFrame.bodyData[bi].bIsTracked = 1;
// transfer body and joints data
bodyFrame.bodyData[bi].liTrackingID = (long)nuiBody.ID;
//bodyIdToIndex[(ushort)nuiBody.ID] = (byte)i;
// z-position of the waist
float waistPosZ = nuiBody.Joints[(int)nuitrack.JointType.Waist].Real.Z / 1000f;
for (int j = 0; j < sensorData.jointCount; j++)
{
KinectInterop.JointData jointData = bodyFrame.bodyData[bi].joint[j];
int nuiJI = !dontHFlipBodyFrame ? BodyJoint2NormalNuitrackJoint[j] : BodyJoint2MirroredNuitrackJoint[j];
if (nuiJI >= 0)
{
nuitrack.Joint nuiJoint = nuiBody.Joints[nuiJI];
if (nuiJoint.Confidence >= 0.5f)
{
jointData.trackingState = KinectInterop.TrackingState.Tracked;
}
else if (nuiJoint.Confidence >= 0.1f)
{
jointData.trackingState = KinectInterop.TrackingState.Inferred;
}
else
{
jointData.trackingState = KinectInterop.TrackingState.NotTracked;
}
Vector3 jointPos = new Vector3((dontHFlipBodyFrame ? nuiJoint.Real.X : -nuiJoint.Real.X) / 1000f, nuiJoint.Real.Y / 1000f, nuiJoint.Real.Z / 1000f);
float jPosZ = (bIgnoreJointZ && j > 0) ? waistPosZ : jointPos.z;
jointData.kinectPos = jointPos;
jointData.position = kinectToWorld.MultiplyPoint3x4(new Vector3(jointPos.x, jointPos.y, jPosZ));
jointData.orientation = Quaternion.identity;
}
else
{
jointData.trackingState = KinectInterop.TrackingState.NotTracked;
}
if (j == 0)
{
bodyFrame.bodyData[bi].position = jointData.position;
bodyFrame.bodyData[bi].orientation = jointData.orientation;
}
bodyFrame.bodyData[bi].joint[j] = jointData;
}
if (handTrackerData != null)
{
nuitrack.UserHands hands = handTrackerData.GetUserHandsByID(nuiBody.ID);
if (hands != null)
{
nuitrack.HandContent?leftHand = !dontHFlipBodyFrame ? hands.LeftHand : hands.RightHand;
bodyFrame.bodyData[bi].leftHandState = leftHand.HasValue && leftHand.Value.Click ? KinectInterop.HandState.Closed : KinectInterop.HandState.Open;
bodyFrame.bodyData[bi].leftHandConfidence = leftHand.HasValue ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low;
nuitrack.HandContent?rightHand = !dontHFlipBodyFrame ? hands.RightHand : hands.LeftHand;
bodyFrame.bodyData[bi].rightHandState = rightHand.HasValue && rightHand.Value.Click ? KinectInterop.HandState.Closed : KinectInterop.HandState.Open;
bodyFrame.bodyData[bi].rightHandConfidence = rightHand.HasValue ? KinectInterop.TrackingConfidence.High : KinectInterop.TrackingConfidence.Low;
}
}
// processes some special joint cases
ProcessBodyFrameSpecialCases(bi, ref bodyFrame);
}
// check for lost users
List <ushort> lostUsers = new List <ushort>();
foreach (ushort uBodyId in bodyIdToTime.Keys)
{
// prevent user removal upon sporadical tracking failures
if ((Time.time - bodyIdToTime[uBodyId]) > waitTimeBeforeRemove)
{
lostUsers.Add(uBodyId);
}
}
// remove the lost users
if (lostUsers.Count > 0)
{
foreach (ushort uBodyId in lostUsers)
{
Debug.Log(" Lost body ID:" + uBodyId + ", index: " + bodyIdToIndex[uBodyId]);
int bi = bodyIdToIndex[uBodyId];
bodyIndexUsed[bi] = false;
bodyIdToIndex.Remove(uBodyId);
bodyIdToTime.Remove(uBodyId);
}
// clean up
lostUsers.Clear();
}
// write bodies-debug info, if needed
if (sbDebugBodies.Length > 0)
{
sbDebugBodies.Append("Time: ").Append(Time.realtimeSinceStartup);
Debug.Log(sbDebugBodies.ToString());
sbDebugBodies.Remove(0, sbDebugBodies.Length);
}
lastBodyCount = bodyCount;
bNewFrame = true;
}
return(bNewFrame);
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if(kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if(sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if(debugText != null)
{
debugText.GetComponent<GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if(sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if(bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
instance = this;
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if(debugText != null)
{
debugText.GetComponent<GUIText>().text = "Ready.";
}
}
catch(DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if(debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
void Start()
{
manager = KinectManager.Instance;
if (manager != null)
{
sensorData = manager.GetSensorData();
depthWidth = manager.GetDepthImageWidth();
depthHeight = manager.GetDepthImageHeight();
sampledWidth = depthWidth / sampleSize;
sampledHeight = depthHeight / sampleSize;
kinectToWorld = manager.GetKinectToWorldMatrix();
//spaceCoords = new Vector3[depthWidth * depthHeight];
if(sensorData.depth2SpaceCoords == null)
{
sensorData.depth2SpaceCoords = new Vector3[depthWidth * depthHeight];
}
vertexType = new byte[sampledWidth * sampledHeight];
vertexIndex = new int[sampledWidth * sampledHeight];
CreateMesh(sampledWidth, sampledHeight);
}
}
