public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
sourceFlags = dwFlags;
NuiInitializeFlags nuiFlags = // NuiInitializeFlags.UsesNone;
NuiInitializeFlags.UsesSkeleton | NuiInitializeFlags.UsesDepthAndPlayerIndex;
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesSkeleton;
}
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesColor;
}
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesDepthAndPlayerIndex;
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
nuiFlags |= (NuiInitializeFlags.UsesColor | (NuiInitializeFlags)0x8000);
}
if ((dwFlags & KinectInterop.FrameSource.TypeAudio) != 0)
{
nuiFlags |= NuiInitializeFlags.UsesAudio;
}
FacetrackingManager[] faceManagers = GameObject.FindObjectsOfType(typeof(FacetrackingManager)) as FacetrackingManager[];
if (faceManagers != null && faceManagers.Length > 0)
{
for (int i = 0; i < faceManagers.Length; i++)
{
if (faceManagers[i].enabled)
{
//Debug.Log("Found FacetrackingManager => UsesColor");
nuiFlags |= NuiInitializeFlags.UsesColor;
break;
}
}
}
SpeechManager[] speechManagers = GameObject.FindObjectsOfType(typeof(SpeechManager)) as SpeechManager[];
if (speechManagers != null && speechManagers.Length > 0)
{
for (int i = 0; i < speechManagers.Length; i++)
{
if (speechManagers[i].enabled)
{
//Debug.Log("Found SpeechManager => UsesAudio");
nuiFlags |= NuiInitializeFlags.UsesAudio;
break;
}
}
}
int hr = InitKinectSensor(nuiFlags, true, (int)Constants.ColorImageResolution, (int)Constants.DepthImageResolution, Constants.IsNearMode);
if (hr == 0)
{
// set sensor angle
SetKinectElevationAngle((int)sensorAngle);
// initialize Kinect interaction
hr = InitKinectInteraction();
if (hr != 0)
{
Debug.LogError(string.Format("Error initializing KinectInteraction: hr=0x{0:X}", hr));
}
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorData.bodyCount = Constants.NuiSkeletonCount;
sensorData.jointCount = Constants.JointCount;
sensorData.depthCameraFOV = 46.6f;
sensorData.colorCameraFOV = 48.6f;
sensorData.depthCameraOffset = 0.01f;
sensorData.faceOverlayOffset = 0.01f;
NuiImageResolutionToSize(Constants.ColorImageResolution, out sensorData.colorImageWidth, out sensorData.colorImageHeight);
// sensorData.colorImageWidth = Constants.ColorImageWidth;
// sensorData.colorImageHeight = Constants.ColorImageHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
//colorStreamHandle = GetColorStreamHandle();
sensorData.colorImage = new byte[sensorData.colorImageWidth * sensorData.colorImageHeight * 4];
}
NuiImageResolutionToSize(Constants.DepthImageResolution, out sensorData.depthImageWidth, out sensorData.depthImageHeight);
// sensorData.depthImageWidth = Constants.DepthImageWidth;
// sensorData.depthImageHeight = Constants.DepthImageHeight;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
//depthStreamHandle = GetDepthStreamHandle();
sensorData.depthImage = new ushort[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
sensorData.bodyIndexImage = new byte[sensorData.depthImageWidth * sensorData.depthImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
sensorData.infraredImage = new ushort[sensorData.colorImageWidth * sensorData.colorImageHeight];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
skeletonFrame = new NuiSkeletonFrame()
{
SkeletonData = new NuiSkeletonData[Constants.NuiSkeletonCount]
};
// default values used to pass to smoothing function
smoothParameters = new NuiTransformSmoothParameters();
smoothParameters.fSmoothing = 0.5f;
smoothParameters.fCorrection = 0.5f;
smoothParameters.fPrediction = 0.5f;
smoothParameters.fJitterRadius = 0.05f;
smoothParameters.fMaxDeviationRadius = 0.04f;
}
return sensorData;
}
else
{
Debug.LogError("InitKinectSensor failed: " + GetNuiErrorString(hr));
}
return null;
}
public KinectInterop.SensorData OpenDefaultSensor(KinectInterop.FrameSource dwFlags, float sensorAngle, bool bUseMultiSource)
{
KinectInterop.SensorData sensorData = new KinectInterop.SensorData();
sensorFlags = dwFlags;
kinectSensor = KinectSensor.GetDefault();
if (kinectSensor == null)
return null;
coordMapper = kinectSensor.CoordinateMapper;
this.bodyCount = kinectSensor.BodyFrameSource.BodyCount;
sensorData.bodyCount = this.bodyCount;
sensorData.jointCount = 25;
sensorData.depthCameraFOV = 60f;
sensorData.colorCameraFOV = 53.8f;
sensorData.depthCameraOffset = -0.05f;
sensorData.faceOverlayOffset = -0.04f;
if ((dwFlags & KinectInterop.FrameSource.TypeBody) != 0)
{
if (!bUseMultiSource)
bodyFrameReader = kinectSensor.BodyFrameSource.OpenReader();
bodyData = new Body[sensorData.bodyCount];
}
var frameDesc = kinectSensor.ColorFrameSource.CreateFrameDescription(ColorImageFormat.Rgba);
sensorData.colorImageWidth = frameDesc.Width;
sensorData.colorImageHeight = frameDesc.Height;
if ((dwFlags & KinectInterop.FrameSource.TypeColor) != 0)
{
if (!bUseMultiSource)
colorFrameReader = kinectSensor.ColorFrameSource.OpenReader();
sensorData.colorImage = new byte[frameDesc.BytesPerPixel * frameDesc.LengthInPixels];
}
sensorData.depthImageWidth = kinectSensor.DepthFrameSource.FrameDescription.Width;
sensorData.depthImageHeight = kinectSensor.DepthFrameSource.FrameDescription.Height;
if ((dwFlags & KinectInterop.FrameSource.TypeDepth) != 0)
{
if (!bUseMultiSource)
depthFrameReader = kinectSensor.DepthFrameSource.OpenReader();
sensorData.depthImage = new ushort[kinectSensor.DepthFrameSource.FrameDescription.LengthInPixels];
}
if ((dwFlags & KinectInterop.FrameSource.TypeBodyIndex) != 0)
{
if (!bUseMultiSource)
bodyIndexFrameReader = kinectSensor.BodyIndexFrameSource.OpenReader();
sensorData.bodyIndexImage = new byte[kinectSensor.BodyIndexFrameSource.FrameDescription.LengthInPixels];
}
if ((dwFlags & KinectInterop.FrameSource.TypeInfrared) != 0)
{
if (!bUseMultiSource)
infraredFrameReader = kinectSensor.InfraredFrameSource.OpenReader();
sensorData.infraredImage = new ushort[kinectSensor.InfraredFrameSource.FrameDescription.LengthInPixels];
}
//if(!kinectSensor.IsOpen)
{
//Debug.Log("Opening sensor, available: " + kinectSensor.IsAvailable);
kinectSensor.Open();
}
float fWaitTime = Time.realtimeSinceStartup + 3f;
while (!kinectSensor.IsAvailable && Time.realtimeSinceStartup < fWaitTime)
{
// wait for sensor to open
}
Debug.Log("K2-sensor " + (kinectSensor.IsOpen ? "opened" : "closed") +
", available: " + kinectSensor.IsAvailable);
if (bUseMultiSource && dwFlags != KinectInterop.FrameSource.TypeNone && kinectSensor.IsOpen)
{
multiSourceFrameReader = kinectSensor.OpenMultiSourceFrameReader((FrameSourceTypes)((int)dwFlags & 0x3F));
}
return sensorData;
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.GetComponent<GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "SM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Speech recognition is not supported!");
}
// Initialize the speech recognizer
string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode);
int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage));
}
if (requiredConfidence > 0)
{
sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence);
}
if (grammarFileName != string.Empty)
{
// copy the grammar file from Resources, if available
//if(!File.Exists(grammarFileName))
{
TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset;
if (textRes != null)
{
string sResText = textRes.text;
File.WriteAllText(grammarFileName, sResText);
}
}
// load the grammar file
rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode, dynamicGrammar);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage);
}
// // test dynamic grammar phrases
// AddGrammarPhrase("addressBook", string.Empty, "Nancy Anderson", true, false);
// AddGrammarPhrase("addressBook", string.Empty, "Cindy White", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "Oliver Lee", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "Alan Brewer", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "April Reagan", false, true);
}
instance = this;
sapiInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent<GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and SAPI installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized.");
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart);
if (bSuccess)
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "BR");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Background removal is not supported!");
}
// Initialize the background removal
bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution);
if (!bSuccess)
{
throw new Exception("Background removal could not be initialized.");
}
// create the foreground image and alpha-image
int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution);
foregroundImage = new byte[imageLength];
// get the needed rectangle
Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution);
// create the foreground texture
foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false);
// calculate the foreground rectangle
if (foregroundCamera != null)
{
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if (rectWidth > rectHeight)
rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height);
else
rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width);
foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
}
instance = this;
isBrInited = true;
//DontDestroyOnLoad(gameObject);
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and BR-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.GetComponent<GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
instance = this;
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent<GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
debugText.GetComponent<GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
debugText.GetComponent<GUIText>().text = ex.Message;
}
}
public void StartKinect()
{
try
{
// try to initialize the default Kinect2 sensor
KinectInterop.FrameSource dwFlags = KinectInterop.FrameSource.TypeBody;
if (computeUserMap)
dwFlags |= KinectInterop.FrameSource.TypeDepth | KinectInterop.FrameSource.TypeBodyIndex;
if (computeColorMap)
dwFlags |= KinectInterop.FrameSource.TypeColor;
if (computeInfraredMap)
dwFlags |= KinectInterop.FrameSource.TypeInfrared;
// if(useAudioSource)
// dwFlags |= KinectInterop.FrameSource.TypeAudio;
// open the default sensor
sensorData = KinectInterop.OpenDefaultSensor(sensorInterfaces, dwFlags, sensorAngle, useMultiSourceReader);
if (sensorData == null)
{
if (sensorInterfaces == null || sensorInterfaces.Count == 0)
{
transform.parent.gameObject.SetActive(false);
//throw new Exception("No sensor found. Make sure you have installed the SDK and the sensor is connected.");
return;
}
else
throw new Exception("OpenDefaultSensor failed.");
}
// enable or disable getting height and angle info
sensorData.hintHeightAngle = (autoHeightAngle != AutoHeightAngle.DontUse);
//create the transform matrix - kinect to world
Quaternion quatTiltAngle = Quaternion.Euler(-sensorAngle, 0.0f, 0.0f);
kinectToWorld.SetTRS(new Vector3(0.0f, sensorHeight, 0.0f), quatTiltAngle, Vector3.one);
}
catch (DllNotFoundException ex)
{
string message = ex.Message + " cannot be loaded. Please check the Kinect SDK installation.";
//Debug.LogError(message);
Debug.LogException(ex);
if (calibrationText != null)
{
calibrationText.GetComponent<GUIText>().text = message;
}
return;
}
catch (Exception ex)
{
string message = ex.Message;
//Debug.LogError(message);
Debug.LogException(ex);
if (calibrationText != null)
{
calibrationText.GetComponent<GUIText>().text = message;
}
return;
}
// set the singleton instance
instance = this;
// init skeleton structures
bodyFrame = new KinectInterop.BodyFrameData(sensorData.bodyCount, KinectInterop.Constants.JointCount); // sensorData.jointCount
bodyFrame.bTurnAnalisys = allowTurnArounds;
KinectInterop.SmoothParameters smoothParameters = new KinectInterop.SmoothParameters();
switch (smoothing)
{
case Smoothing.Default:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.5f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.05f;
smoothParameters.maxDeviationRadius = 0.04f;
break;
case Smoothing.Medium:
smoothParameters.smoothing = 0.5f;
smoothParameters.correction = 0.1f;
smoothParameters.prediction = 0.5f;
smoothParameters.jitterRadius = 0.1f;
smoothParameters.maxDeviationRadius = 0.1f;
break;
case Smoothing.Aggressive:
smoothParameters.smoothing = 0.7f;
smoothParameters.correction = 0.3f;
smoothParameters.prediction = 1.0f;
smoothParameters.jitterRadius = 1.0f;
smoothParameters.maxDeviationRadius = 1.0f;
break;
}
// init data filters
jointPositionFilter = new JointPositionsFilter();
jointPositionFilter.Init(smoothParameters);
// init the bone orientation constraints
if (useBoneOrientationConstraints)
{
boneConstraintsFilter = new BoneOrientationsConstraint();
boneConstraintsFilter.AddDefaultConstraints();
boneConstraintsFilter.SetDebugText(calibrationText);
}
if (computeUserMap)
{
// Initialize depth & label map related stuff
usersLblTex = new Texture2D(sensorData.depthImageWidth, sensorData.depthImageHeight, TextureFormat.ARGB32, false);
usersMapSize = sensorData.depthImageWidth * sensorData.depthImageHeight;
usersHistogramImage = new Color32[usersMapSize];
usersPrevState = new ushort[usersMapSize];
usersHistogramMap = new float[5001];
}
if (computeColorMap)
{
// Initialize color map related stuff
//usersClrTex = new Texture2D(sensorData.colorImageWidth, sensorData.colorImageHeight, TextureFormat.RGBA32, false);
usersClrSize = sensorData.colorImageWidth * sensorData.colorImageHeight;
}
// try to automatically use the available avatar controllers in the scene
if (avatarControllers.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
if (typeof(AvatarController).IsAssignableFrom(monoScript.GetType()) &&
monoScript.enabled)
{
AvatarController avatar = (AvatarController)monoScript;
avatarControllers.Add(avatar);
}
}
}
// try to automatically use the available gesture listeners in the scene
if (gestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
if (typeof(KinectGestures.GestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
monoScript.enabled)
{
//KinectGestures.GestureListenerInterface gl = (KinectGestures.GestureListenerInterface)monoScript;
gestureListeners.Add(monoScript);
}
}
}
// Initialize user list to contain all users.
//alUserIds = new List<Int64>();
//dictUserIdToIndex = new Dictionary<Int64, int>();
kinectInitialized = true;
#if USE_SINGLE_KM_IN_MULTIPLE_SCENES
//DontDestroyOnLoad(gameObject);
#endif
// GUI Text.
if (calibrationText != null)
{
calibrationText.GetComponent<GUIText>().text = "WAITING FOR USERS";
}
Debug.Log("Waiting for users.");
}