public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
//string sGestureText = gesture + " detected";
//if(GestureInfo != null)
//{
// GestureInfo.guiText.text = sGestureText;
//}
if (gesture == KinectGestures.Gestures.Squat) {
int num = 0;
Camera.main.transform.SendMessage ("ActionPerformed",num);
}
else if (gesture == KinectGestures.Gestures.Jump) {
Camera.main.transform.SendMessage ("ActionPerformed",1);
}
else if (gesture == KinectGestures.Gestures.Wave) {
Camera.main.transform.SendMessage ("ActionPerformed",2);
}
else if (gesture == KinectGestures.Gestures.RaiseLeftHand) {
Camera.main.transform.SendMessage ("ActionPerformed",3);
}
else if (gesture == KinectGestures.Gestures.RaiseRightHand) {
Camera.main.transform.SendMessage ("ActionPerformed",4);
}
return true;
}
public void GestureInProgress(long userId, int userIndex, KinectGestures.Gestures gesture,
float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
if((gesture == KinectGestures.Gestures.ZoomOut || gesture == KinectGestures.Gestures.ZoomIn) && progress > 0.5f)
{
string sGestureText = string.Format ("{0} {1:F0}%", gesture, screenPos.z * 100f);
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = sGestureText;
}
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
else if(gesture == KinectGestures.Gestures.Wheel && progress > 0.5f)
{
string sGestureText = string.Format ("{0} {1:F0} deg.", gesture, screenPos.z);
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = sGestureText;
}
//Debug.Log(sGestureText);
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
}
public void GestureInProgress(long userId, int userIndex, KinectGestures.Gestures gesture,
float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
//GestureInfo.guiText.text = string.Format("{0} Progress: {1:F1}%", gesture, (progress * 100));
// if(gesture == KinectGestures.Gestures.Click && progress > 0.3f)
// {
// string sGestureText = string.Format ("{0} {1:F1}% complete", gesture, progress * 100);
// if(GestureInfo != null)
// GestureInfo.guiText.text = sGestureText;
//
// progressDisplayed = true;
// }
// else
if((gesture == KinectGestures.Gestures.ZoomOut || gesture == KinectGestures.Gestures.ZoomIn) && progress > 0.5f)
{
string sGestureText = string.Format ("{0} detected, zoom={1:F1}%", gesture, screenPos.z * 100);
if(GestureInfo != null)
GestureInfo.guiText.text = sGestureText;
progressDisplayed = true;
}
else if(gesture == KinectGestures.Gestures.Wheel && progress > 0.5f)
{
string sGestureText = string.Format ("{0} detected, angle={1:F1} deg", gesture, screenPos.z);
if(GestureInfo != null)
GestureInfo.guiText.text = sGestureText;
progressDisplayed = true;
}
}
public void GestureInProgress(long userId, int userIndex, KinectGestures.Gestures gesture,
float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
// the gestures are allowed for the primary user only
KinectManager manager = KinectManager.Instance;
if(!manager || (userId != manager.GetPrimaryUserID()))
return;
// this function is currently needed only to display gesture progress, skip it otherwise
if(gestureInfo == null)
return;
if((gesture == KinectGestures.Gestures.ZoomOut || gesture == KinectGestures.Gestures.ZoomIn) && progress > 0.5f)
{
string sGestureText = string.Format ("{0} {1:F0}%", gesture, screenPos.z * 100f);
gestureInfo.GetComponent<GUIText>().text = sGestureText;
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
else if(gesture == KinectGestures.Gestures.Wheel && progress > 0.5f)
{
string sGestureText = string.Format ("{0} {1:F0} degrees", gesture, screenPos.z);
gestureInfo.GetComponent<GUIText>().text = sGestureText;
//Debug.Log(sGestureText);
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
}
public void GestureInProgress(long userId, int userIndex, KinectGestures.Gestures gesture,
float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
// the gestures are allowed for the primary user only
KinectManager manager = KinectManager.Instance;
if(!manager || (userId != manager.GetPrimaryUserID()))
return;
if((gesture == KinectGestures.Gestures.ZoomOut || gesture == KinectGestures.Gestures.ZoomIn) && progress > 0.5f)
{
string sGestureText = string.Format ("{0} detected, zoom={1:F1}%", gesture, screenPos.z * 100);
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = sGestureText;
}
//Debug.Log(sGestureText);
progressDisplayed = true;
}
else if(gesture == KinectGestures.Gestures.Wheel && progress > 0.5f)
{
string sGestureText = string.Format ("{0} detected, angle={1:F1} deg", gesture, screenPos.z);
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = sGestureText;
}
//Debug.Log(sGestureText);
progressDisplayed = true;
}
}
//Virtual override of KinectGestures.GestureListenerInterface.GestureCompleted
//Once completed a gesture propogate the gesture to the GestureNavigationMenu
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
string sGestureText = gesture + " detected";
if(GestureInfo != null)
{
GestureInfo.text = sGestureText;
}
//depending on which gesture is performed, call the relevant function in the GestureNavigationMenu
switch (gesture) {
case KinectGestures.Gestures.SwipeUp:
menu.SwipeUp();
break;
case KinectGestures.Gestures.SwipeDown:
menu.SwipeDown();
break;
case KinectGestures.Gestures.SwipeLeft:
menu.SwipeLeft();
break;
case KinectGestures.Gestures.SwipeRight:
menu.SwipeRight();
break;
default:
break;
}
progressDisplayed = false;
return true;
}
// Initialize the filter with a set of TransformSmoothParameters.
public void Init(KinectInterop.SmoothParameters smoothParameters)
{
this.smoothParameters = smoothParameters;
this.Reset();
this.init = true;
}
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
string sGestureText = gesture + " detected";
if(GestureInfo != null)
{
GestureInfo.text = sGestureText;
}
switch (gesture) {
case KinectGestures.Gestures.SwipeUp:
menu.SwipeUp();
break;
case KinectGestures.Gestures.SwipeDown:
menu.SwipeDown();
break;
case KinectGestures.Gestures.SwipeLeft:
menu.SwipeLeft();
break;
case KinectGestures.Gestures.SwipeRight:
menu.SwipeRight();
break;
default:
break;
}
progressDisplayed = false;
return true;
}
public void GestureInProgress(long userId, int userIndex, KinectGestures.Gestures gesture,
float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
/*if((gesture == KinectGestures.Gestures.ZoomOut || gesture == KinectGestures.Gestures.ZoomIn) && progress > 0.5f)
{
string sGestureText = string.Format ("{0} detected, zoom={1:F1}%", gesture, screenPos.z * 100);
if(GestureInfo != null)
{
GestureInfo.text = sGestureText;
}
//Debug.Log(sGestureText);
progressDisplayed = true;
}
else if(gesture == KinectGestures.Gestures.Wheel && progress > 0.5f)
{
string sGestureText = string.Format ("{0} detected, angle={1:F1} deg", gesture, screenPos.z);
if(GestureInfo != null)
{
GestureInfo.text = sGestureText;
}
//Debug.Log(sGestureText);
progressDisplayed = true;
}*/
}
/// <summary>
/// Invoked if a gesture is cancelled.
/// </summary>
/// <returns>true</returns>
/// <c>false</c>
/// <param name="userId">User ID</param>
/// <param name="userIndex">User index</param>
/// <param name="gesture">Gesture type</param>
/// <param name="joint">Joint type</param>
public bool GestureCancelled(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint)
{
// the gestures are allowed for the primary user only
KinectManager manager = KinectManager.Instance;
if(!manager || (userId != manager.GetPrimaryUserID()))
return false;
if(gesture == KinectGestures.Gestures.LeanLeft)
{
leanLeft = false;
}
else if(gesture == KinectGestures.Gestures.LeanRight)
{
leanRight = false;
}
else if(gesture == KinectGestures.Gestures.Wheel)
{
wheel = false;
}
if(gestureInfo != null && progressDisplayed)
{
progressDisplayed = false;
gestureInfo.GetComponent<GUIText>().text = "Lean-left, Lean-right to rotate the Room.";
}
return true;
}
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
if (activeGestures.ContainsKey(userId))
activeGestures[userId].Enqueue(gesture);
return true;
// this true return will clear the gesture info, because once we've completed one gesture,
// we assume that any others started at the same time were actually just misinterpretations.
}
//KinectGestures.GestureListenerInterface virtual implementation
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
string sGestureText = gesture + " detected";
//Propogate the recognised gesture to the cursor controller
if(cursor!=null)cursor.WaveRecognised();
if(cursorDirect!=null)cursorDirect.WaveRecognised();
return true;
}
public bool GestureCancelled(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint)
{
// the gestures are allowed for the primary user only
KinectManager manager = KinectManager.Instance;
if(!manager || (userId != manager.GetPrimaryUserID()))
return false;
// don't do anything here, just reset the gesture state
return true;
}
public void CloseSensor(KinectInterop.SensorData sensorData)
{
if(coordMapper != null)
{
coordMapper = null;
}
if(bodyFrameReader != null)
{
bodyFrameReader.Dispose();
bodyFrameReader = null;
}
if(bodyIndexFrameReader != null)
{
bodyIndexFrameReader.Dispose();
bodyIndexFrameReader = null;
}
if(colorFrameReader != null)
{
colorFrameReader.Dispose();
colorFrameReader = null;
}
if(depthFrameReader != null)
{
depthFrameReader.Dispose();
depthFrameReader = null;
}
if(infraredFrameReader != null)
{
infraredFrameReader.Dispose();
infraredFrameReader = null;
}
if(multiSourceFrameReader != null)
{
multiSourceFrameReader.Dispose();
multiSourceFrameReader = null;
}
if(kinectSensor != null)
{
if (kinectSensor.IsOpen)
{
kinectSensor.Close();
}
kinectSensor = null;
}
}
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
string sGestureText = gesture + " detected";
if(GestureInfo != null)
{
GestureInfo.guiText.text = sGestureText;
}
progressDisplayed = false;
return true;
}
public bool GestureCompleted (long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
string sGestureText = gesture + " detected";
// if(gesture == KinectGestures.Gestures.Click)
// sGestureText += string.Format(" at ({0:F1}, {1:F1})", screenPos.x, screenPos.y);
if(GestureInfo != null)
GestureInfo.guiText.text = sGestureText;
progressDisplayed = false;
return true;
}
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
if(progressDisplayed)
return true;
string sGestureText = gesture + " detected";
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = sGestureText;
}
return true;
}
public bool GestureCancelled(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint)
{
if(progressDisplayed)
{
progressDisplayed = false;
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = String.Empty;
}
}
return true;
}
public bool GestureCancelled(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint)
{
if(progressDisplayed)
{
// clear the progress info
if(GestureInfo != null)
{
GestureInfo.guiText.text = String.Empty;
}
progressDisplayed = false;
}
return true;
}
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
// squat gesture is detected
if (gesture == KinectGestures.Gestures.Squat) {
SquatNum++;
Camera.main.transform.SendMessage ("SquatPerformed");
}
// jump or jumping jack gesture is detected
else if (gesture == KinectGestures.Gestures.Jump) {
JumpNum++;
Camera.main.transform.SendMessage ("JumpPerformed");
}
return true;
}
public bool GestureCancelled(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint)
{
// if(progressDisplayed)
// {
// // clear the progress info
// if(GestureInfo != null)
// {
// GestureInfo.guiText.text = gesture.ToString();
// }
//
// progressDisplayed = false;
// }
return true;
}
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
// the gestures are allowed for the primary user only
KinectManager manager = KinectManager.Instance;
if(!manager || (userId != manager.GetPrimaryUserID()))
return false;
string sGestureText = gesture + " detected";
if(GestureInfo != null)
{
GestureInfo.GetComponent<GUIText>().text = sGestureText;
}
progressDisplayed = false;
return true;
}
/// <summary>
/// Invoked if a gesture is cancelled.
/// </summary>
/// <returns>true</returns>
/// <c>false</c>
/// <param name="userId">User ID</param>
/// <param name="userIndex">User index</param>
/// <param name="gesture">Gesture type</param>
/// <param name="joint">Joint type</param>
public bool GestureCancelled(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint)
{
// the gestures are allowed for the primary user only
KinectManager manager = KinectManager.Instance;
if(!manager || (userId != manager.GetPrimaryUserID()))
return false;
if(progressDisplayed)
{
progressDisplayed = false;
if(gestureInfo != null)
{
gestureInfo.GetComponent<GUIText>().text = String.Empty;
}
}
return true;
}
//public CollectableApplesController applesController;
//public WorkOutController workOutController;
public bool GestureCompleted(long userId, int userIndex, KinectGestures.Gestures gesture,
KinectInterop.JointType joint, Vector3 screenPos)
{
string sGestureText = gesture + " detected";
if(cursor!=null)cursor.WaveRecognised();
/*if (applesController != null)
applesController.WaveRecognised ();
if (workOutController != null)
workOutController.WaveRecognised ();
if (turretAim != null)
turretAim.WaveRecognised ();*/
// if(GestureInfo != null)
// {
// turretGame.FireTurret();
// GestureInfo.guiText.text = sGestureText;
// }
//
// progressDisplayed = false;
//
return true;
}
public void GestureInProgress(long userId, int userIndex, KinectGestures.Gestures gesture,
float progress, KinectInterop.JointType joint, Vector3 screenPos)
{
if((gesture == KinectGestures.Gestures.ZoomOut || gesture == KinectGestures.Gestures.ZoomIn) && progress > 0.5f)
{
if(gestureInfo != null)
{
string sGestureText = string.Format ("{0} - {1:F0}%", gesture, screenPos.z * 100f);
gestureInfo.GetComponent<GUIText>().text = sGestureText;
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
}
else if((gesture == KinectGestures.Gestures.Wheel || gesture == KinectGestures.Gestures.LeanLeft ||
gesture == KinectGestures.Gestures.LeanRight) && progress > 0.5f)
{
if(gestureInfo != null)
{
string sGestureText = string.Format ("{0} - {1:F0} degrees", gesture, screenPos.z);
gestureInfo.GetComponent<GUIText>().text = sGestureText;
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
}
else if(gesture == KinectGestures.Gestures.Run && progress > 0.5f)
{
if(gestureInfo != null)
{
string sGestureText = string.Format ("{0} - progress: {1:F0}%", gesture, progress * 100);
gestureInfo.GetComponent<GUIText>().text = sGestureText;
progressDisplayed = true;
progressGestureTime = Time.realtimeSinceStartup;
}
}
}
// Update the filter with a new frame of data and smooth.
public void UpdateFilter(ref KinectInterop.BodyFrameData bodyFrame)
{
if (this.init == false)
{
this.Init(); // initialize with default parameters
}
KinectInterop.SmoothParameters tempSmoothingParams = new KinectInterop.SmoothParameters();
tempSmoothingParams.smoothing = this.smoothParameters.smoothing;
tempSmoothingParams.correction = this.smoothParameters.correction;
tempSmoothingParams.prediction = this.smoothParameters.prediction;
KinectManager manager = KinectManager.Instance;
int bodyCount = manager.GetBodyCount();
for(int bodyIndex = 0; bodyIndex < bodyCount; bodyIndex++)
{
if(bodyFrame.bodyData[bodyIndex].bIsTracked != 0)
{
FilterBodyJoints(ref bodyFrame.bodyData[bodyIndex], bodyIndex, ref tempSmoothingParams);
}
}
}
public void FinishBackgroundRemoval(KinectInterop.SensorData sensorData)
{
KinectInterop.FinishBackgroundRemoval(sensorData);
bBackgroundRemovalInited = false;
}
public int GetForegroundFrameLength(KinectInterop.SensorData sensorData, bool isHiResPrefered)
{
return(KinectInterop.GetForegroundFrameLength(sensorData, isHiResPrefered));
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Face tracking cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsFaceTrackingAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "FM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Face tracking is not supported!");
}
// Initialize the face tracker
if (!sensorData.sensorInterface.InitFaceTracking(getFaceModelData, displayFaceRect))
{
throw new Exception("Face tracking could not be initialized.");
}
instance = this;
isFacetrackingInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and FT-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
void Update()
{
KinectManager manager = KinectManager.Instance;
// get 1st player
Int64 userID = manager ? manager.GetUserIdByIndex(playerIndex) : 0;
if (userID <= 0)
{
initialPosUserID = 0;
initialPosOffset = Vector3.zero;
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
Vector3 posPointManMP = new Vector3(posPointMan.x, posPointMan.y, !mirroredMovement ? -posPointMan.z : posPointMan.z);
// store the initial position
if (initialPosUserID != userID)
{
initialPosUserID = userID;
//initialPosOffset = transform.position - (verticalMovement ? posPointMan * moveRate : new Vector3(posPointMan.x, 0, posPointMan.z) * moveRate);
initialPosOffset = posPointMan;
}
Vector3 relPosUser = (posPointMan - initialPosOffset);
relPosUser.z = !mirroredMovement ? -relPosUser.z : relPosUser.z;
transform.position = initialPosOffset +
(verticalMovement ? relPosUser * moveRate : new Vector3(relPosUser.x, 0, relPosUser.z) * moveRate);
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = !mirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i);
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !mirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !mirroredMovement);
rotJoint = initialRotation * rotJoint;
posJoint -= posPointManMP;
if (mirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (lines[i] == null && skeletonLine != null)
{
lines[i] = Instantiate((i == 22 || i == 24) && debugLine ? debugLine : skeletonLine) as LineRenderer;
lines[i].transform.parent = transform;
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !mirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
}
else
{
bones[i].gameObject.SetActive(false);
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
public bool IsBackgroundRemovalAvailable(ref bool bNeedRestart)
{
bBackgroundRemovalInited = KinectInterop.IsOpenCvAvailable(ref bNeedRestart);
return(bBackgroundRemovalInited);
}
void Update()
{
if (faceManager == null)
{
faceManager = FacetrackingManager.Instance;
}
if (!kinectManager || !kinectManager.IsInitialized())
{
return;
}
if (!faceManager || !faceManager.IsFaceTrackingInitialized())
{
return;
}
long userId = kinectManager.GetUserIdByIndex(faceManager.playerIndex);
if (userId == 0)
{
return;
}
if (!backManager)
{
backManager = BackgroundRemovalManager.Instance;
if (backManager)
{
// re-initialize the texture
colorTex = null;
}
}
if (backManager && backManager.IsBackgroundRemovalInitialized())
{
// use foreground image
if (!colorTex)
{
colorTex = new Texture2D(kinectManager.GetColorImageWidth(), kinectManager.GetColorImageHeight(), TextureFormat.ARGB32, false);
}
try
{
RenderTexture foregroundTex = (RenderTexture)backManager.GetForegroundTex();
KinectInterop.RenderTex2Tex2D(foregroundTex, ref colorTex);
}
catch (System.Exception)
{
colorTex = (Texture2D)backManager.GetForegroundTex();
}
}
else
{
// use color camera image
colorTex = kinectManager.GetUsersClrTex();
}
//faceRect = faceManager.GetFaceColorRect(userId);
faceRect = GetHeadJointFaceRect(userId);
if (colorTex && faceRect.width > 0 && faceRect.height > 0)
{
int faceX = (int)faceRect.x;
int faceY = (int)faceRect.y;
int faceW = (int)faceRect.width;
int faceH = (int)faceRect.height;
if (faceX < 0)
{
faceX = 0;
}
if (faceY < 0)
{
faceY = 0;
}
if ((faceX + faceW) > colorTex.width)
{
faceW = colorTex.width - faceX;
}
if ((faceY + faceH) > colorTex.height)
{
faceH = colorTex.height - faceY;
}
if (faceTex.width != faceW || faceTex.height != faceH)
{
faceTex.Resize(faceW, faceH);
}
Color[] colorPixels = colorTex.GetPixels(faceX, faceY, faceW, faceH, 0);
faceTex.SetPixels(colorPixels);
faceTex.Apply();
if (targetObject && !targetObject.activeSelf)
{
targetObject.SetActive(true);
}
if (targetRenderer && targetRenderer.material)
{
targetRenderer.material.mainTexture = faceTex;
}
}
else
{
if (targetObject && targetObject.activeSelf)
{
targetObject.SetActive(false);
}
if (targetRenderer && targetRenderer.material && targetRenderer.material.mainTexture != null)
{
targetRenderer.material.mainTexture = null;
}
}
}
/// <summary>
/// Gets the special index by two joint types.
/// </summary>
/// <returns>The spec index by joint.</returns>
/// <param name="joint1">Joint 1 type.</param>
/// <param name="joint2">Joint 2 type.</param>
/// <param name="bMirrored">If set to <c>true</c> gets the mirrored joint index.</param>
public int GetSpecIndexByJoint(KinectInterop.JointType joint1, KinectInterop.JointType joint2, bool bMirrored)
{
int boneIndex = -1;
if((joint1 == KinectInterop.JointType.ShoulderLeft && joint2 == KinectInterop.JointType.SpineShoulder) ||
(joint2 == KinectInterop.JointType.ShoulderLeft && joint1 == KinectInterop.JointType.SpineShoulder))
{
return (!bMirrored ? 25 : 26);
}
else if((joint1 == KinectInterop.JointType.ShoulderRight && joint2 == KinectInterop.JointType.SpineShoulder) ||
(joint2 == KinectInterop.JointType.ShoulderRight && joint1 == KinectInterop.JointType.SpineShoulder))
{
return (!bMirrored ? 26 : 25);
}
return boneIndex;
}
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.text = "Please, wait...";
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "SM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Speech recognition is not supported!");
}
// Initialize the speech recognizer
string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode);
int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage));
}
if (requiredConfidence > 0)
{
sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence);
}
if (grammarFileName != string.Empty)
{
// copy the grammar file from Resources, if available
//if(!File.Exists(grammarFileName))
{
TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset;
if (textRes != null)
{
string sResText = textRes.text;
#if !NETFX_CORE
File.WriteAllText(grammarFileName, sResText);
#else
System.Threading.Tasks.Task task = null;
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
task = CopyGrammarFileToStorageAsync(grammarFileName, sResText);
}, true);
while (task != null && !task.IsCompleted && !task.IsFaulted)
{
task.Wait(100);
}
if (task != null)
{
if (task == null)
{
throw new Exception("Could not create task for CopyGrammarFileToStorageAsync()");
}
else if (task.IsFaulted)
{
throw task.Exception;
}
}
#endif
}
else
{
throw new Exception("Couldn't find grammar resource: " + grammarFileName + ".txt");
}
}
// load the grammar file
rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode, dynamicGrammar);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage);
}
// // test dynamic grammar phrases
// AddGrammarPhrase("addressBook", string.Empty, "Nancy Anderson", true, false);
// AddGrammarPhrase("addressBook", string.Empty, "Cindy White", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "Oliver Lee", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "Alan Brewer", false, false);
// AddGrammarPhrase("addressBook", string.Empty, "April Reagan", false, true);
}
sapiInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.text = "Speech recognizer is ready.";
}
// try to automatically detect the available speech recognition listeners in the scene
if (speechRecognitionListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
if ((monoScript is SpeechRecognitionInterface) && monoScript.enabled)
{
speechRecognitionListeners.Add(monoScript);
}
}
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = "Please check the Kinect and SAPI installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = ex.Message;
}
}
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
instance = this;
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Background removal cannot be started, because KinectManager is missing or not initialized.");
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
bool bSuccess = sensorData.sensorInterface.IsBackgroundRemovalAvailable(ref bNeedRestart);
if (bSuccess)
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "BR");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Background removal is not supported!");
}
// Initialize the background removal
bSuccess = sensorData.sensorInterface.InitBackgroundRemoval(sensorData, colorCameraResolution);
if (!bSuccess)
{
throw new Exception("Background removal could not be initialized.");
}
// create the foreground image and alpha-image
int imageLength = sensorData.sensorInterface.GetForegroundFrameLength(sensorData, colorCameraResolution);
foregroundImage = new byte[imageLength];
// get the needed rectangle
Rect neededFgRect = sensorData.sensorInterface.GetForegroundFrameRect(sensorData, colorCameraResolution);
// create the foreground texture
foregroundTex = new Texture2D((int)neededFgRect.width, (int)neededFgRect.height, TextureFormat.RGBA32, false);
// calculate the foreground rectangle
if (foregroundCamera != null)
{
Rect cameraRect = foregroundCamera.pixelRect;
float rectHeight = cameraRect.height;
float rectWidth = cameraRect.width;
if (rectWidth > rectHeight)
{
rectWidth = Mathf.Round(rectHeight * neededFgRect.width / neededFgRect.height);
}
else
{
rectHeight = Mathf.Round(rectWidth * neededFgRect.height / neededFgRect.width);
}
foregroundRect = new Rect((cameraRect.width - rectWidth) / 2, cameraRect.height - (cameraRect.height - rectHeight) / 2, rectWidth, -rectHeight);
}
isBrInited = true;
//DontDestroyOnLoad(gameObject);
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = "Please check the Kinect and BR-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = ex.Message;
}
}
}
public bool InitSensorInterface(bool bCopyLibs, ref bool bNeedRestart)
{
bool bOneCopied = false, bAllCopied = true;
bool bArchX64 = KinectInterop.Is64bitArchitecture();
string sTargetPath = KinectInterop.GetTargetDllPath(".", bArchX64) + "/";
//string sTargetPath = "./";
if (!bCopyLibs)
{
// check if the native library is there
string sTargetLib = sTargetPath + "UnityInterface2.dll";
bNeedRestart = false;
string sZipFileName = !bArchX64 ? "OpenNI2UnityInterface.x86.zip" : "OpenNI2UnityInterface.x64.zip";
long iTargetSize = KinectInterop.GetUnzippedEntrySize(sZipFileName, "UnityInterface2.dll");
// FileInfo targetFile = new FileInfo(sTargetLib);
// return targetFile.Exists && targetFile.Length == iTargetSize;
return(KinectInterop.IsFileExists(sTargetLib, iTargetSize));
}
// check openni directory and resources
string sOpenNIPath = KinectInterop.GetEnvironmentVariable(!bArchX64 ? "OPENNI2_REDIST" : "OPENNI2_REDIST64");
if (sOpenNIPath == String.Empty || !Directory.Exists(sOpenNIPath))
{
Debug.LogWarning("OpenNI2-folder not found (check OPENNI2_REDIST).");
return(false);
}
sOpenNIPath = sOpenNIPath.Replace('\\', '/');
if (sOpenNIPath.EndsWith("/"))
{
sOpenNIPath = sOpenNIPath.Substring(0, sOpenNIPath.Length - 1);
}
// check nite directory and resources
string sNiTEPath = KinectInterop.GetEnvironmentVariable(!bArchX64 ? "NITE2_REDIST" : "NITE2_REDIST64");
if (sNiTEPath == String.Empty || !Directory.Exists(sNiTEPath))
{
Debug.LogWarning("NiTE2-folder not found (check NITE2_REDIST).");
return(false);
}
sNiTEPath = sNiTEPath.Replace('\\', '/');
if (sNiTEPath.EndsWith("/"))
{
sNiTEPath = sNiTEPath.Substring(0, sNiTEPath.Length - 1);
}
string sUnityInterfaceLib = "UnityInterface2.dll";
string sOpenNI2Lib = "OpenNI2.dll";
string sNiTE22Lib = "NiTE2.dll";
#if UNITY_EDITOR_OSX || UNITY_STANDALONE_OSX
sUnityInterfaceLib = "UnityInterface2.dll";
sOpenNI2Lib = "libOpenNI2.dylib";
sNiTE22Lib = "libNiTE2.dylib";
#endif
KinectInterop.CopyFile(sOpenNIPath + "/" + sOpenNI2Lib, sTargetPath + sOpenNI2Lib, ref bOneCopied, ref bAllCopied);
KinectInterop.CopyFile(sNiTEPath + "/" + sNiTE22Lib, sTargetPath + sNiTE22Lib, ref bOneCopied, ref bAllCopied);
if (!bArchX64)
{
Dictionary <string, string> dictFilesToUnzip = new Dictionary <string, string>();
dictFilesToUnzip[sUnityInterfaceLib] = sTargetPath + sUnityInterfaceLib;
//dictFilesToUnzip["OpenNI2.dll"] = sTargetPath + "OpenNI2.dll";
//dictFilesToUnzip["NiTE2.dll"] = sTargetPath + "NiTE2.dll";
dictFilesToUnzip["OpenNI.ini"] = sTargetPath + "OpenNI.ini";
dictFilesToUnzip["NiTE.ini"] = sTargetPath + "NiTE.ini";
#if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN
dictFilesToUnzip["msvcp100.dll"] = sTargetPath + "msvcp100.dll";
dictFilesToUnzip["msvcr100.dll"] = sTargetPath + "msvcr100.dll";
#endif
KinectInterop.UnzipResourceFiles(dictFilesToUnzip, "OpenNI2UnityInterface.x86.zip", ref bOneCopied, ref bAllCopied);
}
else
{
Dictionary <string, string> dictFilesToUnzip = new Dictionary <string, string>();
dictFilesToUnzip[sUnityInterfaceLib] = sTargetPath + sUnityInterfaceLib;
//dictFilesToUnzip["OpenNI2.dll"] = sTargetPath + "OpenNI2.dll";
//dictFilesToUnzip["NiTE2.dll"] = sTargetPath + "NiTE2.dll";
dictFilesToUnzip["OpenNI.ini"] = sTargetPath + "OpenNI.ini";
dictFilesToUnzip["NiTE.ini"] = sTargetPath + "NiTE.ini";
#if UNITY_EDITOR_WIN || UNITY_STANDALONE_WIN
dictFilesToUnzip["msvcp100.dll"] = sTargetPath + "msvcp100.dll";
dictFilesToUnzip["msvcr100.dll"] = sTargetPath + "msvcr100.dll";
#endif
KinectInterop.UnzipResourceFiles(dictFilesToUnzip, "OpenNI2UnityInterface.x64.zip", ref bOneCopied, ref bAllCopied);
}
if (File.Exists(sTargetPath + "OpenNI.ini"))
{
string sFileContent = File.ReadAllText(sTargetPath + "OpenNI.ini");
sFileContent = sFileContent.Replace("%OPENNI_REDIST_DIR%", sOpenNIPath);
File.WriteAllText(sTargetPath + "OpenNI.ini", sFileContent);
}
if (File.Exists(sTargetPath + "NiTE.ini"))
{
string sFileContent = File.ReadAllText(sTargetPath + "NiTE.ini");
sFileContent = sFileContent.Replace("%NITE_REDIST_DIR%", sNiTEPath);
File.WriteAllText(sTargetPath + "NiTE.ini", sFileContent);
}
bNeedRestart = (bOneCopied && bAllCopied);
return(true);
}
public bool PollForegroundFrame(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, bool bLimitedUsers, ICollection <int> alTrackedIndexes, ref byte[] foregroundImage)
{
return(KinectInterop.PollForegroundFrame(sensorData, isHiResPrefered, defaultColor, bLimitedUsers, alTrackedIndexes, ref foregroundImage));
}
public bool InitBackgroundRemoval(KinectInterop.SensorData sensorData, bool isHiResPrefered)
{
return(KinectInterop.InitBackgroundRemoval(sensorData, isHiResPrefered));
}
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed
private void UpdateTextureWithNewFrame()
{
if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamBodyIndexImage == null || sensorData.colorCamDepthImage == null)
{
return;
}
// get body index frame
if (lastColorCamDepthFrameTime != sensorData.lastColorCamDepthFrameTime || lastColorCamBodyIndexFrameTime != sensorData.lastColorCamBodyIndexFrameTime)
{
lastColorCamDepthFrameTime = sensorData.lastColorCamDepthFrameTime;
lastColorCamBodyIndexFrameTime = sensorData.lastColorCamBodyIndexFrameTime;
if (bodyImageTexture.width != sensorData.colorImageWidth || bodyImageTexture.height != sensorData.colorImageHeight)
{
bodyImageTexture = KinectInterop.CreateRenderTexture(bodyImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight);
}
Array.Clear(depthBodyBufferData, 0, depthBodyBufferData.Length);
Array.Clear(equalBodyBufferData, 0, equalBodyBufferData.Length);
bodyHistTotalPoints = 0;
// get configured min & max distances
float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance;
float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance;
int depthMinDistance = (int)(minDistance * 1000f);
int depthMaxDistance = (int)(maxDistance * 1000f);
int frameLen = sensorData.colorCamDepthImage.Length;
for (int i = 0; i < frameLen; i++)
{
int depth = sensorData.colorCamDepthImage[i];
int limDepth = (depth >= depthMinDistance && depth <= depthMaxDistance) ? depth : 0;
if (/**rawBodyIndexImage[i] != 255 &&*/ limDepth > 0)
{
depthBodyBufferData[limDepth]++;
bodyHistTotalPoints++;
}
}
if (bodyHistTotalPoints > 0)
{
equalBodyBufferData[0] = depthBodyBufferData[0];
for (int i = 1; i < depthBodyBufferData.Length; i++)
{
equalBodyBufferData[i] = equalBodyBufferData[i - 1] + depthBodyBufferData[i];
}
}
int bodyIndexBufferLength = sensorData.colorCamBodyIndexImage.Length >> 2;
if (bodyIndexBuffer == null || bodyIndexBuffer.count != bodyIndexBufferLength)
{
bodyIndexBuffer = KinectInterop.CreateComputeBuffer(bodyIndexBuffer, bodyIndexBufferLength, sizeof(uint));
}
KinectInterop.SetComputeBufferData(bodyIndexBuffer, sensorData.colorCamBodyIndexImage, bodyIndexBufferLength, sizeof(uint));
int depthBufferLength = sensorData.colorCamDepthImage.Length >> 1;
if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength)
{
depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint));
}
KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.colorCamDepthImage, depthBufferLength, sizeof(uint));
if (bodyHistBuffer != null)
{
KinectInterop.SetComputeBufferData(bodyHistBuffer, equalBodyBufferData, equalBodyBufferData.Length, sizeof(int));
}
float minDist = kinectManager.minUserDistance != 0f ? kinectManager.minUserDistance : minDistance;
float maxDist = kinectManager.maxUserDistance != 0f ? kinectManager.maxUserDistance : maxDistance;
bodyImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth);
bodyImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight);
bodyImageMaterial.SetInt("_MinDepth", (int)(minDist * 1000f));
bodyImageMaterial.SetInt("_MaxDepth", (int)(maxDist * 1000f));
bodyImageMaterial.SetBuffer("_BodyIndexMap", bodyIndexBuffer);
bodyImageMaterial.SetBuffer("_DepthMap", depthImageBuffer);
bodyImageMaterial.SetBuffer("_HistMap", bodyHistBuffer);
bodyImageMaterial.SetInt("_TotalPoints", bodyHistTotalPoints);
Color[] bodyIndexColors = kinectManager.GetBodyIndexColors();
bodyImageMaterial.SetColorArray("_BodyIndexColors", bodyIndexColors);
Graphics.Blit(null, bodyImageTexture, bodyImageMaterial);
}
}
void Update()
{
KinectManager manager = KinectManager.Instance;
// if(manager.GetUserIdByIndex(playerIndex) != userID || manager.GetUserIdByIndex(playerIndex) == -1)
// {
// Destroy(gameObject);
// CharacterManager.Instance.numCharacters --;
// }
userID = KinectManager.Instance ? KinectManager.Instance.GetUserIdByIndex(playerIndex) : 0;
if (userID <= 0)
{
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (skeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
//TODO: verify two dancer interaction
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
posPointMan.z = !mirroredMovement ? -posPointMan.z : posPointMan.z;
// store the initial position
// if(initialPosUserID != userID)
// {
// initialPosUserID = userID;
// initialPosOffset = transform.position - (verticalMovement ? posPointMan * moveRate : new Vector3(posPointMan.x, 0, posPointMan.z) * moveRate);
// }
transform.position =
(verticalMovement ? posPointMan * moveRate : new Vector3(posPointMan.x, 0, posPointMan.z) * moveRate);
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = (int)manager.GetJointAtIndex(
!mirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i));
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !mirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !mirroredMovement);
rotJoint = rotJoint;
posJoint -= posPointMan;
if (mirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (skeletonLine)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !mirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
}
else
{
bones[i].gameObject.SetActive(false);
if (skeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
private void DrawLine(Texture2D a_Texture, Vector2 ptStart, Vector2 ptEnd, UnityEngine.Color a_Color)
{
KinectInterop.DrawLine(a_Texture, (int)ptStart.x, (int)ptStart.y, (int)ptEnd.x, (int)ptEnd.y, a_Color);
}
public bool UpdateBackgroundRemoval(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, bool bAlphaTexOnly)
{
return(KinectInterop.UpdateBackgroundRemoval(sensorData, isHiResPrefered, defaultColor, bAlphaTexOnly));
}
// updates the textures and buffers, needed by the mesh renderer
private void UpdateTexturesAndBuffers()
{
if (sensorData == null || sensorInt == null)
{
return;
}
// get image resolution
Vector2Int imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
if (colorTextureCreated && (colorTexture == null ||
colorTexture.width != imageRes.x || colorTexture.height != imageRes.y))
{
colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
sensorInt.pointCloudColorTexture = colorTexture;
//Debug.Log("Created pointCloudColorTexture with resolution " + imageRes);
}
// create depth image buffer
if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution)
{
int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight >> 1;
if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength)
{
depthImageCopy = new ushort[depthBufferLength << 1];
depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint));
//Debug.Log("Created depthImageBuffer with length " + depthBufferLength);
}
//meshShaderMat.SetBuffer("_DepthMap", depthImageBuffer);
}
else
{
int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight >> 1;
if (sensorData.colorDepthBuffer == null || sensorData.colorDepthBuffer.count != bufferLength)
{
sensorData.colorDepthBuffer = KinectInterop.CreateComputeBuffer(sensorData.colorDepthBuffer, bufferLength, sizeof(uint));
//Debug.Log("Created colorDepthBuffer with length " + bufferLength);
}
//meshShaderMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer);
if (meshTexture == MeshTextureType.InfraredTexture)
{
if (sensorData.colorInfraredBuffer == null || sensorData.colorInfraredBuffer.count != bufferLength)
{
sensorData.colorInfraredBuffer = KinectInterop.CreateComputeBuffer(sensorData.colorInfraredBuffer, bufferLength, sizeof(uint));
//Debug.Log("Created colorInfraredBuffer with length " + bufferLength);
}
}
}
if (colorTextureCopy == null || colorTextureCopy.width != imageRes.x || colorTextureCopy.height != imageRes.y)
{
// create copy texture
colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
//Debug.Log("Created colorTextureCopy with resolution " + imageRes);
}
if (spaceTableBuffer == null || imageWidth != imageRes.x || imageHeight != imageRes.y)
{
// create space table
spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ?
sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData);
int spaceBufferLength = imageRes.x * imageRes.y * 3;
spaceTableBuffer = KinectInterop.CreateComputeBuffer(spaceTableBuffer, spaceBufferLength, sizeof(float));
spaceTableBuffer.SetData(spaceTable);
spaceTable = null;
//Debug.Log("Created spaceTable for resolution " + imageRes);
}
// create mesh vertices & indices
if (imageWidth != imageRes.x || imageHeight != imageRes.y)
{
CreateMeshVertInd(imageRes.x, imageRes.y);
}
// image width & height
imageWidth = imageRes.x;
imageHeight = imageRes.y;
}
void Update()
{
KinectManager manager = KinectManager.Instance;
// get 1st player
Int64 userID = manager ? manager.GetUserIdByIndex(playerIndex) : 0;
if (userID <= 0)
{
initialPosUserID = 0;
initialPosOffset = Vector3.zero;
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
Vector3 posPointManMP = new Vector3(posPointMan.x, posPointMan.y, !mirroredMovement ? -posPointMan.z : posPointMan.z);
// store the initial position
if (initialPosUserID != userID)
{
initialPosUserID = userID;
//initialPosOffset = transform.position - (verticalMovement ? posPointMan * moveRate : new Vector3(posPointMan.x, 0, posPointMan.z) * moveRate);
initialPosOffset = posPointMan;
}
Vector3 relPosUser = (posPointMan - initialPosOffset);
relPosUser.z = !mirroredMovement ? -relPosUser.z : relPosUser.z;
transform.position = initialPosOffset + (verticalMovement ? relPosUser * moveRate : new Vector3(relPosUser.x, 0, relPosUser.z) * moveRate);
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = !mirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i);
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !mirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !mirroredMovement);
rotJoint = initialRotation * rotJoint;
posJoint -= posPointManMP;
if (mirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (lines[i] == null && skeletonLine != null)
{
lines[i] = Instantiate((i == 22 || i == 24) && debugLine ? debugLine : skeletonLine) as LineRenderer;
lines[i].transform.parent = transform;
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !mirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
///MED7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if (i == 6)
{
Vector3 HipLeftPos = bones[12].transform.localPosition;
Vector3 HipRightPos = bones[16].transform.localPosition;
dist = Math.Abs(HipLeftPos.y - HipRightPos.y);
lowPassFilterVal = minFreq * Math.Pow((Math.Pow((maxFreq / minFreq), (1 / interval))), (dist));
theMixer.SetFloat("Torso_CutOffFreqLP", (float)lowPassFilterVal);
//Debug.Log("dist: " + dist + " lowPassFilterVal: " + lowPassFilterVal);
//---------------------------------------------------------------------------
//Quaternion HipCenterRota = bones[0].transform.rotation;
//Debug.Log("Rotation CHip: " + HipCenterRota.w +" " + HipCenterRota.x + " " + HipCenterRota.y + " " + HipCenterRota.z);
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
}
else
{
bones[i].gameObject.SetActive(false);
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
// Apply the rotations tracked by kinect to a special joint
protected void TransformSpecialBone(Int64 userId, KinectInterop.JointType joint, KinectInterop.JointType jointParent, int boneIndex, Vector3 baseDir, bool flip)
{
Transform boneTransform = bones[boneIndex];
if(boneTransform == null || kinectManager == null)
return;
if(!kinectManager.IsJointTracked(userId, (int)joint) ||
!kinectManager.IsJointTracked(userId, (int)jointParent))
{
return;
}
Vector3 jointDir = kinectManager.GetJointDirection(userId, (int)joint, false, true);
Quaternion jointRotation = jointDir != Vector3.zero ? Quaternion.FromToRotation(baseDir, jointDir) : Quaternion.identity;
if(!flip)
{
Vector3 mirroredAngles = jointRotation.eulerAngles;
mirroredAngles.y = -mirroredAngles.y;
mirroredAngles.z = -mirroredAngles.z;
jointRotation = Quaternion.Euler(mirroredAngles);
}
if(jointRotation != Quaternion.identity)
{
// Smoothly transition to the new rotation
Quaternion newRotation = Kinect2AvatarRot(jointRotation, boneIndex);
if(smoothFactor != 0f)
boneTransform.rotation = Quaternion.Slerp(boneTransform.rotation, newRotation, smoothFactor * Time.deltaTime);
else
boneTransform.rotation = newRotation;
}
}
public Rect GetForegroundFrameRect(KinectInterop.SensorData sensorData, bool isHiResPrefered)
{
return(KinectInterop.GetForegroundFrameRect(sensorData, isHiResPrefered));
}
void Update()
{
if (!shaderRectInited)
{
PortraitBackground portraitBack = PortraitBackground.Instance;
if (portraitBack && portraitBack.IsInitialized())
{
shaderUvRect = portraitBack.GetShaderUvRect();
}
if (userBlendMat != null)
{
userBlendMat.SetFloat("_ColorOfsX", shaderUvRect.x);
userBlendMat.SetFloat("_ColorMulX", shaderUvRect.width);
userBlendMat.SetFloat("_ColorOfsY", shaderUvRect.y);
userBlendMat.SetFloat("_ColorMulY", shaderUvRect.height);
}
shaderRectInited = true;
}
if (kinectManager && kinectManager.IsInitialized())
{
KinectInterop.SensorData sensorData = kinectManager.GetSensorData();
if (sensorData != null && sensorData.depthImage != null && sensorData.colorImageTexture &&
userBlendMat != null && lastDepthFrameTime != sensorData.lastDepthFrameTime)
{
lastDepthFrameTime = sensorData.lastDepthFrameTime;
//userBlendMat.SetTexture("_ColorTex", sensorData.colorImageTexture);
//if (kinectManager.autoHeightAngle == KinectManager.AutoHeightAngle.AutoUpdate || kinectManager.autoHeightAngle == KinectManager.AutoHeightAngle.AutoUpdateAndShowInfo)
//{
// depthFactor = 1f + Mathf.Sin (Mathf.Abs (kinectManager.sensorAngle) * Mathf.Deg2Rad);
// userBlendMat.SetFloat("_DepthFactor", depthFactor);
//}
if (KinectInterop.MapColorFrameToDepthCoords(sensorData, ref color2DepthCoords))
{
color2DepthBuffer.SetData(color2DepthCoords);
}
// buffer for depths
for (int i = 0; i < sensorData.depthImage.Length; i++)
{
ushort depth = sensorData.depthImage[i];
depthImageBufferData[i] = (float)depth;
}
depthImageBuffer.SetData(depthImageBufferData);
if (backgroundImage)
{
userBlendMat.SetTexture("_BackTex", backgroundImage.texture);
}
// color camera texture
Texture colorTex = backManager && sensorData.color2DepthTexture ? (Texture)sensorData.color2DepthTexture : (Texture)sensorData.colorImageTexture;
userBlendMat.SetTexture("_ColorTex", colorTex);
}
}
}
void Update()
{
//MED7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if (!isAttenReset)
{
theMixer.SetFloat("Torso_Attenuation", -50);
isAttenReset = true;
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
KinectManager manager = KinectManager.Instance;
// get 1st player
Int64 userID = manager ? manager.GetUserIdByIndex(playerIndex) : 0;
if (userID <= 0)
{
initialPosUserID = 0;
initialPosOffset = Vector3.zero;
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
Vector3 posPointManMP = new Vector3(posPointMan.x, posPointMan.y, !mirroredMovement ? -posPointMan.z : posPointMan.z);
// store the initial position
if (initialPosUserID != userID)
{
initialPosUserID = userID;
//initialPosOffset = transform.position - (verticalMovement ? posPointMan * moveRate : new Vector3(posPointMan.x, 0, posPointMan.z) * moveRate);
initialPosOffset = posPointMan;
}
Vector3 relPosUser = (posPointMan - initialPosOffset);
relPosUser.z = !mirroredMovement ? -relPosUser.z : relPosUser.z;
transform.position = initialPosOffset + (verticalMovement ? relPosUser * moveRate : new Vector3(relPosUser.x, 0, relPosUser.z) * moveRate);
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = !mirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i);
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !mirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !mirroredMovement);
rotJoint = initialRotation * rotJoint;
posJoint -= posPointManMP;
if (mirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (lines[i] == null && skeletonLine != null)
{
lines[i] = Instantiate((i == 22 || i == 24) && debugLine ? debugLine : skeletonLine) as LineRenderer;
lines[i].transform.parent = transform;
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !mirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
///MED7 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
if (i == 6)
{
Vector3 posHipCenter = bones[0].transform.localPosition;
Vector3 posShoulderCenter = bones[1].transform.localPosition;
Vector3 posNeck = bones[2].transform.localPosition;
distHipShoulder = CalXZdist(posShoulderCenter, posHipCenter);
distShoulderNeck = CalXZdist(posShoulderCenter, posNeck);
totalDist = distHipShoulder + distShoulderNeck;
totalAttenuation = ScalingBetween(totalDist, minAtten, maxAtten, minDist, maxDist);
//testing this...
float linear = totalAttenuation;
double hepson = (70 * Math.Log10(((double)totalAttenuation / 2) + 42)) - 80;
totalAttenuation = (float)hepson + 20;
Debug.Log("OFF linear: " + linear + " ON log: " + hepson);
if (totalAttenuation > maxAtten)
{
totalAttenuation = maxAtten;
}
else if (totalAttenuation < minAtten)
{
totalAttenuation = minAtten;
}
theMixer.SetFloat("Torso_Attenuation", totalAttenuation);
//Debug.Log("totalDist: " + totalDist + " totalAttenuation: " + totalAttenuation);
//---------------------------------------------------------------------------
//current problem: only 2 points...
float AxisZ = posHipCenter.z - posShoulderCenter.z;
float AxisX = posHipCenter.x - posShoulderCenter.x;
if (AxisZ < 0)
{
NegaScaledAxisZ = ScalingBetween(-AxisZ, 0.3f, 1f, minAxis, maxAxis);
theMixer.SetFloat("Torso_EqFreqGain_00", NegaScaledAxisZ);
}
else if (AxisZ >= 0)
{
PosiScaledAxisZ = ScalingBetween(AxisZ, 0.3f, 1f, minAxis, maxAxis);
theMixer.SetFloat("Torso_EqFreqGain_01", PosiScaledAxisZ);
}
if (AxisX < 0)
{
NegaScaledAxisX = ScalingBetween(-AxisX, 0.3f, 1f, minAxis, maxAxis);
theMixer.SetFloat("Torso_EqFreqGain_02", NegaScaledAxisX);
}
else if (AxisX >= 0)
{
PosiScaledAxisX = ScalingBetween(AxisX, 0.3f, 1f, minAxis, maxAxis);
theMixer.SetFloat("Torso_EqFreqGain_03", PosiScaledAxisX);
}
//Debug.Log("negZ: " + NegaScaledAxisZ + " posZ: " + PosiScaledAxisZ + " negX: " + NegaScaledAxisX + " posX: " + PosiScaledAxisX);
//-------------------------------------------------------------------------------
//Score
score = 100 - ScalingBetween(totalDist, 0, 100, minDist, maxDist);
if (score < 0)
{
score = 0;
}
else if (score > 100)
{
score = 100;
}
//Debug.Log("score: " + score);
}
//+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
}
else
{
bones[i].gameObject.SetActive(false);
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
void Update()
{
if (!manager || !manager.IsInitialized())
{
return;
}
// get required player
long userId = manager.GetUserIdByIndex(playerIndex);
if (trackedUserId != userId)
{
// user lost
trackedUserId = 0;
}
if (trackedUserId == 0 && userId != 0)
{
// new user found
trackedUserId = userId;
}
if (trackedUserId != 0 && sensorData.bodyIndexImage != null && sensorData.depthImage != null &&
sensorData.lastDepthFrameTime != lastDepthFrameTime)
{
lastDepthFrameTime = sensorData.lastDepthFrameTime;
userBodyIndex = (byte)manager.GetBodyIndexByUserId(trackedUserId);
TrackDepthAroundJoint((int)KinectInterop.JointType.HandLeft, ref dposHandLeft, ref rectObjectHandLeft, ref depthMinMaxHL, ref fillRatioLeftHand);
TrackDepthAroundJoint((int)KinectInterop.JointType.HandRight, ref dposHandRight, ref rectObjectHandRight, ref depthMinMaxHR, ref fillRatioRightHand);
// CalculateObjectSize(dposHandLeft, rectObjectHandLeft, depthMinMaxHL, ref sizeObjectHandLeft);
// CalculateObjectSize(dposHandRight, rectObjectHandRight, depthMinMaxHR, ref sizeObjectHandRight);
Texture2D texDepth = manager.GetUsersLblTex();
bool bRectDrawn = false;
if (rectObjectHandLeft.width != 0f && rectObjectHandLeft.height != 0f && dposHandLeft != Vector2.zero)
{
KinectInterop.DrawRect(texDepth, rectObjectHandLeft, fillRatioLeftHand > fillThreshold ? Color.green : Color.yellow);
bRectDrawn = true;
}
if (rectObjectHandRight.width != 0f && rectObjectHandRight.height != 0f && dposHandRight != Vector2.zero)
{
KinectInterop.DrawRect(texDepth, rectObjectHandRight, fillRatioRightHand > fillThreshold ? Color.green : Color.yellow);
bRectDrawn = true;
}
if (bRectDrawn)
{
texDepth.Apply();
}
StringBuilder sbStatusText = new StringBuilder();
sbStatusText.AppendFormat("LH-Fill: {0:F1}%", fillRatioLeftHand * 100f);
if (fillRatioLeftHand > fillThreshold)
{
sbStatusText.Append(" - Object Found");
}
sbStatusText.AppendLine();
sbStatusText.AppendFormat("RF-Fill: {0:F1}%", fillRatioRightHand * 100f);
if (fillRatioRightHand > fillThreshold)
{
sbStatusText.Append(" - Object Found");
}
sbStatusText.AppendLine();
// if (!float.IsNaN(sizeObjectHandLeft.x) && !float.IsNaN(sizeObjectHandLeft.y) && !float.IsNaN(sizeObjectHandLeft.z))
// {
// sbStatusText.AppendFormat("L: ({0:F2}, {1:F2}, {2:F2}), {3:F1}%\n", sizeObjectHandLeft.x, sizeObjectHandLeft.y, sizeObjectHandLeft.z, fillRatioLeftHand * 100f);
// }
//
// if (!float.IsNaN(sizeObjectHandRight.x) && !float.IsNaN(sizeObjectHandRight.y) && !float.IsNaN(sizeObjectHandRight.z))
// {
// sbStatusText.AppendFormat("R: ({0:F2}, {1:F2}, {2:F2}), {3:F1}%\n", sizeObjectHandRight.x, sizeObjectHandRight.y, sizeObjectHandRight.z, fillRatioRightHand * 100f);
// }
//
// if (fillRatioLeftHand > fillThreshold)
// sbStatusText.Append("Found object in the left hand.\n");
// if (fillRatioRightHand > fillThreshold)
// sbStatusText.Append("Found object in the right hand.\n");
if (statusText)
{
statusText.text = sbStatusText.ToString();
}
}
}
// updates the mesh according to current depth frame
private void UpdateMesh()
{
if (bMeshInited && sensorData.depthImage != null && sensorData.depthCamIntr != null && meshShaderMat != null &&
lastDepthFrameTime != sensorData.lastDepthFrameTime && (Time.time - lastMeshUpdateTime) >= updateMeshInterval)
{
lastDepthFrameTime = sensorData.lastDepthFrameTime;
lastMeshUpdateTime = Time.time;
int paramsCache = coarseFactor + (showAsPointCloud ? 10 : 0);
if (meshParamsCache != paramsCache)
{
//Debug.Log("Mesh params changed. Recreating...");
CreateMeshVertInd();
}
if (sensorData.depthImageBuffer != null && sensorData.depthImage != null && depthBufferCreated)
{
int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2;
KinectInterop.SetComputeBufferData(sensorData.depthImageBuffer, sensorData.depthImage, depthBufferLength, sizeof(uint));
}
switch (meshTexture)
{
case MeshTextureType.ColorTexture:
if (colorTexture != null)
{
Graphics.CopyTexture(colorTexture, colorTextureCopy);
}
break;
case MeshTextureType.InfraredTexture:
Texture infraredTexture = kinectManager.GetInfraredImageTex(sensorIndex);
if (infraredTexture != null)
{
Graphics.CopyTexture(infraredTexture, colorTextureCopy);
}
break;
}
if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution)
{
meshShaderMat.SetBuffer("_DepthMap", sensorData.depthImageBuffer);
}
else
{
meshShaderMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer);
sensorData.usedColorDepthBufferTime = sensorData.lastColorDepthBufferTime;
}
meshShaderMat.SetInt("_CoarseFactor", coarseFactor);
meshShaderMat.SetMatrix("_Transform", sensorInt.GetSensorToWorldMatrix());
meshShaderMat.SetVector("_PosMin", new Vector3(xMin, yMin, zMin));
meshShaderMat.SetVector("_PosMax", new Vector3(xMax, yMax, zMax));
meshShaderMat.SetVector("_TexRes", new Vector2(imageWidth, imageHeight));
meshShaderMat.SetVector("_Cxy", new Vector2(sensorData.depthCamIntr.ppx, sensorData.depthCamIntr.ppy));
meshShaderMat.SetVector("_Fxy", new Vector2(sensorData.depthCamIntr.fx, sensorData.depthCamIntr.fy));
meshShaderMat.SetTexture("_ColorTex", colorTextureCopy);
meshShaderMat.SetVector("_SpaceScale", kinectManager.GetSensorSpaceScale(sensorIndex)); // kinectManager.GetDepthImageScale(sensorIndex)
meshShaderMat.SetBuffer("_SpaceTable", spaceTableBuffer);
// mesh bounds
Vector3 boundsCenter = new Vector3((xMax - xMin) / 2f, (yMax - yMin) / 2f, (zMax - zMin) / 2f);
Vector3 boundsSize = new Vector3((xMax - xMin), (yMax - yMin), (zMax - zMin));
mesh.bounds = new Bounds(boundsCenter, boundsSize);
if (updateColliderInterval > 0 && (Time.time - lastColliderUpdateTime) >= updateColliderInterval)
{
lastColliderUpdateTime = Time.time;
MeshCollider meshCollider = GetComponent <MeshCollider>();
if (meshCollider)
{
meshCollider.sharedMesh = null;
meshCollider.sharedMesh = mesh;
}
}
}
}
public bool PollForegroundFrame(KinectInterop.SensorData sensorData, bool isHiResPrefered, Color32 defaultColor, ref byte[] foregroundImage)
{
return(KinectInterop.PollForegroundFrame(sensorData, isHiResPrefered, defaultColor, ref foregroundImage));
}
void Update()
{
KinectManager manager = KinectManager.Instance;
// get 1st player
Int64 userID = manager ? manager.GetUserIdByIndex(playerIndex) : 0;
if (userID <= 0)
{
initialPosUserID = 0;
initialPosOffset = Vector3.zero;
initialPosSetY = true;
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
bones[i].gameObject.SetActive(false);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
Vector3 posPointManWorld = new Vector3(posPointMan.x, posPointMan.y, invertedZMovement ? -posPointMan.z : posPointMan.z) + originPosition;
Vector3 posPointManHips = new Vector3(posPointMan.x, posPointMan.y, !mirroredMovement ? -posPointMan.z : posPointMan.z) + originPosition;
// store the initial position
if (initialPosUserID != userID)
{
initialPosUserID = userID;
initialPosOffset = posPointManWorld;
}
if (!verticalMovement && !initialPosSetY)
{
float fFootPosY = 0f;
if (manager.IsJointTracked(userID, (int)KinectInterop.JointType.FootLeft))
{
fFootPosY = manager.GetJointPosition(userID, (int)KinectInterop.JointType.FootLeft).y;
}
else if (manager.IsJointTracked(userID, (int)KinectInterop.JointType.FootRight))
{
fFootPosY = manager.GetJointPosition(userID, (int)KinectInterop.JointType.FootRight).y;
}
initialPosOffset.y = posPointManWorld.y - (fFootPosY + originPosition.y);
initialPosSetY = true;
}
Vector3 relPosUser = (posPointManWorld - initialPosOffset);
//relPosUser.z = invertedZMovement ? -relPosUser.z : relPosUser.z;
transform.position = initialPosOffset +
(verticalMovement ? relPosUser * moveRate : new Vector3(relPosUser.x, 0, relPosUser.z) * moveRate);
if (manager.IsJointTracked(userID, (int)KinectInterop.JointType.Head))
{
float fHeadPosY = manager.GetJointPosition(userID, (int)KinectInterop.JointType.Head).y + originPosition.y;
float halfHeight = Mathf.Abs(fHeadPosY - posPointManWorld.y);
// CapsuleCollider collider = GetComponent<CapsuleCollider>();
// if(collider != null)
// {
// collider.height = 2f * halfHeight;
// }
}
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] == null && bodyJoint != null)
{
bones[i] = Instantiate(bodyJoint) as GameObject;
bones[i].transform.parent = transform;
bones[i].name = ((KinectInterop.JointType)i).ToString();
// if(Array.IndexOf(ColliderJoints, i) >= 0)
// {
// bones[i].GetComponent<SphereCollider>().radius = 1f;
// }
}
if (bones[i] != null)
{
int joint = !mirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i);
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !mirroredMovement ? -posJoint.z : posJoint.z;
posJoint += originPosition;
posJoint -= posPointManHips;
if (mirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !mirroredMovement);
rotJoint = initialRotation * rotJoint;
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (lines[i] == null && i > 0 && skeletonLine != null)
{
lines[i] = Instantiate(skeletonLine) as GameObject;
lines[i].transform.parent = transform;
lines[i].name = ((KinectInterop.JointType)i).ToString() + "_Line";
}
if (lines[i] != null && i > 0)
{
lines[i].gameObject.SetActive(true);
int jParent = (int)manager.GetParentJoint((KinectInterop.JointType)joint);
Vector3 posJoint2 = manager.GetJointPosition(userID, jParent);
posJoint2.z = !mirroredMovement ? -posJoint2.z : posJoint2.z;
posJoint2 += originPosition;
posJoint2 -= posPointManHips;
if (mirroredMovement)
{
posJoint2.x = -posJoint2.x;
posJoint2.z = -posJoint2.z;
}
//Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
//dirFromParent.z = !mirroredMovement ? -dirFromParent.z : dirFromParent.z;
//Vector3 posParent = posJoint2 - dirFromParent;
Vector3 dirFromParent = posJoint - posJoint2;
// //lines[i].SetVertexCount(2);
// lines[i].SetPosition(0, posParent);
// lines[i].SetPosition(1, posJoint2);
lines[i].transform.localPosition = posJoint2 + dirFromParent / 2f;
lines[i].transform.up = transform.rotation * dirFromParent.normalized;
Vector3 lineScale = lines[i].transform.localScale;
lines[i].transform.localScale = new Vector3(lineScale.x, dirFromParent.magnitude / 2f, lineScale.z);
}
}
else
{
if (bones[i] != null)
{
bones[i].gameObject.SetActive(false);
}
if (lines[i] != null)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
void Update()
{
KinectManager manager = KinectManager.Instance;
// get 1st player
Int64 userID = manager ? manager.GetPrimaryUserID() : 0;
if (userID <= 0)
{
// reset the pointman position and rotation
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (SkeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
Vector3 posPointMan = manager.GetUserPosition(userID);
posPointMan.z = !MirroredMovement ? -posPointMan.z : posPointMan.z;
// store the initial position
if (initialPosUserID != userID)
{
initialPosUserID = userID;
initialPosOffset = transform.position - (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
}
transform.position = initialPosOffset + (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
// update the local positions of the bones
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = (int)manager.GetJointAtIndex(
!MirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i));
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !MirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !MirroredMovement);
rotJoint = initialRotation * rotJoint;
posJoint -= posPointMan;
if (MirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.rotation = rotJoint;
if (SkeletonLine)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !MirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
}
else
{
bones[i].gameObject.SetActive(false);
if (SkeletonLine)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
//----------------------------------- end of public functions --------------------------------------//
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
if (kinectManager && kinectManager.IsInitialized())
{
sensorData = kinectManager.GetSensorData();
}
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Speech recognition cannot be started, because KinectManager is missing or not initialized.");
}
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please, wait...";
}
// ensure the needed dlls are in place and speech recognition is available for this interface
bool bNeedRestart = false;
if (sensorData.sensorInterface.IsSpeechRecognitionAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "SM");
return;
}
}
else
{
string sInterfaceName = sensorData.sensorInterface.GetType().Name;
throw new Exception(sInterfaceName + ": Speech recognition is not supported!");
}
// Initialize the speech recognizer
string sCriteria = String.Format("Language={0:X};Kinect=True", languageCode);
int rc = sensorData.sensorInterface.InitSpeechRecognition(sCriteria, true, false);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception(String.Format("Error initializing Kinect/SAPI: " + sErrorMessage));
}
if (requiredConfidence > 0)
{
sensorData.sensorInterface.SetSpeechConfidence(requiredConfidence);
}
if (grammarFileName != string.Empty)
{
// copy the grammar file from Resources, if available
if (!File.Exists(grammarFileName))
{
TextAsset textRes = Resources.Load(grammarFileName, typeof(TextAsset)) as TextAsset;
if (textRes != null)
{
string sResText = textRes.text;
File.WriteAllText(grammarFileName, sResText);
}
}
// load the grammar file
rc = sensorData.sensorInterface.LoadSpeechGrammar(grammarFileName, (short)languageCode, dynamicGrammar);
if (rc < 0)
{
string sErrorMessage = (new SpeechErrorHandler()).GetSapiErrorMessage(rc);
throw new Exception("Error loading grammar file " + grammarFileName + ": " + sErrorMessage);
}
// // test dynamic grammar phrases
// rc = sensorData.sensorInterface.AddGrammarPhrase("addressBook", string.Empty, "Nancy Anderson", true, false);
// rc = sensorData.sensorInterface.AddGrammarPhrase("addressBook", string.Empty, "Cindy White", false, false);
// rc = sensorData.sensorInterface.AddGrammarPhrase("addressBook", string.Empty, "Oliver Lee", false, false);
// rc = sensorData.sensorInterface.AddGrammarPhrase("addressBook", string.Empty, "Alan Brewer", false, false);
// rc = sensorData.sensorInterface.AddGrammarPhrase("addressBook", string.Empty, "April Reagan", false, true);
}
instance = this;
sapiInitialized = true;
//DontDestroyOnLoad(gameObject);
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Ready.";
}
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = "Please check the Kinect and SAPI installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.GetComponent <GUIText>().text = ex.Message;
}
}
}
void Start()
{
try
{
// get sensor data
KinectManager kinectManager = KinectManager.Instance;
KinectInterop.SensorData sensorData = kinectManager != null?kinectManager.GetSensorData() : null;
if (sensorData == null || sensorData.sensorInterface == null)
{
throw new Exception("Visual gesture tracking cannot be started, because the KinectManager is missing or not initialized.");
}
if (sensorData.sensorInterface.GetSensorPlatform() != KinectInterop.DepthSensorPlatform.KinectSDKv2)
{
throw new Exception("Visual gesture tracking is only supported by Kinect SDK v2");
}
// ensure the needed dlls are in place and face tracking is available for this interface
bool bNeedRestart = false;
if (IsVisualGesturesAvailable(ref bNeedRestart))
{
if (bNeedRestart)
{
KinectInterop.RestartLevel(gameObject, "VG");
return;
}
}
else
{
throw new Exception("Visual gesture tracking is not supported!");
}
// initialize visual gesture tracker
if (!InitVisualGestures())
{
throw new Exception("Visual gesture tracking could not be initialized.");
}
// try to automatically detect the available gesture listeners in the scene
if (visualGestureListeners.Count == 0)
{
MonoBehaviour[] monoScripts = FindObjectsOfType(typeof(MonoBehaviour)) as MonoBehaviour[];
foreach (MonoBehaviour monoScript in monoScripts)
{
// if(typeof(VisualGestureListenerInterface).IsAssignableFrom(monoScript.GetType()) &&
// monoScript.enabled)
if ((monoScript is VisualGestureListenerInterface) && monoScript.enabled)
{
visualGestureListeners.Add(monoScript);
}
}
}
isVisualGestureInitialized = true;
}
catch (DllNotFoundException ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = "Please check the Kinect and VGB-Library installations.";
}
}
catch (Exception ex)
{
Debug.LogError(ex.ToString());
if (debugText != null)
{
debugText.text = ex.Message;
}
}
}
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed
private void UpdateTextureWithNewFrame()
{
if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamDepthImage == null)
{
return;
}
// get the updated depth frame
if (lastColorCamDepthFrameTime != sensorData.lastColorCamDepthFrameTime)
{
lastColorCamDepthFrameTime = sensorData.lastColorCamDepthFrameTime;
if (depthImageTexture.width != sensorData.colorImageWidth || depthImageTexture.height != sensorData.colorImageHeight)
{
depthImageTexture = KinectInterop.CreateRenderTexture(depthImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight);
}
Array.Clear(depthHistBufferData, 0, depthHistBufferData.Length);
Array.Clear(equalHistBufferData, 0, equalHistBufferData.Length);
depthHistTotalPoints = 0;
// get configured min & max distances
float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance;
float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance;
int depthMinDistance = (int)(minDistance * 1000f);
int depthMaxDistance = (int)(maxDistance * 1000f);
int frameLen = sensorData.colorCamDepthImage.Length;
for (int i = 0; i < frameLen; i++)
{
int depth = sensorData.colorCamDepthImage[i];
int limDepth = (depth <= DepthSensorBase.MAX_DEPTH_DISTANCE_MM) ? depth : 0;
if (limDepth > 0)
{
depthHistBufferData[limDepth]++;
depthHistTotalPoints++;
}
}
equalHistBufferData[0] = depthHistBufferData[0];
for (int i = 1; i < depthHistBufferData.Length; i++)
{
equalHistBufferData[i] = equalHistBufferData[i - 1] + depthHistBufferData[i];
}
// make depth 0 equal to the max-depth
equalHistBufferData[0] = equalHistBufferData[equalHistBufferData.Length - 1];
int depthBufferLength = sensorData.colorCamDepthImage.Length >> 1;
if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength)
{
depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint));
}
KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.colorCamDepthImage, depthBufferLength, sizeof(uint));
if (depthHistBuffer != null)
{
KinectInterop.SetComputeBufferData(depthHistBuffer, equalHistBufferData, equalHistBufferData.Length, sizeof(int));
}
depthImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth);
depthImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight);
depthImageMaterial.SetInt("_MinDepth", depthMinDistance);
depthImageMaterial.SetInt("_MaxDepth", depthMaxDistance);
depthImageMaterial.SetBuffer("_DepthMap", depthImageBuffer);
depthImageMaterial.SetBuffer("_HistMap", depthHistBuffer);
depthImageMaterial.SetInt("_TotalPoints", depthHistTotalPoints);
Graphics.Blit(null, depthImageTexture, depthImageMaterial);
}
}
// Apply the rotations tracked by kinect to the joints.
protected void TransformBone(Int64 userId, KinectInterop.JointType joint, int boneIndex, bool flip)
{
Transform boneTransform = bones[boneIndex];
if(boneTransform == null || kinectManager == null)
return;
int iJoint = (int)joint;
if(iJoint < 0 || !kinectManager.IsJointTracked(userId, iJoint))
return;
// Get Kinect joint orientation
Quaternion jointRotation = kinectManager.GetJointOrientation(userId, iJoint, flip);
if(jointRotation == Quaternion.identity)
return;
// calculate the new orientation
Quaternion newRotation = Kinect2AvatarRot(jointRotation, boneIndex);
if(externalRootMotion)
{
newRotation = transform.rotation * newRotation;
}
// Smoothly transition to the new rotation
if(smoothFactor != 0f)
boneTransform.rotation = Quaternion.Slerp(boneTransform.rotation, newRotation, smoothFactor * Time.deltaTime);
else
boneTransform.rotation = newRotation;
}
void Update()
{
//获取KinectManager静态实例
KinectManager manager = KinectManager.Instance;
// get 1st player
//获取到摄像头照射到的人物ID
Int64 userID = manager ? manager.GetPrimaryUserID() : 0;
if (userID <= 0)
{
// reset the pointman position and rotation
//重置点人位置和旋转
if (transform.position != initialPosition)
{
transform.position = initialPosition;
}
if (transform.rotation != initialRotation)
{
transform.rotation = initialRotation;
}
//显示骨骼线
for (int i = 0; i < bones.Length; i++)
{
bones[i].gameObject.SetActive(true);
bones[i].transform.localPosition = Vector3.zero;
bones[i].transform.localRotation = Quaternion.identity;
if (LinePrefab)
{
lines[i].gameObject.SetActive(false);
}
}
return;
}
// set the position in space
//设置空间的位置
Vector3 posPointMan = manager.GetUserPosition(userID);
//根据镜像运行来设置z轴的方向
posPointMan.z = !MirroredMovement ? -posPointMan.z : posPointMan.z;
// store the initial position
// 存储初始位置 如果当前用户ID与实时获取到的ID不一致，则重新赋值
if (initialPosUserID != userID)
{
initialPosUserID = userID;
//设置偏移量
initialPosOffset = transform.position - (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
}
//设置位置
transform.position = initialPosOffset + (MoveVertically ? posPointMan : new Vector3(posPointMan.x, 0, posPointMan.z));
// update the local positions of the bones
// 更新骨骼的本地位置
for (int i = 0; i < bones.Length; i++)
{
if (bones[i] != null)
{
int joint = (int)manager.GetJointAtIndex(
!MirroredMovement ? i : (int)KinectInterop.GetMirrorJoint((KinectInterop.JointType)i));
if (joint < 0)
{
continue;
}
if (manager.IsJointTracked(userID, joint))
{
bones[i].gameObject.SetActive(true);
Vector3 posJoint = manager.GetJointPosition(userID, joint);
posJoint.z = !MirroredMovement ? -posJoint.z : posJoint.z;
Quaternion rotJoint = manager.GetJointOrientation(userID, joint, !MirroredMovement);
posJoint -= posPointMan;
if (MirroredMovement)
{
posJoint.x = -posJoint.x;
posJoint.z = -posJoint.z;
}
bones[i].transform.localPosition = posJoint;
bones[i].transform.localRotation = rotJoint;
if (LinePrefab)
{
lines[i].gameObject.SetActive(true);
Vector3 posJoint2 = bones[i].transform.position;
Vector3 dirFromParent = manager.GetJointDirection(userID, joint, false, false);
dirFromParent.z = !MirroredMovement ? -dirFromParent.z : dirFromParent.z;
Vector3 posParent = posJoint2 - dirFromParent;
//lines[i].SetVertexCount(2);
lines[i].SetPosition(0, posParent);
lines[i].SetPosition(1, posJoint2);
}
#region 注释
// KinectInterop.BodyData bodyData = manager.GetUserBodyData(userID);
// if(lineTLeft != null && bodyData.liTrackingID != 0 && joint == (int)JointType.HandLeft)
// {
// Vector3 leftTDir = bodyData.leftThumbDirection.normalized;
// leftTDir.z = !MirroredMovement ? -leftTDir.z : leftTDir.z;
//
// Vector3 posTStart = bones[i].transform.position;
// Vector3 posTEnd = posTStart + leftTDir;
//
// lineTLeft.SetPosition(0, posTStart);
// lineTLeft.SetPosition(1, posTEnd);
//
// if(lineFLeft != null)
// {
// Vector3 leftFDir = bodyData.leftThumbForward.normalized;
// leftFDir.z = !MirroredMovement ? -leftFDir.z : leftFDir.z;
//
// Vector3 posFStart = bones[i].transform.position;
// Vector3 posFEnd = posTStart + leftFDir;
//
// lineFLeft.SetPosition(0, posFStart);
// lineFLeft.SetPosition(1, posFEnd);
// }
// }
//
// if(lineTRight != null && bodyData.liTrackingID != 0 && joint == (int)JointType.HandRight)
// {
// Vector3 rightTDir = bodyData.rightThumbDirection.normalized;
// rightTDir.z = !MirroredMovement ? -rightTDir.z : rightTDir.z;
//
// Vector3 posTStart = bones[i].transform.position;
// Vector3 posTEnd = posTStart + rightTDir;
//
// lineTRight.SetPosition(0, posTStart);
// lineTRight.SetPosition(1, posTEnd);
//
// if(lineFRight != null)
// {
// Vector3 rightFDir = bodyData.rightThumbForward.normalized;
// rightFDir.z = !MirroredMovement ? -rightFDir.z : rightFDir.z;
//
// Vector3 posFStart = bones[i].transform.position;
// Vector3 posFEnd = posTStart + rightFDir;
//
// lineFRight.SetPosition(0, posFStart);
// lineFRight.SetPosition(1, posFEnd);
// }
// }
#endregion
}
else
{
bones[i].gameObject.SetActive(false);
if (LinePrefab)
{
lines[i].gameObject.SetActive(false);
}
}
}
}
}
// ApplyBoneOrientationConstraints and constrain rotations.
public void Constrain(ref KinectInterop.BodyData bodyData)
{
KinectManager manager = KinectManager.Instance;
for (int i = 0; i < this.jointConstraints.Count; i++)
{
BoneOrientationConstraint jc = this.jointConstraints[i];
if (jc.thisJoint == (int)KinectInterop.JointType.SpineBase || bodyData.joint[jc.thisJoint].normalRotation == Quaternion.identity)
continue;
if (bodyData.joint[jc.thisJoint].trackingState == KinectInterop.TrackingState.NotTracked)
continue;
int prevJoint = (int)manager.GetParentJoint((KinectInterop.JointType)jc.thisJoint);
if (bodyData.joint[prevJoint].trackingState == KinectInterop.TrackingState.NotTracked)
continue;
Quaternion rotJointN = bodyData.joint[jc.thisJoint].normalRotation;
Quaternion rotParentN = bodyData.joint[prevJoint].normalRotation;
Quaternion rotLocalN = Quaternion.Inverse(rotParentN) * rotJointN;
Vector3 eulerAnglesN = rotLocalN.eulerAngles;
Quaternion rotJointM = bodyData.joint[jc.thisJoint].mirroredRotation;
Quaternion rotParentM = bodyData.joint[prevJoint].mirroredRotation;
Quaternion rotLocalM = Quaternion.Inverse(rotParentM) * rotJointM;
Vector3 eulerAnglesM = rotLocalM.eulerAngles;
// if(debugText != null && jc.thisJoint == (int)KinectInterop.JointType.SpineShoulder)
// {
// debugText.guiText.text = string.Format("{0} - ({1:000}, {2:000}, {3:000})", ((KinectInterop.JointType)jc.thisJoint).ToString(),
// eulerAnglesN.x, eulerAnglesN.y, eulerAnglesN.z);
// }
bool isConstrained = false;
for(int a = 0; a < jc.axisConstrainrs.Count; a++)
{
AxisOrientationConstraint ac = jc.axisConstrainrs[a];
Quaternion axisRotation = Quaternion.AngleAxis(eulerAnglesN[ac.axis], ac.rotateAround);
float angleFromMin = Quaternion.Angle(axisRotation, ac.minQuaternion);
float angleFromMax = Quaternion.Angle(axisRotation, ac.maxQuaternion);
if(!(angleFromMin <= ac.angleRange && angleFromMax <= ac.angleRange))
{
// correct the axis that has fallen out of range.
if(angleFromMin > angleFromMax)
{
eulerAnglesN[ac.axis] = ac.angleMax;
}
else
{
eulerAnglesN[ac.axis] = ac.angleMin;
}
// fix mirrored rotation as well
if(ac.axis == 0)
{
eulerAnglesM[ac.axis] = eulerAnglesN[ac.axis];
}
else
{
eulerAnglesM[ac.axis] = -eulerAnglesN[ac.axis];
}
isConstrained = true;
}
}
if(isConstrained)
{
rotLocalN = Quaternion.Euler(eulerAnglesN);
rotJointN = rotParentN * rotLocalN;
rotLocalM = Quaternion.Euler(eulerAnglesM);
rotJointM = rotParentM * rotLocalM;
// Put it back into the bone directions
bodyData.joint[jc.thisJoint].normalRotation = rotJointN;
bodyData.joint[jc.thisJoint].mirroredRotation = rotJointM;
// dirJoint = constrainedRotation * dirParent;
// bodyData.joint[jc.thisJoint].direction = dirJoint;
}
}
}
// inits the mesh and related data
private void InitMesh()
{
// create mesh
mesh = new Mesh
{
name = "SceneMesh-Sensor" + sensorIndex,
indexFormat = UnityEngine.Rendering.IndexFormat.UInt32
};
MeshFilter meshFilter = GetComponent <MeshFilter>();
if (meshFilter != null)
{
meshFilter.mesh = mesh;
}
else
{
Debug.LogWarning("MeshFilter not found! You may not see the mesh on screen");
}
// create point cloud color texture
if (sensorData != null && sensorData.sensorInterface != null)
{
sensorInt = (DepthSensorBase)sensorData.sensorInterface;
Vector2Int imageRes = Vector2Int.zero;
if (sensorInt.pointCloudColorTexture == null)
{
sensorInt.pointCloudResolution = sourceImageResolution;
imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
sensorInt.pointCloudColorTexture = colorTexture;
colorTextureCreated = true;
}
else
{
sourceImageResolution = sensorInt.pointCloudResolution;
imageRes = sensorInt.GetPointCloudTexResolution(sensorData);
colorTexture = sensorInt.pointCloudColorTexture;
colorTextureCreated = false;
}
// create depth image buffer
if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution)
{
if (sensorData.depthImageBuffer == null)
{
int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2;
sensorData.depthImageBuffer = KinectInterop.CreateComputeBuffer(sensorData.depthImageBuffer, depthBufferLength, sizeof(uint));
depthBufferCreated = true;
}
}
else
{
if (sensorData.colorDepthBuffer == null && sensorData.colorImageWidth > 0 && sensorData.colorImageHeight > 0)
{
int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight / 2;
sensorData.colorDepthBuffer = new ComputeBuffer(bufferLength, sizeof(uint));
depthBufferCreated = true;
}
}
// create space table
spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ?
sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData);
int spaceBufferLength = imageRes.x * imageRes.y * 3;
spaceTableBuffer = new ComputeBuffer(spaceBufferLength, sizeof(float));
spaceTableBuffer.SetData(spaceTable);
spaceTable = null;
// create copy texture
colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32);
// set the color texture
Renderer meshRenderer = GetComponent <Renderer>();
if (meshRenderer && meshRenderer.material /**&& meshRenderer.material.mainTexture == null*/)
{
meshShaderMat = meshRenderer.material;
}
// get reference to the transform
trans = GetComponent <Transform>();
// image width & height
imageWidth = imageRes.x;
imageHeight = imageRes.y;
// create mesh vertices & indices
CreateMeshVertInd();
bMeshInited = true;
}
}
