void ColorizeUser(nuitrack.UserFrame frame)
{
if (frame.Users.Length > 0)
{
msg = "User found";
}
else
{
msg = "User not found";
}
for (int i = 0; i < (cols * rows); i++)
{
Color32 currentColor = colorsList[frame[i]];
int ptr = i * 4;
outSegment[ptr] = currentColor.a;
outSegment[ptr + 1] = currentColor.r;
outSegment[ptr + 2] = currentColor.g;
outSegment[ptr + 3] = currentColor.b;
}
segmentTexture.LoadRawTextureData(outSegment);
segmentTexture.Apply();
segmentSprite = Sprite.Create(segmentTexture, imageRect, Vector3.one * 0.5f, 100f, 0, SpriteMeshType.FullRect);
segmentOut.sprite = segmentSprite;
gameColliders.UpdateFrame(frame);
}
/*
* Process the recieved frames and check the presence of the user in front of the sensor.
* Declare the msg variable for displaying 'found' or 'not found'
*/
void ColorizeUser(nuitrack.UserFrame frame)
{
if (frame.Users.Length > 0)
{
msg = "User found!";
}
else
{
msg = "User not found!";
}
for (int i = 0; i < (cols * rows); i++)
{
Color32 currentColor = colorsList[frame[i]];
int ptr = i * 4;
dataSegment[ptr] = currentColor.a;
dataSegment[ptr + 1] = currentColor.r;
dataSegment[ptr + 2] = currentColor.g;
dataSegment[ptr + 3] = currentColor.b;
}
//Pass an array for texture filling and apply
segmentTexture.LoadRawTextureData(dataSegment);
segmentTexture.Apply();
//Apply texture to sprite
segmentSprite = Sprite.Create(segmentTexture, imageRect, Vector3.one * 0.5f, 100f, 0, SpriteMeshType.FullRect);
//Apply Sprite to Image
segmentOut.sprite = segmentSprite;
//Call to Colliders
collider.UpdateFrame(frame);
}
void ColorizeUser(nuitrack.UserFrame frame)
{
//float minZ = 100000;
//float maxZ = 0;
for (int i = 0; i < (cols * rows); i++)
{
outSegment[i] = 0;
if (frame[i] > 0)
{
outSegment[i] = 1;
//if (PointCloudGPU.Instance.particles[i].z < minZ)
// minZ = PointCloudGPU.Instance.particles[i].z;
//else if (PointCloudGPU.Instance.particles[i].z > maxZ)
// maxZ = PointCloudGPU.Instance.particles[i].z;
}
}
//minZArray.Add(minZ);
//maxZArray.Add(maxZ);
//if (minZArray.Count > samples)
//{
// minZArray.RemoveAt(0);
// maxZArray.RemoveAt(0);
//}
//PointCloudGPU.Instance.matPointCloud.SetFloat("_MinZ", minZArray.Average());
//PointCloudGPU.Instance.matPointCloud.SetFloat("_MaxZ", maxZArray.Average());
segmentBuffer.SetData(outSegment);
}
public static Texture2D ToTexture2D(this nuitrack.UserFrame frame, params Color32[] customListColors)
{
Color32[] currentColorList = customListColors ?? defaultColors;
byte[] outSegment = new byte[frame.Cols * frame.Rows * 4];
for (int i = 0; i < (frame.Cols * frame.Rows); i++)
{
Color32 currentColor = (frame[i] == 0) ? transparentColor : currentColorList[frame[i]];
int ptr = i * 4;
outSegment[ptr] = currentColor.a;
outSegment[ptr + 1] = currentColor.r;
outSegment[ptr + 2] = currentColor.g;
outSegment[ptr + 3] = currentColor.b;
}
Texture2D segmentTexture = new Texture2D(frame.Cols, frame.Rows, TextureFormat.ARGB32, false);
segmentTexture.LoadRawTextureData(outSegment);
segmentTexture.Apply();
Resources.UnloadUnusedAssets();
return(segmentTexture);
}
void DrawUserSegment(nuitrack.UserFrame frame)
{
if (frame.Users.Length <= 0)
{
return;
}
for (int i = 0; i < (frameWidth * frameHeight); i++)
{
int ptr = i * 3; // RGB24
if (frame[i] == 0)
{
maskedFrame[ptr] = colorFrame[ptr];
maskedFrame[ptr + 1] = colorFrame[ptr + 1];
maskedFrame[ptr + 2] = colorFrame[ptr + 2];
}
else
{
maskedFrame[ptr] = maskColor.r;
maskedFrame[ptr + 1] = maskColor.g;
maskedFrame[ptr + 2] = maskColor.b;
}
}
frameTexture.LoadRawTextureData(maskedFrame);
frameTexture.Apply();
}
void Update()
{
bool haveNewFrame = false;
if (!nuitrackModules.nuitrackInitialized)
{
return;
}
if ((nuitrackModules.DepthFrame != null) && active)
{
if (depthFrame != null)
{
haveNewFrame = (depthFrame != nuitrackModules.DepthFrame);
}
depthFrame = nuitrackModules.DepthFrame;
colorFrame = nuitrackModules.ColorFrame;
userFrame = nuitrackModules.UserFrame;
if (haveNewFrame)
{
ProcessFrame(depthFrame, colorFrame, userFrame);
}
}
else
{
HideVisualization();
}
}
void NuitrackManager_onUserTrackerUpdate(nuitrack.UserFrame frame)
{
if (renderTexture == null || renderTexture.width != frame.Cols || renderTexture.height != frame.Rows)
{
renderTexture = new RenderTexture(frame.Cols, frame.Rows, 0, RenderTextureFormat.ARGB32);
renderTexture.enableRandomWrite = true;
renderTexture.Create();
rawImage.texture = renderTexture;
segment2Texture.SetInt("textureWidth", renderTexture.width);
segment2Texture.SetTexture(kernelIndex, "Result", renderTexture);
/*
* We put the source data in the buffer, but the buffer does not support types
* that take up less than 4 bytes(instead of ushot(Int16), we specify uint(Int32)).
*
* For optimization, we specify a length half the original length,
* since the data is correctly projected into memory
* (sizeof(ushot) * sourceDataBuffer / 2 == sizeof(uint) * sourceDataBuffer / 2)
*/
sourceDataBuffer = new ComputeBuffer(frame.DataSize / 2, sizeof(uint));
segmentDataArray = new byte[frame.DataSize];
}
Marshal.Copy(frame.Data, segmentDataArray, 0, frame.DataSize);
sourceDataBuffer.SetData(segmentDataArray);
segment2Texture.SetBuffer(kernelIndex, "UserIndexes", sourceDataBuffer);
segment2Texture.Dispatch(kernelIndex, renderTexture.width / (int)x, renderTexture.height / (int)y, (int)z);
}
void Update()
{
if (!NuitrackManager.Instance.nuitrackInitialized)
{
return;
}
if ((NuitrackManager.DepthFrame != null) && active)
{
nuitrack.DepthFrame depthFrame = NuitrackManager.DepthFrame;
nuitrack.ColorFrame colorFrame = NuitrackManager.ColorFrame;
nuitrack.UserFrame userFrame = NuitrackManager.UserFrame;
bool haveNewFrame = (lastFrameTimestamp != depthFrame.Timestamp);
if (haveNewFrame)
{
ProcessFrame(depthFrame, colorFrame, userFrame);
lastFrameTimestamp = depthFrame.Timestamp;
}
}
else
{
HideVisualization();
}
}
void HandleOnUserTrackerUpdateEvent(nuitrack.UserFrame frame)
{
userFrame = frame;
if (onUserTrackerUpdate != null)
{
onUserTrackerUpdate(userFrame);
}
}
void HandleOnUserTrackerUpdateEvent(nuitrack.UserFrame frame)
{
if (userFrame != null)
{
userFrame.Dispose();
}
userFrame = (nuitrack.UserFrame)frame.Clone();
onUserTrackerUpdate?.Invoke(userFrame);
}
private void HandleOnUserTrackerUpdateEvent(nuitrack.UserFrame frame)
{
userFrame = frame;
if (bMultiSource && userFrame != null)
{
bMultiFrameBodyIndex = true;
SyncAllFrames();
}
}
public void CloseUserGen()
{
try
{
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
}
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
depthSensor = null;
colorSensor = null;
userTracker = null;
skeletonTracker = null;
gestureRecognizer = null;
handTracker = null;
nuitrack.Nuitrack.Release();
Debug.Log("CloseUserGen");
nuitrackInitialized = false;
}
catch (System.Exception ex)
{
Debug.LogError(ex.ToString());
}
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.UserFrame userFrame)
{
Color pointColor;
for (int i = 0, pointIndex = 0; i < depthFrame.Rows; i += frameStep)
{
for (int j = 0; j < depthFrame.Cols; j += frameStep, ++pointIndex)
{
nuitrack.Vector3 p = depthSensor.ConvertProjToRealCoords(j, i, depthFrame[i, j]);
uint userId = userFrame[i * userFrame.Rows / depthFrame.Rows,
j *userFrame.Cols / depthFrame.Cols];
pointColor = userCols[userId % userCols.Length];
VerticesUpdate(pointIndex, new Vector3(p.X * 0.001f, p.Y * 0.001f, p.Z * 0.001f), pointColor);
}
}
MeshesUpdate();
}
public void CloseUserGen()
{
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
}
// Debug.Log ("preRelease");
nuitrack.Nuitrack.Release();
// Debug.Log ("postRelease");
depthSensor = null;
colorSensor = null;
userTracker = null;
skeletonTracker = null;
gestureRecognizer = null;
handTracker = null;
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
}
void HandleOnUserTrackerUpdateEvent(nuitrack.UserFrame frame)
{
if (UserFrame != null)
{
UserFrame.Dispose();
}
UserFrame = (nuitrack.UserFrame)frame.Clone();
try
{
onUserTrackerUpdate?.Invoke(UserFrame);
}
catch (Exception ex)
{
Debug.LogException(ex);
}
Floor = new Plane(UserFrame.FloorNormal.ToVector3().normalized, UserFrame.Floor.ToVector3() * 0.001f);
}
public void UpdateFrame(nuitrack.UserFrame frame)
{
for (int c = 0; c < cols; c++)
{
for (int r = 0; r < rows; r++)
{
ushort userId = frame[(int)(r / colliderDetails), (int)(c / colliderDetails)];
if (userId == 0)
{
colliderObjects[c, r].SetActive(false);
}
else
{
colliderObjects[c, r].SetActive(true);
}
}
}
}
/*
* If a user is in the frame, the game objects for displaying are activated
* otherwise they are hidden
*/
public void UpdateFrame(nuitrack.UserFrame frame) //update the frame
{
for (int x = 0; x < cols; x++)
{
for (int y = 0; y < rows; y++)
{
ushort userID = frame[(int)(x / colliderDetails), (int)(y / colliderDetails)]; //Request a user id according to colliderDetails
//Not found
if (userID == 0)
{
colliderObjects[x, y].SetActive(false);
}
//Found
else
{
colliderObjects[x, y].SetActive(true);
}
}
}
}
void OnUserTrackerUpdate(nuitrack.UserFrame frame)
{
userFrame = frame;
}
public void CloseSensor(KinectInterop.SensorData sensorData)
{
if (colorWebCam)
{
colorWebCam.Stop();
colorWebCam = null;
}
if (coordMapper != null)
{
coordMapper.CleanUp();
coordMapper = null;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
colorSensor = null;
colorFrame = null;
}
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
depthSensor = null;
depthFrame = null;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
userTracker = null;
userFrame = null;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
skeletonTracker = null;
skeletonData = null;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
handTracker = null;
handTrackerData = null;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
gestureRecognizer = null;
}
if (bNuitrackInited)
{
bNuitrackInited = false;
nuitrack.Nuitrack.onIssueUpdateEvent -= OnIssuesUpdate;
NuitrackTerm();
}
Debug.Log("Nuitrack sensor closed");
}
void ChangeModulsState(bool skel, bool hand, bool depth, bool color, bool gest, bool user)
{
// Debug.Log ("" + skel + hand + depth + gest + user);
if (skeletonTracker == null)
{
return;
}
if (prevSkel != skel)
{
skeletonData = null;
prevSkel = skel;
if (skel)
{
skeletonTracker.OnSkeletonUpdateEvent += HandleOnSkeletonUpdateEvent;
}
else
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
}
}
if (prevHand != hand)
{
handTrackerData = null;
prevHand = hand;
if (hand)
{
handTracker.OnUpdateEvent += HandleOnHandsUpdateEvent;
}
else
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
}
}
if (prevGest != gest)
{
prevGest = gest;
if (gest)
{
gestureRecognizer.OnNewGesturesEvent += OnNewGestures;
}
else
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
}
}
if (prevDepth != depth)
{
depthFrame = null;
prevDepth = depth;
if (depth)
{
depthSensor.OnUpdateEvent += HandleOnDepthSensorUpdateEvent;
}
else
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
}
}
if (prevColor != color)
{
colorFrame = null;
prevColor = color;
if (color)
{
colorSensor.OnUpdateEvent += HandleOnColorSensorUpdateEvent;
}
else
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
}
}
if (prevUser != user)
{
userFrame = null;
prevUser = user;
if (user)
{
userTracker.OnUpdateEvent += HandleOnUserTrackerUpdateEvent;
}
else
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
}
}
}
private void InitTrackers(bool depthOn, bool colorOn, bool userOn, bool skeletonOn, bool handsOn, bool gesturesOn)
{
if (!nuitrackInitialized)
{
nuitrack.Nuitrack.Init();
Debug.Log("init ok");
depthSensorInit = nuitrack.DepthSensor.Create();
colorSensorInit = nuitrack.ColorSensor.Create();
userTrackerInit = nuitrack.UserTracker.Create();
skeletonTrackerInit = nuitrack.SkeletonTracker.Create();
handTrackerInit = nuitrack.HandTracker.Create();
gestureRecognizerInit = nuitrack.GestureRecognizer.Create();
nuitrack.Nuitrack.Run();
Debug.Log("run ok");
nuitrackInitialized = true;
}
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
gesturesData = null;
//
// depthSensor = null;
// userTracker = null;
// skeletonTracker = null;
// handTracker = null;
// gestureRecognizer = null;
//if (issuesProcessor != null) Destroy(issuesProcessor);
if (prevDepth != depthOn)
{
prevDepth = depthOn;
if (depthOn)
{
depthSensor = depthSensorInit;
depthSensorInit.OnUpdateEvent += DepthUpdate;
}
else
{
depthSensorInit.OnUpdateEvent -= DepthUpdate;
depthSensor = null;
}
}
if (prevColor != colorOn)
{
prevColor = colorOn;
if (colorOn)
{
colorSensor = colorSensorInit;
colorSensorInit.OnUpdateEvent += ColorUpdate;
}
else
{
colorSensorInit.OnUpdateEvent -= ColorUpdate;
colorSensor = null;
}
}
if (prevUser != userOn)
{
prevUser = userOn;
if (userOn)
{
userTracker = userTrackerInit;
userTrackerInit.OnUpdateEvent += UserUpdate;
}
else
{
userTrackerInit.OnUpdateEvent -= UserUpdate;
userTracker = null;
}
}
if (skeletonOn != prevSkel)
{
prevSkel = skeletonOn;
if (skeletonOn)
{
skeletonTracker = skeletonTrackerInit;
skeletonTrackerInit.OnSkeletonUpdateEvent += SkeletonsUpdate;
}
else
{
skeletonTrackerInit.OnSkeletonUpdateEvent -= SkeletonsUpdate;
skeletonTracker = null;
}
}
if (prevHand != handsOn)
{
prevHand = handsOn;
if (handsOn)
{
handTracker = handTrackerInit;
handTrackerInit.OnUpdateEvent += HandTrackerUpdate;
}
else
{
handTrackerInit.OnUpdateEvent -= HandTrackerUpdate;
handTracker = null;
}
}
if (prevGesture != gesturesOn)
{
prevGesture = gesturesOn;
if (gesturesOn)
{
gestureRecognizer = gestureRecognizerInit;
gestureRecognizerInit.OnNewGesturesEvent += GesturesUpdate;
}
else
{
gestureRecognizerInit.OnNewGesturesEvent -= GesturesUpdate;
gestureRecognizer = null;
}
}
StopThread();
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < parts; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
Color pointColor = Color.white;
if (userFrame != null)
{
if (issuesProcessor.userIssues != null)
{
for (int i = 1; i < userCurrentCols.Length; i++)
{
if (issuesProcessor.userIssues.ContainsKey(i))
{
userCurrentCols[i] =
(issuesProcessor.userIssues[i].isOccluded ||
issuesProcessor.userIssues[i].onBorderLeft ||
issuesProcessor.userIssues[i].onBorderRight ||
issuesProcessor.userIssues[i].onBorderTop) ?
occludedUserCols[i] : userCols[i];
}
}
}
}
for (int i = 0, pointIndex = 0, rgbOffset = 0; i < depthFrame.Rows; i += frameStep)
{
for (int j = 0; j < depthFrame.Cols; j += frameStep, ++pointIndex, rgbOffset += 3)
{
depthColors[pointIndex].r = depthFrame[i, j] / 16384f;
uint userId = 0u;
if (userFrame != null)
{
userId = userFrame[i * userFrame.Rows / depthFrame.Rows,
j *userFrame.Cols / depthFrame.Cols];
}
#region RGB coloring
//int rgbOffset = 3 * (i * depthFrame.Cols + j);
Color rgbCol = defaultColor;
if (colorFrame != null)
{
rgbCol = new Color32(colorFrame[i, j].Red, colorFrame[i, j].Green, colorFrame[i, j].Blue, 255);
}
//new Color32(depthFrame.rgb[rgbOffset + 2], depthFrame.rgb[rgbOffset + 1], depthFrame.rgb[rgbOffset + 0], 255);
pointColor = rgbCol;// * userCurrentCols[userId];
if (userId == 0)
{
pointColor *= userCols[0];
}
Color segmColor = userCurrentCols[userId];
//Debug.Log(j.ToString() + ", " + i.ToString() + " : " + rgbCol);
#endregion
// pointColor = userCurrentCols[userId]; //user segmentation coloring
rgbColors[pointIndex] = pointColor;
segmColors[pointIndex] = segmColor;
}
}
depthTexture.SetPixels(depthColors);
rgbTexture.SetPixels(rgbColors);
segmentationTexture.SetPixels(segmColors);
depthTexture.Apply();
rgbTexture.Apply();
segmentationTexture.Apply();
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < visualizationParts.Length; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
System.DateTime t1 = System.DateTime.Now;
for (int i = 0, pointIndex = 0, rgbOffset = 0; i < depthFrame.Rows; i++)
{
for (int j = 0; j < depthFrame.Cols; j++, ++pointIndex, rgbOffset += 3)
{
int userId = 0;
if (userFrame != null)
{
userId = userFrame[i, j];
}
Color rgbCol = defaultColor;
if (colorFrame != null)
{
rgbCol = new Color32(colorFrame[i, j].Red, colorFrame[i, j].Green, colorFrame[i, j].Blue, 255);
}
//new Color32(depthFrame.rgb[rgbOffset + 2], depthFrame.rgb[rgbOffset + 1], depthFrame.rgb[rgbOffset + 0], 255);
Color segmColor = userCols[userId];
depthColors[pointIndex].r = depthFrame[i, j] / 16384f;
rgbColors[pointIndex] = rgbCol;
segmColors[pointIndex] = segmColor;
}
}
System.DateTime t2 = System.DateTime.Now;
depthTexture.SetPixels(depthColors);
rgbTexture.SetPixels(rgbColors);
segmentationTexture.SetPixels(segmColors);
System.DateTime t3 = System.DateTime.Now;
depthTexture.Apply();
rgbTexture.Apply();
segmentationTexture.Apply();
System.DateTime t4 = System.DateTime.Now;
//Debug.Log(
// "Loop time : " + (t2 - t1).TotalMilliseconds.ToString() +
// "; Set pixels: " + (t3 - t2).TotalMilliseconds.ToString() +
// "; Texture.Apply: " + (t4 - t3).TotalMilliseconds.ToString()
//);
}
public void ChangeModules(bool depthOn, bool colorOn, bool userOn, bool skeletonOn, bool handsOn, bool gesturesOn)
{
try
{
// if (depthSensor != null) depthSensor.OnUpdateEvent -= DepthUpdate;
// if (userTracker != null) userTracker.OnUpdateEvent -= UserUpdate;
// if (skeletonTracker != null) skeletonTracker.OnSkeletonUpdateEvent -= SkeletonsUpdate;
// if (handTracker != null) handTracker.OnUpdateEvent -= HandTrackerUpdate;
// if (gestureRecognizer != null) gestureRecognizer.OnNewGesturesEvent -= GesturesUpdate;
//*
if (!nuitrackInitialized)
{
#if UNITY_IOS
nuitrack.Nuitrack.Init("", nuitrack.Nuitrack.NuitrackMode.DEBUG);
#else
nuitrack.Nuitrack.Init();
#endif
Debug.Log("init ok");
depthSensorInit = nuitrack.DepthSensor.Create();
colorSensorInit = nuitrack.ColorSensor.Create();
userTrackerInit = nuitrack.UserTracker.Create();
skeletonTrackerInit = nuitrack.SkeletonTracker.Create();
handTrackerInit = nuitrack.HandTracker.Create();
gestureRecognizerInit = nuitrack.GestureRecognizer.Create();
nuitrack.Nuitrack.Run();
Debug.Log("run ok");
nuitrackInitialized = true;
}
// */
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
gesturesData = null;
//
// depthSensor = null;
// userTracker = null;
// skeletonTracker = null;
// handTracker = null;
// gestureRecognizer = null;
//if (issuesProcessor != null) Destroy(issuesProcessor);
if (prevDepth != depthOn)
{
prevDepth = depthOn;
if (depthOn)
{
depthSensor = depthSensorInit;
depthSensorInit.OnUpdateEvent += DepthUpdate;
}
else
{
depthSensorInit.OnUpdateEvent -= DepthUpdate;
depthSensor = null;
}
}
if (prevColor != colorOn)
{
prevColor = colorOn;
if (colorOn)
{
colorSensor = colorSensorInit;
colorSensorInit.OnUpdateEvent += ColorUpdate;
}
else
{
colorSensorInit.OnUpdateEvent -= ColorUpdate;
colorSensor = null;
}
}
if (prevUser != userOn)
{
prevUser = userOn;
if (userOn)
{
userTracker = userTrackerInit;
userTrackerInit.OnUpdateEvent += UserUpdate;
}
else
{
userTrackerInit.OnUpdateEvent -= UserUpdate;
userTracker = null;
}
}
if (skeletonOn != prevSkel)
{
prevSkel = skeletonOn;
if (skeletonOn)
{
skeletonTracker = skeletonTrackerInit;
skeletonTrackerInit.OnSkeletonUpdateEvent += SkeletonsUpdate;
}
else
{
skeletonTrackerInit.OnSkeletonUpdateEvent -= SkeletonsUpdate;
skeletonTracker = null;
}
}
if (prevHand != handsOn)
{
prevHand = handsOn;
if (handsOn)
{
handTracker = handTrackerInit;
handTrackerInit.OnUpdateEvent += HandTrackerUpdate;
}
else
{
handTrackerInit.OnUpdateEvent -= HandTrackerUpdate;
handTracker = null;
}
}
if (prevGesture != gesturesOn)
{
prevGesture = gesturesOn;
if (gesturesOn)
{
gestureRecognizer = gestureRecognizerInit;
gestureRecognizerInit.OnNewGesturesEvent += GesturesUpdate;
}
else
{
gestureRecognizerInit.OnNewGesturesEvent -= GesturesUpdate;
gestureRecognizer = null;
}
}
//issuesProcessor = (GameObject)Instantiate(issuesProcessorPrefab);
}
catch (Exception ex)
{
exceptionsLogger.AddEntry(ex.ToString());
}
}
void UserUpdate(nuitrack.UserFrame _userFrame)
{
ProcessFrame(depthFrame, _userFrame);
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < parts; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
if (userFrame != null)
{
if (issuesProcessor.userIssues != null)
{
for (int i = 1; i < userCurrentCols.Length; i++)
{
if (issuesProcessor.userIssues.ContainsKey(i))
{
userCurrentCols[i] =
(issuesProcessor.userIssues[i].isOccluded ||
issuesProcessor.userIssues[i].onBorderLeft ||
issuesProcessor.userIssues[i].onBorderRight ||
issuesProcessor.userIssues[i].onBorderTop) ?
occludedUserCols[i] : userCols[i];
}
}
}
}
if (colorFrame == null)
{
rgbTexture = depthFrame.ToRenderTexture();
}
else
{
rgbTexture = colorFrame.ToRenderTexture();
}
depthTexture = depthFrame.ToRenderTexture(depthGradient, depthCache);
segmentationTexture = userFrame.ToRenderTexture(userCurrentCols, textureCache);
if (!showBackground)
{
FrameUtils.TextureUtils.Cut(rgbTexture, segmentationTexture, ref rgbRenderTexture);
visMat.SetTexture("_RGBTex", rgbRenderTexture);
}
else
{
visMat.SetTexture("_RGBTex", rgbTexture);
}
visMat.SetFloat("_maxSensorDepth", FrameUtils.DepthToTexture.MaxSensorDepth);
visMat.SetTexture("_DepthTex", depthTexture);
visMat.SetTexture("_SegmentationTex", segmentationTexture);
}
void UserUpdate(nuitrack.UserFrame _userFrame)
{
userFrame = _userFrame;
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < visualizationParts.Length; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
if (colorFrame == null)
{
rgbTexture = depthFrame.ToRenderTexture();
}
else
{
rgbTexture = colorFrame.ToRenderTexture();
}
if (!showBackground)
{
FrameUtils.TextureUtils.Cut(rgbTexture, segmentationTexture, ref rgbRenderTexture);
meshMat.SetTexture("_RGBTex", rgbRenderTexture);
}
else
{
meshMat.SetTexture("_RGBTex", rgbTexture);
}
depthTexture = depthFrame.ToRenderTexture(depthGradient, depthCache);
segmentationTexture = userFrame.ToRenderTexture(userCols, textureCache);
meshMat.SetTexture("_DepthTex", depthTexture);
meshMat.SetTexture("_SegmentationTex", segmentationTexture);
meshMat.SetFloat("_maxSensorDepth", FrameUtils.DepthToTexture.MaxSensorDepth);
}
