void Update()
{
bool haveNewFrame = false;
if (!nuitrackModules.nuitrackInitialized)
{
return;
}
if ((nuitrackModules.DepthFrame != null) && active)
{
if (depthFrame != null)
{
haveNewFrame = (depthFrame != nuitrackModules.DepthFrame);
}
depthFrame = nuitrackModules.DepthFrame;
colorFrame = nuitrackModules.ColorFrame;
userFrame = nuitrackModules.UserFrame;
if (haveNewFrame)
{
ProcessFrame(depthFrame, colorFrame, userFrame);
}
}
else
{
HideVisualization();
}
}
void DrawDepth(nuitrack.DepthFrame frame)
{
if (renderTexture == null || renderTexture.width != frame.Cols || renderTexture.height != frame.Rows)
{
renderTexture = new RenderTexture(frame.Cols, frame.Rows, 0, RenderTextureFormat.ARGB32);
renderTexture.enableRandomWrite = true;
renderTexture.Create();
background.texture = renderTexture;
depthToTexture.SetInt("textureWidth", renderTexture.width);
depthToTexture.SetTexture(kernelIndex, "Result", renderTexture);
/*
* We put the source data in the buffer, but the buffer does not support types
* that take up less than 4 bytes(instead of ushot(Int16), we specify uint(Int32)).
*
* For optimization, we specify a length half the original length,
* since the data is correctly projected into memory
* (sizeof(ushot) * sourceDataBuffer / 2 == sizeof(uint) * sourceDataBuffer / 2)
*/
int dataSize = frame.DataSize;
sourceDataBuffer = new ComputeBuffer(dataSize / 2, sizeof(uint));
depthToTexture.SetBuffer(kernelIndex, "DepthFrame", sourceDataBuffer);
depthDataArray = new byte[dataSize];
}
Marshal.Copy(frame.Data, depthDataArray, 0, depthDataArray.Length);
sourceDataBuffer.SetData(depthDataArray);
depthToTexture.SetFloat("contrast", contrast);
depthToTexture.Dispatch(kernelIndex, renderTexture.width / (int)x, renderTexture.height / (int)y, (int)z);
}
public static Texture2D ToTexture2D(this nuitrack.DepthFrame frame, float contrast = 0.9f)
{
byte[] outDepth = new byte[(frame.DataSize / 2) * 3];
int de = 1 + 255 - (int)(contrast * 255);
for (int i = 0; i < frame.DataSize / 2; i++)
{
byte depth = (byte)(frame[i] / de);
Color32 currentColor = new Color32(depth, depth, depth, 255);
int ptr = i * 3;
outDepth[ptr] = currentColor.r;
outDepth[ptr + 1] = currentColor.g;
outDepth[ptr + 2] = currentColor.b;
}
Texture2D depthTexture = new Texture2D(frame.Cols, frame.Rows, TextureFormat.RGB24, false);
depthTexture.LoadRawTextureData(outDepth);
depthTexture.Apply();
Resources.UnloadUnusedAssets();
return(depthTexture);
}
void Update()
{
if (!NuitrackManager.Instance.nuitrackInitialized)
{
return;
}
if ((NuitrackManager.DepthFrame != null) && active)
{
nuitrack.DepthFrame depthFrame = NuitrackManager.DepthFrame;
nuitrack.ColorFrame colorFrame = NuitrackManager.ColorFrame;
nuitrack.UserFrame userFrame = NuitrackManager.UserFrame;
bool haveNewFrame = (lastFrameTimestamp != depthFrame.Timestamp);
if (haveNewFrame)
{
ProcessFrame(depthFrame, colorFrame, userFrame);
lastFrameTimestamp = depthFrame.Timestamp;
}
}
else
{
HideVisualization();
}
}
void HandleOnDepthSensorUpdateEvent(nuitrack.DepthFrame frame)
{
// Debug.Log("Depth Update");
depthFrame = frame;
if (onDepthUpdate != null)
{
onDepthUpdate(depthFrame);
}
}
private void HandleOnDepthSensorUpdateEvent(nuitrack.DepthFrame frame)
{
depthFrame = frame;
if (bMultiSource && depthFrame != null)
{
bMultiFrameDepth = true;
SyncAllFrames();
}
}
void HandleOnDepthSensorUpdateEvent(nuitrack.DepthFrame frame)
{
if (depthFrame != null)
{
depthFrame.Dispose();
}
depthFrame = (nuitrack.DepthFrame)frame.Clone();
//Debug.Log("Depth Update");
onDepthUpdate?.Invoke(depthFrame);
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < parts; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
if (userFrame != null)
{
if (issuesProcessor.userIssues != null)
{
for (int i = 1; i < userCurrentCols.Length; i++)
{
if (issuesProcessor.userIssues.ContainsKey(i))
{
userCurrentCols[i] =
(issuesProcessor.userIssues[i].isOccluded ||
issuesProcessor.userIssues[i].onBorderLeft ||
issuesProcessor.userIssues[i].onBorderRight ||
issuesProcessor.userIssues[i].onBorderTop) ?
occludedUserCols[i] : userCols[i];
}
}
}
}
if (colorFrame == null)
{
rgbTexture = depthFrame.ToRenderTexture();
}
else
{
rgbTexture = colorFrame.ToRenderTexture();
}
depthTexture = depthFrame.ToRenderTexture(depthGradient, depthCache);
segmentationTexture = userFrame.ToRenderTexture(userCurrentCols, textureCache);
if (!showBackground)
{
FrameUtils.TextureUtils.Cut(rgbTexture, segmentationTexture, ref rgbRenderTexture);
visMat.SetTexture("_RGBTex", rgbRenderTexture);
}
else
{
visMat.SetTexture("_RGBTex", rgbTexture);
}
visMat.SetFloat("_maxSensorDepth", FrameUtils.DepthToTexture.MaxSensorDepth);
visMat.SetTexture("_DepthTex", depthTexture);
visMat.SetTexture("_SegmentationTex", segmentationTexture);
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < visualizationParts.Length; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
System.DateTime t1 = System.DateTime.Now;
for (int i = 0, pointIndex = 0, rgbOffset = 0; i < depthFrame.Rows; i++)
{
for (int j = 0; j < depthFrame.Cols; j++, ++pointIndex, rgbOffset += 3)
{
int userId = 0;
if (userFrame != null)
{
userId = userFrame[i, j];
}
Color rgbCol = defaultColor;
if (colorFrame != null)
{
rgbCol = new Color32(colorFrame[i, j].Red, colorFrame[i, j].Green, colorFrame[i, j].Blue, 255);
}
//new Color32(depthFrame.rgb[rgbOffset + 2], depthFrame.rgb[rgbOffset + 1], depthFrame.rgb[rgbOffset + 0], 255);
Color segmColor = userCols[userId];
depthColors[pointIndex].r = depthFrame[i, j] / 16384f;
rgbColors[pointIndex] = rgbCol;
segmColors[pointIndex] = segmColor;
}
}
System.DateTime t2 = System.DateTime.Now;
depthTexture.SetPixels(depthColors);
rgbTexture.SetPixels(rgbColors);
segmentationTexture.SetPixels(segmColors);
System.DateTime t3 = System.DateTime.Now;
depthTexture.Apply();
rgbTexture.Apply();
segmentationTexture.Apply();
System.DateTime t4 = System.DateTime.Now;
//Debug.Log(
// "Loop time : " + (t2 - t1).TotalMilliseconds.ToString() +
// "; Set pixels: " + (t3 - t2).TotalMilliseconds.ToString() +
// "; Texture.Apply: " + (t4 - t3).TotalMilliseconds.ToString()
//);
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame)
{
int pointIndex = 0;
for (int i = 0; i < depthFrame.Rows; i += frameStep)
{
for (int j = 0; j < depthFrame.Cols; j += frameStep)
{
//take depth from the frame and put it into the depthColors array
depthColors[pointIndex].r = depthFrame[i, j] / 16384f;
//take from the rgb frame a color and put it into an array rgbColors
Color rgbCol = defaultColor;
if (colorFrame != null)
{
rgbCol = new Color32(colorFrame[i, j].Red, colorFrame[i, j].Green, colorFrame[i, j].Blue, 255);
}
rgbColors[pointIndex] = rgbCol;
points[pointIndex].GetComponent <Renderer>().material.color = rgbCol;
//change of position of the point (cube) in Z (depth)
Vector3 newPos = NuitrackManager.DepthSensor.ConvertProjToRealCoords(j, i, depthFrame[i, j]).ToVector3();
//NuitrackManager.depthSensor.ConvertRealToProjCoords(j - depthFrame.Cols, i, depthFrame[i, j]).ToVector3();
//Hide the "point" if its depth is 0
if (depthFrame[i, j] == 0)
{
points[pointIndex].SetActive(false);
}
else
{
points[pointIndex].SetActive(true);
points[pointIndex].transform.position = newPos;
float distancePoints = Vector3.Distance(newPos, NuitrackManager.DepthSensor.ConvertProjToRealCoords(j + 1, i, depthFrame[i, j]).ToVector3()); //Distance between points
depthToScale = distancePoints * depthFrame.Cols / hRes; //calculate the size of cubes as a function of depth
points[pointIndex].transform.localScale = Vector3.one * meshScaling * depthToScale;
}
++pointIndex;
}
Debug.Log(pointIndex);
}
Debug.Log("BREAK1");
depthTexture.SetPixels(depthColors);
rgbTexture.SetPixels(rgbColors);
Debug.Log("Break2");
depthTexture.Apply();
rgbTexture.Apply();
}
public void CloseUserGen()
{
try
{
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
}
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
depthSensor = null;
colorSensor = null;
userTracker = null;
skeletonTracker = null;
gestureRecognizer = null;
handTracker = null;
nuitrack.Nuitrack.Release();
Debug.Log("CloseUserGen");
nuitrackInitialized = false;
}
catch (System.Exception ex)
{
Debug.LogError(ex.ToString());
}
}
void Update()
{
if (NuitrackManager.DepthFrame != null)
{
nuitrack.DepthFrame depthFrame = NuitrackManager.DepthFrame;
nuitrack.ColorFrame colorFrame = NuitrackManager.ColorFrame;
bool haveNewFrame = (lastFrameID != depthFrame.ID);
if (haveNewFrame)
{
ProcessFrame(depthFrame, colorFrame);
lastFrameID = depthFrame.ID;
}
}
}
void HandleOnDepthSensorUpdateEvent(nuitrack.DepthFrame frame)
{
if (DepthFrame != null)
{
DepthFrame.Dispose();
}
DepthFrame = (nuitrack.DepthFrame)frame.Clone();
try
{
onDepthUpdate?.Invoke(DepthFrame);
}
catch (Exception ex)
{
Debug.LogException(ex);
}
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.UserFrame userFrame)
{
Color pointColor;
for (int i = 0, pointIndex = 0; i < depthFrame.Rows; i += frameStep)
{
for (int j = 0; j < depthFrame.Cols; j += frameStep, ++pointIndex)
{
nuitrack.Vector3 p = depthSensor.ConvertProjToRealCoords(j, i, depthFrame[i, j]);
uint userId = userFrame[i * userFrame.Rows / depthFrame.Rows,
j *userFrame.Cols / depthFrame.Cols];
pointColor = userCols[userId % userCols.Length];
VerticesUpdate(pointIndex, new Vector3(p.X * 0.001f, p.Y * 0.001f, p.Z * 0.001f), pointColor);
}
}
MeshesUpdate();
}
void Update()
{
bool haveNewFrame = false;
if ((NuitrackManager.DepthFrame != null))
{
if (depthFrame != null)
{
haveNewFrame = (depthFrame != NuitrackManager.DepthFrame);
}
depthFrame = NuitrackManager.DepthFrame;
colorFrame = NuitrackManager.ColorFrame;
if (haveNewFrame)
{
ProcessFrame(depthFrame, colorFrame);
}
}
}
public void CloseUserGen()
{
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
}
// Debug.Log ("preRelease");
nuitrack.Nuitrack.Release();
// Debug.Log ("postRelease");
depthSensor = null;
colorSensor = null;
userTracker = null;
skeletonTracker = null;
gestureRecognizer = null;
handTracker = null;
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
}
// void HandleOnColorSensorUpdateEvent(nuitrack.ColorFrame frame)
// {
// if (texture == null)
// {
// nuitrack.OutputMode ouput = NuitrackManager.ColorSensor.GetOutputMode();
// texture = new Texture2D(ouput.XRes, ouput.YRes, TextureFormat.RGB24, false);
// matPointCloud.SetTexture("_MainTex", texture);
// matPointCloud.SetInt("_WidthTex", ouput.XRes);
// matPointCloud.SetInt("_HeightTex", ouput.YRes);
// }
// texture.LoadRawTextureData(frame.Data);
// texture.Apply();
// }
// Update is called once per frame
void HandleOnDepthSensorUpdateEvent(nuitrack.DepthFrame frame)
{
if (buffer == null)
{
Debug.Log("Initialize compute buffer");
particles = new Vector3[frame.Cols * frame.Rows];
buffer = new ComputeBuffer(frame.Cols * frame.Rows, 12);
width = frame.Cols;
height = frame.Rows;
matPointCloud.SetBuffer("particleBuffer", buffer);
matPointCloud.SetInt("_Width", width);
matPointCloud.SetInt("_Height", height);
}
for (int i = 0; i < frame.Rows; i++)
{
for (int j = 0; j < frame.Cols; j++)
{
particles[i * frame.Cols + j] = NuitrackManager.DepthSensor.ConvertProjToRealCoords(j, i, frame[i, j]).ToVector3();
}
}
buffer.SetData(particles);
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < visualizationParts.Length; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
if (colorFrame == null)
{
rgbTexture = depthFrame.ToRenderTexture();
}
else
{
rgbTexture = colorFrame.ToRenderTexture();
}
if (!showBackground)
{
FrameUtils.TextureUtils.Cut(rgbTexture, segmentationTexture, ref rgbRenderTexture);
meshMat.SetTexture("_RGBTex", rgbRenderTexture);
}
else
{
meshMat.SetTexture("_RGBTex", rgbTexture);
}
depthTexture = depthFrame.ToRenderTexture(depthGradient, depthCache);
segmentationTexture = userFrame.ToRenderTexture(userCols, textureCache);
meshMat.SetTexture("_DepthTex", depthTexture);
meshMat.SetTexture("_SegmentationTex", segmentationTexture);
meshMat.SetFloat("_maxSensorDepth", FrameUtils.DepthToTexture.MaxSensorDepth);
}
void Update()
{
bool haveNewFrame = false;
if ((NuitrackManager.DepthFrame != null))
{
if (depthFrame != null)
{
haveNewFrame = (depthFrame != NuitrackManager.DepthFrame);
}
depthFrame = NuitrackManager.DepthFrame;
if (haveNewFrame)
{
ProcessFrame(depthFrame);
}
}
//FRAMERATE COUNTER
}
void DepthUpdate(nuitrack.DepthFrame _depthFrame)
{
depthFrame = _depthFrame;
}
public void ChangeModules(bool depthOn, bool colorOn, bool userOn, bool skeletonOn, bool handsOn, bool gesturesOn)
{
try
{
// if (depthSensor != null) depthSensor.OnUpdateEvent -= DepthUpdate;
// if (userTracker != null) userTracker.OnUpdateEvent -= UserUpdate;
// if (skeletonTracker != null) skeletonTracker.OnSkeletonUpdateEvent -= SkeletonsUpdate;
// if (handTracker != null) handTracker.OnUpdateEvent -= HandTrackerUpdate;
// if (gestureRecognizer != null) gestureRecognizer.OnNewGesturesEvent -= GesturesUpdate;
//*
if (!nuitrackInitialized)
{
#if UNITY_IOS
nuitrack.Nuitrack.Init("", nuitrack.Nuitrack.NuitrackMode.DEBUG);
#else
nuitrack.Nuitrack.Init();
#endif
Debug.Log("init ok");
depthSensorInit = nuitrack.DepthSensor.Create();
colorSensorInit = nuitrack.ColorSensor.Create();
userTrackerInit = nuitrack.UserTracker.Create();
skeletonTrackerInit = nuitrack.SkeletonTracker.Create();
handTrackerInit = nuitrack.HandTracker.Create();
gestureRecognizerInit = nuitrack.GestureRecognizer.Create();
nuitrack.Nuitrack.Run();
Debug.Log("run ok");
nuitrackInitialized = true;
}
// */
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
gesturesData = null;
//
// depthSensor = null;
// userTracker = null;
// skeletonTracker = null;
// handTracker = null;
// gestureRecognizer = null;
//if (issuesProcessor != null) Destroy(issuesProcessor);
if (prevDepth != depthOn)
{
prevDepth = depthOn;
if (depthOn)
{
depthSensor = depthSensorInit;
depthSensorInit.OnUpdateEvent += DepthUpdate;
}
else
{
depthSensorInit.OnUpdateEvent -= DepthUpdate;
depthSensor = null;
}
}
if (prevColor != colorOn)
{
prevColor = colorOn;
if (colorOn)
{
colorSensor = colorSensorInit;
colorSensorInit.OnUpdateEvent += ColorUpdate;
}
else
{
colorSensorInit.OnUpdateEvent -= ColorUpdate;
colorSensor = null;
}
}
if (prevUser != userOn)
{
prevUser = userOn;
if (userOn)
{
userTracker = userTrackerInit;
userTrackerInit.OnUpdateEvent += UserUpdate;
}
else
{
userTrackerInit.OnUpdateEvent -= UserUpdate;
userTracker = null;
}
}
if (skeletonOn != prevSkel)
{
prevSkel = skeletonOn;
if (skeletonOn)
{
skeletonTracker = skeletonTrackerInit;
skeletonTrackerInit.OnSkeletonUpdateEvent += SkeletonsUpdate;
}
else
{
skeletonTrackerInit.OnSkeletonUpdateEvent -= SkeletonsUpdate;
skeletonTracker = null;
}
}
if (prevHand != handsOn)
{
prevHand = handsOn;
if (handsOn)
{
handTracker = handTrackerInit;
handTrackerInit.OnUpdateEvent += HandTrackerUpdate;
}
else
{
handTrackerInit.OnUpdateEvent -= HandTrackerUpdate;
handTracker = null;
}
}
if (prevGesture != gesturesOn)
{
prevGesture = gesturesOn;
if (gesturesOn)
{
gestureRecognizer = gestureRecognizerInit;
gestureRecognizerInit.OnNewGesturesEvent += GesturesUpdate;
}
else
{
gestureRecognizerInit.OnNewGesturesEvent -= GesturesUpdate;
gestureRecognizer = null;
}
}
//issuesProcessor = (GameObject)Instantiate(issuesProcessorPrefab);
}
catch (Exception ex)
{
exceptionsLogger.AddEntry(ex.ToString());
}
}
void ChangeModulsState(bool skel, bool hand, bool depth, bool color, bool gest, bool user)
{
// Debug.Log ("" + skel + hand + depth + gest + user);
if (skeletonTracker == null)
{
return;
}
if (prevSkel != skel)
{
skeletonData = null;
prevSkel = skel;
if (skel)
{
skeletonTracker.OnSkeletonUpdateEvent += HandleOnSkeletonUpdateEvent;
}
else
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
}
}
if (prevHand != hand)
{
handTrackerData = null;
prevHand = hand;
if (hand)
{
handTracker.OnUpdateEvent += HandleOnHandsUpdateEvent;
}
else
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
}
}
if (prevGest != gest)
{
prevGest = gest;
if (gest)
{
gestureRecognizer.OnNewGesturesEvent += OnNewGestures;
}
else
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
}
}
if (prevDepth != depth)
{
depthFrame = null;
prevDepth = depth;
if (depth)
{
depthSensor.OnUpdateEvent += HandleOnDepthSensorUpdateEvent;
}
else
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
}
}
if (prevColor != color)
{
colorFrame = null;
prevColor = color;
if (color)
{
colorSensor.OnUpdateEvent += HandleOnColorSensorUpdateEvent;
}
else
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
}
}
if (prevUser != user)
{
userFrame = null;
prevUser = user;
if (user)
{
userTracker.OnUpdateEvent += HandleOnUserTrackerUpdateEvent;
}
else
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
}
}
}
public void CloseSensor(KinectInterop.SensorData sensorData)
{
if (colorWebCam)
{
colorWebCam.Stop();
colorWebCam = null;
}
if (coordMapper != null)
{
coordMapper.CleanUp();
coordMapper = null;
}
if (colorSensor != null)
{
colorSensor.OnUpdateEvent -= HandleOnColorSensorUpdateEvent;
colorSensor = null;
colorFrame = null;
}
if (depthSensor != null)
{
depthSensor.OnUpdateEvent -= HandleOnDepthSensorUpdateEvent;
depthSensor = null;
depthFrame = null;
}
if (userTracker != null)
{
userTracker.OnUpdateEvent -= HandleOnUserTrackerUpdateEvent;
userTracker = null;
userFrame = null;
}
if (skeletonTracker != null)
{
skeletonTracker.OnSkeletonUpdateEvent -= HandleOnSkeletonUpdateEvent;
skeletonTracker = null;
skeletonData = null;
}
if (handTracker != null)
{
handTracker.OnUpdateEvent -= HandleOnHandsUpdateEvent;
handTracker = null;
handTrackerData = null;
}
if (gestureRecognizer != null)
{
gestureRecognizer.OnNewGesturesEvent -= OnNewGestures;
gestureRecognizer = null;
}
if (bNuitrackInited)
{
bNuitrackInited = false;
nuitrack.Nuitrack.onIssueUpdateEvent -= OnIssuesUpdate;
NuitrackTerm();
}
Debug.Log("Nuitrack sensor closed");
}
void ProcessFrame(nuitrack.DepthFrame depthFrame, nuitrack.ColorFrame colorFrame, nuitrack.UserFrame userFrame)
{
for (int i = 0; i < parts; i++)
{
if (!visualizationParts[i].activeSelf)
{
visualizationParts[i].SetActive(true);
}
}
Color pointColor = Color.white;
if (userFrame != null)
{
if (issuesProcessor.userIssues != null)
{
for (int i = 1; i < userCurrentCols.Length; i++)
{
if (issuesProcessor.userIssues.ContainsKey(i))
{
userCurrentCols[i] =
(issuesProcessor.userIssues[i].isOccluded ||
issuesProcessor.userIssues[i].onBorderLeft ||
issuesProcessor.userIssues[i].onBorderRight ||
issuesProcessor.userIssues[i].onBorderTop) ?
occludedUserCols[i] : userCols[i];
}
}
}
}
for (int i = 0, pointIndex = 0, rgbOffset = 0; i < depthFrame.Rows; i += frameStep)
{
for (int j = 0; j < depthFrame.Cols; j += frameStep, ++pointIndex, rgbOffset += 3)
{
depthColors[pointIndex].r = depthFrame[i, j] / 16384f;
uint userId = 0u;
if (userFrame != null)
{
userId = userFrame[i * userFrame.Rows / depthFrame.Rows,
j *userFrame.Cols / depthFrame.Cols];
}
#region RGB coloring
//int rgbOffset = 3 * (i * depthFrame.Cols + j);
Color rgbCol = defaultColor;
if (colorFrame != null)
{
rgbCol = new Color32(colorFrame[i, j].Red, colorFrame[i, j].Green, colorFrame[i, j].Blue, 255);
}
//new Color32(depthFrame.rgb[rgbOffset + 2], depthFrame.rgb[rgbOffset + 1], depthFrame.rgb[rgbOffset + 0], 255);
pointColor = rgbCol;// * userCurrentCols[userId];
if (userId == 0)
{
pointColor *= userCols[0];
}
Color segmColor = userCurrentCols[userId];
//Debug.Log(j.ToString() + ", " + i.ToString() + " : " + rgbCol);
#endregion
// pointColor = userCurrentCols[userId]; //user segmentation coloring
rgbColors[pointIndex] = pointColor;
segmColors[pointIndex] = segmColor;
}
}
depthTexture.SetPixels(depthColors);
rgbTexture.SetPixels(rgbColors);
segmentationTexture.SetPixels(segmColors);
depthTexture.Apply();
rgbTexture.Apply();
segmentationTexture.Apply();
}
private void InitTrackers(bool depthOn, bool colorOn, bool userOn, bool skeletonOn, bool handsOn, bool gesturesOn)
{
if (!nuitrackInitialized)
{
nuitrack.Nuitrack.Init();
Debug.Log("init ok");
depthSensorInit = nuitrack.DepthSensor.Create();
colorSensorInit = nuitrack.ColorSensor.Create();
userTrackerInit = nuitrack.UserTracker.Create();
skeletonTrackerInit = nuitrack.SkeletonTracker.Create();
handTrackerInit = nuitrack.HandTracker.Create();
gestureRecognizerInit = nuitrack.GestureRecognizer.Create();
nuitrack.Nuitrack.Run();
Debug.Log("run ok");
nuitrackInitialized = true;
}
depthFrame = null;
colorFrame = null;
userFrame = null;
skeletonData = null;
handTrackerData = null;
gesturesData = null;
//
// depthSensor = null;
// userTracker = null;
// skeletonTracker = null;
// handTracker = null;
// gestureRecognizer = null;
//if (issuesProcessor != null) Destroy(issuesProcessor);
if (prevDepth != depthOn)
{
prevDepth = depthOn;
if (depthOn)
{
depthSensor = depthSensorInit;
depthSensorInit.OnUpdateEvent += DepthUpdate;
}
else
{
depthSensorInit.OnUpdateEvent -= DepthUpdate;
depthSensor = null;
}
}
if (prevColor != colorOn)
{
prevColor = colorOn;
if (colorOn)
{
colorSensor = colorSensorInit;
colorSensorInit.OnUpdateEvent += ColorUpdate;
}
else
{
colorSensorInit.OnUpdateEvent -= ColorUpdate;
colorSensor = null;
}
}
if (prevUser != userOn)
{
prevUser = userOn;
if (userOn)
{
userTracker = userTrackerInit;
userTrackerInit.OnUpdateEvent += UserUpdate;
}
else
{
userTrackerInit.OnUpdateEvent -= UserUpdate;
userTracker = null;
}
}
if (skeletonOn != prevSkel)
{
prevSkel = skeletonOn;
if (skeletonOn)
{
skeletonTracker = skeletonTrackerInit;
skeletonTrackerInit.OnSkeletonUpdateEvent += SkeletonsUpdate;
}
else
{
skeletonTrackerInit.OnSkeletonUpdateEvent -= SkeletonsUpdate;
skeletonTracker = null;
}
}
if (prevHand != handsOn)
{
prevHand = handsOn;
if (handsOn)
{
handTracker = handTrackerInit;
handTrackerInit.OnUpdateEvent += HandTrackerUpdate;
}
else
{
handTrackerInit.OnUpdateEvent -= HandTrackerUpdate;
handTracker = null;
}
}
if (prevGesture != gesturesOn)
{
prevGesture = gesturesOn;
if (gesturesOn)
{
gestureRecognizer = gestureRecognizerInit;
gestureRecognizerInit.OnNewGesturesEvent += GesturesUpdate;
}
else
{
gestureRecognizerInit.OnNewGesturesEvent -= GesturesUpdate;
gestureRecognizer = null;
}
}
StopThread();
}
void ProcessFrame(nuitrack.DepthFrame depthFrame)
{
int pointIndex = 0;
depthArray = new int[depthFrame.Rows, depthFrame.Cols];
for (int i = 0; i < depthFrame.Rows; i += 1)
{
for (int j = 0; j < depthFrame.Cols; j += 1)
{
//take depth from the frame and put it into the depthColors array
depthArray[i, j] = depthFrame[i, j];
float value = depthFrame[i, j] / 16384f;
depthColors[pointIndex].r = value;
depthColors[pointIndex].g = value;
depthColors[pointIndex].b = value;
depthColors[pointIndex].a = 1;
++pointIndex;
}
}
depthTexture.SetPixels(depthColors);
//depthTexture.Apply();
Debug.Log(pointIndex);
if (record)
{
string filename = "/" + DateTime.Now.Hour + "-" + DateTime.Now.Minute + "-" + DateTime.Now.Second + "-" + DateTime.Now.Millisecond + ".png";
byte[] bytes = depthTexture.EncodeToPNG();
File.WriteAllBytes(recordDirectory + filename, bytes);
message = "recording";
//writeline for skeletal data will need to go in this block
//SKELINGTON STUFF
//SKELINGTON UPDATE
frameNum++;
if (CurrentUserTracker.CurrentUser != 0)
{
string newData = "";
newData += frameNum + "," + System.DateTime.Now.Hour + "" + System.DateTime.Now.Minute + System.DateTime.Now.Second + System.DateTime.Now.Millisecond;
nuitrack.Skeleton skeleton = CurrentUserTracker.CurrentSkeleton;
for (int q = 0; q < typeJoint.Length; q++)
{
nuitrack.Joint joint = skeleton.GetJoint(typeJoint[q]);
Vector3 newPosition = 0.001f * joint.ToVector3();
CreatedJoint[q].transform.localPosition = newPosition;
newData += "," + joint.Confidence;
newData += "," + joint.ToVector3().x;
newData += "," + joint.ToVector3().y;
newData += "," + joint.ToVector3().z;
}
file.WriteLine(newData);
}
}
}
