void Start() { kinectManager = KinectManager.Instance; if (kinectManager && kinectManager.IsInitialized()) { Shader depthFilterShader = Shader.Find("Kinect/DepthIrFilterShader"); sensorData = kinectManager.GetSensorData(sensorIndex); if (depthFilterShader != null && sensorData != null) { depthFilterMat = new Material(depthFilterShader); } if (depthImageBuffer == null) { int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } if (depthFilterTex == null) { depthFilterTex = KinectInterop.CreateRenderTexture(depthFilterTex, sensorData.depthImageWidth, sensorData.depthImageHeight); if (backgroundImage) { backgroundImage.texture = depthFilterTex; backgroundImage.rectTransform.localScale = kinectManager.GetDepthImageScale(sensorIndex); backgroundImage.color = Color.white; } } } }
void Start() { if (backgroundImage == null) { backgroundImage = GetComponent <UnityEngine.UI.RawImage>(); } kinectManager = KinectManager.Instance; sensorData = kinectManager != null?kinectManager.GetSensorData(sensorIndex) : null; if (sensorData != null) { // enable the color camera aligned depth frames sensorData.sensorInterface.EnableColorCameraDepthFrame(sensorData, true); // create the output texture and needed buffers depthImageTexture = KinectInterop.CreateRenderTexture(depthImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); depthImageMaterial = new Material(Shader.Find("Kinect/DepthHistImageShader")); //int depthBufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight >> 1; //depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); depthHistBuffer = KinectInterop.CreateComputeBuffer(depthHistBuffer, DepthSensorBase.MAX_DEPTH_DISTANCE_MM + 1, sizeof(int)); depthHistBufferData = new int[DepthSensorBase.MAX_DEPTH_DISTANCE_MM + 1]; equalHistBufferData = new int[DepthSensorBase.MAX_DEPTH_DISTANCE_MM + 1]; } }
void LateUpdate() { if (kinectManager && kinectManager.IsInitialized() && depthFilterMat != null) { if (sensorData != null && sensorData.infraredImageTexture != null && sensorData.depthImage != null && lastDepthFrameTime != sensorData.lastDepthFrameTime) { lastDepthFrameTime = sensorData.lastDepthFrameTime; int depthBufferLength = (sensorData.depthImageWidth * sensorData.depthImageHeight) >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } if (depthFilterTex == null || depthFilterTex.width != sensorData.depthImageWidth || depthFilterTex.height != sensorData.depthImageHeight) { depthFilterTex = KinectInterop.CreateRenderTexture(depthFilterTex, sensorData.depthImageWidth, sensorData.depthImageHeight); if (backgroundImage) { backgroundImage.texture = depthFilterTex; backgroundImage.rectTransform.localScale = kinectManager.GetDepthImageScale(sensorIndex); backgroundImage.color = Color.white; } } float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; depthFilterMat.SetInt("_TexResX", sensorData.depthImageWidth); depthFilterMat.SetInt("_TexResY", sensorData.depthImageHeight); depthFilterMat.SetInt("_MinDepth", (int)(minDistance * 1000f)); depthFilterMat.SetInt("_MaxDepth", (int)(maxDistance * 1000f)); KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.depthImage, depthBufferLength, sizeof(uint)); depthFilterMat.SetBuffer("_DepthMap", depthImageBuffer); depthFilterMat.SetTexture("_IrTex", sensorData.infraredImageTexture); Graphics.Blit(null, depthFilterTex, depthFilterMat); } // check for resolution change float cameraWidth = backgroundCamera ? backgroundCamera.pixelRect.width : 0f; float cameraHeight = backgroundCamera ? backgroundCamera.pixelRect.height : 0f; if (backgroundImage && (lastCamRectW != cameraWidth || lastCamRectH != cameraHeight)) { SetImageResolution(cameraWidth, cameraHeight); } } }
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed private void UpdateTextureWithNewFrame() { if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamInfraredImage == null) { return; } // get the updated infrared if (lastColorCamInfraredFrameTime != sensorData.lastColorCamInfraredFrameTime) { lastColorCamInfraredFrameTime = sensorData.lastColorCamInfraredFrameTime; if (infraredImageTexture.width != sensorData.colorImageWidth || infraredImageTexture.height != sensorData.colorImageHeight) { infraredImageTexture = KinectInterop.CreateRenderTexture(infraredImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); } int infraredBufferLength = sensorData.colorCamInfraredImage.Length >> 1; if (infraredImageBuffer == null || infraredImageBuffer.count != infraredBufferLength) { infraredImageBuffer = KinectInterop.CreateComputeBuffer(infraredImageBuffer, infraredBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(infraredImageBuffer, sensorData.colorCamInfraredImage, infraredBufferLength, sizeof(uint)); float minInfraredValue = ((DepthSensorBase)sensorData.sensorInterface).GetMinInfraredValue(); float maxInfraredValue = ((DepthSensorBase)sensorData.sensorInterface).GetMaxInfraredValue(); infraredImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth); infraredImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight); infraredImageMaterial.SetFloat("_MinValue", minInfraredValue); infraredImageMaterial.SetFloat("_MaxValue", maxInfraredValue); infraredImageMaterial.SetBuffer("_InfraredMap", infraredImageBuffer); Graphics.Blit(null, infraredImageTexture, infraredImageMaterial); } }
void Update() { if (matRenderer == null || sensorInt == null) { return; } if (alphaTex == null) { // alpha texture alphaTex = backgroundRemovalManager.GetAlphaTex(); if (alphaTex != null) { matRenderer.SetTexture("_AlphaTex", alphaTex); } } if (colorTex == null) { // color texture colorTex = !backgroundRemovalManager.computeAlphaMaskOnly ? backgroundRemovalManager.GetForegroundTex() : alphaTex; // sensorInt.pointCloudColorTexture if (colorTex != null) { matRenderer.SetInt("_TexResX", colorTex.width); matRenderer.SetInt("_TexResY", colorTex.height); matRenderer.SetTexture("_ColorTex", colorTex); } } if (colorTex == null || alphaTex == null /**|| foregroundCamera == null*/) { return; } if (sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution) { if (depthImageBuffer == null) { //int depthImageLength = sensorData.depthImageWidth * sensorData.depthImageHeight; //depthImageCopy = new ushort[depthImageLength]; int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); matRenderer.SetBuffer("_DepthMap", depthImageBuffer); ScaleRendererTransform(colorTex); } if (depthImageBuffer != null && sensorData.depthImage != null) { //KinectInterop.CopyBytes(sensorData.depthImage, sizeof(ushort), depthImageCopy, sizeof(ushort)); int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.depthImage, depthBufferLength, sizeof(uint)); } //Debug.Log("ForegroundBlendRenderer DepthFrameTime: " + lastDepthFrameTime); } else { if (sensorData.colorDepthBuffer == null) { int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight / 2; sensorData.colorDepthBuffer = new ComputeBuffer(bufferLength, sizeof(uint)); matRenderer.SetBuffer("_DepthMap", sensorData.colorDepthBuffer); ScaleRendererTransform(colorTex); } //Debug.Log("ForegroundBlendRenderer ColorDepthBufferTime: " + sensorData.lastColorDepthBufferTime); } matRenderer.SetFloat("_DepthDistance", 0f); matRenderer.SetFloat("_InvDepthVal", invalidDepthValue); // update lighting parameters lighting.UpdateLighting(matRenderer, applyLighting); }
// inits the mesh and related data private void InitMesh() { // create mesh mesh = new Mesh { name = "User" + playerIndex + "Mesh-S" + sensorIndex, indexFormat = UnityEngine.Rendering.IndexFormat.UInt32 }; MeshFilter meshFilter = GetComponent <MeshFilter>(); if (meshFilter != null) { meshFilter.mesh = mesh; } else { Debug.LogWarning("MeshFilter not found! You may not see the mesh on screen"); } // create depth image buffer if (depthImageBuffer == null) { int depthImageLength = sensorData.depthImageWidth * sensorData.depthImageHeight; depthImageCopy = new ushort[depthImageLength]; int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); //depthBufferCreated = true; } // create body index buffer if (bodyIndexBuffer == null) { int bodyIndexLength = sensorData.depthImageWidth * sensorData.depthImageHeight; bodyIndexCopy = new byte[bodyIndexLength]; int bodyIndexBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 4; bodyIndexBuffer = KinectInterop.CreateComputeBuffer(bodyIndexBuffer, bodyIndexBufferLength, sizeof(uint)); //bodyIndexBufferCreated = true; } // create point cloud color texture if (sensorData != null && sensorData.sensorInterface != null) { sensorInt = (DepthSensorBase)sensorData.sensorInterface; Vector2Int imageRes = Vector2Int.zero; if (sensorInt.pointCloudColorTexture == null) { sensorInt.pointCloudResolution = sourceImageResolution; imageRes = sensorInt.GetPointCloudTexResolution(sensorData); colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32); sensorInt.pointCloudColorTexture = colorTexture; colorTextureCreated = true; } else { sourceImageResolution = sensorInt.pointCloudResolution; imageRes = sensorInt.GetPointCloudTexResolution(sensorData); colorTexture = sensorInt.pointCloudColorTexture; colorTextureCreated = false; } // create space table spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ? sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData); int spaceBufferLength = imageRes.x * imageRes.y * 3; spaceTableBuffer = new ComputeBuffer(spaceBufferLength, sizeof(float)); spaceTableBuffer.SetData(spaceTable); spaceTable = null; // create copy texture colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32); // set the color texture Renderer meshRenderer = GetComponent <Renderer>(); if (meshRenderer && meshRenderer.material /**&& meshRenderer.material.mainTexture == null*/) { meshShaderMat = meshRenderer.material; } // get reference to the transform trans = GetComponent <Transform>(); // image width & height imageWidth = imageRes.x; imageHeight = imageRes.y; // create mesh vertices & indices CreateMeshVertInd(); bMeshInited = true; } }
// updates the textures and buffers, needed by the mesh renderer private void UpdateTexturesAndBuffers() { if (sensorData == null || sensorInt == null) { return; } // get image resolution Vector2Int imageRes = sensorInt.GetPointCloudTexResolution(sensorData); if (colorTextureCreated && (colorTexture == null || colorTexture.width != imageRes.x || colorTexture.height != imageRes.y)) { colorTexture = KinectInterop.CreateRenderTexture(colorTexture, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32); sensorInt.pointCloudColorTexture = colorTexture; //Debug.Log("Created pointCloudColorTexture with resolution " + imageRes); } // create depth image buffer if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution) { int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageCopy = new ushort[depthBufferLength << 1]; depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); //Debug.Log("Created depthImageBuffer with length " + depthBufferLength); } //meshShaderMat.SetBuffer("_DepthMap", depthImageBuffer); } else { int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight >> 1; if (sensorData.colorDepthBuffer == null || sensorData.colorDepthBuffer.count != bufferLength) { sensorData.colorDepthBuffer = KinectInterop.CreateComputeBuffer(sensorData.colorDepthBuffer, bufferLength, sizeof(uint)); //Debug.Log("Created colorDepthBuffer with length " + bufferLength); } //meshShaderMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer); if (meshTexture == MeshTextureType.InfraredTexture) { if (sensorData.colorInfraredBuffer == null || sensorData.colorInfraredBuffer.count != bufferLength) { sensorData.colorInfraredBuffer = KinectInterop.CreateComputeBuffer(sensorData.colorInfraredBuffer, bufferLength, sizeof(uint)); //Debug.Log("Created colorInfraredBuffer with length " + bufferLength); } } } // create body index buffer if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution) { int bodyIndexBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight >> 2; if (bodyIndexBuffer == null || bodyIndexBuffer.count != bodyIndexBufferLength) { bodyIndexCopy = new byte[bodyIndexBufferLength << 2]; bodyIndexBuffer = KinectInterop.CreateComputeBuffer(bodyIndexBuffer, bodyIndexBufferLength, sizeof(uint)); //Debug.Log("Created bodyIndexBuffer with length " + bodyIndexBufferLength); } //meshShaderMat.SetBuffer("_BodyIndexMap", bodyIndexBuffer); } else { int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight >> 2; if (sensorData.colorBodyIndexBuffer == null || sensorData.colorBodyIndexBuffer.count != bufferLength) { sensorData.colorBodyIndexBuffer = new ComputeBuffer(bufferLength, sizeof(uint)); //Debug.Log("Created colorBodyIndexBuffer with length " + bufferLength); } //meshShaderMat.SetBuffer("_BodyIndexMap", sensorData.colorBodyIndexBuffer); } if (colorTextureCopy == null || colorTextureCopy.width != imageRes.x || colorTextureCopy.height != imageRes.y) { // create copy texture colorTextureCopy = KinectInterop.CreateRenderTexture(colorTextureCopy, imageRes.x, imageRes.y, RenderTextureFormat.ARGB32); //Debug.Log("Created colorTextureCopy with resolution " + imageRes); } if (spaceTableBuffer == null || imageWidth != imageRes.x || imageHeight != imageRes.y) { // create space table spaceTable = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ? sensorInt.GetDepthCameraSpaceTable(sensorData) : sensorInt.GetColorCameraSpaceTable(sensorData); int spaceBufferLength = imageRes.x * imageRes.y * 3; spaceTableBuffer = KinectInterop.CreateComputeBuffer(spaceTableBuffer, spaceBufferLength, sizeof(float)); spaceTableBuffer.SetData(spaceTable); spaceTable = null; //Debug.Log("Created spaceTable for resolution " + imageRes); } // create mesh vertices & indices if (imageWidth != imageRes.x || imageHeight != imageRes.y) { CreateMeshVertInd(imageRes.x, imageRes.y); } // image width & height imageWidth = imageRes.x; imageHeight = imageRes.y; }
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed private void UpdateTextureWithNewFrame() { if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamDepthImage == null) { return; } // get the updated depth frame if (lastColorCamDepthFrameTime != sensorData.lastColorCamDepthFrameTime) { lastColorCamDepthFrameTime = sensorData.lastColorCamDepthFrameTime; if (depthImageTexture.width != sensorData.colorImageWidth || depthImageTexture.height != sensorData.colorImageHeight) { depthImageTexture = KinectInterop.CreateRenderTexture(depthImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); } Array.Clear(depthHistBufferData, 0, depthHistBufferData.Length); Array.Clear(equalHistBufferData, 0, equalHistBufferData.Length); depthHistTotalPoints = 0; // get configured min & max distances float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; int depthMinDistance = (int)(minDistance * 1000f); int depthMaxDistance = (int)(maxDistance * 1000f); int frameLen = sensorData.colorCamDepthImage.Length; for (int i = 0; i < frameLen; i++) { int depth = sensorData.colorCamDepthImage[i]; int limDepth = (depth <= DepthSensorBase.MAX_DEPTH_DISTANCE_MM) ? depth : 0; if (limDepth > 0) { depthHistBufferData[limDepth]++; depthHistTotalPoints++; } } equalHistBufferData[0] = depthHistBufferData[0]; for (int i = 1; i < depthHistBufferData.Length; i++) { equalHistBufferData[i] = equalHistBufferData[i - 1] + depthHistBufferData[i]; } // make depth 0 equal to the max-depth equalHistBufferData[0] = equalHistBufferData[equalHistBufferData.Length - 1]; int depthBufferLength = sensorData.colorCamDepthImage.Length >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.colorCamDepthImage, depthBufferLength, sizeof(uint)); if (depthHistBuffer != null) { KinectInterop.SetComputeBufferData(depthHistBuffer, equalHistBufferData, equalHistBufferData.Length, sizeof(int)); } depthImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth); depthImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight); depthImageMaterial.SetInt("_MinDepth", depthMinDistance); depthImageMaterial.SetInt("_MaxDepth", depthMaxDistance); depthImageMaterial.SetBuffer("_DepthMap", depthImageBuffer); depthImageMaterial.SetBuffer("_HistMap", depthHistBuffer); depthImageMaterial.SetInt("_TotalPoints", depthHistTotalPoints); Graphics.Blit(null, depthImageTexture, depthImageMaterial); } }
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed private void UpdateTextureWithNewFrame() { if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamBodyIndexImage == null || sensorData.colorCamDepthImage == null) { return; } // get body index frame if (lastColorCamDepthFrameTime != sensorData.lastColorCamDepthFrameTime || lastColorCamBodyIndexFrameTime != sensorData.lastColorCamBodyIndexFrameTime) { lastColorCamDepthFrameTime = sensorData.lastColorCamDepthFrameTime; lastColorCamBodyIndexFrameTime = sensorData.lastColorCamBodyIndexFrameTime; if (bodyImageTexture.width != sensorData.colorImageWidth || bodyImageTexture.height != sensorData.colorImageHeight) { bodyImageTexture = KinectInterop.CreateRenderTexture(bodyImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); } Array.Clear(depthBodyBufferData, 0, depthBodyBufferData.Length); Array.Clear(equalBodyBufferData, 0, equalBodyBufferData.Length); bodyHistTotalPoints = 0; // get configured min & max distances float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; int depthMinDistance = (int)(minDistance * 1000f); int depthMaxDistance = (int)(maxDistance * 1000f); int frameLen = sensorData.colorCamDepthImage.Length; for (int i = 0; i < frameLen; i++) { int depth = sensorData.colorCamDepthImage[i]; int limDepth = (depth >= depthMinDistance && depth <= depthMaxDistance) ? depth : 0; if (/**rawBodyIndexImage[i] != 255 &&*/ limDepth > 0) { depthBodyBufferData[limDepth]++; bodyHistTotalPoints++; } } if (bodyHistTotalPoints > 0) { equalBodyBufferData[0] = depthBodyBufferData[0]; for (int i = 1; i < depthBodyBufferData.Length; i++) { equalBodyBufferData[i] = equalBodyBufferData[i - 1] + depthBodyBufferData[i]; } } int bodyIndexBufferLength = sensorData.colorCamBodyIndexImage.Length >> 2; if (bodyIndexBuffer == null || bodyIndexBuffer.count != bodyIndexBufferLength) { bodyIndexBuffer = KinectInterop.CreateComputeBuffer(bodyIndexBuffer, bodyIndexBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(bodyIndexBuffer, sensorData.colorCamBodyIndexImage, bodyIndexBufferLength, sizeof(uint)); int depthBufferLength = sensorData.colorCamDepthImage.Length >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.colorCamDepthImage, depthBufferLength, sizeof(uint)); if (bodyHistBuffer != null) { KinectInterop.SetComputeBufferData(bodyHistBuffer, equalBodyBufferData, equalBodyBufferData.Length, sizeof(int)); } float minDist = kinectManager.minUserDistance != 0f ? kinectManager.minUserDistance : minDistance; float maxDist = kinectManager.maxUserDistance != 0f ? kinectManager.maxUserDistance : maxDistance; bodyImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth); bodyImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight); bodyImageMaterial.SetInt("_MinDepth", (int)(minDist * 1000f)); bodyImageMaterial.SetInt("_MaxDepth", (int)(maxDist * 1000f)); bodyImageMaterial.SetBuffer("_BodyIndexMap", bodyIndexBuffer); bodyImageMaterial.SetBuffer("_DepthMap", depthImageBuffer); bodyImageMaterial.SetBuffer("_HistMap", bodyHistBuffer); bodyImageMaterial.SetInt("_TotalPoints", bodyHistTotalPoints); Color[] bodyIndexColors = kinectManager.GetBodyIndexColors(); bodyImageMaterial.SetColorArray("_BodyIndexColors", bodyIndexColors); Graphics.Blit(null, bodyImageTexture, bodyImageMaterial); } }