void LateUpdate() { if (kinectManager && kinectManager.IsInitialized() && depthFilterMat != null) { if (sensorData != null && sensorData.infraredImageTexture != null && sensorData.depthImage != null && lastDepthFrameTime != sensorData.lastDepthFrameTime) { lastDepthFrameTime = sensorData.lastDepthFrameTime; int depthBufferLength = (sensorData.depthImageWidth * sensorData.depthImageHeight) >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } if (depthFilterTex == null || depthFilterTex.width != sensorData.depthImageWidth || depthFilterTex.height != sensorData.depthImageHeight) { depthFilterTex = KinectInterop.CreateRenderTexture(depthFilterTex, sensorData.depthImageWidth, sensorData.depthImageHeight); if (backgroundImage) { backgroundImage.texture = depthFilterTex; backgroundImage.rectTransform.localScale = kinectManager.GetDepthImageScale(sensorIndex); backgroundImage.color = Color.white; } } float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; depthFilterMat.SetInt("_TexResX", sensorData.depthImageWidth); depthFilterMat.SetInt("_TexResY", sensorData.depthImageHeight); depthFilterMat.SetInt("_MinDepth", (int)(minDistance * 1000f)); depthFilterMat.SetInt("_MaxDepth", (int)(maxDistance * 1000f)); KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.depthImage, depthBufferLength, sizeof(uint)); depthFilterMat.SetBuffer("_DepthMap", depthImageBuffer); depthFilterMat.SetTexture("_IrTex", sensorData.infraredImageTexture); Graphics.Blit(null, depthFilterTex, depthFilterMat); } // check for resolution change float cameraWidth = backgroundCamera ? backgroundCamera.pixelRect.width : 0f; float cameraHeight = backgroundCamera ? backgroundCamera.pixelRect.height : 0f; if (backgroundImage && (lastCamRectW != cameraWidth || lastCamRectH != cameraHeight)) { SetImageResolution(cameraWidth, cameraHeight); } } }
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed private void UpdateTextureWithNewFrame() { if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamInfraredImage == null) { return; } // get the updated infrared if (lastColorCamInfraredFrameTime != sensorData.lastColorCamInfraredFrameTime) { lastColorCamInfraredFrameTime = sensorData.lastColorCamInfraredFrameTime; if (infraredImageTexture.width != sensorData.colorImageWidth || infraredImageTexture.height != sensorData.colorImageHeight) { infraredImageTexture = KinectInterop.CreateRenderTexture(infraredImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); } int infraredBufferLength = sensorData.colorCamInfraredImage.Length >> 1; if (infraredImageBuffer == null || infraredImageBuffer.count != infraredBufferLength) { infraredImageBuffer = KinectInterop.CreateComputeBuffer(infraredImageBuffer, infraredBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(infraredImageBuffer, sensorData.colorCamInfraredImage, infraredBufferLength, sizeof(uint)); float minInfraredValue = ((DepthSensorBase)sensorData.sensorInterface).GetMinInfraredValue(); float maxInfraredValue = ((DepthSensorBase)sensorData.sensorInterface).GetMaxInfraredValue(); infraredImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth); infraredImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight); infraredImageMaterial.SetFloat("_MinValue", minInfraredValue); infraredImageMaterial.SetFloat("_MaxValue", maxInfraredValue); infraredImageMaterial.SetBuffer("_InfraredMap", infraredImageBuffer); Graphics.Blit(null, infraredImageTexture, infraredImageMaterial); } }
void LateUpdate() { if (kinectManager && kinectManager.IsInitialized() && depthFilterMat != null && depthImageBuffer != null) { if (sensorData != null && sensorData.infraredImageTexture != null && sensorData.depthImage != null && lastDepthFrameTime != sensorData.lastDepthFrameTime) { lastDepthFrameTime = sensorData.lastDepthFrameTime; float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; depthFilterMat.SetInt("_TexResX", sensorData.depthImageWidth); depthFilterMat.SetInt("_TexResY", sensorData.depthImageHeight); depthFilterMat.SetInt("_MinDepth", (int)(minDistance * 1000f)); depthFilterMat.SetInt("_MaxDepth", (int)(maxDistance * 1000f)); int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.depthImage, depthBufferLength, sizeof(uint)); depthFilterMat.SetBuffer("_DepthMap", depthImageBuffer); depthFilterMat.SetTexture("_IrTex", sensorData.infraredImageTexture); Graphics.Blit(null, depthFilterTex, depthFilterMat); } // check for resolution change float cameraWidth = backgroundCamera ? backgroundCamera.pixelRect.width : 0f; float cameraHeight = backgroundCamera ? backgroundCamera.pixelRect.height : 0f; if (backgroundImage && (lastCamRectW != cameraWidth || lastCamRectH != cameraHeight)) { SetImageResolution(cameraWidth, cameraHeight); } } }
void Update() { if (matRenderer == null || sensorInt == null) { return; } if (alphaTex == null) { // alpha texture alphaTex = backgroundRemovalManager.GetAlphaTex(); if (alphaTex != null) { matRenderer.SetTexture("_AlphaTex", alphaTex); } } if (colorTex == null) { // color texture colorTex = !backgroundRemovalManager.computeAlphaMaskOnly ? backgroundRemovalManager.GetForegroundTex() : alphaTex; // sensorInt.pointCloudColorTexture if (colorTex != null) { matRenderer.SetInt("_TexResX", colorTex.width); matRenderer.SetInt("_TexResY", colorTex.height); matRenderer.SetTexture("_ColorTex", colorTex); } } if (colorTex == null || alphaTex == null /**|| foregroundCamera == null*/) { return; } if (sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution) { if (depthImageBuffer == null) { //int depthImageLength = sensorData.depthImageWidth * sensorData.depthImageHeight; //depthImageCopy = new ushort[depthImageLength]; int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); matRenderer.SetBuffer("_DepthMap", depthImageBuffer); ScaleRendererTransform(colorTex); } if (depthImageBuffer != null && sensorData.depthImage != null) { //KinectInterop.CopyBytes(sensorData.depthImage, sizeof(ushort), depthImageCopy, sizeof(ushort)); int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.depthImage, depthBufferLength, sizeof(uint)); } //Debug.Log("ForegroundBlendRenderer DepthFrameTime: " + lastDepthFrameTime); } else { if (sensorData.colorDepthBuffer == null) { int bufferLength = sensorData.colorImageWidth * sensorData.colorImageHeight / 2; sensorData.colorDepthBuffer = new ComputeBuffer(bufferLength, sizeof(uint)); matRenderer.SetBuffer("_DepthMap", sensorData.colorDepthBuffer); ScaleRendererTransform(colorTex); } //Debug.Log("ForegroundBlendRenderer ColorDepthBufferTime: " + sensorData.lastColorDepthBufferTime); } matRenderer.SetFloat("_DepthDistance", 0f); matRenderer.SetFloat("_InvDepthVal", invalidDepthValue); // update lighting parameters lighting.UpdateLighting(matRenderer, applyLighting); }
// updates the mesh according to current depth frame private void UpdateMesh() { if (bMeshInited && colorTexture != null && sensorData.depthImage != null && sensorData.depthCamIntr != null && meshShaderMat != null && lastDepthFrameTime != sensorData.lastDepthFrameTime && (Time.time - lastMeshUpdateTime) >= updateMeshInterval) { lastDepthFrameTime = sensorData.lastDepthFrameTime; lastMeshUpdateTime = Time.time; int paramsCache = coarseFactor + (showAsPointCloud ? 10 : 0); if (meshParamsCache != paramsCache) { //Debug.Log("Mesh params changed. Recreating..."); CreateMeshVertInd(); } if (depthImageBuffer != null && sensorData.depthImage != null /**&& depthBufferCreated*/) { KinectInterop.CopyBytes(sensorData.depthImage, sizeof(ushort), depthImageCopy, sizeof(ushort)); int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; KinectInterop.SetComputeBufferData(depthImageBuffer, depthImageCopy, depthBufferLength, sizeof(uint)); } if (bodyIndexBuffer != null && sensorData.bodyIndexImage != null /**&& bodyIndexBufferCreated*/) { KinectInterop.CopyBytes(sensorData.bodyIndexImage, sizeof(byte), bodyIndexCopy, sizeof(byte)); int bodyIndexBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 4; KinectInterop.SetComputeBufferData(bodyIndexBuffer, bodyIndexCopy, bodyIndexBufferLength, sizeof(uint)); } switch (meshTexture) { case MeshTextureType.ColorTexture: if (colorTexture != null) { Graphics.CopyTexture(colorTexture, colorTextureCopy); } break; case MeshTextureType.InfraredTexture: Texture infraredTexture = kinectManager.GetInfraredImageTex(sensorIndex); if (infraredTexture != null) { Graphics.CopyTexture(infraredTexture, colorTextureCopy); } break; } meshShaderMat.SetBuffer("_DepthMap", depthImageBuffer); meshShaderMat.SetBuffer("_BodyIndexMap", bodyIndexBuffer); meshShaderMat.SetInt("_CoarseFactor", coarseFactor); meshShaderMat.SetInt("_IsPointCloud", showAsPointCloud ? 1 : 0); meshShaderMat.SetInt("_UserBodyIndex", userBodyIndex); meshShaderMat.SetVector("_UserBodyPos", userBodyPos); meshShaderMat.SetMatrix("_Transform", sensorInt.GetSensorToWorldMatrix()); meshShaderMat.SetVector("_TexRes", new Vector2(imageWidth, imageHeight)); meshShaderMat.SetVector("_Cxy", new Vector2(sensorData.depthCamIntr.ppx, sensorData.depthCamIntr.ppy)); meshShaderMat.SetVector("_Fxy", new Vector2(sensorData.depthCamIntr.fx, sensorData.depthCamIntr.fy)); meshShaderMat.SetTexture("_ColorTex", colorTextureCopy); meshShaderMat.SetVector("_SpaceScale", kinectManager.GetSensorSpaceScale(sensorIndex)); // kinectManager.GetDepthImageScale(sensorIndex) meshShaderMat.SetBuffer("_SpaceTable", spaceTableBuffer); // mesh bounds if (kinectManager.GetUserBoundingBox(userId, null, sensorIndex, Rect.zero, out Vector3 posMin, out Vector3 posMax)) { Vector3 boundsCenter = new Vector3((posMax.x - posMin.x) / 2f, (posMax.y - posMin.y) / 2f, (posMax.z - posMin.z) / 2f); Vector3 boundsSize = new Vector3((posMax.x - posMin.x), (posMax.y - posMin.y), (posMax.z - posMin.z)); mesh.bounds = new Bounds(boundsCenter, boundsSize); } if (updateColliderInterval > 0 && (Time.time - lastColliderUpdateTime) >= updateColliderInterval) { lastColliderUpdateTime = Time.time; MeshCollider meshCollider = GetComponent <MeshCollider>(); if (meshCollider) { meshCollider.sharedMesh = null; meshCollider.sharedMesh = mesh; } } } }
// updates the mesh according to current depth frame private void UpdateMesh() { if (bMeshInited && sensorData.depthImage != null && sensorData.depthCamIntr != null && meshShaderMat != null && lastDepthFrameTime != sensorData.lastDepthFrameTime && (Time.time - lastMeshUpdateTime) >= updateMeshInterval) { lastDepthFrameTime = sensorData.lastDepthFrameTime; lastMeshUpdateTime = Time.time; int paramsCache = coarseFactor + (showAsPointCloud ? 10 : 0); if (meshParamsCache != paramsCache) { //Debug.Log("Mesh params changed. Recreating..."); CreateMeshVertInd(); } if (depthImageBuffer != null && sensorData.depthImage != null /**&& depthBufferCreated*/) { KinectInterop.CopyBytes(sensorData.depthImage, sizeof(ushort), depthImageCopy, sizeof(ushort)); int depthBufferLength = sensorData.depthImageWidth * sensorData.depthImageHeight / 2; KinectInterop.SetComputeBufferData(depthImageBuffer, depthImageCopy, depthBufferLength, sizeof(uint)); } switch (meshTexture) { case MeshTextureType.ColorTexture: if (colorTexture != null) { Graphics.CopyTexture(colorTexture, colorTextureCopy); } break; case MeshTextureType.InfraredTexture: Texture infraredTexture = sensorInt.pointCloudResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution ? sensorData.infraredImageTexture : sensorData.colorInfraredTexture; if (infraredTexture != null) { Graphics.CopyTexture(infraredTexture, colorTextureCopy); } break; } if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution) { //Debug.Log("SceneMeshGpu DepthFrameTime: " + lastDepthFrameTime); } else { //sensorData.usedColorDepthBufferTime = sensorData.lastColorDepthBufferTime; //Debug.Log("SceneMeshGpu ColorDepthBufferTime: " + sensorData.lastColorDepthBufferTime); } if (sourceImageResolution == DepthSensorBase.PointCloudResolution.DepthCameraResolution) { meshShaderMat.SetBuffer("_DepthMap", depthImageBuffer); } else { meshShaderMat.SetBuffer("_DepthMap", sensorData.colorDepthBuffer); } meshShaderMat.SetTexture("_ColorTex", colorTextureCopy); meshShaderMat.SetVector("_SpaceScale", sensorSpaceScale); meshShaderMat.SetBuffer("_SpaceTable", spaceTableBuffer); meshShaderMat.SetVector("_TexRes", new Vector2(imageWidth, imageHeight)); //meshShaderMat.SetVector("_Cxy", new Vector2(sensorData.depthCamIntr.ppx, sensorData.depthCamIntr.ppy)); //meshShaderMat.SetVector("_Fxy", new Vector2(sensorData.depthCamIntr.fx, sensorData.depthCamIntr.fy)); meshShaderMat.SetInt("_CoarseFactor", coarseFactor); meshShaderMat.SetInt("_IsPointCloud", showAsPointCloud ? 1 : 0); meshShaderMat.SetMatrix("_Transform", sensorInt.GetSensorToWorldMatrix()); meshShaderMat.SetVector("_PosMin", new Vector3(xMin, yMin, zMin)); meshShaderMat.SetVector("_PosMax", new Vector3(xMax, yMax, zMax)); // mesh bounds Vector3 boundsCenter = new Vector3((xMax - xMin) / 2f, (yMax - yMin) / 2f, (zMax - zMin) / 2f); Vector3 boundsSize = new Vector3((xMax - xMin), (yMax - yMin), (zMax - zMin)); mesh.bounds = new Bounds(boundsCenter, boundsSize); // update lighting parameters lighting.UpdateLighting(meshShaderMat, applyLighting); if (updateColliderInterval > 0 && (Time.time - lastColliderUpdateTime) >= updateColliderInterval) { lastColliderUpdateTime = Time.time; MeshCollider meshCollider = GetComponent <MeshCollider>(); if (meshCollider) { meshCollider.sharedMesh = null; meshCollider.sharedMesh = mesh; } } } }
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed private void UpdateTextureWithNewFrame() { if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamDepthImage == null) { return; } // get the updated depth frame if (lastColorCamDepthFrameTime != sensorData.lastColorCamDepthFrameTime) { lastColorCamDepthFrameTime = sensorData.lastColorCamDepthFrameTime; if (depthImageTexture.width != sensorData.colorImageWidth || depthImageTexture.height != sensorData.colorImageHeight) { depthImageTexture = KinectInterop.CreateRenderTexture(depthImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); } Array.Clear(depthHistBufferData, 0, depthHistBufferData.Length); Array.Clear(equalHistBufferData, 0, equalHistBufferData.Length); depthHistTotalPoints = 0; // get configured min & max distances float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; int depthMinDistance = (int)(minDistance * 1000f); int depthMaxDistance = (int)(maxDistance * 1000f); int frameLen = sensorData.colorCamDepthImage.Length; for (int i = 0; i < frameLen; i++) { int depth = sensorData.colorCamDepthImage[i]; int limDepth = (depth <= DepthSensorBase.MAX_DEPTH_DISTANCE_MM) ? depth : 0; if (limDepth > 0) { depthHistBufferData[limDepth]++; depthHistTotalPoints++; } } equalHistBufferData[0] = depthHistBufferData[0]; for (int i = 1; i < depthHistBufferData.Length; i++) { equalHistBufferData[i] = equalHistBufferData[i - 1] + depthHistBufferData[i]; } // make depth 0 equal to the max-depth equalHistBufferData[0] = equalHistBufferData[equalHistBufferData.Length - 1]; int depthBufferLength = sensorData.colorCamDepthImage.Length >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.colorCamDepthImage, depthBufferLength, sizeof(uint)); if (depthHistBuffer != null) { KinectInterop.SetComputeBufferData(depthHistBuffer, equalHistBufferData, equalHistBufferData.Length, sizeof(int)); } depthImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth); depthImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight); depthImageMaterial.SetInt("_MinDepth", depthMinDistance); depthImageMaterial.SetInt("_MaxDepth", depthMaxDistance); depthImageMaterial.SetBuffer("_DepthMap", depthImageBuffer); depthImageMaterial.SetBuffer("_HistMap", depthHistBuffer); depthImageMaterial.SetInt("_TotalPoints", depthHistTotalPoints); Graphics.Blit(null, depthImageTexture, depthImageMaterial); } }
// checks for new color-camera aligned frames, and composes an updated body-index texture, if needed private void UpdateTextureWithNewFrame() { if (sensorData == null || sensorData.sensorInterface == null || sensorData.colorCamBodyIndexImage == null || sensorData.colorCamDepthImage == null) { return; } // get body index frame if (lastColorCamDepthFrameTime != sensorData.lastColorCamDepthFrameTime || lastColorCamBodyIndexFrameTime != sensorData.lastColorCamBodyIndexFrameTime) { lastColorCamDepthFrameTime = sensorData.lastColorCamDepthFrameTime; lastColorCamBodyIndexFrameTime = sensorData.lastColorCamBodyIndexFrameTime; if (bodyImageTexture.width != sensorData.colorImageWidth || bodyImageTexture.height != sensorData.colorImageHeight) { bodyImageTexture = KinectInterop.CreateRenderTexture(bodyImageTexture, sensorData.colorImageWidth, sensorData.colorImageHeight); } Array.Clear(depthBodyBufferData, 0, depthBodyBufferData.Length); Array.Clear(equalBodyBufferData, 0, equalBodyBufferData.Length); bodyHistTotalPoints = 0; // get configured min & max distances float minDistance = ((DepthSensorBase)sensorData.sensorInterface).minDistance; float maxDistance = ((DepthSensorBase)sensorData.sensorInterface).maxDistance; int depthMinDistance = (int)(minDistance * 1000f); int depthMaxDistance = (int)(maxDistance * 1000f); int frameLen = sensorData.colorCamDepthImage.Length; for (int i = 0; i < frameLen; i++) { int depth = sensorData.colorCamDepthImage[i]; int limDepth = (depth >= depthMinDistance && depth <= depthMaxDistance) ? depth : 0; if (/**rawBodyIndexImage[i] != 255 &&*/ limDepth > 0) { depthBodyBufferData[limDepth]++; bodyHistTotalPoints++; } } if (bodyHistTotalPoints > 0) { equalBodyBufferData[0] = depthBodyBufferData[0]; for (int i = 1; i < depthBodyBufferData.Length; i++) { equalBodyBufferData[i] = equalBodyBufferData[i - 1] + depthBodyBufferData[i]; } } int bodyIndexBufferLength = sensorData.colorCamBodyIndexImage.Length >> 2; if (bodyIndexBuffer == null || bodyIndexBuffer.count != bodyIndexBufferLength) { bodyIndexBuffer = KinectInterop.CreateComputeBuffer(bodyIndexBuffer, bodyIndexBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(bodyIndexBuffer, sensorData.colorCamBodyIndexImage, bodyIndexBufferLength, sizeof(uint)); int depthBufferLength = sensorData.colorCamDepthImage.Length >> 1; if (depthImageBuffer == null || depthImageBuffer.count != depthBufferLength) { depthImageBuffer = KinectInterop.CreateComputeBuffer(depthImageBuffer, depthBufferLength, sizeof(uint)); } KinectInterop.SetComputeBufferData(depthImageBuffer, sensorData.colorCamDepthImage, depthBufferLength, sizeof(uint)); if (bodyHistBuffer != null) { KinectInterop.SetComputeBufferData(bodyHistBuffer, equalBodyBufferData, equalBodyBufferData.Length, sizeof(int)); } float minDist = kinectManager.minUserDistance != 0f ? kinectManager.minUserDistance : minDistance; float maxDist = kinectManager.maxUserDistance != 0f ? kinectManager.maxUserDistance : maxDistance; bodyImageMaterial.SetInt("_TexResX", sensorData.colorImageWidth); bodyImageMaterial.SetInt("_TexResY", sensorData.colorImageHeight); bodyImageMaterial.SetInt("_MinDepth", (int)(minDist * 1000f)); bodyImageMaterial.SetInt("_MaxDepth", (int)(maxDist * 1000f)); bodyImageMaterial.SetBuffer("_BodyIndexMap", bodyIndexBuffer); bodyImageMaterial.SetBuffer("_DepthMap", depthImageBuffer); bodyImageMaterial.SetBuffer("_HistMap", bodyHistBuffer); bodyImageMaterial.SetInt("_TotalPoints", bodyHistTotalPoints); Color[] bodyIndexColors = kinectManager.GetBodyIndexColors(); bodyImageMaterial.SetColorArray("_BodyIndexColors", bodyIndexColors); Graphics.Blit(null, bodyImageTexture, bodyImageMaterial); } }