public static Stream AsStream(this IBuffer source) { if (source == null) { throw new ArgumentNullException(nameof(source)); } byte[] dataArr; int dataOffs; if (source.TryGetUnderlyingData(out dataArr, out dataOffs)) { Debug.Assert(source.Capacity < int.MaxValue); return(new MemoryStream(dataArr, dataOffs, (int)source.Capacity, true)); } unsafe { IBufferByteAccess bufferByteAccess = (IBufferByteAccess)source; return(new WindowsRuntimeBufferUnmanagedMemoryStream(source, (byte *)bufferByteAccess.GetBuffer())); } }
unsafe private void ShowMappedBodyFrame(int depthWidth, int depthHeight, IBuffer bodyIndexFrameData, IBufferByteAccess bodyIndexByteAccess) { bodyIndexByteAccess = (IBufferByteAccess)bodyIndexFrameData; byte* bodyIndexBytes = null; bodyIndexByteAccess.Buffer(out bodyIndexBytes); fixed (DepthSpacePoint* colorMappedToDepthPointsPointer = this.colorMappedToDepthPoints) { IBufferByteAccess bitmapBackBufferByteAccess = (IBufferByteAccess)this.bitmap.PixelBuffer; byte* bitmapBackBufferBytes = null; bitmapBackBufferByteAccess.Buffer(out bitmapBackBufferBytes); // Treat the color data as 4-byte pixels uint* bitmapPixelsPointer = (uint*)bitmapBackBufferBytes; // Loop over each row and column of the color image // Zero out any pixels that don't correspond to a body index int colorMappedLength = this.colorMappedToDepthPoints.Length; for (int colorIndex = 0; colorIndex < colorMappedLength; ++colorIndex) { float colorMappedToDepthX = colorMappedToDepthPointsPointer[colorIndex].X; float colorMappedToDepthY = colorMappedToDepthPointsPointer[colorIndex].Y; // The sentinel value is -inf, -inf, meaning that no depth pixel corresponds to this color pixel. if (!float.IsNegativeInfinity(colorMappedToDepthX) && !float.IsNegativeInfinity(colorMappedToDepthY)) { // Make sure the depth pixel maps to a valid point in color space int depthX = (int)(colorMappedToDepthX + 0.5f); int depthY = (int)(colorMappedToDepthY + 0.5f); // If the point is not valid, there is no body index there. if ((depthX >= 0) && (depthX < depthWidth) && (depthY >= 0) && (depthY < depthHeight)) { int depthIndex = (depthY * depthWidth) + depthX; // If we are tracking a body for the current pixel, do not zero out the pixel if (bodyIndexBytes[depthIndex] != 0xff) { // this bodyIndexByte is good and is a body, loop again. continue; } } } // this pixel does not correspond to a body so make it black and transparent bitmapPixelsPointer[colorIndex] = 0; } } this.bitmap.Invalidate(); FrameDisplayImage.Source = this.bitmap; }
unsafe private void ShowMappedBodyFrame(int depthWidth, int depthHeight, IBuffer bodyIndexFrameData, IBufferByteAccess bodyIndexByteAccess) { bodyIndexByteAccess = (IBufferByteAccess)bodyIndexFrameData; byte *bodyIndexBytes = null; bodyIndexByteAccess.Buffer(out bodyIndexBytes); fixed(DepthSpacePoint *colorMappedToDepthPointsPointer = this.colorMappedToDepthPoints) { IBufferByteAccess bitmapBackBufferByteAccess = (IBufferByteAccess)this.bitmap.PixelBuffer; byte *bitmapBackBufferBytes = null; bitmapBackBufferByteAccess.Buffer(out bitmapBackBufferBytes); // Treat the color data as 4-byte pixels uint *bitmapPixelsPointer = (uint *)bitmapBackBufferBytes; // Loop over each row and column of the color image // Zero out any pixels that don't correspond to a body index int colorMappedLength = this.colorMappedToDepthPoints.Length; for (int colorIndex = 0; colorIndex < colorMappedLength; ++colorIndex) { float colorMappedToDepthX = colorMappedToDepthPointsPointer[colorIndex].X; float colorMappedToDepthY = colorMappedToDepthPointsPointer[colorIndex].Y; // The sentinel value is -inf, -inf, meaning that no depth pixel corresponds to this color pixel. if (!float.IsNegativeInfinity(colorMappedToDepthX) && !float.IsNegativeInfinity(colorMappedToDepthY)) { // Make sure the depth pixel maps to a valid point in color space int depthX = (int)(colorMappedToDepthX + 0.5f); int depthY = (int)(colorMappedToDepthY + 0.5f); // If the point is not valid, there is no body index there. if ((depthX >= 0) && (depthX < depthWidth) && (depthY >= 0) && (depthY < depthHeight)) { int depthIndex = (depthY * depthWidth) + depthX; // If we are tracking a body for the current pixel, do not zero out the pixel if (bodyIndexBytes[depthIndex] != 0xff) { // this bodyIndexByte is good and is a body, loop again. continue; } } } // this pixel does not correspond to a body so make it black and transparent bitmapPixelsPointer[colorIndex] = 0; } } this.bitmap.Invalidate(); FrameDisplayImage.Source = this.bitmap; }
private void Reader_MultiSourceFrameArrived(MultiSourceFrameReader sender, MultiSourceFrameArrivedEventArgs e) { MultiSourceFrame multiSourceFrame = e.FrameReference.AcquireFrame(); // If the Frame has expired by the time we process this event, return. if (multiSourceFrame == null) { return; } DepthFrame depthFrame = null; ColorFrame colorFrame = null; InfraredFrame infraredFrame = null; BodyFrame bodyFrame = null; BodyIndexFrame bodyIndexFrame = null; IBuffer depthFrameDataBuffer = null; IBuffer bodyIndexFrameData = null; // Com interface for unsafe byte manipulation IBufferByteAccess bufferByteAccess = null; switch (CurrentDisplayFrameType) { case DisplayFrameType.Infrared: using (infraredFrame = multiSourceFrame.InfraredFrameReference.AcquireFrame()) { ShowInfraredFrame(infraredFrame); } break; case DisplayFrameType.Color: using (colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame()) { ShowColorFrame(colorFrame); } break; case DisplayFrameType.Depth: using (depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame()) { ShowDepthFrame(depthFrame); } break; case DisplayFrameType.BodyMask: // Put in a try catch to utilise finally() and clean up frames try { depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame(); bodyIndexFrame = multiSourceFrame.BodyIndexFrameReference.AcquireFrame(); colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame(); if ((depthFrame == null) || (colorFrame == null) || (bodyIndexFrame == null)) { return; } // Access the depth frame data directly via LockImageBuffer to avoid making a copy depthFrameDataBuffer = depthFrame.LockImageBuffer(); this.coordinateMapper.MapColorFrameToDepthSpaceUsingIBuffer(depthFrameDataBuffer, this.colorMappedToDepthPoints); // Process Color colorFrame.CopyConvertedFrameDataToBuffer(this.bitmap.PixelBuffer, ColorImageFormat.Bgra); // Access the body index frame data directly via LockImageBuffer to avoid making a copy bodyIndexFrameData = bodyIndexFrame.LockImageBuffer(); ShowMappedBodyFrame(depthFrame.FrameDescription.Width, depthFrame.FrameDescription.Height, bodyIndexFrameData, bufferByteAccess); } finally { if (depthFrame != null) { depthFrame.Dispose(); } if (colorFrame != null) { colorFrame.Dispose(); } if (bodyIndexFrame != null) { bodyIndexFrame.Dispose(); } if (depthFrameDataBuffer != null) { // We must force a release of the IBuffer in order to ensure that we have dropped all references to it. System.Runtime.InteropServices.Marshal.ReleaseComObject(depthFrameDataBuffer); } if (bodyIndexFrameData != null) { System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexFrameData); } if (bufferByteAccess != null) { System.Runtime.InteropServices.Marshal.ReleaseComObject(bufferByteAccess); } } break; case DisplayFrameType.BodyJoints: using (bodyFrame = multiSourceFrame.BodyFrameReference.AcquireFrame()) { ShowBodyJoints(bodyFrame); } break; case DisplayFrameType.BackgroundRemoved: // Put in a try catch to utilise finally() and clean up frames try { depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame(); colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame(); if ((depthFrame == null) || (colorFrame == null)) { return; } depthFrame.CopyFrameDataToArray(depthFrameData); this.coordinateMapper.MapColorFrameToDepthSpace(depthFrameData, this.colorMappedToDepthPoints); // Process Color. colorFrame.CopyConvertedFrameDataToBuffer(this.bitmap.PixelBuffer, ColorImageFormat.Bgra); ShowMappedColorBackgroundRemoved(colorMappedToDepthPoints, depthFrameData, depthFrame.FrameDescription); } finally { if (depthFrame != null) { depthFrame.Dispose(); } if (colorFrame != null) { colorFrame.Dispose(); } } break; default: break; } }
/// <summary> /// Handles the depth/color/body index frame data arriving from the sensor. /// </summary> /// <param name="sender">object sending the event</param> /// <param name="e">event arguments</param> private void Reader_MultiSourceFrameArrived(object sender, MultiSourceFrameArrivedEventArgs e) { int depthWidth = 0; int depthHeight = 0; DepthFrame depthFrame = null; IBuffer depthFrameData = null; ColorFrame colorFrame = null; BodyIndexFrame bodyIndexFrame = null; IBuffer bodyIndexFrameData = null; IBufferByteAccess bodyIndexByteAccess = null; MultiSourceFrame multiSourceFrame = e.FrameReference.AcquireFrame(); // If the Frame has expired by the time we process this event, return. if (multiSourceFrame == null) { return; } // We use a try/finally to ensure that we clean up before we exit the function. // This includes calling Dispose on any Frame objects that we may have and unlocking the bitmap back buffer. try { depthFrame = multiSourceFrame.DepthFrameReference.AcquireFrame(); colorFrame = multiSourceFrame.ColorFrameReference.AcquireFrame(); bodyIndexFrame = multiSourceFrame.BodyIndexFrameReference.AcquireFrame(); // If any frame has expired by the time we process this event, return. // The "finally" statement will Dispose any that are not null. if ((depthFrame == null) || (colorFrame == null) || (bodyIndexFrame == null)) { return; } // Process Depth FrameDescription depthFrameDescription = depthFrame.FrameDescription; depthWidth = depthFrameDescription.Width; depthHeight = depthFrameDescription.Height; // Access the depth frame data directly via LockImageBuffer to avoid making a copy depthFrameData = depthFrame.LockImageBuffer(); this.coordinateMapper.MapColorFrameToDepthSpaceUsingIBuffer(depthFrameData, this.colorMappedToDepthPoints); // We're done with the DepthFrame depthFrame.Dispose(); depthFrame = null; // We must force a release of the IBuffer in order to ensure that we have dropped all references to it. System.Runtime.InteropServices.Marshal.ReleaseComObject(depthFrameData); depthFrameData = null; // Process Color colorFrame.CopyConvertedFrameDataToBuffer(this.bitmap.PixelBuffer, ColorImageFormat.Bgra); // We're done with the ColorFrame colorFrame.Dispose(); colorFrame = null; FrameDescription bodyIndexFrameDescription = bodyIndexFrame.FrameDescription; // Access the body index frame data directly via LockImageBuffer to avoid making a copy bodyIndexFrameData = bodyIndexFrame.LockImageBuffer(); int colorMappedToDepthPointCount = this.colorMappedToDepthPoints.Length; unsafe { bodyIndexByteAccess = (IBufferByteAccess)bodyIndexFrameData; byte *bodyIndexBytes = null; bodyIndexByteAccess.Buffer(out bodyIndexBytes); fixed(DepthSpacePoint *colorMappedToDepthPointsPointer = this.colorMappedToDepthPoints) { IBufferByteAccess bitmapBackBufferByteAccess = (IBufferByteAccess)this.bitmap.PixelBuffer; byte *bitmapBackBufferBytes = null; bitmapBackBufferByteAccess.Buffer(out bitmapBackBufferBytes); // Treat the color data as 4-byte pixels uint *bitmapPixelsPointer = (uint *)bitmapBackBufferBytes; // Loop over each row and column of the color image // Zero out any pixels that don't correspond to a body index for (int colorIndex = 0; colorIndex < colorMappedToDepthPointCount; ++colorIndex) { float colorMappedToDepthX = colorMappedToDepthPointsPointer[colorIndex].X; float colorMappedToDepthY = colorMappedToDepthPointsPointer[colorIndex].Y; // The sentinel value is -inf, -inf, meaning that no depth pixel corresponds to this color pixel. if (!float.IsNegativeInfinity(colorMappedToDepthX) && !float.IsNegativeInfinity(colorMappedToDepthY)) { // Make sure the depth pixel maps to a valid point in color space int depthX = (int)(colorMappedToDepthX + 0.5f); int depthY = (int)(colorMappedToDepthY + 0.5f); // If the point is not valid, there is no body index there. if ((depthX >= 0) && (depthX < depthWidth) && (depthY >= 0) && (depthY < depthHeight)) { int depthIndex = (depthY * depthWidth) + depthX; // If we are tracking a body for the current pixel, do not zero out the pixel if (bodyIndexBytes[depthIndex] != 0xff) { continue; } } } bitmapPixelsPointer[colorIndex] = 0; } } this.bitmap.Invalidate(); } } finally { if (depthFrame != null) { depthFrame.Dispose(); } if (depthFrameData != null) { // We must force a release of the IBuffer in order to ensure that we have dropped all references to it. System.Runtime.InteropServices.Marshal.ReleaseComObject(depthFrameData); } if (colorFrame != null) { colorFrame.Dispose(); } if (bodyIndexFrame != null) { bodyIndexFrame.Dispose(); } if (bodyIndexFrameData != null) { // We must force a release of the IBuffer in order to ensure that we have dropped all references to it. System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexFrameData); } if (bodyIndexByteAccess != null) { // We must force a release of the IBuffer in order to ensure that we have dropped all references to it. System.Runtime.InteropServices.Marshal.ReleaseComObject(bodyIndexByteAccess); } } }