// Update is called once per frame
void Update()
{
if (_Sensor == null || _ColorSourceManager == null || _DepthSourceManager == null)
{
return;
}
Mat colorMat = _ColorSourceManager.GetColorMat();
ushort[] depthData = _DepthSourceManager.GetData();
_colorTexture = _ColorSourceManager.GetColorTexture();
if (_mapperMat == null)
{
_mapperMat = new Mat(_colorTexture.height, _colorTexture.width, CvType.CV_8UC3);
}
//mapper
colorSpace = new ColorSpacePoint[depthData.Length];
depthSpace = new DepthSpacePoint[_colorTexture.width * _colorTexture.height];
_Mapper.MapDepthFrameToColorSpace(depthData, colorSpace);
_Mapper.MapColorFrameToDepthSpace(depthData, depthSpace);
//_mapperMat = colorMat;
}
// ******************************************************Body frame event************************************************************
void SendDatatoController()
{
//emit a event to bodydataeventcontroller
if (OnBodyData != null)
{
coordinateMapper.MapColorFrameToDepthSpace(depthData, depthSpacePoints);
OnBodyData(this, new KinectBodyDataEvent(depthSpacePoints, bodyIndexData, colorData));
}
//emit a event to bodydataeventcontroller
}
private void DrawColorCoodinate()
{
// カラー画像の解像度でデータを作る
var colorImageBuffer = new byte[colorFrameDesc.LengthInPixels *
colorFrameDesc.BytesPerPixel];
// カラー座標系に対応するDepth座標系の一覧を取得する
var depthSpace = new DepthSpacePoint[colorFrameDesc.LengthInPixels];
mapper.MapColorFrameToDepthSpace(depthBuffer, depthSpace);
// 並列で処理する
Parallel.For(0, colorFrameDesc.LengthInPixels, i =>
{
int depthX = (int)depthSpace[i].X;
int depthY = (int)depthSpace[i].Y;
if ((depthX < 0) || (depthFrameDesc.Width <= depthX) ||
(depthY < 0) || (depthFrameDesc.Height <= depthY))
{
return;
}
// Depth座標系のインデックス
int depthIndex = (depthY * depthFrameDesc.Width) + depthX;
int bodyIndex = bodyIndexBuffer[depthIndex];
// 人を検出した位置だけ色を付ける
if (bodyIndex == 255)
{
return;
}
// カラー画像を設定する
int colorImageIndex = (int)(i * colorFrameDesc.BytesPerPixel);
colorImageBuffer[colorImageIndex + 0] = colorBuffer[colorImageIndex + 0];
colorImageBuffer[colorImageIndex + 1] = colorBuffer[colorImageIndex + 1];
colorImageBuffer[colorImageIndex + 2] = colorBuffer[colorImageIndex + 2];
});
ImageColor.Source = BitmapSource.Create(
colorFrameDesc.Width, colorFrameDesc.Height, 96, 96,
PixelFormats.Bgr32, null, colorImageBuffer,
(int)(colorFrameDesc.Width * colorFrameDesc.BytesPerPixel));
}
public static void Test_GetCameraPoints(DepthMetaData myDepthMetaData, CoordinateMapper myCoordinateMapper)
{
//test
CameraSpacePoint[] cameraMappedPoints = new CameraSpacePoint[ColorMetaData.XColorMaxKinect * ColorMetaData.YColorMaxKinect];
myCoordinateMapper.MapColorFrameToCameraSpace(myDepthMetaData.FrameData, cameraMappedPoints);
List <CameraSpacePoint> myMappedPoints = new List <CameraSpacePoint>();
for (int i = 0; i < cameraMappedPoints.Length; ++i)
{
double colorMappedToDepthX = cameraMappedPoints[i].X;
double colorMappedToDepthY = cameraMappedPoints[i].Y;
if (!double.IsNegativeInfinity(colorMappedToDepthX) &&
!double.IsNegativeInfinity(colorMappedToDepthY))
{
myMappedPoints.Add(cameraMappedPoints[i]);
//System.Diagnostics.Debug.WriteLine("--> Camera Points : " + cameraMappedPoints[colorIndex].X.ToString() + " : " + cameraMappedPoints[colorIndex].Y.ToString() + " : " + cameraMappedPoints[colorIndex].Z.ToString());
}
}
System.Diagnostics.Debug.WriteLine("--> Number of ColorSpacePoints: " + myMappedPoints.Count.ToString());
//System.Diagnostics.Debug.WriteLine("--> Old.Camera Points : " + cameraMappedPoints[0].X.ToString() + " : " + cameraMappedPoints[0].X.ToString() + " : " + cameraMappedPoints[0].Z.ToString() + " : ");
DepthSpacePoint[] colorMappedToDepthPoints = new DepthSpacePoint[ColorMetaData.XColorMaxKinect * ColorMetaData.YColorMaxKinect];
myCoordinateMapper.MapColorFrameToDepthSpace(myDepthMetaData.FrameData, colorMappedToDepthPoints);
List <DepthSpacePoint> myMappedPointsDepth = new List <DepthSpacePoint>();
for (int i = 0; i < colorMappedToDepthPoints.Length; ++i)
{
double colorMappedToDepthX = colorMappedToDepthPoints[i].X;
double colorMappedToDepthY = colorMappedToDepthPoints[i].Y;
if (!double.IsNegativeInfinity(colorMappedToDepthX) &&
!double.IsNegativeInfinity(colorMappedToDepthY))
{
myMappedPointsDepth.Add(colorMappedToDepthPoints[i]);
//System.Diagnostics.Debug.WriteLine("--> Camera Points : " + cameraMappedPoints[colorIndex].X.ToString() + " : " + cameraMappedPoints[colorIndex].Y.ToString() + " : " + cameraMappedPoints[colorIndex].Z.ToString());
}
}
System.Diagnostics.Debug.WriteLine("--> Number of ColorDepthPoints: " + myMappedPointsDepth.Count.ToString());
//System.Diagnostics.Debug.WriteLine("--> Depth Points : " + colorMappedToDepthPoints[0].X.ToString() + " : " + colorMappedToDepthPoints[0].X.ToString() + " : " );
}
public unsafe void Update(ushort[] depthData, byte[] bodyIndexData, Body body)
{
double handLength = 0.0;
double barLength = 0.0;
double barHeight = 0.0;
double angle = 0.0;
Joint shoulderLeft = body.Joints[JointType.ShoulderLeft];
Joint shoulderRight = body.Joints[JointType.ShoulderRight];
Joint chest = body.Joints[JointType.SpineShoulder];
Joint waist = body.Joints[JointType.SpineBase];
Joint elbowLeft = body.Joints[JointType.ElbowLeft];
Joint elbowRight = body.Joints[JointType.ElbowRight];
Joint handLeft = body.Joints[JointType.HandLeft];
Joint handRight = body.Joints[JointType.HandRight];
Joint footLeft = body.Joints[JointType.FootLeft];
Joint footRight = body.Joints[JointType.FootRight];
if (waist.TrackingState == TrackingState.NotTracked)
{
return;
}
if (waist.Position.Z < 2.0f || waist.Position.Z > 4.5f)
{
return;
}
if (shoulderLeft.TrackingState != TrackingState.NotTracked && shoulderRight.TrackingState != TrackingState.NotTracked &&
elbowLeft.TrackingState != TrackingState.NotTracked && elbowRight.TrackingState != TrackingState.NotTracked &&
handLeft.TrackingState != TrackingState.NotTracked && handRight.TrackingState != TrackingState.NotTracked)
{
handLength =
shoulderLeft.Position.Length(shoulderRight.Position) +
shoulderLeft.Position.Length(elbowLeft.Position) +
shoulderRight.Position.Length(elbowRight.Position) +
elbowLeft.Position.Length(handLeft.Position) +
elbowRight.Position.Length(handRight.Position);
}
CoordinateMapper.MapColorFrameToDepthSpace(depthData, _depthPoints);
fixed(DepthSpacePoint *colorMappedToDepthPointsPointer = _depthPoints)
{
int minimumX = int.MaxValue;
int maximumX = int.MinValue;
int minimumY = int.MaxValue;
int maximumY = int.MinValue;
ushort minimumDistance = 0;
ushort maximumdistance = 0;
for (int colorIndex = 0; colorIndex < _depthPoints.Length; ++colorIndex)
{
float colorMappedToDepthX = colorMappedToDepthPointsPointer[colorIndex].X;
float colorMappedToDepthY = colorMappedToDepthPointsPointer[colorIndex].Y;
if (!float.IsNegativeInfinity(colorMappedToDepthX) &&
!float.IsNegativeInfinity(colorMappedToDepthY))
{
int depthX = (int)(colorMappedToDepthX + 0.5f);
int depthY = (int)(colorMappedToDepthY + 0.5f);
if ((depthX >= 0) && (depthX < DepthWidth) && (depthY >= 0) && (depthY < DepthHeight))
{
int depthIndex = (depthY * DepthWidth) + depthX;
ushort depth = depthData[depthIndex];
if (bodyIndexData[depthIndex] != 0xff)
{
if (depthX < minimumX)
{
minimumX = depthX;
minimumY = depthY;
minimumDistance = depth;
}
if (depthX > maximumX)
{
maximumX = depthX;
maximumY = depthY;
maximumdistance = depth;
}
continue;
}
}
}
}
DepthSpacePoint depthMinimum = new DepthSpacePoint
{
X = minimumX,
Y = minimumY
};
DepthSpacePoint depthMaximum = new DepthSpacePoint
{
X = maximumX,
Y = maximumY
};
CameraSpacePoint cameraMinimum = CoordinateMapper.MapDepthPointToCameraSpace(depthMinimum, minimumDistance);
CameraSpacePoint cameraMaximum = CoordinateMapper.MapDepthPointToCameraSpace(depthMaximum, maximumdistance);
ColorSpacePoint colorMinimum = CoordinateMapper.MapDepthPointToColorSpace(depthMinimum, minimumDistance);
ColorSpacePoint colorMaximum = CoordinateMapper.MapDepthPointToColorSpace(depthMaximum, maximumdistance);
CameraSpacePoint cameraTrail = new CameraSpacePoint
{
X = (cameraMinimum.X + cameraMaximum.X) / 2f,
Y = (cameraMinimum.Y + cameraMaximum.Y) / 2f,
Z = (cameraMinimum.Z + cameraMaximum.Z) / 2f
};
ColorSpacePoint colorTrail = new ColorSpacePoint
{
X = (colorMinimum.X + colorMaximum.X) / 2f,
Y = (colorMinimum.Y + colorMaximum.Y) / 2f
};
DepthSpacePoint depthTrail = new DepthSpacePoint
{
X = (depthMinimum.X + depthMaximum.X) / 2f,
Y = (depthMinimum.Y + depthMaximum.Y) / 2f
};
CameraSpacePoint feet = new CameraSpacePoint
{
X = (footLeft.Position.X + footRight.Position.X) / 2f,
Y = (footLeft.Position.Y + footRight.Position.Y) / 2f,
Z = (footLeft.Position.Z + footRight.Position.Z) / 2f
};
CameraSpacePoint projection = new CameraSpacePoint
{
X = cameraTrail.X,
Y = feet.Y,
Z = cameraTrail.Z
};
barLength = cameraMinimum.Length(cameraMaximum);
barHeight = cameraTrail.Length(projection);
angle = cameraMinimum.Angle(cameraMaximum, new CameraSpacePoint {
X = cameraMaximum.X, Y = cameraMinimum.Y, Z = (cameraMaximum.Z + cameraMinimum.Z) / 2f
});
if (angle > 180.0)
{
angle = 360.0 - angle;
}
if (cameraMinimum.Y < cameraMaximum.Y)
{
angle = -angle;
}
if (barLength > handLength)
{
BarDetectionResult result = new BarDetectionResult
{
Minimum = new MultiPoint
{
CameraPoint = cameraMinimum,
ColorPoint = colorMinimum,
DepthPoint = depthMinimum
},
Maximum = new MultiPoint
{
CameraPoint = cameraMaximum,
ColorPoint = colorMaximum,
DepthPoint = depthMaximum
},
Trail = new MultiPoint
{
CameraPoint = cameraTrail,
ColorPoint = colorTrail,
DepthPoint = depthTrail
},
BarHeight = barHeight + HEIGHT_DIFFERENCE,
BarLength = barLength,
Angle = angle
};
BarDetected?.Invoke(this, result);
}
}
}
/// <summary>
/// Updates the bitmap with new frame data.
/// </summary>
/// <param name="depthFrame">The specified depth frame.</param>
/// <param name="colorFrame">The specified color frame.</param>
/// <param name="bodyIndexFrame">The specified body index frame.</param>
override public void Update(ColorFrame colorFrame, DepthFrame depthFrame, BodyIndexFrame bodyIndexFrame)
{
int colorWidth = colorFrame.FrameDescription.Width;
int colorHeight = colorFrame.FrameDescription.Height;
int depthWidth = depthFrame.FrameDescription.Width;
int depthHeight = depthFrame.FrameDescription.Height;
int bodyIndexWidth = bodyIndexFrame.FrameDescription.Width;
int bodyIndexHeight = bodyIndexFrame.FrameDescription.Height;
if (Bitmap == null)
{
InitBuffers(colorFrame.FrameDescription, depthFrame.FrameDescription, bodyIndexFrame.FrameDescription);
}
if (((depthWidth * depthHeight) == _depthData.Length) && ((colorWidth * colorHeight * Constants.BYTES_PER_PIXEL) == Pixels.Length) && ((bodyIndexWidth * bodyIndexHeight) == _bodyData.Length))
{
depthFrame.CopyFrameDataToArray(_depthData);
if (colorFrame.RawColorImageFormat == ColorImageFormat.Bgra)
{
colorFrame.CopyRawFrameDataToArray(Pixels);
}
else
{
colorFrame.CopyConvertedFrameDataToArray(Pixels, ColorImageFormat.Bgra);
}
bodyIndexFrame.CopyFrameDataToArray(_bodyData);
CoordinateMapper.MapColorFrameToDepthSpace(_depthData, _depthPoints);
// Loop over each row and column of the color image
// Zero out any pixels that don't correspond to a body index
for (int i = 0, ci = 0; i < _depthPoints.Length; ++i, ci += Constants.BYTES_PER_PIXEL)
{
float colorToDepthX = _depthPoints[i].X;
float colorToDepthY = _depthPoints[i].Y;
// The sentinel value is -inf, -inf, meaning that no depth pixel corresponds to this color pixel.
if (!float.IsNegativeInfinity(colorToDepthX) &&
!float.IsNegativeInfinity(colorToDepthY))
{
// Make sure the depth pixel maps to a valid point in color space
int depthX = (int)(colorToDepthX + 0.5f);
int depthY = (int)(colorToDepthY + 0.5f);
// If the point is not valid, there is no body index there.
if ((depthX >= 0) && (depthX < depthWidth) && (depthY >= 0) && (depthY < depthHeight))
{
int depthIndex = (depthY * depthWidth) + depthX;
// If we are tracking a body for the current pixel, do not zero out the pixel
if (_bodyData[depthIndex] != 0xff)
{
continue;
}
}
}
for (int b = 0; b < Constants.BYTES_PER_PIXEL; ++b)
{
Pixels[ci + b] = 0;
}
}
UpdateBitmap();
}
}
private void DoMapping() => coordinateMapper.MapColorFrameToDepthSpace(uDepthPixels, mappedDepth);
// --- color
private void ProcessColorMapping(MultiFrame multiFrame)
{
// --- color mapping from kinect
long ticks1 = DateTime.Now.Ticks;
DepthSpacePoint[] colorMappedToDepthPoints = new DepthSpacePoint[colorWidth * colorHeight];
coordMapper.MapColorFrameToDepthSpace(multiFrame.DepthData, colorMappedToDepthPoints);
Utils.UpdateTimer("(ColorMappingCoord)", ticks1);
// --- mapped colorAsDepth -> depth
long ticks2 = DateTime.Now.Ticks;
byte[] colorMappedBytes = new byte[colorWidth * colorHeight * 4];
unsafe
{
fixed(byte *fixedColorMapped = colorMappedBytes)
fixed(ushort *fixedDepthData = multiFrame.DepthData)
fixed(byte *fixedBodyIndexData = multiFrame.BodyIndexData)
{
byte * ptrColorMapped = fixedColorMapped;
ushort *ptrDepthData = fixedDepthData;
byte * ptrBodyIndexData = fixedBodyIndexData;
// 8 bit
if (Context.Use8bitDepth)
{
for (int i = 0; i < colorPixelSize; i++)
{
// checking infinity before adding + 0.5f is about 5x faster (!!)
float xTmp = colorMappedToDepthPoints[i].X;
float yTmp = colorMappedToDepthPoints[i].Y;
int x = float.IsInfinity(xTmp) ? -1 : (int)(xTmp + 0.5f);
int y = float.IsInfinity(yTmp) ? -1 : (int)(yTmp + 0.5f);
if (x >= 0 && x < depthWidth && y >= 0 && y < depthHeight)
{
int idx = x + y * depthWidth;
byte val = (ptrBodyIndexData[idx] < 6 ? byte.MaxValue : (byte)(ptrDepthData[idx] / depthToByte));
* ptrColorMapped++ = val;
* ptrColorMapped++ = val;
* ptrColorMapped++ = val;
* ptrColorMapped++ = 255; // alpha
}
else
{
ptrColorMapped += 4; // 0 is default
}
}
}
// full depth (13 bit)
else
{
for (int i = 0; i < colorPixelSize; i++)
{
// checking infinity before adding + 0.5f is about 5x faster (!!)
float xTmp = colorMappedToDepthPoints[i].X;
float yTmp = colorMappedToDepthPoints[i].Y;
int x = float.IsInfinity(xTmp) ? -1 : (int)(xTmp + 0.5f);
int y = float.IsInfinity(yTmp) ? -1 : (int)(yTmp + 0.5f);
if (x >= 0 && x < depthWidth && y >= 0 && y < depthHeight)
{
int idx = x + y * depthWidth;
ushort val = ptrDepthData[idx];
* ptrColorMapped++ = (byte)(val % 256);
* ptrColorMapped++ = (byte)(val / 256);
* ptrColorMapped++ = ptrBodyIndexData[idx];
* ptrColorMapped++ = 255; // alpha
}
else
{
ptrColorMapped += 4; // 0 is default
}
}
}
}
}
multiFrame.ColorMappedBytes = colorMappedBytes;
Utils.UpdateTimer("(ColorMappingBytes)", ticks2);
}
void Reader_MultiSourceFrameArrived(object sender, MultiSourceFrameArrivedEventArgs e)
{
var reference = e.FrameReference.AcquireFrame();
// Infrared
using (var infraFrame = reference.InfraredFrameReference.AcquireFrame())
{
if (infraFrame != null)
{
RenderInfraredPixels(infraFrame);
}
}
// Color
using (var colorFrame = reference.ColorFrameReference.AcquireFrame())
{
// Depth
using (var depthFrame = reference.DepthFrameReference.AcquireFrame())
{
if (colorFrame != null && depthFrame != null)
{
var _colorWidth = colorFrame.ColorFrameSource.FrameDescription.Width;
var _colorHeight = colorFrame.ColorFrameSource.FrameDescription.Height;
var _depthWidth = depthFrame.DepthFrameSource.FrameDescription.Width;
var _depthHeight = depthFrame.DepthFrameSource.FrameDescription.Height;
using (Microsoft.Kinect.KinectBuffer depthBuffer = depthFrame.LockImageBuffer())
{
// verify data and write the color data to the display bitmap
if (((this.depthFrameDescription.Width * this.depthFrameDescription.Height) == (depthBuffer.Size / this.depthFrameDescription.BytesPerPixel)) &&
(this.depthFrameDescription.Width == this.depthBitmap.PixelWidth) && (this.depthFrameDescription.Height == this.depthBitmap.PixelHeight))
{
// Note: In order to see the full range of depth (including the less reliable far field depth)
// we are setting maxDepth to the extreme potential depth threshold
ushort maxDepth = ushort.MaxValue;
// If you wish to filter by reliable depth distance, uncomment the following line:
//// maxDepth = depthFrame.DepthMaxReliableDistance
this.ProcessDepthFrameData(depthBuffer.UnderlyingBuffer, depthBuffer.Size, depthFrame.DepthMinReliableDistance, maxDepth);
this.RenderDepthPixels();
}
}
using (KinectBuffer colorBuffer = colorFrame.LockRawImageBuffer())
{
this.colorBitmap.Lock();
// verify data and write the new color frame data to the display bitmap
if ((_colorWidth == this.colorBitmap.PixelWidth) && (_colorHeight == this.colorBitmap.PixelHeight))
{
colorFrame.CopyConvertedFrameDataToIntPtr(
this.colorBitmap.BackBuffer,
(uint)(_colorWidth * _colorHeight * 4),
ColorImageFormat.Bgra);
this.colorBitmap.AddDirtyRect(new Int32Rect(0, 0, this.colorBitmap.PixelWidth, this.colorBitmap.PixelHeight));
}
this.colorBitmap.Unlock();
}
if ((takeScreenshot || dumpPpms) && !robot.IsMoving())
{
ushort[] depths = new ushort[_depthHeight * _depthWidth];
DepthSpacePoint[] mappedColor = new DepthSpacePoint[_colorHeight * _colorWidth];
depthFrame.CopyFrameDataToArray(depths);
cm.MapColorFrameToDepthSpace(depths, mappedColor);
byte[] colors = new byte[_colorHeight * _colorWidth * 4];
//this is the byte array that is converted into a ppm in the end, make it rgba form
colorFrame.CopyConvertedFrameDataToArray(colors, ColorImageFormat.Rgba);
this.mappedColor = mappedColor;
this.depths = depths;
this.colors = colors;
if (takeScreenshot)
{
ScreenshotSaveFile();
takeScreenshot = capturePanorama || false;
}
else if (dumpPpms)
{
ScreenshotSaveFile();
//DumpPpms();
dumpPpms = false;
}
// Kick off another rotation if capturing a panorama
if (capturePanorama)
{
if (numRotations < MAX_ROTATIONS)
{
numRotations++;
RotateCW();
StopMoving(rotateTime);
Thread.Sleep(STABILIZE_TIME);
}
else
{
this.capturePanorama = false;
this.takeScreenshot = false;
this.panoramaNum++;
this.imageNum = 0;
this.numRotations = 0;
}
}
}
depthCamera.Source = this.depthBitmap;
colorCamera.Source = this.colorBitmap;
infraCamera.Source = this.infraBitmap;
}
}
}
}
void Update()
{
if (reader != null)
{
MultiSourceFrame frame = reader.AcquireLatestFrame();
if (frame != null)
{
using (ColorFrame colorFrame = frame.ColorFrameReference.AcquireFrame()) {
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(colorData, ColorImageFormat.Rgba);
}
}
using (DepthFrame depthFrame = frame.DepthFrameReference.AcquireFrame()) {
if (depthFrame != null)
{
//Debug.Log ("bodyIndexFrame not null");
depthFrame.CopyFrameDataToArray(depthData);
}
}
using (BodyIndexFrame bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame()) {
if (bodyIndexFrame != null)
{
//Debug.Log ("bodyIndexFrame not null");
bodyIndexFrame.CopyFrameDataToArray(bodyIndexData);
}
}
frame = null;
}
}
else
{
return;
}
Utils.copyToMat(colorData, outputMat);
Utils.copyToMat(colorData, rgbaMat);
coordinateMapper.MapColorFrameToDepthSpace(depthData, depthSpacePoints);
int width = rgbaMat.width();
int height = rgbaMat.height();
int depthWidth = 512;
byte[] maskOn = new byte[] { 0 };
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
int index = x + y * width;
int tmp = ((int)depthSpacePoints [index].X + (int)depthSpacePoints [index].Y * depthWidth < 0) ? 0 : (int)depthSpacePoints [index].X + (int)depthSpacePoints [index].Y * depthWidth;
if (bodyIndexData [tmp] == 255)
{
maskData [index] = 0;
}
else
{
maskData [index] = 255;
}
}
}
Utils.copyToMat(maskData, maskMat);
if (mode == modeType.original)
{
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "ORIGINAL MODE " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
else if (mode == modeType.sepia)
{
Core.transform(rgbaMat, rgbaMat, sepiaKernel);
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "SEPIA MODE " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
else if (mode == modeType.pixelize)
{
Imgproc.resize(rgbaMat, pixelizeIntermediateMat, pixelizeSize0, 0.1, 0.1, Imgproc.INTER_NEAREST);
Imgproc.resize(pixelizeIntermediateMat, rgbaMat, rgbaMat.size(), 0.0, 0.0, Imgproc.INTER_NEAREST);
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "PIXELIZE MODE" + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
else if (mode == modeType.comic)
{
Imgproc.cvtColor(rgbaMat, comicGrayMat, Imgproc.COLOR_RGBA2GRAY);
comicBgMat.copyTo(comicDstMat);
Imgproc.GaussianBlur(comicGrayMat, comicLineMat, new Size(3, 3), 0);
Utils.copyFromMat(comicGrayMat, comicGrayPixels);
for (int i = 0; i < comicGrayPixels.Length; i++)
{
comicMaskPixels [i] = 0;
if (comicGrayPixels [i] < 70)
{
comicGrayPixels [i] = 0;
comicMaskPixels [i] = 1;
}
else if (70 <= comicGrayPixels [i] && comicGrayPixels [i] < 120)
{
comicGrayPixels [i] = 100;
}
else
{
comicGrayPixels [i] = 255;
comicMaskPixels [i] = 1;
}
}
Utils.copyToMat(comicGrayPixels, comicGrayMat);
Utils.copyToMat(comicMaskPixels, comicMaskMat);
comicGrayMat.copyTo(comicDstMat, comicMaskMat);
Imgproc.Canny(comicLineMat, comicLineMat, 20, 120);
comicLineMat.copyTo(comicMaskMat);
Core.bitwise_not(comicLineMat, comicLineMat);
comicLineMat.copyTo(comicDstMat, comicMaskMat);
Imgproc.cvtColor(comicDstMat, rgbaMat, Imgproc.COLOR_GRAY2RGBA);
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "COMIC MODE " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
Utils.matToTexture(outputMat, texture);
}
public WriteableBitmap ColorBitmap2(ColorMetaData myColorMetaData, DepthMetaData myDepthMetaData, BodyMetaData myBodyMetaData)
{
if (myDepthMetaData == null || myBodyMetaData == null || myColorMetaData == null)
{
return(null);
}
this.colorArrayWithDepthInfo = new DepthSpacePoint[ColorMetaData.XColorMaxKinect * ColorMetaData.YColorMaxKinect];
coordinateMapper.MapColorFrameToDepthSpace(myDepthMetaData.FrameData, this.colorArrayWithDepthInfo);
if (colorArrayWithDepthInfo == null)
{
return(null);
}
this.colorArrayWithDepthInfo = new DepthSpacePoint[ColorMetaData.XColorMaxKinect * ColorMetaData.YColorMaxKinect];
// Loop over each row and column of the color image
// Zero out any pixels that don't correspond to a body index
myColorMetaData.WriteableBitmapColor.Lock();
unsafe
{
int colorMappedToDepthPointCount = this.colorArrayWithDepthInfo.Length;
fixed(DepthSpacePoint *colorMappedToDepthPointsPointer = this.colorArrayWithDepthInfo)
{
for (int colorIndex = 0; colorIndex < colorMappedToDepthPointCount; ++colorIndex)
{
double colorMappedToDepthX = colorMappedToDepthPointsPointer[colorIndex].X;
double colorMappedToDepthY = colorMappedToDepthPointsPointer[colorIndex].Y;
if (!double.IsNegativeInfinity(colorMappedToDepthX) &&
!double.IsNegativeInfinity(colorMappedToDepthY))
{
// Make sure the depth pixel maps to a valid point in color space
int depthX = (int)(colorMappedToDepthX + 0.5f);
int depthY = (int)(colorMappedToDepthY + 0.5f);
// If the point is not valid, there is no body index there.
if ((depthX >= 0) && (depthX < DepthMetaData.XDepthMaxKinect) && (depthY >= 0) && (depthY < DepthMetaData.YDepthMaxKinect))
{
int depthIndex = (depthY * DepthMetaData.XDepthMaxKinect) + depthX;
byte player = myBodyMetaData.Pixels[depthIndex];
if (player != 0xff)
{
continue;
}
}
}
int displayIndex = colorIndex * BYTES_PER_PIXEL;
myColorMetaData.Pixels[colorIndex + 1] = 0;
myColorMetaData.Pixels[colorIndex + 2] = 0;
myColorMetaData.Pixels[colorIndex + 3] = 0xff;
}
myColorMetaData.WriteableBitmapColor.AddDirtyRect(new Int32Rect(0, 0, myColorMetaData.WriteableBitmapColor.PixelWidth, myColorMetaData.WriteableBitmapColor.PixelHeight));
}
}
myColorMetaData.WriteableBitmapColor.Unlock();
return(myColorMetaData.WriteableBitmapColor);
//}
}
void Update()
{
if (reader != null)
{
MultiSourceFrame frame = reader.AcquireLatestFrame();
if (frame != null)
{
using (ColorFrame colorFrame = frame.ColorFrameReference.AcquireFrame())
{
if (colorFrame != null)
{
colorFrame.CopyConvertedFrameDataToArray(colorData, ColorImageFormat.Rgba);
}
}
using (DepthFrame depthFrame = frame.DepthFrameReference.AcquireFrame())
{
if (depthFrame != null)
{
//Debug.Log ("bodyIndexFrame not null");
depthFrame.CopyFrameDataToArray(depthData);
}
}
using (BodyIndexFrame bodyIndexFrame = frame.BodyIndexFrameReference.AcquireFrame())
{
if (bodyIndexFrame != null)
{
//Debug.Log ("bodyIndexFrame not null");
bodyIndexFrame.CopyFrameDataToArray(bodyIndexData);
}
}
frame = null;
}
}
else
{
return;
}
MatUtils.copyToMat(colorData, outputMat);
MatUtils.copyToMat(colorData, rgbaMat);
// update mask image from bodyIndexData.
coordinateMapper.MapColorFrameToDepthSpace(depthData, depthSpacePoints);
for (int colorY = 0; colorY < colorFrameHeight; colorY++)
{
for (int colorX = 0; colorX < colorFrameWidth; colorX++)
{
int colorIndex = colorY * colorFrameWidth + colorX;
int depthX = (int)(depthSpacePoints[colorIndex].X);
int depthY = (int)(depthSpacePoints[colorIndex].Y);
if ((0 <= depthX) && (depthX < depthFrameWidth) && (0 <= depthY) && (depthY < depthFrameHeight))
{
int depthIndex = depthY * depthFrameWidth + depthX;
if (bodyIndexData[depthIndex] == 255)
{
maskData[colorIndex] = 0;
}
else
{
maskData[colorIndex] = 255;
}
}
}
}
MatUtils.copyToMat(maskData, maskMat);
if (filterType == FilterTypePreset.NONE)
{
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "Filter Type: NONE " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
else if (filterType == FilterTypePreset.SEPIA)
{
Core.transform(rgbaMat, rgbaMat, sepiaKernel);
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "Filter Type: SEPIA " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
else if (filterType == FilterTypePreset.PIXELIZE)
{
Imgproc.resize(rgbaMat, pixelizeIntermediateMat, pixelizeSize0, 0.1, 0.1, Imgproc.INTER_NEAREST);
Imgproc.resize(pixelizeIntermediateMat, rgbaMat, rgbaMat.size(), 0.0, 0.0, Imgproc.INTER_NEAREST);
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "Filter Type: PIXELIZE " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
else if (filterType == FilterTypePreset.COMIC)
{
comicFilter.Process(rgbaMat, rgbaMat);
rgbaMat.copyTo(outputMat, maskMat);
Imgproc.putText(outputMat, "Filter Type: COMIC " + texture.width + "x" + texture.height, new Point(5, texture.height - 5), Imgproc.FONT_HERSHEY_PLAIN, 4.0, new Scalar(255, 0, 0, 255), 3);
}
Utils.matToTexture2D(outputMat, texture);
}
