private BitmapImage ConvertBitmap(Bitmap bitmap)
{
BitmapImage bitmapImage = null;
System.Drawing.Pen fancyPen = new System.Drawing.Pen(System.Drawing.Color.Cyan, 5);
if (bitmap != null)
{
if (blobData.QueryNumberOfBlobs() == 2)
{
for (int i = 0; i < 2; i++)
{
using (var graphics = Graphics.FromImage(bitmap))
{
//graphics.DrawRectangle(fancyPen, blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x, blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y, 5, 5);
//graphics.DrawRectangle(fancyPen, blobCoordinates[(int)hand.LEFT, (int)cord.X] - 50, blobCoordinates[(int)hand.LEFT, (int)cord.Y], 5, 5);
//graphics.DrawRectangle(fancyPen, blobCoordinates[(int)hand.RIGHT, (int)cord.X] - 50, blobCoordinates[(int)hand.RIGHT, (int)cord.Y], 5, 5);
}
}
}
bitmap.RotateFlip(RotateFlipType.Rotate180FlipY);
MemoryStream memoryStream = new MemoryStream();
bitmap.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Bmp);
memoryStream.Position = 0;
bitmapImage = new BitmapImage();
bitmapImage.BeginInit();
bitmapImage.StreamSource = memoryStream;
bitmapImage.CacheOption = BitmapCacheOption.OnLoad;
bitmapImage.EndInit();
bitmapImage.Freeze();
}
return(bitmapImage);
}
/// <summary>
/// Grab the blob data every frame
/// </summary>
void Update()
{
if (instance.AcquireFrame(true) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/* To store all blob points */
blobPointsPos = new List <Vector2>();
PXCMCapture.Sample sample = instance.QuerySample();
if (sample != null && sample.depth != null)
{
PXCMImage.ImageInfo info = sample.depth.QueryInfo();
if (blobData != null)
{
blobData.Update();
int numblobs = blobData.QueryNumberOfBlobs();
for (int i = 0; i <= numblobs; i++)
{
PXCMBlobData.IBlob pBlob;
if (blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out pBlob) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Vector3 centerPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER);
Vector3 topPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST);
Vector3 bottomPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST);
Vector3 leftPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST);
Vector3 rightPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST);
Vector3 closestPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST);
blobPointsPos.Add(new Vector2(centerPoint.x * -1, centerPoint.y * -1));
blobPointsPos.Add(new Vector2(topPoint.x * -1, topPoint.y * -1));
blobPointsPos.Add(new Vector2(bottomPoint.x * -1, bottomPoint.y * -1));
blobPointsPos.Add(new Vector2(leftPoint.x * -1, leftPoint.y * -1));
blobPointsPos.Add(new Vector2(rightPoint.x * -1, rightPoint.y * -1));
blobPointsPos.Add(new Vector2(closestPoint.x * -1, closestPoint.y * -1));
DisplayPoints();
if (pBlob.QueryContourPoints(0, out pointOuter[i]) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayContour(pointOuter[i], i, numblobs);
}
}
}
}
}
instance.ReleaseFrame();
}
}
private void UpdateBlobImage(PXCMImage depthFrame)
{
if (depthFrame == null)
{
return;
}
// Blobを更新する
var sts = blobData.Update();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
// Blobのための画像オブジェクトを作成する
var depthInfo = depthFrame.QueryInfo();
depthInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8;
var session = senseManager.QuerySession();
var blobImage = session.CreateImage(depthInfo);
// 表示用画像を初期化する
Array.Clear(imageBuffer, 0, imageBuffer.Length);
CanvasHandParts.Children.Clear();
// Blobを取得する
int numOfBlobs = blobData.QueryNumberOfBlobs();
for (int i = 0; i < numOfBlobs; ++i)
{
// Blobデータを取得する
PXCMBlobData.IBlob blob;
sts = blobData.QueryBlobByAccessOrder(i,
PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
sts = blob.QuerySegmentationImage(out blobImage);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// Blob画像を取得する
PXCMImage.ImageData data;
sts = blobImage.AcquireAccess(PXCMImage.Access.ACCESS_READ,
depthInfo.format, out data);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// データをコピーする
var buffer = data.ToByteArray(0, data.pitches[0] * depthInfo.height);
for (int j = 0; j < depthInfo.height * depthInfo.width; ++j)
{
if (buffer[j] != 0)
{
imageBuffer[j] = (byte)((i + 1) * 64);
}
}
// Blob画像を解放する
blobImage.ReleaseAccess(data);
// Blobの輪郭を表示する
UpdateContoursImage(blob, i);
}
// Blob画像オブジェクトを解放する
blobImage.Dispose();
// ピクセルデータを更新する
imageBitmap.WritePixels(imageRect, imageBuffer,
DEPTH_WIDTH * BYTE_PER_PIXEL, 0);
}
private void Update()
{
// Start AcquireFrame-ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
blobData.Update();
/*while (blobData.QueryNumberOfBlobs() < 2)
* {
* trackingDistance += 100;
* blobConfig.SetMaxDistance(trackingDistance);
* blobConfig.ApplyChanges();
* blobData.Update();
*
* senseManager.ReleaseFrame();
*
* if(trackingDistance > 3000)
* trackingDistance = 600;
* }
* {
* }
*/
for (int i = 0; i < 2; i++)
{
blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blobList[i]);
}
if (blobData.QueryNumberOfBlobs() == 2)
{
if (blobCoordinates[3, (int)cord.Y] == -1)
{
//smoothing
}
if (blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x > blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x)
{
blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
}
else
{
blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
}
}
// Tonausgabe: aktuelle Tonausgabe beenden und neue beginnen
frequency = (int)(blobCoordinates[(int)hand.LEFT, (int)cord.Y] * 1.8);
volume = (500 - blobCoordinates[(int)hand.RIGHT, (int)cord.Y]) / 500;
sineWaveProvider.Frequency = frequency;
sineWaveProvider.Amplitude = volume;
// Get color image data
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Update UI
Render(colorBitmap);
// Release frame
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
private void Update()
{
// Start AcquireFrame-ReleaseFrame loop
while (sm.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Acquire color image data
PXCMCapture.Sample sample = sm.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Create an instance of MyTrackedPerson
MyTrackedPerson myTrackedPerson = new MyTrackedPerson();
MyBlobs myBlobs = new MyBlobs();
// Acquire person tracking data
personData = personModule.QueryOutput();
myTrackedPerson.PersonsDetected = personData.QueryNumberOfPeople();
if (myTrackedPerson.PersonsDetected == 1)
{
PXCMPersonTrackingData.Person trackedPerson = personData.QueryPersonData(PXCMPersonTrackingData.AccessOrderType.ACCESS_ORDER_BY_ID, 0);
PXCMPersonTrackingData.PersonTracking trackedPersonData = trackedPerson.QueryTracking();
PXCMPersonTrackingData.BoundingBox2D personBox = trackedPersonData.Query2DBoundingBox();
myTrackedPerson.X = personBox.rect.x;
myTrackedPerson.Y = personBox.rect.y;
myTrackedPerson.H = personBox.rect.h;
myTrackedPerson.W = personBox.rect.w;
/*
* PXCMPersonTrackingData.PersonJoints personJoints = trackedPerson.QuerySkeletonJoints();
* PXCMPersonTrackingData.PersonJoints.SkeletonPoint[] skeletonPoints = new PXCMPersonTrackingData.PersonJoints.SkeletonPoint[personJoints.QueryNumJoints()];
* trackedPerson.QuerySkeletonJoints().QueryJoints(skeletonPoints);
* if (skeletonPoints.Length > 0)
* skeletonPoints[0].GetType();
*/
}
// Acquire face tracking data
faceData.Update();
myTrackedPerson.FacesDetected = faceData.QueryNumberOfDetectedFaces();
if (myTrackedPerson.FacesDetected == 1)
{
PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
PXCMFaceData.DetectionData faceDetectionData = face.QueryDetection();
PXCMRectI32 faceRectangle;
faceDetectionData.QueryBoundingRect(out faceRectangle);
myTrackedPerson.FaceH = faceRectangle.h;
myTrackedPerson.FaceW = faceRectangle.w;
myTrackedPerson.FaceX = faceRectangle.x;
myTrackedPerson.FaceY = faceRectangle.y;
float faceDepth;
faceDetectionData.QueryFaceAverageDepth(out faceDepth);
myTrackedPerson.FaceDepth = faceDepth;
}
blobData.Update();
int numBlobs = blobData.QueryNumberOfBlobs();
myBlobs.numBlobs = numBlobs;
myBlobs.blobs = new List <List <PXCMPointI32> >(numBlobs);
myBlobs.closestPoints = new List <PXCMPoint3DF32>(4);
for (int i = 0; i < numBlobs; i++)
{
PXCMBlobData.IBlob blob;
pxcmStatus result1 = blobData.QueryBlob(i, PXCMBlobData.SegmentationImageType.SEGMENTATION_IMAGE_DEPTH, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob);
if (result1 == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMPoint3DF32 closestPoint = blob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST);
myBlobs.closestPoints.Add(closestPoint);
int numContours = blob.QueryNumberOfContours();
if (numContours > 0)
{
// only deal with outer contour
for (int j = 0; j < numContours; j++)
{
PXCMBlobData.IContour contour;
pxcmStatus result2 = blob.QueryContour(j, out contour);
if (result2 == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (contour.IsOuter())
{
PXCMPointI32[] points;
pxcmStatus result3 = contour.QueryPoints(out points);
if (result3 == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
int numPoints = points.Length;
myBlobs.blobs.Add(points.ToList <PXCMPointI32>());
}
}
}
}
}
}
}
// Update UI
Render(colorBitmap, myTrackedPerson, myBlobs);
// serialize to json and send all clients
var personJson = JsonConvert.SerializeObject(myTrackedPerson);
personSockets.ToList().ForEach(s => s.Send(personJson));
var blobJson = JsonConvert.SerializeObject(myBlobs);
blobSockets.ToList().ForEach(s => s.Send(blobJson));
// deserialize json as follows
//MyTrackedPerson deserializedProduct = JsonConvert.DeserializeObject<MyTrackedPerson>(json);
// Release resources
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
sm.ReleaseFrame();
}
}
/* Displaying Mask Images */
private unsafe void DisplayPicture(PXCMImage depth, PXCMBlobData blobData)
{
if (depth == null)
return;
PXCMImage image = depth;
PXCMImage.ImageInfo info = image.QueryInfo();
int numOfBlobs = blobData.QueryNumberOfBlobs();
if (_maxBlobToShow > numOfBlobs)
{
_maxBlobToShow = numOfBlobs;
}
PXCMBlobData.IBlob[] blobList = new PXCMBlobData.IBlob[_maxBlobToShow];
PXCMPointI32[][] pointOuter = new PXCMPointI32[_maxBlobToShow][];
PXCMPointI32[][] pointInner = new PXCMPointI32[_maxBlobToShow][];
Bitmap picture = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb);
PXCMImage.ImageData bdata;
pxcmStatus results = pxcmStatus.PXCM_STATUS_NO_ERROR;
PXCMBlobData.AccessOrderType accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_LARGE_TO_SMALL;
int accessOrderBy = form.GetAccessOrder();
switch (accessOrderBy)
{
case 1:
accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR;
break;
case 2:
accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_RIGHT_TO_LEFT;
break;
case 0:
default:
accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_LARGE_TO_SMALL;
break;
}
Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height);
BitmapData bitmapdata = picture.LockBits(rect, ImageLockMode.ReadWrite, picture.PixelFormat);
for (int j = 0; j < _maxBlobToShow; j++)
{
byte tmp1 = (Byte)(255 - (255 / (_maxBlobToShow) * j));
results = blobData.QueryBlobByAccessOrder(j, accessOrder, out blobList[j]);
if (results == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
bool isSegmentationImage = true;
results = blobList[j].QuerySegmentationImage(out image);
if (results != pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMImage.ImageInfo imgInfo = new PXCMImage.ImageInfo();
imgInfo.width = 640;
imgInfo.height = 480;
imgInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8;
PXCMSession session = PXCMSession.CreateInstance();
if (session != null)
{
image = session.CreateImage(info);
if (image == null) return;
}
image.AcquireAccess(PXCMImage.Access.ACCESS_WRITE, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out bdata);
byte* numPtr = (byte*)bitmapdata.Scan0; //dst
byte* numPtr2 = (byte*)bdata.planes[0]; //row
int imagesize = image.info.width * image.info.height;
byte tmp;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
tmp = (byte)(0);
numPtr[0] = tmp;
numPtr[1] = tmp;
numPtr[2] = tmp;
numPtr[3] = tmp;
}
image.ReleaseAccess(bdata);
isSegmentationImage = false;
}
results = image.AcquireAccess(PXCMImage.Access.ACCESS_READ_WRITE, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out bdata);
if (form.GetBlobState() && isSegmentationImage == true)
{
byte* numPtr = (byte*)bitmapdata.Scan0; //dst
byte* numPtr2 = (byte*)bdata.planes[0]; //row
int imagesize = image.info.width * image.info.height;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
byte tmp = (Byte)(numPtr2[0] == 0 ? 0 : tmp1);
tmp |= numPtr[0];
numPtr[0] = tmp;
numPtr[1] = tmp;
numPtr[2] = tmp;
numPtr[3] = 0xff;
}
}
if ((form.GetContourState()))
{
int contourNumber = blobList[j].QueryNumberOfContours();
if (contourNumber > 0)
{
for (int k = 0; k < contourNumber; ++k)
{
int contourSize = blobList[j].QueryContourSize(k);
if (blobList[j].IsContourOuter(k) == true)
blobList[j].QueryContourPoints(k, out pointOuter[j]);
else
{
blobList[j].QueryContourPoints(k, out pointInner[j]);
}
}
if (results == pxcmStatus.PXCM_STATUS_NO_ERROR && form.GetBlobState() == false)
{
byte* numPtr = (byte*)bitmapdata.Scan0; //dst
byte* numPtr2 = (byte*)bdata.planes[0]; //row
int imagesize = image.info.width * image.info.height;
byte tmp;
for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++)
{
tmp = (byte)(0);
numPtr[0] = tmp;
numPtr[1] = tmp;
numPtr[2] = tmp;
numPtr[3] = tmp;
}
}
}
}
image.ReleaseAccess(bdata);
image.Dispose();
}
}
picture.UnlockBits(bitmapdata);
form.DisplayBitmap(picture);
///////// that is my polygon zone
Bitmap imageInstance2 = picture;
i++;
Bitmap croppedImage = null;
if (contourMostRight.x > 0) {
int rectWidth = (int)(contourMostRight.x - contourMostLeft.x);
int rectHeight = (int)(contourMostTop.y - contourMostBottom.y);
Rectangle sourceRectangle = new Rectangle(new Point((int)contourMostLeft.x,
(int)contourMostBottom.y),
new Size(rectWidth, rectHeight));
croppedImage = CropImage(imageInstance2, sourceRectangle);
}
String[] origArray = {
"d:\\origG.jpeg",
"d:\\origX.jpeg",
"d:\\origY.jpeg"
};
for (int i = 0; i < origArray.Length; i++)
{
Bitmap orig = null;
if (File.Exists(origArray[i]))
orig = new Bitmap(@origArray[i]);
if (orig != null && croppedImage != null)
{
float diff = 0;
orig = ScaleImage(orig, 150, 100);
croppedImage = ScaleImage(croppedImage, 150, 100);
bool isImageBlank = true;
for (int y = 0; y < orig.Height; y++)
{
for (int x = 0; x < orig.Width; x++)
{
if(croppedImage.GetPixel(x, y).R > 1 && croppedImage.GetPixel(x, y).B > 1
&& croppedImage.GetPixel(x, y).G > 1)
{
isImageBlank = false;
break;
}
}
}
if (!isImageBlank && orig.Size.Width == croppedImage.Size.Width)
{
for (int y = 0; y < orig.Height; y++)
{
for (int x = 0; x < orig.Width; x++)
{
diff += (float)Math.Abs(orig.GetPixel(x, y).R -
croppedImage.GetPixel(x, y).R) / 255;
diff += (float)Math.Abs(orig.GetPixel(x, y).G -
croppedImage.GetPixel(x, y).G) / 255;
diff += (float)Math.Abs(orig.GetPixel(x, y).B -
croppedImage.GetPixel(x, y).B) / 255;
}
}
}
float percentMatch = 100 * diff / (orig.Width * orig.Height * 2);
Console.WriteLine("diff: {0} %", percentMatch);
if (percentMatch >= 90){
Console.WriteLine(origArray[i].ToString());
Environment.Exit(0);
}
}
}
/*
if (croppedImage != null)
{
croppedImage.Save("d:\\cropedImage.jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
// croppedImage.Save("d:\\origG.jpeg", System.Drawing.Imaging.ImageFormat.Jpeg);
}*/
if (i == 100)
Environment.Exit(0);
picture.Dispose();
for (int i = 0; i < _maxBlobToShow; i++)
{
if (form.GetContourState())
{
if (pointOuter[i] != null && pointOuter[i].Length > 0)
form.DisplayContour(pointOuter[i], i);
if (pointInner[i] != null && pointInner[i].Length > 0)
form.DisplayContour(pointInner[i], i);
}
if (form.GetBlobDataPointsState())
{
form.DisplayBlobDataPoints(blobList[i], i + 1);
}
PXCMPoint3DF32 point = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER);
contourMostRight = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST);
contourMostLeft = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST);
contourMostBottom = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST);
contourMostTop = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST);
form.DisplayBlobNumber(point, i + 1);
}
}
