private BitmapImage ConvertBitmap(Bitmap bitmap) { BitmapImage bitmapImage = null; System.Drawing.Pen fancyPen = new System.Drawing.Pen(System.Drawing.Color.Cyan, 5); if (bitmap != null) { if (blobData.QueryNumberOfBlobs() == 2) { for (int i = 0; i < 2; i++) { using (var graphics = Graphics.FromImage(bitmap)) { //graphics.DrawRectangle(fancyPen, blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x, blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y, 5, 5); //graphics.DrawRectangle(fancyPen, blobCoordinates[(int)hand.LEFT, (int)cord.X] - 50, blobCoordinates[(int)hand.LEFT, (int)cord.Y], 5, 5); //graphics.DrawRectangle(fancyPen, blobCoordinates[(int)hand.RIGHT, (int)cord.X] - 50, blobCoordinates[(int)hand.RIGHT, (int)cord.Y], 5, 5); } } } bitmap.RotateFlip(RotateFlipType.Rotate180FlipY); MemoryStream memoryStream = new MemoryStream(); bitmap.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Bmp); memoryStream.Position = 0; bitmapImage = new BitmapImage(); bitmapImage.BeginInit(); bitmapImage.StreamSource = memoryStream; bitmapImage.CacheOption = BitmapCacheOption.OnLoad; bitmapImage.EndInit(); bitmapImage.Freeze(); } return(bitmapImage); }
/// <summary> /// Grab the blob data every frame /// </summary> void Update() { if (instance.AcquireFrame(true) == pxcmStatus.PXCM_STATUS_NO_ERROR) { /* To store all blob points */ blobPointsPos = new List <Vector2>(); PXCMCapture.Sample sample = instance.QuerySample(); if (sample != null && sample.depth != null) { PXCMImage.ImageInfo info = sample.depth.QueryInfo(); if (blobData != null) { blobData.Update(); int numblobs = blobData.QueryNumberOfBlobs(); for (int i = 0; i <= numblobs; i++) { PXCMBlobData.IBlob pBlob; if (blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out pBlob) == pxcmStatus.PXCM_STATUS_NO_ERROR) { Vector3 centerPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER); Vector3 topPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST); Vector3 bottomPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST); Vector3 leftPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST); Vector3 rightPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST); Vector3 closestPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST); blobPointsPos.Add(new Vector2(centerPoint.x * -1, centerPoint.y * -1)); blobPointsPos.Add(new Vector2(topPoint.x * -1, topPoint.y * -1)); blobPointsPos.Add(new Vector2(bottomPoint.x * -1, bottomPoint.y * -1)); blobPointsPos.Add(new Vector2(leftPoint.x * -1, leftPoint.y * -1)); blobPointsPos.Add(new Vector2(rightPoint.x * -1, rightPoint.y * -1)); blobPointsPos.Add(new Vector2(closestPoint.x * -1, closestPoint.y * -1)); DisplayPoints(); if (pBlob.QueryContourPoints(0, out pointOuter[i]) == pxcmStatus.PXCM_STATUS_NO_ERROR) { DisplayContour(pointOuter[i], i, numblobs); } } } } } instance.ReleaseFrame(); } }
private void UpdateBlobImage(PXCMImage depthFrame) { if (depthFrame == null) { return; } // Blobを更新する var sts = blobData.Update(); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } // Blobのための画像オブジェクトを作成する var depthInfo = depthFrame.QueryInfo(); depthInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8; var session = senseManager.QuerySession(); var blobImage = session.CreateImage(depthInfo); // 表示用画像を初期化する Array.Clear(imageBuffer, 0, imageBuffer.Length); CanvasHandParts.Children.Clear(); // Blobを取得する int numOfBlobs = blobData.QueryNumberOfBlobs(); for (int i = 0; i < numOfBlobs; ++i) { // Blobデータを取得する PXCMBlobData.IBlob blob; sts = blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } sts = blob.QuerySegmentationImage(out blobImage); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // Blob画像を取得する PXCMImage.ImageData data; sts = blobImage.AcquireAccess(PXCMImage.Access.ACCESS_READ, depthInfo.format, out data); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // データをコピーする var buffer = data.ToByteArray(0, data.pitches[0] * depthInfo.height); for (int j = 0; j < depthInfo.height * depthInfo.width; ++j) { if (buffer[j] != 0) { imageBuffer[j] = (byte)((i + 1) * 64); } } // Blob画像を解放する blobImage.ReleaseAccess(data); // Blobの輪郭を表示する UpdateContoursImage(blob, i); } // Blob画像オブジェクトを解放する blobImage.Dispose(); // ピクセルデータを更新する imageBitmap.WritePixels(imageRect, imageBuffer, DEPTH_WIDTH * BYTE_PER_PIXEL, 0); }
private void Update() { // Start AcquireFrame-ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMCapture.Sample sample = senseManager.QuerySample(); Bitmap colorBitmap; PXCMImage.ImageData colorData; blobData.Update(); /*while (blobData.QueryNumberOfBlobs() < 2) * { * trackingDistance += 100; * blobConfig.SetMaxDistance(trackingDistance); * blobConfig.ApplyChanges(); * blobData.Update(); * * senseManager.ReleaseFrame(); * * if(trackingDistance > 3000) * trackingDistance = 600; * } * { * } */ for (int i = 0; i < 2; i++) { blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blobList[i]); } if (blobData.QueryNumberOfBlobs() == 2) { if (blobCoordinates[3, (int)cord.Y] == -1) { //smoothing } if (blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x > blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x) { blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; } else { blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; } } // Tonausgabe: aktuelle Tonausgabe beenden und neue beginnen frequency = (int)(blobCoordinates[(int)hand.LEFT, (int)cord.Y] * 1.8); volume = (500 - blobCoordinates[(int)hand.RIGHT, (int)cord.Y]) / 500; sineWaveProvider.Frequency = frequency; sineWaveProvider.Amplitude = volume; // Get color image data sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); // Update UI Render(colorBitmap); // Release frame colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); senseManager.ReleaseFrame(); } }
private void Update() { // Start AcquireFrame-ReleaseFrame loop while (sm.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { // Acquire color image data PXCMCapture.Sample sample = sm.QuerySample(); Bitmap colorBitmap; PXCMImage.ImageData colorData; sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); // Create an instance of MyTrackedPerson MyTrackedPerson myTrackedPerson = new MyTrackedPerson(); MyBlobs myBlobs = new MyBlobs(); // Acquire person tracking data personData = personModule.QueryOutput(); myTrackedPerson.PersonsDetected = personData.QueryNumberOfPeople(); if (myTrackedPerson.PersonsDetected == 1) { PXCMPersonTrackingData.Person trackedPerson = personData.QueryPersonData(PXCMPersonTrackingData.AccessOrderType.ACCESS_ORDER_BY_ID, 0); PXCMPersonTrackingData.PersonTracking trackedPersonData = trackedPerson.QueryTracking(); PXCMPersonTrackingData.BoundingBox2D personBox = trackedPersonData.Query2DBoundingBox(); myTrackedPerson.X = personBox.rect.x; myTrackedPerson.Y = personBox.rect.y; myTrackedPerson.H = personBox.rect.h; myTrackedPerson.W = personBox.rect.w; /* * PXCMPersonTrackingData.PersonJoints personJoints = trackedPerson.QuerySkeletonJoints(); * PXCMPersonTrackingData.PersonJoints.SkeletonPoint[] skeletonPoints = new PXCMPersonTrackingData.PersonJoints.SkeletonPoint[personJoints.QueryNumJoints()]; * trackedPerson.QuerySkeletonJoints().QueryJoints(skeletonPoints); * if (skeletonPoints.Length > 0) * skeletonPoints[0].GetType(); */ } // Acquire face tracking data faceData.Update(); myTrackedPerson.FacesDetected = faceData.QueryNumberOfDetectedFaces(); if (myTrackedPerson.FacesDetected == 1) { PXCMFaceData.Face face = faceData.QueryFaceByIndex(0); PXCMFaceData.DetectionData faceDetectionData = face.QueryDetection(); PXCMRectI32 faceRectangle; faceDetectionData.QueryBoundingRect(out faceRectangle); myTrackedPerson.FaceH = faceRectangle.h; myTrackedPerson.FaceW = faceRectangle.w; myTrackedPerson.FaceX = faceRectangle.x; myTrackedPerson.FaceY = faceRectangle.y; float faceDepth; faceDetectionData.QueryFaceAverageDepth(out faceDepth); myTrackedPerson.FaceDepth = faceDepth; } blobData.Update(); int numBlobs = blobData.QueryNumberOfBlobs(); myBlobs.numBlobs = numBlobs; myBlobs.blobs = new List <List <PXCMPointI32> >(numBlobs); myBlobs.closestPoints = new List <PXCMPoint3DF32>(4); for (int i = 0; i < numBlobs; i++) { PXCMBlobData.IBlob blob; pxcmStatus result1 = blobData.QueryBlob(i, PXCMBlobData.SegmentationImageType.SEGMENTATION_IMAGE_DEPTH, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob); if (result1 == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMPoint3DF32 closestPoint = blob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST); myBlobs.closestPoints.Add(closestPoint); int numContours = blob.QueryNumberOfContours(); if (numContours > 0) { // only deal with outer contour for (int j = 0; j < numContours; j++) { PXCMBlobData.IContour contour; pxcmStatus result2 = blob.QueryContour(j, out contour); if (result2 == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (contour.IsOuter()) { PXCMPointI32[] points; pxcmStatus result3 = contour.QueryPoints(out points); if (result3 == pxcmStatus.PXCM_STATUS_NO_ERROR) { int numPoints = points.Length; myBlobs.blobs.Add(points.ToList <PXCMPointI32>()); } } } } } } } // Update UI Render(colorBitmap, myTrackedPerson, myBlobs); // serialize to json and send all clients var personJson = JsonConvert.SerializeObject(myTrackedPerson); personSockets.ToList().ForEach(s => s.Send(personJson)); var blobJson = JsonConvert.SerializeObject(myBlobs); blobSockets.ToList().ForEach(s => s.Send(blobJson)); // deserialize json as follows //MyTrackedPerson deserializedProduct = JsonConvert.DeserializeObject<MyTrackedPerson>(json); // Release resources colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); sm.ReleaseFrame(); } }
/* Displaying Mask Images */ private unsafe void DisplayPicture(PXCMImage depth, PXCMBlobData blobData) { if (depth == null) return; PXCMImage image = depth; PXCMImage.ImageInfo info = image.QueryInfo(); int numOfBlobs = blobData.QueryNumberOfBlobs(); if (_maxBlobToShow > numOfBlobs) { _maxBlobToShow = numOfBlobs; } PXCMBlobData.IBlob[] blobList = new PXCMBlobData.IBlob[_maxBlobToShow]; PXCMPointI32[][] pointOuter = new PXCMPointI32[_maxBlobToShow][]; PXCMPointI32[][] pointInner = new PXCMPointI32[_maxBlobToShow][]; Bitmap picture = new Bitmap(image.info.width, image.info.height, PixelFormat.Format32bppRgb); PXCMImage.ImageData bdata; pxcmStatus results = pxcmStatus.PXCM_STATUS_NO_ERROR; PXCMBlobData.AccessOrderType accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_LARGE_TO_SMALL; int accessOrderBy = form.GetAccessOrder(); switch (accessOrderBy) { case 1: accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR; break; case 2: accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_RIGHT_TO_LEFT; break; case 0: default: accessOrder = PXCMBlobData.AccessOrderType.ACCESS_ORDER_LARGE_TO_SMALL; break; } Rectangle rect = new Rectangle(0, 0, image.info.width, image.info.height); BitmapData bitmapdata = picture.LockBits(rect, ImageLockMode.ReadWrite, picture.PixelFormat); for (int j = 0; j < _maxBlobToShow; j++) { byte tmp1 = (Byte)(255 - (255 / (_maxBlobToShow) * j)); results = blobData.QueryBlobByAccessOrder(j, accessOrder, out blobList[j]); if (results == pxcmStatus.PXCM_STATUS_NO_ERROR) { bool isSegmentationImage = true; results = blobList[j].QuerySegmentationImage(out image); if (results != pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMImage.ImageInfo imgInfo = new PXCMImage.ImageInfo(); imgInfo.width = 640; imgInfo.height = 480; imgInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8; PXCMSession session = PXCMSession.CreateInstance(); if (session != null) { image = session.CreateImage(info); if (image == null) return; } image.AcquireAccess(PXCMImage.Access.ACCESS_WRITE, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out bdata); byte* numPtr = (byte*)bitmapdata.Scan0; //dst byte* numPtr2 = (byte*)bdata.planes[0]; //row int imagesize = image.info.width * image.info.height; byte tmp; for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++) { tmp = (byte)(0); numPtr[0] = tmp; numPtr[1] = tmp; numPtr[2] = tmp; numPtr[3] = tmp; } image.ReleaseAccess(bdata); isSegmentationImage = false; } results = image.AcquireAccess(PXCMImage.Access.ACCESS_READ_WRITE, PXCMImage.PixelFormat.PIXEL_FORMAT_Y8, out bdata); if (form.GetBlobState() && isSegmentationImage == true) { byte* numPtr = (byte*)bitmapdata.Scan0; //dst byte* numPtr2 = (byte*)bdata.planes[0]; //row int imagesize = image.info.width * image.info.height; for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++) { byte tmp = (Byte)(numPtr2[0] == 0 ? 0 : tmp1); tmp |= numPtr[0]; numPtr[0] = tmp; numPtr[1] = tmp; numPtr[2] = tmp; numPtr[3] = 0xff; } } if ((form.GetContourState())) { int contourNumber = blobList[j].QueryNumberOfContours(); if (contourNumber > 0) { for (int k = 0; k < contourNumber; ++k) { int contourSize = blobList[j].QueryContourSize(k); if (blobList[j].IsContourOuter(k) == true) blobList[j].QueryContourPoints(k, out pointOuter[j]); else { blobList[j].QueryContourPoints(k, out pointInner[j]); } } if (results == pxcmStatus.PXCM_STATUS_NO_ERROR && form.GetBlobState() == false) { byte* numPtr = (byte*)bitmapdata.Scan0; //dst byte* numPtr2 = (byte*)bdata.planes[0]; //row int imagesize = image.info.width * image.info.height; byte tmp; for (int i = 0; i < imagesize; i++, numPtr += 4, numPtr2++) { tmp = (byte)(0); numPtr[0] = tmp; numPtr[1] = tmp; numPtr[2] = tmp; numPtr[3] = tmp; } } } } image.ReleaseAccess(bdata); image.Dispose(); } } picture.UnlockBits(bitmapdata); form.DisplayBitmap(picture); ///////// that is my polygon zone Bitmap imageInstance2 = picture; i++; Bitmap croppedImage = null; if (contourMostRight.x > 0) { int rectWidth = (int)(contourMostRight.x - contourMostLeft.x); int rectHeight = (int)(contourMostTop.y - contourMostBottom.y); Rectangle sourceRectangle = new Rectangle(new Point((int)contourMostLeft.x, (int)contourMostBottom.y), new Size(rectWidth, rectHeight)); croppedImage = CropImage(imageInstance2, sourceRectangle); } String[] origArray = { "d:\\origG.jpeg", "d:\\origX.jpeg", "d:\\origY.jpeg" }; for (int i = 0; i < origArray.Length; i++) { Bitmap orig = null; if (File.Exists(origArray[i])) orig = new Bitmap(@origArray[i]); if (orig != null && croppedImage != null) { float diff = 0; orig = ScaleImage(orig, 150, 100); croppedImage = ScaleImage(croppedImage, 150, 100); bool isImageBlank = true; for (int y = 0; y < orig.Height; y++) { for (int x = 0; x < orig.Width; x++) { if(croppedImage.GetPixel(x, y).R > 1 && croppedImage.GetPixel(x, y).B > 1 && croppedImage.GetPixel(x, y).G > 1) { isImageBlank = false; break; } } } if (!isImageBlank && orig.Size.Width == croppedImage.Size.Width) { for (int y = 0; y < orig.Height; y++) { for (int x = 0; x < orig.Width; x++) { diff += (float)Math.Abs(orig.GetPixel(x, y).R - croppedImage.GetPixel(x, y).R) / 255; diff += (float)Math.Abs(orig.GetPixel(x, y).G - croppedImage.GetPixel(x, y).G) / 255; diff += (float)Math.Abs(orig.GetPixel(x, y).B - croppedImage.GetPixel(x, y).B) / 255; } } } float percentMatch = 100 * diff / (orig.Width * orig.Height * 2); Console.WriteLine("diff: {0} %", percentMatch); if (percentMatch >= 90){ Console.WriteLine(origArray[i].ToString()); Environment.Exit(0); } } } /* if (croppedImage != null) { croppedImage.Save("d:\\cropedImage.jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); // croppedImage.Save("d:\\origG.jpeg", System.Drawing.Imaging.ImageFormat.Jpeg); }*/ if (i == 100) Environment.Exit(0); picture.Dispose(); for (int i = 0; i < _maxBlobToShow; i++) { if (form.GetContourState()) { if (pointOuter[i] != null && pointOuter[i].Length > 0) form.DisplayContour(pointOuter[i], i); if (pointInner[i] != null && pointInner[i].Length > 0) form.DisplayContour(pointInner[i], i); } if (form.GetBlobDataPointsState()) { form.DisplayBlobDataPoints(blobList[i], i + 1); } PXCMPoint3DF32 point = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER); contourMostRight = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST); contourMostLeft = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST); contourMostBottom = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST); contourMostTop = blobList[i].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST); form.DisplayBlobNumber(point, i + 1); } }