/// <summary> /// Grab the blob data every frame /// </summary> void Update() { if (instance.AcquireFrame(true) == pxcmStatus.PXCM_STATUS_NO_ERROR) { /* To store all blob points */ blobPointsPos = new List <Vector2>(); PXCMCapture.Sample sample = instance.QuerySample(); if (sample != null && sample.depth != null) { PXCMImage.ImageInfo info = sample.depth.QueryInfo(); if (blobData != null) { blobData.Update(); int numblobs = blobData.QueryNumberOfBlobs(); for (int i = 0; i <= numblobs; i++) { PXCMBlobData.IBlob pBlob; if (blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out pBlob) == pxcmStatus.PXCM_STATUS_NO_ERROR) { Vector3 centerPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER); Vector3 topPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST); Vector3 bottomPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST); Vector3 leftPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST); Vector3 rightPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST); Vector3 closestPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST); blobPointsPos.Add(new Vector2(centerPoint.x * -1, centerPoint.y * -1)); blobPointsPos.Add(new Vector2(topPoint.x * -1, topPoint.y * -1)); blobPointsPos.Add(new Vector2(bottomPoint.x * -1, bottomPoint.y * -1)); blobPointsPos.Add(new Vector2(leftPoint.x * -1, leftPoint.y * -1)); blobPointsPos.Add(new Vector2(rightPoint.x * -1, rightPoint.y * -1)); blobPointsPos.Add(new Vector2(closestPoint.x * -1, closestPoint.y * -1)); DisplayPoints(); if (pBlob.QueryContourPoints(0, out pointOuter[i]) == pxcmStatus.PXCM_STATUS_NO_ERROR) { DisplayContour(pointOuter[i], i, numblobs); } } } } } instance.ReleaseFrame(); } }
private void UpdateBlobImage(PXCMImage depthFrame) { if (depthFrame == null) { return; } // Blobを更新する var sts = blobData.Update(); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { return; } // Blobのための画像オブジェクトを作成する var depthInfo = depthFrame.QueryInfo(); depthInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8; var session = senseManager.QuerySession(); var blobImage = session.CreateImage(depthInfo); // 表示用画像を初期化する Array.Clear(imageBuffer, 0, imageBuffer.Length); CanvasHandParts.Children.Clear(); // Blobを取得する int numOfBlobs = blobData.QueryNumberOfBlobs(); for (int i = 0; i < numOfBlobs; ++i) { // Blobデータを取得する PXCMBlobData.IBlob blob; sts = blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } sts = blob.QuerySegmentationImage(out blobImage); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // Blob画像を取得する PXCMImage.ImageData data; sts = blobImage.AcquireAccess(PXCMImage.Access.ACCESS_READ, depthInfo.format, out data); if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) { continue; } // データをコピーする var buffer = data.ToByteArray(0, data.pitches[0] * depthInfo.height); for (int j = 0; j < depthInfo.height * depthInfo.width; ++j) { if (buffer[j] != 0) { imageBuffer[j] = (byte)((i + 1) * 64); } } // Blob画像を解放する blobImage.ReleaseAccess(data); // Blobの輪郭を表示する UpdateContoursImage(blob, i); } // Blob画像オブジェクトを解放する blobImage.Dispose(); // ピクセルデータを更新する imageBitmap.WritePixels(imageRect, imageBuffer, DEPTH_WIDTH * BYTE_PER_PIXEL, 0); }
void Update() { //Dynamically Pause/Enable Modules int numberOfEnabledModules = 0; foreach (var option in _senseOptions) { if (option.RefCounter == 0 && option.Enabled) { if (option.ModuleCUID > 0) { SenseManager.PauseModule(option.ModuleCUID, true); } option.Enabled = false; } else if (option.RefCounter > 0 && !option.Enabled) { if (!option.Initialized) { OnDisable(); OnEnable(); Start(); } if (option.ModuleCUID > 0) { SenseManager.PauseModule(option.ModuleCUID, false); } option.Enabled = true; } if (option.Enabled) { numberOfEnabledModules++; } } //Update Speech commands if changed if (_speechCommandsChanged) { UpdateSpeechCommands(); SpeechManager.Reset(); } // Every frame update all the data if (Initialized && numberOfEnabledModules > 0) { _sts = SenseManager.AcquireFrame(true, 0); if (_sts == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled) { if (ImageRgbOutput != null) { ImageRgbOutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.color != null) { ImageRgbOutput = _captureSample.color; ImageRgbOutput.AddRef(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled || _senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled) { if (ImageDepthOutput != null) { ImageDepthOutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.depth != null) { ImageDepthOutput = _captureSample.depth; ImageDepthOutput.AddRef(); /* GZ * if (!_isInitBlob) * { * PXCMImage.ImageInfo info = ImageDepthOutput.QueryInfo(); * BlobExtractor.Init(info); * BlobExtractor.SetMaxBlobs(MaxBlobsToDetect); * _isInitBlob = true; * } */ if (PointCloud == null) { PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled) { if (PointCloud == null) { PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } _sts = Projection.QueryVertices(ImageDepthOutput, PointCloud); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.UVMap).Enabled) { if (UvMap == null) { UvMap = new PXCMPointF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height]; } Projection.QueryUVMap(ImageDepthOutput, UvMap); } } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled) { if (ImageIROutput != null) { ImageIROutput.Dispose(); } if (_captureSample == null) { _captureSample = SenseManager.QuerySample(); } if (_captureSample.ir != null) { ImageIROutput = _captureSample.ir; ImageIROutput.AddRef(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled) { if (Image3DSegmentationOutput != null) { Image3DSegmentationOutput.Dispose(); } PXCM3DSeg seg = SenseManager.Query3DSeg(); if (seg != null) { Image3DSegmentationOutput = seg.AcquireSegmentedImage(); } } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled) { FaceModuleOutput.Update(); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled) { HandDataOutput.Update(); } if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled) { BlobDataOutput.Update(); } _captureSample = null; SenseManager.ReleaseFrame(); } //Speech if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled) { SpeechManager.QueryRecognizedCommands(out SpeechOutput); } } }
private void Update() { // Start AcquireFrame-ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMCapture.Sample sample = senseManager.QuerySample(); Bitmap colorBitmap; PXCMImage.ImageData colorData; blobData.Update(); /*while (blobData.QueryNumberOfBlobs() < 2) * { * trackingDistance += 100; * blobConfig.SetMaxDistance(trackingDistance); * blobConfig.ApplyChanges(); * blobData.Update(); * * senseManager.ReleaseFrame(); * * if(trackingDistance > 3000) * trackingDistance = 600; * } * { * } */ for (int i = 0; i < 2; i++) { blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blobList[i]); } if (blobData.QueryNumberOfBlobs() == 2) { if (blobCoordinates[3, (int)cord.Y] == -1) { //smoothing } if (blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x > blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x) { blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; } else { blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; } } // Tonausgabe: aktuelle Tonausgabe beenden und neue beginnen frequency = (int)(blobCoordinates[(int)hand.LEFT, (int)cord.Y] * 1.8); volume = (500 - blobCoordinates[(int)hand.RIGHT, (int)cord.Y]) / 500; sineWaveProvider.Frequency = frequency; sineWaveProvider.Amplitude = volume; // Get color image data sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); // Update UI Render(colorBitmap); // Release frame colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); senseManager.ReleaseFrame(); } }
private void Update() { // Start AcquireFrame-ReleaseFrame loop while (sm.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { // Acquire color image data PXCMCapture.Sample sample = sm.QuerySample(); Bitmap colorBitmap; PXCMImage.ImageData colorData; sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); // Create an instance of MyTrackedPerson MyTrackedPerson myTrackedPerson = new MyTrackedPerson(); MyBlobs myBlobs = new MyBlobs(); // Acquire person tracking data personData = personModule.QueryOutput(); myTrackedPerson.PersonsDetected = personData.QueryNumberOfPeople(); if (myTrackedPerson.PersonsDetected == 1) { PXCMPersonTrackingData.Person trackedPerson = personData.QueryPersonData(PXCMPersonTrackingData.AccessOrderType.ACCESS_ORDER_BY_ID, 0); PXCMPersonTrackingData.PersonTracking trackedPersonData = trackedPerson.QueryTracking(); PXCMPersonTrackingData.BoundingBox2D personBox = trackedPersonData.Query2DBoundingBox(); myTrackedPerson.X = personBox.rect.x; myTrackedPerson.Y = personBox.rect.y; myTrackedPerson.H = personBox.rect.h; myTrackedPerson.W = personBox.rect.w; /* * PXCMPersonTrackingData.PersonJoints personJoints = trackedPerson.QuerySkeletonJoints(); * PXCMPersonTrackingData.PersonJoints.SkeletonPoint[] skeletonPoints = new PXCMPersonTrackingData.PersonJoints.SkeletonPoint[personJoints.QueryNumJoints()]; * trackedPerson.QuerySkeletonJoints().QueryJoints(skeletonPoints); * if (skeletonPoints.Length > 0) * skeletonPoints[0].GetType(); */ } // Acquire face tracking data faceData.Update(); myTrackedPerson.FacesDetected = faceData.QueryNumberOfDetectedFaces(); if (myTrackedPerson.FacesDetected == 1) { PXCMFaceData.Face face = faceData.QueryFaceByIndex(0); PXCMFaceData.DetectionData faceDetectionData = face.QueryDetection(); PXCMRectI32 faceRectangle; faceDetectionData.QueryBoundingRect(out faceRectangle); myTrackedPerson.FaceH = faceRectangle.h; myTrackedPerson.FaceW = faceRectangle.w; myTrackedPerson.FaceX = faceRectangle.x; myTrackedPerson.FaceY = faceRectangle.y; float faceDepth; faceDetectionData.QueryFaceAverageDepth(out faceDepth); myTrackedPerson.FaceDepth = faceDepth; } blobData.Update(); int numBlobs = blobData.QueryNumberOfBlobs(); myBlobs.numBlobs = numBlobs; myBlobs.blobs = new List <List <PXCMPointI32> >(numBlobs); myBlobs.closestPoints = new List <PXCMPoint3DF32>(4); for (int i = 0; i < numBlobs; i++) { PXCMBlobData.IBlob blob; pxcmStatus result1 = blobData.QueryBlob(i, PXCMBlobData.SegmentationImageType.SEGMENTATION_IMAGE_DEPTH, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob); if (result1 == pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMPoint3DF32 closestPoint = blob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST); myBlobs.closestPoints.Add(closestPoint); int numContours = blob.QueryNumberOfContours(); if (numContours > 0) { // only deal with outer contour for (int j = 0; j < numContours; j++) { PXCMBlobData.IContour contour; pxcmStatus result2 = blob.QueryContour(j, out contour); if (result2 == pxcmStatus.PXCM_STATUS_NO_ERROR) { if (contour.IsOuter()) { PXCMPointI32[] points; pxcmStatus result3 = contour.QueryPoints(out points); if (result3 == pxcmStatus.PXCM_STATUS_NO_ERROR) { int numPoints = points.Length; myBlobs.blobs.Add(points.ToList <PXCMPointI32>()); } } } } } } } // Update UI Render(colorBitmap, myTrackedPerson, myBlobs); // serialize to json and send all clients var personJson = JsonConvert.SerializeObject(myTrackedPerson); personSockets.ToList().ForEach(s => s.Send(personJson)); var blobJson = JsonConvert.SerializeObject(myBlobs); blobSockets.ToList().ForEach(s => s.Send(blobJson)); // deserialize json as follows //MyTrackedPerson deserializedProduct = JsonConvert.DeserializeObject<MyTrackedPerson>(json); // Release resources colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); sm.ReleaseFrame(); } }
private void Update() { // Start AcquireFrame-ReleaseFrame loop while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR) { PXCMCapture.Sample sample = senseManager.QuerySample(); PXCMImage.ImageData colorData; blobData.Update(); /*while (blobData.QueryNumberOfBlobs() < 2) * { * trackingDistance += 100; * blobConfig.SetMaxDistance(trackingDistance); * blobConfig.ApplyChanges(); * blobData.Update(); * * senseManager.ReleaseFrame(); * * if(trackingDistance > 3000) * trackingDistance = 600; * } * { * } */ /* * for (int i = 0; i < 2; i++) * { * blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blobList[i]); * } * if (blobData.QueryNumberOfBlobs() == 2) * { * if (blobCoordinates[3, (int)cord.Y] == -1) * { * //smoothing * } * if (blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x > blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x) * { * blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; * blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; * blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; * blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; * } * else * { * blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; * blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y; * blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; * blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x; * } * } */ // Tonausgabe: aktuelle Tonausgabe beenden und neue beginnen frequency = (int)(blobCoordinates[(int)hand.LEFT, (int)cord.Y] * 1.8); volume = (500 - blobCoordinates[(int)hand.RIGHT, (int)cord.Y]) / 500; sineWaveProvider.Frequency = frequency; sineWaveProvider.Amplitude = volume; // Get color image data sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData); colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height); checkBitmap = new Bitmap(colorBitmap); if ((DateTime.Now.Ticks - startTime.Ticks) < 50000000) { using (var graphics = Graphics.FromImage(colorBitmap)) { System.Drawing.Pen fancyPen = new System.Drawing.Pen(System.Drawing.Color.Cyan, 5); graphics.DrawRectangle(fancyPen, colorBitmap.Width / 5, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3); graphics.DrawRectangle(fancyPen, (colorBitmap.Width / 5) * 3, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3); } } else if (!templatesSet) { Bitmap template1 = new Bitmap(colorBitmap.Width / 5, colorBitmap.Height / 3); Bitmap template2 = new Bitmap(colorBitmap.Width / 5, colorBitmap.Height / 3); Rectangle rect1 = new Rectangle(colorBitmap.Width / 5, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3); Rectangle rect2 = new Rectangle((colorBitmap.Width / 5) * 3, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3); Rectangle dest = new Rectangle(0, 0, template1.Width, template1.Height); using (var graphics = Graphics.FromImage(template1)) { graphics.DrawImage(colorBitmap, dest, rect1, GraphicsUnit.Pixel); } using (var graphics = Graphics.FromImage(template2)) { graphics.DrawImage(colorBitmap, dest, rect2, GraphicsUnit.Pixel); } template1.Save("hand1.bmp"); template2.Save("hand2.bmp"); templatesSet = true; } else { // Update UI colorBitmap.Save("bitmap.bmp", ImageFormat.Bmp); //Mat img = new Mat("bitmap.bmp", Emgu.CV.CvEnum.LoadImageType.AnyColor); Mat img = CvInvoke.Imread("bitmap.bmp", Emgu.CV.CvEnum.LoadImageType.Color); Mat template = CvInvoke.Imread("hand1.bmp", Emgu.CV.CvEnum.LoadImageType.Color); templateMatch(img, template, false); template = CvInvoke.Imread("hand2.bmp", Emgu.CV.CvEnum.LoadImageType.Color); Mat img2 = CvInvoke.Imread("checkBitmap.bmp", Emgu.CV.CvEnum.LoadImageType.Color); templateMatch(img2, template, true); } Render(colorBitmap); // Release frame colorBitmap.Dispose(); sample.color.ReleaseAccess(colorData); senseManager.ReleaseFrame(); } }