/// <summary>
/// Grab the blob data every frame
/// </summary>
void Update()
{
if (instance.AcquireFrame(true) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
/* To store all blob points */
blobPointsPos = new List <Vector2>();
PXCMCapture.Sample sample = instance.QuerySample();
if (sample != null && sample.depth != null)
{
PXCMImage.ImageInfo info = sample.depth.QueryInfo();
if (blobData != null)
{
blobData.Update();
int numblobs = blobData.QueryNumberOfBlobs();
for (int i = 0; i <= numblobs; i++)
{
PXCMBlobData.IBlob pBlob;
if (blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out pBlob) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
Vector3 centerPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER);
Vector3 topPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_TOP_MOST);
Vector3 bottomPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_BOTTOM_MOST);
Vector3 leftPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_LEFT_MOST);
Vector3 rightPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_RIGHT_MOST);
Vector3 closestPoint = pBlob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST);
blobPointsPos.Add(new Vector2(centerPoint.x * -1, centerPoint.y * -1));
blobPointsPos.Add(new Vector2(topPoint.x * -1, topPoint.y * -1));
blobPointsPos.Add(new Vector2(bottomPoint.x * -1, bottomPoint.y * -1));
blobPointsPos.Add(new Vector2(leftPoint.x * -1, leftPoint.y * -1));
blobPointsPos.Add(new Vector2(rightPoint.x * -1, rightPoint.y * -1));
blobPointsPos.Add(new Vector2(closestPoint.x * -1, closestPoint.y * -1));
DisplayPoints();
if (pBlob.QueryContourPoints(0, out pointOuter[i]) == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
DisplayContour(pointOuter[i], i, numblobs);
}
}
}
}
}
instance.ReleaseFrame();
}
}
private void UpdateBlobImage(PXCMImage depthFrame)
{
if (depthFrame == null)
{
return;
}
// Blobを更新する
var sts = blobData.Update();
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
return;
}
// Blobのための画像オブジェクトを作成する
var depthInfo = depthFrame.QueryInfo();
depthInfo.format = PXCMImage.PixelFormat.PIXEL_FORMAT_Y8;
var session = senseManager.QuerySession();
var blobImage = session.CreateImage(depthInfo);
// 表示用画像を初期化する
Array.Clear(imageBuffer, 0, imageBuffer.Length);
CanvasHandParts.Children.Clear();
// Blobを取得する
int numOfBlobs = blobData.QueryNumberOfBlobs();
for (int i = 0; i < numOfBlobs; ++i)
{
// Blobデータを取得する
PXCMBlobData.IBlob blob;
sts = blobData.QueryBlobByAccessOrder(i,
PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
sts = blob.QuerySegmentationImage(out blobImage);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// Blob画像を取得する
PXCMImage.ImageData data;
sts = blobImage.AcquireAccess(PXCMImage.Access.ACCESS_READ,
depthInfo.format, out data);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR)
{
continue;
}
// データをコピーする
var buffer = data.ToByteArray(0, data.pitches[0] * depthInfo.height);
for (int j = 0; j < depthInfo.height * depthInfo.width; ++j)
{
if (buffer[j] != 0)
{
imageBuffer[j] = (byte)((i + 1) * 64);
}
}
// Blob画像を解放する
blobImage.ReleaseAccess(data);
// Blobの輪郭を表示する
UpdateContoursImage(blob, i);
}
// Blob画像オブジェクトを解放する
blobImage.Dispose();
// ピクセルデータを更新する
imageBitmap.WritePixels(imageRect, imageBuffer,
DEPTH_WIDTH * BYTE_PER_PIXEL, 0);
}
void Update()
{
//Dynamically Pause/Enable Modules
int numberOfEnabledModules = 0;
foreach (var option in _senseOptions)
{
if (option.RefCounter == 0 && option.Enabled)
{
if (option.ModuleCUID > 0)
{
SenseManager.PauseModule(option.ModuleCUID, true);
}
option.Enabled = false;
}
else if (option.RefCounter > 0 && !option.Enabled)
{
if (!option.Initialized)
{
OnDisable();
OnEnable();
Start();
}
if (option.ModuleCUID > 0)
{
SenseManager.PauseModule(option.ModuleCUID, false);
}
option.Enabled = true;
}
if (option.Enabled)
{
numberOfEnabledModules++;
}
}
//Update Speech commands if changed
if (_speechCommandsChanged)
{
UpdateSpeechCommands();
SpeechManager.Reset();
}
// Every frame update all the data
if (Initialized && numberOfEnabledModules > 0)
{
_sts = SenseManager.AcquireFrame(true, 0);
if (_sts == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoColorStream).Enabled)
{
if (ImageRgbOutput != null)
{
ImageRgbOutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.color != null)
{
ImageRgbOutput = _captureSample.color;
ImageRgbOutput.AddRef();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoDepthStream).Enabled ||
_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled)
{
if (ImageDepthOutput != null)
{
ImageDepthOutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.depth != null)
{
ImageDepthOutput = _captureSample.depth;
ImageDepthOutput.AddRef();
/* GZ
* if (!_isInitBlob)
* {
* PXCMImage.ImageInfo info = ImageDepthOutput.QueryInfo();
* BlobExtractor.Init(info);
* BlobExtractor.SetMaxBlobs(MaxBlobsToDetect);
* _isInitBlob = true;
* }
*/
if (PointCloud == null)
{
PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.PointCloud).Enabled)
{
if (PointCloud == null)
{
PointCloud = new PXCMPoint3DF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
_sts = Projection.QueryVertices(ImageDepthOutput, PointCloud);
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.UVMap).Enabled)
{
if (UvMap == null)
{
UvMap = new PXCMPointF32[ImageDepthOutput.info.width * ImageDepthOutput.info.height];
}
Projection.QueryUVMap(ImageDepthOutput, UvMap);
}
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoIRStream).Enabled)
{
if (ImageIROutput != null)
{
ImageIROutput.Dispose();
}
if (_captureSample == null)
{
_captureSample = SenseManager.QuerySample();
}
if (_captureSample.ir != null)
{
ImageIROutput = _captureSample.ir;
ImageIROutput.AddRef();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.VideoSegmentation).Enabled)
{
if (Image3DSegmentationOutput != null)
{
Image3DSegmentationOutput.Dispose();
}
PXCM3DSeg seg = SenseManager.Query3DSeg();
if (seg != null)
{
Image3DSegmentationOutput = seg.AcquireSegmentedImage();
}
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Face).Enabled)
{
FaceModuleOutput.Update();
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Hand).Enabled)
{
HandDataOutput.Update();
}
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Blob).Enabled)
{
BlobDataOutput.Update();
}
_captureSample = null;
SenseManager.ReleaseFrame();
}
//Speech
if (_senseOptions.Find(i => i.ID == SenseOption.SenseOptionID.Speech).Enabled)
{
SpeechManager.QueryRecognizedCommands(out SpeechOutput);
}
}
}
private void Update()
{
// Start AcquireFrame-ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.Sample sample = senseManager.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
blobData.Update();
/*while (blobData.QueryNumberOfBlobs() < 2)
* {
* trackingDistance += 100;
* blobConfig.SetMaxDistance(trackingDistance);
* blobConfig.ApplyChanges();
* blobData.Update();
*
* senseManager.ReleaseFrame();
*
* if(trackingDistance > 3000)
* trackingDistance = 600;
* }
* {
* }
*/
for (int i = 0; i < 2; i++)
{
blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blobList[i]);
}
if (blobData.QueryNumberOfBlobs() == 2)
{
if (blobCoordinates[3, (int)cord.Y] == -1)
{
//smoothing
}
if (blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x > blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x)
{
blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
}
else
{
blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
}
}
// Tonausgabe: aktuelle Tonausgabe beenden und neue beginnen
frequency = (int)(blobCoordinates[(int)hand.LEFT, (int)cord.Y] * 1.8);
volume = (500 - blobCoordinates[(int)hand.RIGHT, (int)cord.Y]) / 500;
sineWaveProvider.Frequency = frequency;
sineWaveProvider.Amplitude = volume;
// Get color image data
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Update UI
Render(colorBitmap);
// Release frame
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
private void Update()
{
// Start AcquireFrame-ReleaseFrame loop
while (sm.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
// Acquire color image data
PXCMCapture.Sample sample = sm.QuerySample();
Bitmap colorBitmap;
PXCMImage.ImageData colorData;
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
// Create an instance of MyTrackedPerson
MyTrackedPerson myTrackedPerson = new MyTrackedPerson();
MyBlobs myBlobs = new MyBlobs();
// Acquire person tracking data
personData = personModule.QueryOutput();
myTrackedPerson.PersonsDetected = personData.QueryNumberOfPeople();
if (myTrackedPerson.PersonsDetected == 1)
{
PXCMPersonTrackingData.Person trackedPerson = personData.QueryPersonData(PXCMPersonTrackingData.AccessOrderType.ACCESS_ORDER_BY_ID, 0);
PXCMPersonTrackingData.PersonTracking trackedPersonData = trackedPerson.QueryTracking();
PXCMPersonTrackingData.BoundingBox2D personBox = trackedPersonData.Query2DBoundingBox();
myTrackedPerson.X = personBox.rect.x;
myTrackedPerson.Y = personBox.rect.y;
myTrackedPerson.H = personBox.rect.h;
myTrackedPerson.W = personBox.rect.w;
/*
* PXCMPersonTrackingData.PersonJoints personJoints = trackedPerson.QuerySkeletonJoints();
* PXCMPersonTrackingData.PersonJoints.SkeletonPoint[] skeletonPoints = new PXCMPersonTrackingData.PersonJoints.SkeletonPoint[personJoints.QueryNumJoints()];
* trackedPerson.QuerySkeletonJoints().QueryJoints(skeletonPoints);
* if (skeletonPoints.Length > 0)
* skeletonPoints[0].GetType();
*/
}
// Acquire face tracking data
faceData.Update();
myTrackedPerson.FacesDetected = faceData.QueryNumberOfDetectedFaces();
if (myTrackedPerson.FacesDetected == 1)
{
PXCMFaceData.Face face = faceData.QueryFaceByIndex(0);
PXCMFaceData.DetectionData faceDetectionData = face.QueryDetection();
PXCMRectI32 faceRectangle;
faceDetectionData.QueryBoundingRect(out faceRectangle);
myTrackedPerson.FaceH = faceRectangle.h;
myTrackedPerson.FaceW = faceRectangle.w;
myTrackedPerson.FaceX = faceRectangle.x;
myTrackedPerson.FaceY = faceRectangle.y;
float faceDepth;
faceDetectionData.QueryFaceAverageDepth(out faceDepth);
myTrackedPerson.FaceDepth = faceDepth;
}
blobData.Update();
int numBlobs = blobData.QueryNumberOfBlobs();
myBlobs.numBlobs = numBlobs;
myBlobs.blobs = new List <List <PXCMPointI32> >(numBlobs);
myBlobs.closestPoints = new List <PXCMPoint3DF32>(4);
for (int i = 0; i < numBlobs; i++)
{
PXCMBlobData.IBlob blob;
pxcmStatus result1 = blobData.QueryBlob(i, PXCMBlobData.SegmentationImageType.SEGMENTATION_IMAGE_DEPTH, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blob);
if (result1 == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMPoint3DF32 closestPoint = blob.QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CLOSEST);
myBlobs.closestPoints.Add(closestPoint);
int numContours = blob.QueryNumberOfContours();
if (numContours > 0)
{
// only deal with outer contour
for (int j = 0; j < numContours; j++)
{
PXCMBlobData.IContour contour;
pxcmStatus result2 = blob.QueryContour(j, out contour);
if (result2 == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
if (contour.IsOuter())
{
PXCMPointI32[] points;
pxcmStatus result3 = contour.QueryPoints(out points);
if (result3 == pxcmStatus.PXCM_STATUS_NO_ERROR)
{
int numPoints = points.Length;
myBlobs.blobs.Add(points.ToList <PXCMPointI32>());
}
}
}
}
}
}
}
// Update UI
Render(colorBitmap, myTrackedPerson, myBlobs);
// serialize to json and send all clients
var personJson = JsonConvert.SerializeObject(myTrackedPerson);
personSockets.ToList().ForEach(s => s.Send(personJson));
var blobJson = JsonConvert.SerializeObject(myBlobs);
blobSockets.ToList().ForEach(s => s.Send(blobJson));
// deserialize json as follows
//MyTrackedPerson deserializedProduct = JsonConvert.DeserializeObject<MyTrackedPerson>(json);
// Release resources
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
sm.ReleaseFrame();
}
}
private void Update()
{
// Start AcquireFrame-ReleaseFrame loop
while (senseManager.AcquireFrame(true) >= pxcmStatus.PXCM_STATUS_NO_ERROR)
{
PXCMCapture.Sample sample = senseManager.QuerySample();
PXCMImage.ImageData colorData;
blobData.Update();
/*while (blobData.QueryNumberOfBlobs() < 2)
* {
* trackingDistance += 100;
* blobConfig.SetMaxDistance(trackingDistance);
* blobConfig.ApplyChanges();
* blobData.Update();
*
* senseManager.ReleaseFrame();
*
* if(trackingDistance > 3000)
* trackingDistance = 600;
* }
* {
* }
*/
/*
* for (int i = 0; i < 2; i++)
* {
* blobData.QueryBlobByAccessOrder(i, PXCMBlobData.AccessOrderType.ACCESS_ORDER_NEAR_TO_FAR, out blobList[i]);
* }
* if (blobData.QueryNumberOfBlobs() == 2)
* {
* if (blobCoordinates[3, (int)cord.Y] == -1)
* {
* //smoothing
* }
* if (blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x > blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x)
* {
* blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
* blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
* blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
* blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
* }
* else
* {
* blobCoordinates[(int)hand.RIGHT, (int)cord.Y] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
* blobCoordinates[(int)hand.LEFT, (int)cord.Y] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).y;
* blobCoordinates[(int)hand.RIGHT, (int)cord.X] = blobList[0].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
* blobCoordinates[(int)hand.LEFT, (int)cord.X] = blobList[1].QueryExtremityPoint(PXCMBlobData.ExtremityType.EXTREMITY_CENTER).x;
* }
* }
*/
// Tonausgabe: aktuelle Tonausgabe beenden und neue beginnen
frequency = (int)(blobCoordinates[(int)hand.LEFT, (int)cord.Y] * 1.8);
volume = (500 - blobCoordinates[(int)hand.RIGHT, (int)cord.Y]) / 500;
sineWaveProvider.Frequency = frequency;
sineWaveProvider.Amplitude = volume;
// Get color image data
sample.color.AcquireAccess(PXCMImage.Access.ACCESS_READ, PXCMImage.PixelFormat.PIXEL_FORMAT_RGB32, out colorData);
colorBitmap = colorData.ToBitmap(0, sample.color.info.width, sample.color.info.height);
checkBitmap = new Bitmap(colorBitmap);
if ((DateTime.Now.Ticks - startTime.Ticks) < 50000000)
{
using (var graphics = Graphics.FromImage(colorBitmap))
{
System.Drawing.Pen fancyPen = new System.Drawing.Pen(System.Drawing.Color.Cyan, 5);
graphics.DrawRectangle(fancyPen, colorBitmap.Width / 5, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3);
graphics.DrawRectangle(fancyPen, (colorBitmap.Width / 5) * 3, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3);
}
}
else if (!templatesSet)
{
Bitmap template1 = new Bitmap(colorBitmap.Width / 5, colorBitmap.Height / 3);
Bitmap template2 = new Bitmap(colorBitmap.Width / 5, colorBitmap.Height / 3);
Rectangle rect1 = new Rectangle(colorBitmap.Width / 5, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3);
Rectangle rect2 = new Rectangle((colorBitmap.Width / 5) * 3, colorBitmap.Height / 3, colorBitmap.Width / 5, colorBitmap.Height / 3);
Rectangle dest = new Rectangle(0, 0, template1.Width, template1.Height);
using (var graphics = Graphics.FromImage(template1))
{
graphics.DrawImage(colorBitmap, dest, rect1, GraphicsUnit.Pixel);
}
using (var graphics = Graphics.FromImage(template2))
{
graphics.DrawImage(colorBitmap, dest, rect2, GraphicsUnit.Pixel);
}
template1.Save("hand1.bmp");
template2.Save("hand2.bmp");
templatesSet = true;
}
else
{
// Update UI
colorBitmap.Save("bitmap.bmp", ImageFormat.Bmp);
//Mat img = new Mat("bitmap.bmp", Emgu.CV.CvEnum.LoadImageType.AnyColor);
Mat img = CvInvoke.Imread("bitmap.bmp", Emgu.CV.CvEnum.LoadImageType.Color);
Mat template = CvInvoke.Imread("hand1.bmp", Emgu.CV.CvEnum.LoadImageType.Color);
templateMatch(img, template, false);
template = CvInvoke.Imread("hand2.bmp", Emgu.CV.CvEnum.LoadImageType.Color);
Mat img2 = CvInvoke.Imread("checkBitmap.bmp", Emgu.CV.CvEnum.LoadImageType.Color);
templateMatch(img2, template, true);
}
Render(colorBitmap);
// Release frame
colorBitmap.Dispose();
sample.color.ReleaseAccess(colorData);
senseManager.ReleaseFrame();
}
}
