//////////////////////////////////////////////////
#region Private Methods
void buildPipeline(string deviceName, string gestureModuleName)
{
_pipeline = new UtilMPipeline();
_pipeline.QueryCapture().SetFilter(deviceName);
_pipeline.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB24);
_pipeline.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_VERTICES);
_pipeline.EnableGesture(gestureModuleName);
if (!_pipeline.Init())
{
throw new GestureCameraException("Failed to initialize pipeline.");
}
float unit;
GestureCameraUtil.Assert(
_pipeline.QueryCapture().device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_UNIT, out unit),
"Failed to query sensor depth unit.");
_unit_m = unit / 1000000;
GestureCameraUtil.Assert(
_pipeline.QueryCapture().device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out _depthLowConf),
"Failed to query sensor depth low confidence value.");
GestureCameraUtil.Assert(
_pipeline.QueryCapture().device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out _depthSaturation),
"Failed to query sensor depth saturation value.");
}
public CaptureHand(CaptureForm form) : base()
{
pp = new UtilMPipeline();
pp.EnableGesture();
pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
if (pp.Init())
{
pp.QueryCapture().SetFilter(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, 100);
do
{
if (!pp.AcquireFrame(true))
{
MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
break;
}
gesture = pp.QueryGesture();
gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);
gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);
bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ, out data);
info = bimage.imageInfo;
color = (int)bdata.labelLeftHand;
form.setTextBox(color.ToString());
pp.ReleaseFrame();
} while (color == -1);
Coord[] locations = handLocation(data, (int)(info.width * info.height), (int)info.width);
bitmap = createBitmap(locations, (int)info.width, (int)info.height);
form.setImage(bitmap);
form.bitmap = bitmap;
}
}
public CaptureHand(CaptureForm form)
: base()
{
pp = new UtilMPipeline();
pp.EnableGesture();
pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
if (pp.Init())
{
pp.QueryCapture().SetFilter(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, 100);
do
{
if (!pp.AcquireFrame(true))
{
MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
break;
}
gesture = pp.QueryGesture();
gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);
gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);
bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ,out data);
info = bimage.imageInfo;
color =(int) bdata.labelLeftHand;
form.setTextBox(color.ToString());
pp.ReleaseFrame();
} while (color == -1);
Coord[] locations = handLocation(data, (int)(info.width * info.height),(int) info.width);
bitmap = createBitmap(locations, (int)info.width, (int)info.height);
form.setImage(bitmap);
form.bitmap = bitmap;
}
}
/// <summary>
/// Initializes the camera and returns true on success, false otherwise.
/// </summary>
/// <returns>Initialization successful (true) or failed (false)</returns>
private bool InitializeCamera()
{
/* UtilMPipeline works best for synchronous color and depth streaming */
_pp = new UtilMPipeline();
/* Set Input Source */
_pp.capture.SetFilter("DepthSense Device 325V2");
/* Set Color & Depth Resolution */
var cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
_pp.capture.SetFilter(ref cinfo); // only needed to set FPS
var dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
_pp.capture.SetFilter(ref dinfo2); // only needed to set FPS
/* Initialization */
if (!_pp.Init())
{
LogFormat("Could not initialize Senz3D hardware");
HasErrorState = true;
return(false);
}
var capture = _pp.capture;
_device = capture.device;
_device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);
return(true);
}
public unsafe MainWindow()
{
InitializeComponent();
InitializeServer();
this.Closing += new CancelEventHandler(OnWindowClosing);
System.Windows.Threading.Dispatcher uiDispatcher = this.Dispatcher;
UtilMPipeline pp = new UtilMPipeline();
BackgroundWorker worker = new BackgroundWorker();
worker.DoWork += delegate(object s, DoWorkEventArgs args)
{
pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
pp.Init();
for (; ; )
{
if (!pp.AcquireFrame(true)) break;
{
PXCMImage image = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
PXCMImage.ImageData ddata;
pxcmStatus sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ, out ddata);
if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) break;
short* depth = (short*)ddata.buffer.planes[0];
short minZ = 32001;
int minX = 0;
int minY = 0;
short maxX = 320;
for (int y = 0; y < 240; y++)
{
for (int x = 0; x < 320; x++)
{
if (depth[(y * 320) + x] < minZ)
{
minZ = depth[(y * 320) + x];
minX = x;
minY = y;
}
}
}
/////////////////////
//normalize extremes//
if (minZ < DistanceFromScreenEdge) minZ = DistanceFromScreenEdge;
//match origin//
minX -= 160;
minY -= 120;
/////////////////////
maxX = (short)(Math.Tan(FOV / 2) * minZ);
short discardedX = (short)(Math.Tan(FOV / 2) * (minZ - DistanceFromScreenEdge));
short screenMaxX = (short)(maxX - discardedX);
float xRatio = minX / screenMaxX;
UpdateGridDelegate update = new UpdateGridDelegate(UpdateGrid);
uiDispatcher.BeginInvoke(update, minX, minY, minZ, maxX, discardedX, screenMaxX, xRatio);
image.ReleaseAccess(ref ddata);
pp.ReleaseFrame();
}
}
pp.Close();
pp.Dispose();
};
worker.RunWorkerAsync();
}
/// <summary>
/// Initializes the camera and returns true on success, false otherwise.
/// </summary>
/// <returns>Initialization successful (true) or failed (false)</returns>
private bool InitializeCamera()
{
/* UtilMPipeline works best for synchronous color and depth streaming */
_pp = new UtilMPipeline();
/* Set Input Source */
_pp.capture.SetFilter("DepthSense Device 325V2");
/* Set Color & Depth Resolution */
var cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
_pp.capture.SetFilter(ref cinfo); // only needed to set FPS
var dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
_pp.capture.SetFilter(ref dinfo2); // only needed to set FPS
/* Initialization */
if (!_pp.Init())
{
LogFormat("Could not initialize Senz3D hardware");
HasErrorState = true;
return false;
}
var capture = _pp.capture;
_device = capture.device;
_device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);
return true;
}
public GesturePipeline(Form1 form)
: base()
{
//------------------------------
// Creates datastructures for
// the PCSDK
//------------------------------
UtilMPipeline pp; //The pipeline
PXCMGesture.Blob bdata; //The intrepred blobs
PXCMImage bimage; //The frame from the camera
PXCMImage.ImageData data; //The data contained in the frame.
PXCMImage.ImageInfo info; //Metadata about the frame.
int color; //The "color" for the hand in question.
//------------------------------
//------------------------------
// Initilize the datastructures
// defined above.
//------------------------------
Coord[] locations;
pp = new UtilMPipeline();
pp.EnableGesture();
pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
//-----------------------------
//initilize the pointer for the WMC window.
IntPtr handle = GetDesktopWindow();
//----------------------------------
// Create and initilize the bitmaps
// that the frames will be comapred
// against.
//---------------------------------
Bitmap pause = convertBitmap(KwisStandalone.Properties.Resources.STOP);
Bitmap fastForward = convertBitmap(KwisStandalone.Properties.Resources.FAST_FORWARD);
Bitmap rewind = convertBitmap(KwisStandalone.Properties.Resources.REWIND);
Bitmap play = convertBitmap(KwisStandalone.Properties.Resources.PLAY);
Bitmap menu = convertBitmap(KwisStandalone.Properties.Resources.MENU);
Bitmap volUp = convertBitmap(KwisStandalone.Properties.Resources.VOL_UP);
Bitmap volDown = convertBitmap(KwisStandalone.Properties.Resources.VOL_DOWN);
//--------------------------------
// initial the template matching algorithm.
templateMatching = new ExhaustiveTemplateMatching(0.75f);
/* If we were able to aquire a frame from the camera,
* start the main application logic.
* */
if (pp.Init())
{
//Loop forever as long as the form is still open.
while (!form.closing)
{
if (!pp.AcquireFrame(true))
{
MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OKCancel, MessageBoxIcon.Asterisk);
break;
}
//Aquire the gesture data for the frame.
using (PXCMGesture gesture = pp.QueryGesture())
{
//Load gesture data into respective data structures.
gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);
gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);
bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ, out data);
info = bimage.imageInfo;
/* Although this says left hand,
* this actually aquires the right hand.
* This is from trial and error and contradicts
* both the documentation and common sense. A bug
* in the SDK?
* */
color = (int)bdata.labelLeftHand;
//Store the pixels that include the hand specified above.
locations = new Coord[info.width * info.height];
handLocation(ref locations, data, (int)info.width);
//create a new bitmap from this data.
Bitmap bitmap = createBitmap(ref locations, (int)info.width, (int)info.height);
/* Zero is the left hand (actuall right). So, if we detected this,
* then lets see if it matches any of the stored reference images.
* */
if (color == 0)
{
try
{
if (compare(ref bitmap, ref pause))
{
clickAndSendKey(RemoteKey.PLAY_PAUSE);
}
else if (compare(ref bitmap, ref play))
{
clickAndSendKey(RemoteKey.PLAY);
}
else if (compare(ref bitmap, ref fastForward))
{
clickAndSendKey(RemoteKey.FAST_FORWARD);
}
else if (compare(ref bitmap, ref rewind))
{
clickAndSendKey(APPCOMMAND_MEDIA_REWIND, handle);
}
else if (compare(ref bitmap, ref menu))
{
clickAndSendKey(RemoteKey.STOP);
}
else if (compare(ref bitmap, ref volUp))
{
clickAndSendKey(RemoteKey.VOL_UP);
}
else if (compare(ref bitmap, ref volDown))
{
clickAndSendKey(RemoteKey.VOL_DOWN);
}
}
catch (InvalidImagePropertiesException iipe)
{
}
}
//Clean up the datastructures in prep for a new frame.
bitmap.Dispose();
bimage.ReleaseAccess(ref data);
bimage.Dispose();
gesture.Dispose();
}
pp.ReleaseFrame();
}
}
}
public void SimplePipeline()
{
bool sts = true;
UtilMPipeline pp = null;
disconnected = false;
/* Set Source */
if (form.GetRecordState()) {
pp = new UtilMPipeline(form.GetRecordFile(), true);
pp.QueryCapture().SetFilter(form.GetCheckedDevice());
}
else if (form.GetPlaybackState())
{
pp = new UtilMPipeline(form.GetPlaybackFile(), false);
}
else
{
pp = new UtilMPipeline();
pp.QueryCapture().SetFilter(form.GetCheckedDevice());
}
/* Set Module */
pp.EnableFaceLocation();
pp.EnableFaceLandmark();
pp.EnableGesture(form.GetCheckedModule());
/* Initialization */
form.UpdateStatus("Init Started");
if (pp.Init())
{
form.UpdateStatus("Streaming");
while (!form.stop)
{
if (!pp.AcquireFrame(true)) break;
if (!DisplayDeviceConnection(pp.IsDisconnected()))
{
/* Display Results */
PXCMGesture gesture = pp.QueryGesture();
PXCMImage img = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_COLOR);
Bitmap bmp = ConvertImageToBitmap(img);
#region EMGU Stuff
Image<Bgr, Byte> frame = new Image<Bgr, Byte>(bmp); //http://stackoverflow.com/questions/503427/the-type-initializer-for-emgu-cv-cvinvoke-threw-an-exception
Image<Gray, Byte> circleFrame = frame.Convert<Gray, Byte>().PyrDown().PyrUp();
Image<Gray, Byte> smallGrayFrame = circleFrame.PyrDown();
Image<Gray, Byte> smoothedGrayFrame = smallGrayFrame.PyrUp();
Image<Gray, Byte> cannyFrame = smoothedGrayFrame.Canny(100, 60);
//new Bgr(60, 90, 160), new Bgr(255, 255, 255) -- with light reflecting off the orange
//new Bgr(40, 90, 190), new Bgr(120, 190, 255)
//circleFrame = frame.InRange(new Bgr(100, 100, 100), new Bgr(255, 255, 255)); // somehow range for orange related colors
//circleFrame = frame.InRange(new Bgr(100, 110, 220), new Bgr(130, 140, 255)); // somehow range for orange related colors
//circleFrame = frame.InRange(new Bgr(100, 100, 100), new Bgr(255, 255, 255)); // somehow range for orange related colors
//circleFrame = circleFrame.SmoothGaussian(9);
double cannyThreshold = 180.0;
double circleAccumulatorThreshold = 40;
CircleF[] circles = circleFrame.HoughCircles(
new Gray(cannyThreshold),
new Gray(circleAccumulatorThreshold),
2.0, //Resolution of the accumulator used to detect centers of the circles
20.0, //min distance
18, //min radius
30 //max radius
)[0]; //Get the circles from the first channel\
/*
*
*
*
*
*
*
*
* Create a Timer
* Use the timer to space out the HoughCircles calls, so they aren't happening every frame
*
* Check for the puck more frequently
* Check for the fruits less frequently, and then only remove the info from the List after a certain time the fruit has been missing (like 50ms)
*
* Never do more than one HoughCircls call per frame.
*
*
*
*
*
*/
Image<Bgr, Byte> circleImage = frame;
/*
foreach (CircleF circle in circles)
{
circleImage.Draw(circle, new Bgr(Color.Brown), 2);
Console.WriteLine(circle.Radius);
}
*/
if (circles.Length > 0)
{
foreach (CircleF circle in circles)
{
circleImage.Draw(circle, new Bgr(Color.Brown), 2);
Point center = new Point((int)circle.Center.X, (int)circle.Center.Y);
if (frame[center].Red > Object1RGBLower[0] && frame[center].Red <= Object1RGBUpper[0] &&
frame[center].Green > Object1RGBLower[1] && frame[center].Green <= Object1RGBUpper[1] &&
frame[center].Blue > Object1RGBLower[2] && frame[center].Blue <= Object1RGBUpper[2])
{
//orange center of circle
Console.WriteLine("orange");
if (RecognizedObjects.Contains("orange") == false)
RecognizedObjects.Add("orange");
Projected.TimeSinceLastSeenOrange = Projected.TimeElapsed;
}
else
{
RecognizedObjects.Remove("orange");
}
if (frame[center].Red > Object2RGBLower[0] && frame[center].Red <= Object2RGBUpper[0] &&
frame[center].Green > Object2RGBLower[1] && frame[center].Green <= Object2RGBUpper[1] &&
frame[center].Blue > Object2RGBLower[2] && frame[center].Blue <= Object2RGBUpper[2])
{
//orange center of circle
Console.WriteLine("onion");
if (RecognizedObjects.Contains("onion") == false)
RecognizedObjects.Add("onion");
Projected.TimeSinceLastSeenOnion = Projected.TimeElapsed;
}
else
{
RecognizedObjects.Remove("onion");
}
if (frame[center].Red > PuckRGBLower[0] && frame[center].Red <= PuckRGBUpper[0] &&
frame[center].Green > PuckRGBLower[1] && frame[center].Green <= PuckRGBUpper[1] &&
frame[center].Blue > PuckRGBLower[2] && frame[center].Blue <= PuckRGBUpper[2])
{
//orange center of circle
Console.WriteLine("puck");
if (RecognizedObjects.Contains("puck") == false)
{
//RecognizedObjects.Add("puck"); //--don't add it to the object list, ruins the count.
}
if (Program.VerticallPuck)
{
PuckY = circle.Center.X;
}
else
{
PuckY = circle.Center.Y;
}
Projected.TimeSinceLastSeenPuck = Projected.TimeElapsed;
}
else
{
RecognizedObjects.Remove("puck");
}
}
}
else
{
PuckY = -1;
}
#endregion
DisplayPicture(ConvertToPXCMImage(frame.Bitmap), gesture);
DisplayGeoNodes(gesture);
DisplayGesture(gesture);
form.UpdatePanel();
}
pp.ReleaseFrame();
}
}
else
{
form.UpdateStatus("Init Failed");
sts = false;
}
pp.Close();
pp.Dispose();
if (sts) form.UpdateStatus("Stopped");
}
private void DoRendering()
{
_isRunning = true;
/* UtilMPipeline works best for synchronous color and depth streaming */
_pp = new UtilMPipeline();
/* Set Input Source */
_pp.capture.SetFilter("DepthSense Device 325V2");
/* Set Color & Depth Resolution */
PXCMCapture.VideoStream.ProfileInfo cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
_pp.capture.SetFilter(ref cinfo); // only needed to set FPS
PXCMCapture.VideoStream.ProfileInfo dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
_pp.capture.SetFilter(ref dinfo2); // only needed to set FPS
/* Initialization */
if (!_pp.Init())
{
LogFormat("Could not initialize Senz3D hardware");
HasErrorState = true;
return;
}
var capture = _pp.capture;
_device = capture.device;
_device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);
while (_isRunning)
{
/* If raw depth is needed, disable smoothing */
_pp.capture.device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, DepthSmoothing ? 1 : 0);
/* Wait until a frame is ready */
if (!_pp.AcquireFrame(true)) break;
if (_pp.IsDisconnected()) break;
/* Get RGB color image */
Stopwatch sw = Stopwatch.StartNew();
var color = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_COLOR);
var colorBitmap = Senz3DUtils.GetRgb32Pixels(color);
var colorImage = new Image<Rgb, byte>(colorBitmap);
var colorImageCopy = colorImage.Copy();
ColorImageFrameTime = sw.ElapsedMilliseconds;
/* Get depth image */
sw.Restart();
var depth = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
var depthImageAndConfidence = Senz3DUtils.GetHighPrecisionDepthImage(depth, MinDepthValue, MaxDepthValue);
var depthImage = (Image<Gray, float>)depthImageAndConfidence[0];
var depthImageCopy = depthImage.Copy();
var confidenceMapImage = (Image<Rgb, Byte>)depthImageAndConfidence[1];
var confidenceMapImageCopy = confidenceMapImage.Copy();
DepthImageFrameTime = sw.ElapsedMilliseconds;
ConfidenceMapImageFrameTime = 0;
bool getRgbInDepthROI = false;
/* if rgbInDepthROI is undefined get uvmap and rgbofdepth and rgbInDepthROI */
if (_rgbInDepthROI.Left == 0 && _rgbInDepthROI.Right == 0 && _rgbInDepthROI.Width == 0 &&
_rgbInDepthROI.Height == 0)
{
getRgbInDepthROI = true;
}
/* Get UV map */
Image<Rgb, float> uvMapImage, uvMapImageCopy;
if (UvMapChecked || getRgbInDepthROI)
{
sw.Restart();
uvMapImage = Senz3DUtils.GetDepthUvMap(depth);
uvMapImageCopy = uvMapImage.Copy();
UVMapImageFrameTime = sw.ElapsedMilliseconds;
}
else
{
uvMapImage = null;
uvMapImageCopy = null;
UVMapImageFrameTime = -1;
}
/* Get RgbOfDepth */
Image<Rgb, byte> rgbOfDepthImage, rgbOfDepthImageCopy;
if ((RgbOfDepthChecked && uvMapImage != null) || getRgbInDepthROI)
{
sw.Restart();
if (getRgbInDepthROI)
{
rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage, true, ref _rgbInDepthROI);
Stage(new ROI(this, "rgbInDepthROI")
{
RoiRectangle = _rgbInDepthROI
});
Push();
LogFormat("Identified rgbInDepthROI as {0}", _rgbInDepthROI);
}
else
{
rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage);
}
rgbOfDepthImageCopy = rgbOfDepthImage.Copy();
RgbOfDepthImageFrameTime = sw.ElapsedMilliseconds;
}
else
{
rgbOfDepthImage = null;
rgbOfDepthImageCopy = null;
RgbOfDepthImageFrameTime = -1;
}
/* Get DepthOfRGB */
Image<Gray, float> depthOfRgbImage, depthOfRgbImageCopy;
if (DepthOfRgbChecked && uvMapImage != null)
{
sw.Restart();
depthOfRgbImage = Senz3DUtils.GetDepthOfRGBPixels(depthImage, colorImage, uvMapImage);
depthOfRgbImageCopy = depthOfRgbImage.Copy();
DepthOfRgbImageFrameTime = sw.ElapsedMilliseconds;
}
else
{
depthOfRgbImage = null;
depthOfRgbImageCopy = null;
DepthOfRgbImageFrameTime = -1;
}
_pp.ReleaseFrame();
if (IsRenderContent)
{
Task.Factory.StartNew(() =>
{
var bitmap = colorImageCopy.ToBitmapSource(true);
colorImageCopy.Dispose();
return bitmap;
}).ContinueWith(s => ColorImageSource = s.Result);
Task.Factory.StartNew(() =>
{
var bitmap = depthImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
depthImageCopy.Dispose();
return bitmap;
}).ContinueWith(s => DepthImageSource = s.Result);
Task.Factory.StartNew(() =>
{
var bitmap = confidenceMapImageCopy.ToBitmapSource(true);
confidenceMapImageCopy.Dispose();
return bitmap;
}).ContinueWith(s => ConfidenceMapImageSource = s.Result);
/* draw uvmap */
if (uvMapImage != null)
Task.Factory.StartNew(() =>
{
var bitmap = uvMapImageCopy.ToBitmapSource(true);
uvMapImageCopy.Dispose();
return bitmap;
}).ContinueWith(s => UVMapImageSource = s.Result);
/* draw rgbofdepth */
if (rgbOfDepthImage != null)
{
Task.Factory.StartNew(() =>
{
var bitmap = rgbOfDepthImageCopy.ToBitmapSource(true);
rgbOfDepthImageCopy.Dispose();
return bitmap;
}).ContinueWith(s => RgbOfDepthImageSource = s.Result);
}
/* draw depthofrgb */
if (depthOfRgbImage != null)
Task.Factory.StartNew(() =>
{
var bitmap = depthOfRgbImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
depthOfRgbImageCopy.Dispose();
return bitmap;
}).ContinueWith(s => DepthOfRgbImageSource = s.Result);
}
var dc = new DataContainer(++_frameId, DateTime.Now)
{
new RgbImageData(this, "color", colorImage),
new GrayFloatImage(this, "depth", depthImage),
new RgbImageData(this, "confidence", confidenceMapImage),
};
if (uvMapImage != null) dc.Add(new RgbFloatImage(this, "uvmap", uvMapImage));
if (rgbOfDepthImage != null) dc.Add(new RgbImageData(this, "rgbofdepth", rgbOfDepthImage));
if (depthOfRgbImage != null) dc.Add(new GrayFloatImage(this, "depthofrgb", depthOfRgbImage));
Publish(dc);
}
_pp.Close();
_pp.Dispose();
}
public void SimplePipeline()
{
bool sts = true;
UtilMPipeline pp = null;
disconnected = false;
/* Set Source */
if (form.GetRecordState())
{
pp = new UtilMPipeline(form.GetRecordFile(), true);
pp.QueryCapture().SetFilter(form.GetCheckedDevice());
}
else if (form.GetPlaybackState())
{
pp = new UtilMPipeline(form.GetPlaybackFile(), false);
}
else
{
pp = new UtilMPipeline();
pp.QueryCapture().SetFilter(form.GetCheckedDevice());
}
/* Set Module */
pp.EnableGesture(form.GetCheckedModule());
/* Initialization */
form.UpdateStatus("Init Started");
if (pp.Init())
{
form.UpdateStatus("Streaming");
while (!form.stop)
{
if (!pp.AcquireFrame(true))
{
break;
}
if (!DisplayDeviceConnection(pp.IsDisconnected()))
{
/* Display Results */
PXCMGesture gesture = pp.QueryGesture();
PXCMImage depth = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
DisplayPicture(depth, gesture);
DisplayGeoNodes(gesture);
DisplayGesture(gesture);
form.UpdatePanel();
}
pp.ReleaseFrame();
}
}
else
{
form.UpdateStatus("Init Failed");
sts = false;
}
pp.Close();
pp.Dispose();
if (sts)
{
form.UpdateStatus("Stopped");
}
}
public void SimplePipeline()
{
bool sts = true;
UtilMPipeline pp = null;
disconnected = false;
/* Set Source */
if (form.GetRecordState())
{
pp = new UtilMPipeline(form.GetRecordFile(), true);
pp.QueryCapture().SetFilter(form.GetCheckedDevice());
}
else if (form.GetPlaybackState())
{
pp = new UtilMPipeline(form.GetPlaybackFile(), false);
}
else
{
pp = new UtilMPipeline();
pp.QueryCapture().SetFilter(form.GetCheckedDevice());
}
/* Set Module */
pp.EnableGesture(form.GetCheckedModule());
/* Initialization */
form.UpdateStatus("Init Started");
if (pp.Init())
{
form.UpdateStatus("Streaming");
while (!form.stop)
{
if (!pp.AcquireFrame(true)) break;
if (!DisplayDeviceConnection(pp.IsDisconnected()))
{
/* Display Results */
PXCMGesture gesture = pp.QueryGesture();
PXCMImage depth = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
DisplayPicture(depth, gesture);
DisplayGeoNodes(gesture);
DisplayGesture(gesture);
form.UpdatePanel();
}
pp.ReleaseFrame();
}
}
else
{
form.UpdateStatus("Init Failed");
sts = false;
}
pp.Close();
pp.Dispose();
if (sts) form.UpdateStatus("Stopped");
}
private void DoRendering()
{
_isRunning = true;
/* UtilMPipeline works best for synchronous color and depth streaming */
_pp = new UtilMPipeline();
/* Set Input Source */
_pp.capture.SetFilter("DepthSense Device 325V2");
/* Set Color & Depth Resolution */
PXCMCapture.VideoStream.ProfileInfo cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
_pp.capture.SetFilter(ref cinfo); // only needed to set FPS
PXCMCapture.VideoStream.ProfileInfo dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
_pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
_pp.capture.SetFilter(ref dinfo2); // only needed to set FPS
/* Initialization */
if (!_pp.Init())
{
LogFormat("Could not initialize Senz3D hardware");
HasErrorState = true;
return;
}
var capture = _pp.capture;
_device = capture.device;
_device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
_device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);
while (_isRunning)
{
/* If raw depth is needed, disable smoothing */
_pp.capture.device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, DepthSmoothing ? 1 : 0);
/* Wait until a frame is ready */
if (!_pp.AcquireFrame(true))
{
break;
}
if (_pp.IsDisconnected())
{
break;
}
/* Get RGB color image */
Stopwatch sw = Stopwatch.StartNew();
var color = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_COLOR);
var colorBitmap = Senz3DUtils.GetRgb32Pixels(color);
var colorImage = new Image <Rgb, byte>(colorBitmap);
var colorImageCopy = colorImage.Copy();
ColorImageFrameTime = sw.ElapsedMilliseconds;
/* Get depth image */
sw.Restart();
var depth = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
var depthImageAndConfidence = Senz3DUtils.GetHighPrecisionDepthImage(depth, MinDepthValue, MaxDepthValue);
var depthImage = (Image <Gray, float>)depthImageAndConfidence[0];
var depthImageCopy = depthImage.Copy();
var confidenceMapImage = (Image <Rgb, Byte>)depthImageAndConfidence[1];
var confidenceMapImageCopy = confidenceMapImage.Copy();
DepthImageFrameTime = sw.ElapsedMilliseconds;
ConfidenceMapImageFrameTime = 0;
bool getRgbInDepthROI = false;
/* if rgbInDepthROI is undefined get uvmap and rgbofdepth and rgbInDepthROI */
if (_rgbInDepthROI.Left == 0 && _rgbInDepthROI.Right == 0 && _rgbInDepthROI.Width == 0 &&
_rgbInDepthROI.Height == 0)
{
getRgbInDepthROI = true;
}
/* Get UV map */
Image <Rgb, float> uvMapImage, uvMapImageCopy;
if (UvMapChecked || getRgbInDepthROI)
{
sw.Restart();
uvMapImage = Senz3DUtils.GetDepthUvMap(depth);
uvMapImageCopy = uvMapImage.Copy();
UVMapImageFrameTime = sw.ElapsedMilliseconds;
}
else
{
uvMapImage = null;
uvMapImageCopy = null;
UVMapImageFrameTime = -1;
}
/* Get RgbOfDepth */
Image <Rgb, byte> rgbOfDepthImage, rgbOfDepthImageCopy;
if ((RgbOfDepthChecked && uvMapImage != null) || getRgbInDepthROI)
{
sw.Restart();
if (getRgbInDepthROI)
{
rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage, true, ref _rgbInDepthROI);
Stage(new ROI(this, "rgbInDepthROI")
{
RoiRectangle = _rgbInDepthROI
});
Push();
LogFormat("Identified rgbInDepthROI as {0}", _rgbInDepthROI);
}
else
{
rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage);
}
rgbOfDepthImageCopy = rgbOfDepthImage.Copy();
RgbOfDepthImageFrameTime = sw.ElapsedMilliseconds;
}
else
{
rgbOfDepthImage = null;
rgbOfDepthImageCopy = null;
RgbOfDepthImageFrameTime = -1;
}
/* Get DepthOfRGB */
Image <Gray, float> depthOfRgbImage, depthOfRgbImageCopy;
if (DepthOfRgbChecked && uvMapImage != null)
{
sw.Restart();
depthOfRgbImage = Senz3DUtils.GetDepthOfRGBPixels(depthImage, colorImage, uvMapImage);
depthOfRgbImageCopy = depthOfRgbImage.Copy();
DepthOfRgbImageFrameTime = sw.ElapsedMilliseconds;
}
else
{
depthOfRgbImage = null;
depthOfRgbImageCopy = null;
DepthOfRgbImageFrameTime = -1;
}
_pp.ReleaseFrame();
if (IsRenderContent)
{
Task.Factory.StartNew(() =>
{
var bitmap = colorImageCopy.ToBitmapSource(true);
colorImageCopy.Dispose();
return(bitmap);
}).ContinueWith(s => ColorImageSource = s.Result);
Task.Factory.StartNew(() =>
{
var bitmap = depthImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
depthImageCopy.Dispose();
return(bitmap);
}).ContinueWith(s => DepthImageSource = s.Result);
Task.Factory.StartNew(() =>
{
var bitmap = confidenceMapImageCopy.ToBitmapSource(true);
confidenceMapImageCopy.Dispose();
return(bitmap);
}).ContinueWith(s => ConfidenceMapImageSource = s.Result);
/* draw uvmap */
if (uvMapImage != null)
{
Task.Factory.StartNew(() =>
{
var bitmap = uvMapImageCopy.ToBitmapSource(true);
uvMapImageCopy.Dispose();
return(bitmap);
}).ContinueWith(s => UVMapImageSource = s.Result);
}
/* draw rgbofdepth */
if (rgbOfDepthImage != null)
{
Task.Factory.StartNew(() =>
{
var bitmap = rgbOfDepthImageCopy.ToBitmapSource(true);
rgbOfDepthImageCopy.Dispose();
return(bitmap);
}).ContinueWith(s => RgbOfDepthImageSource = s.Result);
}
/* draw depthofrgb */
if (depthOfRgbImage != null)
{
Task.Factory.StartNew(() =>
{
var bitmap = depthOfRgbImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
depthOfRgbImageCopy.Dispose();
return(bitmap);
}).ContinueWith(s => DepthOfRgbImageSource = s.Result);
}
}
var dc = new DataContainer(++_frameId, DateTime.Now)
{
new RgbImageData(this, "color", colorImage),
new GrayFloatImage(this, "depth", depthImage),
new RgbImageData(this, "confidence", confidenceMapImage),
};
if (uvMapImage != null)
{
dc.Add(new RgbFloatImage(this, "uvmap", uvMapImage));
}
if (rgbOfDepthImage != null)
{
dc.Add(new RgbImageData(this, "rgbofdepth", rgbOfDepthImage));
}
if (depthOfRgbImage != null)
{
dc.Add(new GrayFloatImage(this, "depthofrgb", depthOfRgbImage));
}
Publish(dc);
}
_pp.Close();
_pp.Dispose();
}
public GesturePipeline(Form1 form)
: base()
{
//------------------------------
// Creates datastructures for
// the PCSDK
//------------------------------
UtilMPipeline pp; //The pipeline
PXCMGesture.Blob bdata; //The intrepred blobs
PXCMImage bimage; //The frame from the camera
PXCMImage.ImageData data; //The data contained in the frame.
PXCMImage.ImageInfo info; //Metadata about the frame.
int color; //The "color" for the hand in question.
//------------------------------
//------------------------------
// Initilize the datastructures
// defined above.
//------------------------------
Coord[] locations;
pp = new UtilMPipeline();
pp.EnableGesture();
pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
//-----------------------------
//initilize the pointer for the WMC window.
IntPtr handle = GetDesktopWindow();
//----------------------------------
// Create and initilize the bitmaps
// that the frames will be comapred
// against.
//---------------------------------
Bitmap pause = convertBitmap(KwisStandalone.Properties.Resources.STOP);
Bitmap fastForward = convertBitmap(KwisStandalone.Properties.Resources.FAST_FORWARD);
Bitmap rewind = convertBitmap(KwisStandalone.Properties.Resources.REWIND);
Bitmap play = convertBitmap(KwisStandalone.Properties.Resources.PLAY);
Bitmap menu = convertBitmap(KwisStandalone.Properties.Resources.MENU);
Bitmap volUp = convertBitmap(KwisStandalone.Properties.Resources.VOL_UP);
Bitmap volDown = convertBitmap(KwisStandalone.Properties.Resources.VOL_DOWN);
//--------------------------------
// initial the template matching algorithm.
templateMatching = new ExhaustiveTemplateMatching(0.75f);
/* If we were able to aquire a frame from the camera,
* start the main application logic.
* */
if (pp.Init())
{
//Loop forever as long as the form is still open.
while (!form.closing)
{
if (!pp.AcquireFrame(true))
{
MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OKCancel, MessageBoxIcon.Asterisk);
break;
}
//Aquire the gesture data for the frame.
using (PXCMGesture gesture = pp.QueryGesture())
{
//Load gesture data into respective data structures.
gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);
gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);
bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ, out data);
info = bimage.imageInfo;
/* Although this says left hand,
* this actually aquires the right hand.
* This is from trial and error and contradicts
* both the documentation and common sense. A bug
* in the SDK?
* */
color = (int)bdata.labelLeftHand;
//Store the pixels that include the hand specified above.
locations = new Coord[info.width * info.height];
handLocation(ref locations, data, (int)info.width);
//create a new bitmap from this data.
Bitmap bitmap = createBitmap(ref locations, (int)info.width, (int)info.height);
/* Zero is the left hand (actuall right). So, if we detected this,
* then lets see if it matches any of the stored reference images.
* */
if (color == 0)
{
try
{
if (compare(ref bitmap, ref pause))
{
clickAndSendKey(RemoteKey.PLAY_PAUSE);
}
else if (compare(ref bitmap, ref play))
{
clickAndSendKey(RemoteKey.PLAY);
}
else if (compare(ref bitmap, ref fastForward))
{
clickAndSendKey(RemoteKey.FAST_FORWARD);
}
else if (compare(ref bitmap, ref rewind))
{
clickAndSendKey(APPCOMMAND_MEDIA_REWIND, handle);
}
else if (compare(ref bitmap, ref menu))
{
clickAndSendKey(RemoteKey.STOP);
}
else if (compare(ref bitmap, ref volUp))
{
clickAndSendKey(RemoteKey.VOL_UP);
}
else if (compare(ref bitmap, ref volDown))
{
clickAndSendKey(RemoteKey.VOL_DOWN);
}
}
catch (InvalidImagePropertiesException iipe)
{
}
}
//Clean up the datastructures in prep for a new frame.
bitmap.Dispose();
bimage.ReleaseAccess(ref data);
bimage.Dispose();
gesture.Dispose();
}
pp.ReleaseFrame();
}
}
}
