Ejemplo n.º 1
0
        //////////////////////////////////////////////////
        #region Private Methods
        void buildPipeline(string deviceName, string gestureModuleName)
        {
            _pipeline = new UtilMPipeline();
            _pipeline.QueryCapture().SetFilter(deviceName);
            _pipeline.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB24);
            _pipeline.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_VERTICES);
            _pipeline.EnableGesture(gestureModuleName);
            if (!_pipeline.Init())
            {
                throw new GestureCameraException("Failed to initialize pipeline.");
            }

            float unit;

            GestureCameraUtil.Assert(
                _pipeline.QueryCapture().device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_UNIT, out unit),
                "Failed to query sensor depth unit.");
            _unit_m = unit / 1000000;

            GestureCameraUtil.Assert(
                _pipeline.QueryCapture().device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out _depthLowConf),
                "Failed to query sensor depth low confidence value.");
            GestureCameraUtil.Assert(
                _pipeline.QueryCapture().device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out _depthSaturation),
                "Failed to query sensor depth saturation value.");
        }
Ejemplo n.º 2
0
        public CaptureHand(CaptureForm form) : base()
        {
            pp = new UtilMPipeline();
            pp.EnableGesture();
            pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);



            if (pp.Init())
            {
                pp.QueryCapture().SetFilter(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, 100);
                do
                {
                    if (!pp.AcquireFrame(true))
                    {
                        MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
                        break;
                    }

                    gesture = pp.QueryGesture();


                    gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);

                    gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);



                    bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ, out data);
                    info = bimage.imageInfo;


                    color = (int)bdata.labelLeftHand;


                    form.setTextBox(color.ToString());


                    pp.ReleaseFrame();
                } while (color == -1);

                Coord[] locations = handLocation(data, (int)(info.width * info.height), (int)info.width);
                bitmap = createBitmap(locations, (int)info.width, (int)info.height);
                form.setImage(bitmap);
                form.bitmap = bitmap;
            }
        }
Ejemplo n.º 3
0
        public CaptureHand(CaptureForm form)
            : base()
        {
            pp = new UtilMPipeline();
            pp.EnableGesture();
            pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);

            if (pp.Init())
            {

                pp.QueryCapture().SetFilter(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, 100);
                do
                {
                    if (!pp.AcquireFrame(true))
                    {
                        MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OK, MessageBoxIcon.Asterisk);
                        break;
                    }

                    gesture =  pp.QueryGesture();

                    gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);

                    gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);

                    bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ,out  data);
                    info = bimage.imageInfo;

                    color =(int) bdata.labelLeftHand;

                    form.setTextBox(color.ToString());

                    pp.ReleaseFrame();

                } while (color == -1);

                Coord[] locations = handLocation(data, (int)(info.width * info.height),(int) info.width);
                bitmap = createBitmap(locations, (int)info.width, (int)info.height);
                form.setImage(bitmap);
                form.bitmap = bitmap;

            }
        }
Ejemplo n.º 4
0
        /// <summary>
        /// Initializes the camera and returns true on success, false otherwise.
        /// </summary>
        /// <returns>Initialization successful (true) or failed (false)</returns>
        private bool InitializeCamera()
        {
            /* UtilMPipeline works best for synchronous color and depth streaming */
            _pp = new UtilMPipeline();

            /* Set Input Source */
            _pp.capture.SetFilter("DepthSense Device 325V2");

            /* Set Color & Depth Resolution */
            var cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);

            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
            _pp.capture.SetFilter(ref cinfo); // only needed to set FPS

            var dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);

            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
            _pp.capture.SetFilter(ref dinfo2); // only needed to set FPS

            /* Initialization */
            if (!_pp.Init())
            {
                LogFormat("Could not initialize Senz3D hardware");
                HasErrorState = true;
                return(false);
            }

            var capture = _pp.capture;

            _device = capture.device;
            _device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);

            return(true);
        }
        public unsafe MainWindow()
        {
            InitializeComponent();
            InitializeServer();

            this.Closing += new CancelEventHandler(OnWindowClosing);

            System.Windows.Threading.Dispatcher uiDispatcher = this.Dispatcher;

            UtilMPipeline pp = new UtilMPipeline();

            BackgroundWorker worker = new BackgroundWorker();

            worker.DoWork += delegate(object s, DoWorkEventArgs args)
            {
                pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
                pp.Init();

                for (; ; )
                {
                    if (!pp.AcquireFrame(true)) break;
                    {
                        PXCMImage image = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);

                        PXCMImage.ImageData ddata;

                        pxcmStatus sts = image.AcquireAccess(PXCMImage.Access.ACCESS_READ, out ddata);
                        if (sts < pxcmStatus.PXCM_STATUS_NO_ERROR) break;
                        short* depth = (short*)ddata.buffer.planes[0];

                        short minZ = 32001;
                        int minX = 0;
                        int minY = 0;
                        short maxX = 320;

                        for (int y = 0; y < 240; y++)
                        {

                            for (int x = 0; x < 320; x++)
                            {
                                if (depth[(y * 320) + x] < minZ)
                                {
                                    minZ = depth[(y * 320) + x];
                                    minX = x;
                                    minY = y;
                                }

                            }
                        }

                        /////////////////////
                        //normalize extremes//
                        if (minZ < DistanceFromScreenEdge) minZ = DistanceFromScreenEdge;
                        //match origin//
                        minX -= 160;
                        minY -= 120;
                        /////////////////////
                        maxX = (short)(Math.Tan(FOV / 2) * minZ);
                        short discardedX = (short)(Math.Tan(FOV / 2) * (minZ - DistanceFromScreenEdge));
                        short screenMaxX = (short)(maxX - discardedX);
                        float xRatio = minX / screenMaxX;

                        UpdateGridDelegate update = new UpdateGridDelegate(UpdateGrid);
                        uiDispatcher.BeginInvoke(update, minX, minY, minZ, maxX, discardedX, screenMaxX, xRatio);

                        image.ReleaseAccess(ref ddata);
                        pp.ReleaseFrame();
                    }
                }

                pp.Close();
                pp.Dispose();
            };

            worker.RunWorkerAsync();
        }
Ejemplo n.º 6
0
        /// <summary>
        /// Initializes the camera and returns true on success, false otherwise.
        /// </summary>
        /// <returns>Initialization successful (true) or failed (false)</returns>
        private bool InitializeCamera()
        {
            /* UtilMPipeline works best for synchronous color and depth streaming */
            _pp = new UtilMPipeline();

            /* Set Input Source */
            _pp.capture.SetFilter("DepthSense Device 325V2");

            /* Set Color & Depth Resolution */
            var cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
            _pp.capture.SetFilter(ref cinfo); // only needed to set FPS

            var dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
            _pp.capture.SetFilter(ref dinfo2); // only needed to set FPS

            /* Initialization */
            if (!_pp.Init())
            {
                LogFormat("Could not initialize Senz3D hardware");
                HasErrorState = true;
                return false;
            }

            var capture = _pp.capture;
            _device = capture.device;
            _device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);

            return true;
        }
Ejemplo n.º 7
0
        public GesturePipeline(Form1 form)
            : base()
        {
            //------------------------------
            // Creates datastructures for
            // the PCSDK
            //------------------------------
            UtilMPipeline pp; //The pipeline

            PXCMGesture.Blob bdata; //The intrepred blobs
            PXCMImage bimage; //The frame from the camera
            PXCMImage.ImageData data; //The data contained in the frame.
            PXCMImage.ImageInfo info; //Metadata about the frame.
            int color; //The "color" for the hand in question.
            //------------------------------

            //------------------------------
            // Initilize the datastructures
            // defined above.
            //------------------------------
            Coord[] locations;
            pp = new UtilMPipeline();
            pp.EnableGesture();
            pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
            //-----------------------------

            //initilize the pointer for the WMC window.
            IntPtr handle = GetDesktopWindow();

            //----------------------------------
            // Create and initilize the bitmaps
            // that the frames will be comapred
            // against.
            //---------------------------------
            Bitmap pause = convertBitmap(KwisStandalone.Properties.Resources.STOP);
            Bitmap fastForward = convertBitmap(KwisStandalone.Properties.Resources.FAST_FORWARD);
            Bitmap rewind = convertBitmap(KwisStandalone.Properties.Resources.REWIND);
            Bitmap play = convertBitmap(KwisStandalone.Properties.Resources.PLAY);
            Bitmap menu = convertBitmap(KwisStandalone.Properties.Resources.MENU);
            Bitmap volUp = convertBitmap(KwisStandalone.Properties.Resources.VOL_UP);
            Bitmap volDown = convertBitmap(KwisStandalone.Properties.Resources.VOL_DOWN);
            //--------------------------------

            // initial the template matching algorithm.
            templateMatching = new ExhaustiveTemplateMatching(0.75f);

            /* If we were able to aquire a frame from the camera,
             * start the main application logic.
             * */
            if (pp.Init())
            {

                //Loop forever as long as the form is still open.
                while (!form.closing)
                {
                    if (!pp.AcquireFrame(true))
                    {
                        MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OKCancel, MessageBoxIcon.Asterisk);
                        break;
                    }

                    //Aquire the gesture data for the frame.
                    using (PXCMGesture gesture = pp.QueryGesture())
                    {

                        //Load gesture data into respective data structures.
                        gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);
                        gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);
                        bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ, out  data);
                        info = bimage.imageInfo;

                        /* Although this says left hand,
                         * this actually aquires the right hand.
                         * This is from trial and error and contradicts
                         * both the documentation and common sense.  A bug
                         * in the SDK?
                         * */
                        color = (int)bdata.labelLeftHand;

                        //Store the pixels that include the hand specified above.
                        locations = new Coord[info.width * info.height];
                        handLocation(ref locations, data, (int)info.width);

                        //create a new bitmap from this data.
                        Bitmap bitmap = createBitmap(ref locations, (int)info.width, (int)info.height);

                        /* Zero is the left hand (actuall right).  So, if we detected this,
                         * then lets see if it matches any of the stored reference images.
                         * */
                        if (color == 0)
                        {

                            try
                            {
                                if (compare(ref bitmap, ref  pause))
                                {
                                    clickAndSendKey(RemoteKey.PLAY_PAUSE);

                                }
                                else if (compare(ref bitmap, ref play))
                                {

                                    clickAndSendKey(RemoteKey.PLAY);
                                }
                                else if (compare(ref bitmap, ref fastForward))
                                {

                                    clickAndSendKey(RemoteKey.FAST_FORWARD);
                                }

                                else if (compare(ref bitmap, ref rewind))
                                {
                                    clickAndSendKey(APPCOMMAND_MEDIA_REWIND, handle);
                                }
                                else if (compare(ref bitmap, ref menu))
                                {

                                    clickAndSendKey(RemoteKey.STOP);
                                }
                                else if (compare(ref bitmap, ref volUp))
                                {
                                   clickAndSendKey(RemoteKey.VOL_UP);
                                }
                                else if (compare(ref bitmap, ref volDown))
                                {

                                    clickAndSendKey(RemoteKey.VOL_DOWN);
                                }

                            }
                            catch (InvalidImagePropertiesException iipe)
                            {

                            }

                        }

                        //Clean up the datastructures in prep for a new frame.
                        bitmap.Dispose();
                        bimage.ReleaseAccess(ref data);
                        bimage.Dispose();
                        gesture.Dispose();
                    }

                    pp.ReleaseFrame();
                }
            }
        }
        public void SimplePipeline()
        {
            bool sts = true;
            UtilMPipeline pp = null;
            disconnected = false;

            /* Set Source */
            if (form.GetRecordState()) {
                pp = new UtilMPipeline(form.GetRecordFile(), true);
                pp.QueryCapture().SetFilter(form.GetCheckedDevice());
            }
            else if (form.GetPlaybackState())
            {
                pp = new UtilMPipeline(form.GetPlaybackFile(), false);
            }
            else
            {
                pp = new UtilMPipeline();
                pp.QueryCapture().SetFilter(form.GetCheckedDevice());
            }

            /* Set Module */
            pp.EnableFaceLocation();
            pp.EnableFaceLandmark();
            pp.EnableGesture(form.GetCheckedModule());

            /* Initialization */
            form.UpdateStatus("Init Started");
            if (pp.Init())
            {
                form.UpdateStatus("Streaming");

                while (!form.stop)
                {
                    if (!pp.AcquireFrame(true)) break;
                    if (!DisplayDeviceConnection(pp.IsDisconnected()))
                    {
                        /* Display Results */
                        PXCMGesture gesture = pp.QueryGesture();
                        PXCMImage img = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_COLOR);

                        Bitmap bmp = ConvertImageToBitmap(img);

                        #region EMGU Stuff
                        Image<Bgr, Byte> frame = new Image<Bgr, Byte>(bmp); //http://stackoverflow.com/questions/503427/the-type-initializer-for-emgu-cv-cvinvoke-threw-an-exception

                        Image<Gray, Byte> circleFrame = frame.Convert<Gray, Byte>().PyrDown().PyrUp();
                        Image<Gray, Byte> smallGrayFrame = circleFrame.PyrDown();
                        Image<Gray, Byte> smoothedGrayFrame = smallGrayFrame.PyrUp();
                        Image<Gray, Byte> cannyFrame = smoothedGrayFrame.Canny(100, 60);

                        //new Bgr(60, 90, 160), new Bgr(255, 255, 255) -- with light reflecting off the orange
                        //new Bgr(40, 90, 190), new Bgr(120, 190, 255)

                        //circleFrame = frame.InRange(new Bgr(100, 100, 100), new Bgr(255, 255, 255));               // somehow range for orange related colors

                        //circleFrame = frame.InRange(new Bgr(100, 110, 220), new Bgr(130, 140, 255));               // somehow range for orange related colors

                        //circleFrame = frame.InRange(new Bgr(100, 100, 100), new Bgr(255, 255, 255));               // somehow range for orange related colors

                        //circleFrame = circleFrame.SmoothGaussian(9);

                        double cannyThreshold = 180.0;
                        double circleAccumulatorThreshold = 40;

                        CircleF[] circles = circleFrame.HoughCircles(
                            new Gray(cannyThreshold),
                            new Gray(circleAccumulatorThreshold),
                            2.0, //Resolution of the accumulator used to detect centers of the circles
                            20.0, //min distance
                            18, //min radius
                            30 //max radius
                            )[0]; //Get the circles from the first channel\

                        /*
                         *
                         *
                         *
                         *
                         *
                         *
                         *
                         * Create a Timer
                         * Use the timer to space out the HoughCircles calls, so they aren't happening every frame
                         *
                         * Check for the puck more frequently
                         * Check for the fruits less frequently, and then only remove the info from the List after a certain time the fruit has been missing (like 50ms)
                         *
                         * Never do more than one HoughCircls call per frame.
                         *
                         *
                         *
                         *
                         *
                         */

                        Image<Bgr, Byte> circleImage = frame;
                        /*
                        foreach (CircleF circle in circles)
                        {
                            circleImage.Draw(circle, new Bgr(Color.Brown), 2);
                            Console.WriteLine(circle.Radius);
                        }
                        */
                        if (circles.Length > 0)
                        {
                            foreach (CircleF circle in circles)
                            {
                                circleImage.Draw(circle, new Bgr(Color.Brown), 2);

                                Point center = new Point((int)circle.Center.X, (int)circle.Center.Y);

                                if (frame[center].Red > Object1RGBLower[0] && frame[center].Red <= Object1RGBUpper[0] &&
                                    frame[center].Green > Object1RGBLower[1] && frame[center].Green <= Object1RGBUpper[1] &&
                                    frame[center].Blue > Object1RGBLower[2] && frame[center].Blue <= Object1RGBUpper[2])
                                {
                                    //orange center of circle
                                    Console.WriteLine("orange");

                                    if (RecognizedObjects.Contains("orange") == false)
                                        RecognizedObjects.Add("orange");

                                    Projected.TimeSinceLastSeenOrange = Projected.TimeElapsed;
                                }
                                else
                                {
                                    RecognizedObjects.Remove("orange");
                                }
                                if (frame[center].Red > Object2RGBLower[0] && frame[center].Red <= Object2RGBUpper[0] &&
                                    frame[center].Green > Object2RGBLower[1] && frame[center].Green <= Object2RGBUpper[1] &&
                                    frame[center].Blue > Object2RGBLower[2] && frame[center].Blue <= Object2RGBUpper[2])
                                {
                                    //orange center of circle
                                    Console.WriteLine("onion");

                                    if (RecognizedObjects.Contains("onion") == false)
                                        RecognizedObjects.Add("onion");

                                    Projected.TimeSinceLastSeenOnion = Projected.TimeElapsed;
                                }
                                else
                                {
                                    RecognizedObjects.Remove("onion");
                                }
                                if (frame[center].Red > PuckRGBLower[0] && frame[center].Red <= PuckRGBUpper[0] &&
                                    frame[center].Green > PuckRGBLower[1] && frame[center].Green <= PuckRGBUpper[1] &&
                                    frame[center].Blue > PuckRGBLower[2] && frame[center].Blue <= PuckRGBUpper[2])
                                {
                                    //orange center of circle
                                    Console.WriteLine("puck");

                                    if (RecognizedObjects.Contains("puck") == false)
                                    {
                                        //RecognizedObjects.Add("puck"); //--don't add it to the object list, ruins the count.
                                    }

                                    if (Program.VerticallPuck)
                                    {
                                        PuckY = circle.Center.X;
                                    }
                                    else
                                    {
                                        PuckY = circle.Center.Y;
                                    }

                                    Projected.TimeSinceLastSeenPuck = Projected.TimeElapsed;
                                }
                                else
                                {
                                    RecognizedObjects.Remove("puck");
                                }
                            }
                        }
                        else
                        {
                            PuckY = -1;
                        }
                        #endregion

                        DisplayPicture(ConvertToPXCMImage(frame.Bitmap), gesture);
                        DisplayGeoNodes(gesture);
                        DisplayGesture(gesture);
                        form.UpdatePanel();
                    }
                    pp.ReleaseFrame();
                }
            }
            else
            {
                form.UpdateStatus("Init Failed");
                sts = false;
            }

            pp.Close();
            pp.Dispose();
            if (sts) form.UpdateStatus("Stopped");
        }
Ejemplo n.º 9
0
        private void DoRendering()
        {
            _isRunning = true;

            /* UtilMPipeline works best for synchronous color and depth streaming */
            _pp = new UtilMPipeline();

            /* Set Input Source */
            _pp.capture.SetFilter("DepthSense Device 325V2");

            /* Set Color & Depth Resolution */
            PXCMCapture.VideoStream.ProfileInfo cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
            _pp.capture.SetFilter(ref cinfo); // only needed to set FPS

            PXCMCapture.VideoStream.ProfileInfo dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
            _pp.capture.SetFilter(ref dinfo2); // only needed to set FPS

            /* Initialization */
            if (!_pp.Init())
            {
                LogFormat("Could not initialize Senz3D hardware");
                HasErrorState = true;
                return;
            }

            var capture = _pp.capture;
            _device = capture.device;
            _device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);

            while (_isRunning)
            {
                /* If raw depth is needed, disable smoothing */
                _pp.capture.device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, DepthSmoothing ? 1 : 0);

                /* Wait until a frame is ready */
                if (!_pp.AcquireFrame(true)) break;
                if (_pp.IsDisconnected()) break;

                /* Get RGB color image */
                Stopwatch sw = Stopwatch.StartNew();
                var color = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_COLOR);
                var colorBitmap = Senz3DUtils.GetRgb32Pixels(color);
                var colorImage = new Image<Rgb, byte>(colorBitmap);
                var colorImageCopy = colorImage.Copy();
                ColorImageFrameTime = sw.ElapsedMilliseconds;

                /* Get depth image */
                sw.Restart();
                var depth = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
                var depthImageAndConfidence = Senz3DUtils.GetHighPrecisionDepthImage(depth, MinDepthValue, MaxDepthValue);
                var depthImage = (Image<Gray, float>)depthImageAndConfidence[0];
                var depthImageCopy = depthImage.Copy();
                var confidenceMapImage = (Image<Rgb, Byte>)depthImageAndConfidence[1];
                var confidenceMapImageCopy = confidenceMapImage.Copy();
                DepthImageFrameTime = sw.ElapsedMilliseconds;
                ConfidenceMapImageFrameTime = 0;

                bool getRgbInDepthROI = false;
                /* if rgbInDepthROI is undefined get uvmap and rgbofdepth and rgbInDepthROI */
                if (_rgbInDepthROI.Left == 0 && _rgbInDepthROI.Right == 0 && _rgbInDepthROI.Width == 0 &&
                    _rgbInDepthROI.Height == 0)
                {
                    getRgbInDepthROI = true;
                }

                /* Get UV map */
                Image<Rgb, float> uvMapImage, uvMapImageCopy;
                if (UvMapChecked || getRgbInDepthROI)
                {
                    sw.Restart();
                    uvMapImage = Senz3DUtils.GetDepthUvMap(depth);
                    uvMapImageCopy = uvMapImage.Copy();
                    UVMapImageFrameTime = sw.ElapsedMilliseconds;
                }
                else
                {
                    uvMapImage = null;
                    uvMapImageCopy = null;
                    UVMapImageFrameTime = -1;
                }

                /* Get RgbOfDepth */
                Image<Rgb, byte> rgbOfDepthImage, rgbOfDepthImageCopy;
                if ((RgbOfDepthChecked && uvMapImage != null) || getRgbInDepthROI)
                {
                    sw.Restart();
                    if (getRgbInDepthROI)
                    {
                        rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage, true, ref _rgbInDepthROI);
                        Stage(new ROI(this, "rgbInDepthROI")
                        {
                            RoiRectangle = _rgbInDepthROI
                        });
                        Push();

                        LogFormat("Identified rgbInDepthROI as {0}", _rgbInDepthROI);
                    }
                    else
                    {
                        rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage);
                    }

                    rgbOfDepthImageCopy = rgbOfDepthImage.Copy();
                    RgbOfDepthImageFrameTime = sw.ElapsedMilliseconds;
                }
                else
                {
                    rgbOfDepthImage = null;
                    rgbOfDepthImageCopy = null;
                    RgbOfDepthImageFrameTime = -1;
                }

                /* Get DepthOfRGB */
                Image<Gray, float> depthOfRgbImage, depthOfRgbImageCopy;
                if (DepthOfRgbChecked && uvMapImage != null)
                {
                    sw.Restart();
                    depthOfRgbImage = Senz3DUtils.GetDepthOfRGBPixels(depthImage, colorImage, uvMapImage);
                    depthOfRgbImageCopy = depthOfRgbImage.Copy();
                    DepthOfRgbImageFrameTime = sw.ElapsedMilliseconds;
                }
                else
                {
                    depthOfRgbImage = null;
                    depthOfRgbImageCopy = null;
                    DepthOfRgbImageFrameTime = -1;
                }

                _pp.ReleaseFrame();

                if (IsRenderContent)
                {
                    Task.Factory.StartNew(() =>
                    {
                        var bitmap = colorImageCopy.ToBitmapSource(true);
                        colorImageCopy.Dispose();
                        return bitmap;
                    }).ContinueWith(s => ColorImageSource = s.Result);

                    Task.Factory.StartNew(() =>
                    {
                        var bitmap = depthImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
                        depthImageCopy.Dispose();
                        return bitmap;
                    }).ContinueWith(s => DepthImageSource = s.Result);

                    Task.Factory.StartNew(() =>
                    {
                        var bitmap = confidenceMapImageCopy.ToBitmapSource(true);
                        confidenceMapImageCopy.Dispose();
                        return bitmap;
                    }).ContinueWith(s => ConfidenceMapImageSource = s.Result);

                    /* draw uvmap */
                    if (uvMapImage != null)
                        Task.Factory.StartNew(() =>
                            {
                                var bitmap = uvMapImageCopy.ToBitmapSource(true);
                                uvMapImageCopy.Dispose();
                                return bitmap;
                            }).ContinueWith(s => UVMapImageSource = s.Result);

                    /* draw rgbofdepth */
                    if (rgbOfDepthImage != null)
                    {
                        Task.Factory.StartNew(() =>
                        {
                            var bitmap = rgbOfDepthImageCopy.ToBitmapSource(true);
                            rgbOfDepthImageCopy.Dispose();
                            return bitmap;
                        }).ContinueWith(s => RgbOfDepthImageSource = s.Result);
                    }

                    /* draw depthofrgb */
                    if (depthOfRgbImage != null)
                        Task.Factory.StartNew(() =>
                        {
                            var bitmap = depthOfRgbImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
                            depthOfRgbImageCopy.Dispose();
                            return bitmap;
                        }).ContinueWith(s => DepthOfRgbImageSource = s.Result);
                }

                var dc = new DataContainer(++_frameId, DateTime.Now)
                    {
                        new RgbImageData(this, "color", colorImage),
                        new GrayFloatImage(this, "depth", depthImage),
                        new RgbImageData(this, "confidence", confidenceMapImage),
                    };

                if (uvMapImage != null) dc.Add(new RgbFloatImage(this, "uvmap", uvMapImage));
                if (rgbOfDepthImage != null) dc.Add(new RgbImageData(this, "rgbofdepth", rgbOfDepthImage));
                if (depthOfRgbImage != null) dc.Add(new GrayFloatImage(this, "depthofrgb", depthOfRgbImage));
                Publish(dc);
            }

            _pp.Close();
            _pp.Dispose();
        }
Ejemplo n.º 10
0
        public void SimplePipeline()
        {
            bool          sts = true;
            UtilMPipeline pp  = null;

            disconnected = false;

            /* Set Source */
            if (form.GetRecordState())
            {
                pp = new UtilMPipeline(form.GetRecordFile(), true);
                pp.QueryCapture().SetFilter(form.GetCheckedDevice());
            }
            else if (form.GetPlaybackState())
            {
                pp = new UtilMPipeline(form.GetPlaybackFile(), false);
            }
            else
            {
                pp = new UtilMPipeline();
                pp.QueryCapture().SetFilter(form.GetCheckedDevice());
            }

            /* Set Module */
            pp.EnableGesture(form.GetCheckedModule());

            /* Initialization */
            form.UpdateStatus("Init Started");
            if (pp.Init())
            {
                form.UpdateStatus("Streaming");

                while (!form.stop)
                {
                    if (!pp.AcquireFrame(true))
                    {
                        break;
                    }
                    if (!DisplayDeviceConnection(pp.IsDisconnected()))
                    {
                        /* Display Results */
                        PXCMGesture gesture = pp.QueryGesture();
                        PXCMImage   depth   = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
                        DisplayPicture(depth, gesture);
                        DisplayGeoNodes(gesture);
                        DisplayGesture(gesture);
                        form.UpdatePanel();
                    }
                    pp.ReleaseFrame();
                }
            }
            else
            {
                form.UpdateStatus("Init Failed");
                sts = false;
            }

            pp.Close();
            pp.Dispose();
            if (sts)
            {
                form.UpdateStatus("Stopped");
            }
        }
Ejemplo n.º 11
0
        public void SimplePipeline()
        {
            bool sts = true;
            UtilMPipeline pp = null;
            disconnected = false;

            /* Set Source */
            if (form.GetRecordState())
            {
                pp = new UtilMPipeline(form.GetRecordFile(), true);
                pp.QueryCapture().SetFilter(form.GetCheckedDevice());
            }
            else if (form.GetPlaybackState())
            {
                pp = new UtilMPipeline(form.GetPlaybackFile(), false);
            }
            else
            {
                pp = new UtilMPipeline();
                pp.QueryCapture().SetFilter(form.GetCheckedDevice());
            }

            /* Set Module */
            pp.EnableGesture(form.GetCheckedModule());

            /* Initialization */
            form.UpdateStatus("Init Started");
            if (pp.Init())
            {
                form.UpdateStatus("Streaming");

                while (!form.stop)
                {
                    if (!pp.AcquireFrame(true)) break;
                    if (!DisplayDeviceConnection(pp.IsDisconnected()))
                    {
                        /* Display Results */
                        PXCMGesture gesture = pp.QueryGesture();
                        PXCMImage depth = pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
                        DisplayPicture(depth, gesture);
                        DisplayGeoNodes(gesture);
                        DisplayGesture(gesture);
                        form.UpdatePanel();
                    }
                    pp.ReleaseFrame();
                }
            }
            else
            {
                form.UpdateStatus("Init Failed");
                sts = false;
            }

            pp.Close();
            pp.Dispose();
            if (sts) form.UpdateStatus("Stopped");
        }
Ejemplo n.º 12
0
        private void DoRendering()
        {
            _isRunning = true;

            /* UtilMPipeline works best for synchronous color and depth streaming */
            _pp = new UtilMPipeline();

            /* Set Input Source */
            _pp.capture.SetFilter("DepthSense Device 325V2");

            /* Set Color & Depth Resolution */
            PXCMCapture.VideoStream.ProfileInfo cinfo = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32);
            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_RGB32, cinfo.imageInfo.width, cinfo.imageInfo.height);
            _pp.capture.SetFilter(ref cinfo); // only needed to set FPS

            PXCMCapture.VideoStream.ProfileInfo dinfo2 = GetConfiguration(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
            _pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH, dinfo2.imageInfo.width, dinfo2.imageInfo.height);
            _pp.capture.SetFilter(ref dinfo2); // only needed to set FPS

            /* Initialization */
            if (!_pp.Init())
            {
                LogFormat("Could not initialize Senz3D hardware");
                HasErrorState = true;
                return;
            }

            var capture = _pp.capture;

            _device = capture.device;
            _device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_CONFIDENCE_THRESHOLD, DepthConfidenceThreshold);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_LOW_CONFIDENCE_VALUE, out EmguExtensions.LowConfidence);
            _device.QueryProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SATURATION_VALUE, out EmguExtensions.Saturation);

            while (_isRunning)
            {
                /* If raw depth is needed, disable smoothing */
                _pp.capture.device.SetProperty(PXCMCapture.Device.Property.PROPERTY_DEPTH_SMOOTHING, DepthSmoothing ? 1 : 0);

                /* Wait until a frame is ready */
                if (!_pp.AcquireFrame(true))
                {
                    break;
                }
                if (_pp.IsDisconnected())
                {
                    break;
                }

                /* Get RGB color image */
                Stopwatch sw             = Stopwatch.StartNew();
                var       color          = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_COLOR);
                var       colorBitmap    = Senz3DUtils.GetRgb32Pixels(color);
                var       colorImage     = new Image <Rgb, byte>(colorBitmap);
                var       colorImageCopy = colorImage.Copy();
                ColorImageFrameTime = sw.ElapsedMilliseconds;

                /* Get depth image */
                sw.Restart();
                var depth = _pp.QueryImage(PXCMImage.ImageType.IMAGE_TYPE_DEPTH);
                var depthImageAndConfidence = Senz3DUtils.GetHighPrecisionDepthImage(depth, MinDepthValue, MaxDepthValue);
                var depthImage             = (Image <Gray, float>)depthImageAndConfidence[0];
                var depthImageCopy         = depthImage.Copy();
                var confidenceMapImage     = (Image <Rgb, Byte>)depthImageAndConfidence[1];
                var confidenceMapImageCopy = confidenceMapImage.Copy();
                DepthImageFrameTime         = sw.ElapsedMilliseconds;
                ConfidenceMapImageFrameTime = 0;

                bool getRgbInDepthROI = false;
                /* if rgbInDepthROI is undefined get uvmap and rgbofdepth and rgbInDepthROI */
                if (_rgbInDepthROI.Left == 0 && _rgbInDepthROI.Right == 0 && _rgbInDepthROI.Width == 0 &&
                    _rgbInDepthROI.Height == 0)
                {
                    getRgbInDepthROI = true;
                }


                /* Get UV map */
                Image <Rgb, float> uvMapImage, uvMapImageCopy;
                if (UvMapChecked || getRgbInDepthROI)
                {
                    sw.Restart();
                    uvMapImage          = Senz3DUtils.GetDepthUvMap(depth);
                    uvMapImageCopy      = uvMapImage.Copy();
                    UVMapImageFrameTime = sw.ElapsedMilliseconds;
                }
                else
                {
                    uvMapImage          = null;
                    uvMapImageCopy      = null;
                    UVMapImageFrameTime = -1;
                }

                /* Get RgbOfDepth */
                Image <Rgb, byte> rgbOfDepthImage, rgbOfDepthImageCopy;
                if ((RgbOfDepthChecked && uvMapImage != null) || getRgbInDepthROI)
                {
                    sw.Restart();
                    if (getRgbInDepthROI)
                    {
                        rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage, true, ref _rgbInDepthROI);
                        Stage(new ROI(this, "rgbInDepthROI")
                        {
                            RoiRectangle = _rgbInDepthROI
                        });
                        Push();

                        LogFormat("Identified rgbInDepthROI as {0}", _rgbInDepthROI);
                    }
                    else
                    {
                        rgbOfDepthImage = Senz3DUtils.GetRgbOfDepthPixels(depthImage, colorImage, uvMapImage);
                    }

                    rgbOfDepthImageCopy      = rgbOfDepthImage.Copy();
                    RgbOfDepthImageFrameTime = sw.ElapsedMilliseconds;
                }
                else
                {
                    rgbOfDepthImage          = null;
                    rgbOfDepthImageCopy      = null;
                    RgbOfDepthImageFrameTime = -1;
                }

                /* Get DepthOfRGB */
                Image <Gray, float> depthOfRgbImage, depthOfRgbImageCopy;
                if (DepthOfRgbChecked && uvMapImage != null)
                {
                    sw.Restart();
                    depthOfRgbImage          = Senz3DUtils.GetDepthOfRGBPixels(depthImage, colorImage, uvMapImage);
                    depthOfRgbImageCopy      = depthOfRgbImage.Copy();
                    DepthOfRgbImageFrameTime = sw.ElapsedMilliseconds;
                }
                else
                {
                    depthOfRgbImage          = null;
                    depthOfRgbImageCopy      = null;
                    DepthOfRgbImageFrameTime = -1;
                }

                _pp.ReleaseFrame();

                if (IsRenderContent)
                {
                    Task.Factory.StartNew(() =>
                    {
                        var bitmap = colorImageCopy.ToBitmapSource(true);
                        colorImageCopy.Dispose();
                        return(bitmap);
                    }).ContinueWith(s => ColorImageSource = s.Result);

                    Task.Factory.StartNew(() =>
                    {
                        var bitmap = depthImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
                        depthImageCopy.Dispose();
                        return(bitmap);
                    }).ContinueWith(s => DepthImageSource = s.Result);

                    Task.Factory.StartNew(() =>
                    {
                        var bitmap = confidenceMapImageCopy.ToBitmapSource(true);
                        confidenceMapImageCopy.Dispose();
                        return(bitmap);
                    }).ContinueWith(s => ConfidenceMapImageSource = s.Result);

                    /* draw uvmap */
                    if (uvMapImage != null)
                    {
                        Task.Factory.StartNew(() =>
                        {
                            var bitmap = uvMapImageCopy.ToBitmapSource(true);
                            uvMapImageCopy.Dispose();
                            return(bitmap);
                        }).ContinueWith(s => UVMapImageSource = s.Result);
                    }

                    /* draw rgbofdepth */
                    if (rgbOfDepthImage != null)
                    {
                        Task.Factory.StartNew(() =>
                        {
                            var bitmap = rgbOfDepthImageCopy.ToBitmapSource(true);
                            rgbOfDepthImageCopy.Dispose();
                            return(bitmap);
                        }).ContinueWith(s => RgbOfDepthImageSource = s.Result);
                    }

                    /* draw depthofrgb */
                    if (depthOfRgbImage != null)
                    {
                        Task.Factory.StartNew(() =>
                        {
                            var bitmap = depthOfRgbImageCopy.ToGradientBitmapSource(true, EmguExtensions.LowConfidence, EmguExtensions.Saturation);
                            depthOfRgbImageCopy.Dispose();
                            return(bitmap);
                        }).ContinueWith(s => DepthOfRgbImageSource = s.Result);
                    }
                }

                var dc = new DataContainer(++_frameId, DateTime.Now)
                {
                    new RgbImageData(this, "color", colorImage),
                    new GrayFloatImage(this, "depth", depthImage),
                    new RgbImageData(this, "confidence", confidenceMapImage),
                };

                if (uvMapImage != null)
                {
                    dc.Add(new RgbFloatImage(this, "uvmap", uvMapImage));
                }
                if (rgbOfDepthImage != null)
                {
                    dc.Add(new RgbImageData(this, "rgbofdepth", rgbOfDepthImage));
                }
                if (depthOfRgbImage != null)
                {
                    dc.Add(new GrayFloatImage(this, "depthofrgb", depthOfRgbImage));
                }
                Publish(dc);
            }

            _pp.Close();
            _pp.Dispose();
        }
Ejemplo n.º 13
0
        public GesturePipeline(Form1 form)
            : base()
        {
            //------------------------------
            // Creates datastructures for
            // the PCSDK
            //------------------------------
            UtilMPipeline pp;         //The pipeline

            PXCMGesture.Blob bdata;   //The intrepred blobs
            PXCMImage        bimage;  //The frame from the camera

            PXCMImage.ImageData data; //The data contained in the frame.
            PXCMImage.ImageInfo info; //Metadata about the frame.
            int color;                //The "color" for the hand in question.

            //------------------------------


            //------------------------------
            // Initilize the datastructures
            // defined above.
            //------------------------------
            Coord[] locations;
            pp = new UtilMPipeline();
            pp.EnableGesture();
            pp.EnableImage(PXCMImage.ColorFormat.COLOR_FORMAT_DEPTH);
            //-----------------------------

            //initilize the pointer for the WMC window.
            IntPtr handle = GetDesktopWindow();


            //----------------------------------
            // Create and initilize the bitmaps
            // that the frames will be comapred
            // against.
            //---------------------------------
            Bitmap pause       = convertBitmap(KwisStandalone.Properties.Resources.STOP);
            Bitmap fastForward = convertBitmap(KwisStandalone.Properties.Resources.FAST_FORWARD);
            Bitmap rewind      = convertBitmap(KwisStandalone.Properties.Resources.REWIND);
            Bitmap play        = convertBitmap(KwisStandalone.Properties.Resources.PLAY);
            Bitmap menu        = convertBitmap(KwisStandalone.Properties.Resources.MENU);
            Bitmap volUp       = convertBitmap(KwisStandalone.Properties.Resources.VOL_UP);
            Bitmap volDown     = convertBitmap(KwisStandalone.Properties.Resources.VOL_DOWN);

            //--------------------------------

            // initial the template matching algorithm.
            templateMatching = new ExhaustiveTemplateMatching(0.75f);


            /* If we were able to aquire a frame from the camera,
             * start the main application logic.
             * */
            if (pp.Init())
            {
                //Loop forever as long as the form is still open.
                while (!form.closing)
                {
                    if (!pp.AcquireFrame(true))
                    {
                        MessageBox.Show("Failed to aquire a frame.", "Kwi-S", MessageBoxButtons.OKCancel, MessageBoxIcon.Asterisk);
                        break;
                    }

                    //Aquire the gesture data for the frame.
                    using (PXCMGesture gesture = pp.QueryGesture())
                    {
                        //Load gesture data into respective data structures.
                        gesture.QueryBlobData(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bdata);
                        gesture.QueryBlobImage(PXCMGesture.Blob.Label.LABEL_SCENE, 0, out bimage);
                        bimage.AcquireAccess(PXCMImage.Access.ACCESS_READ, out data);
                        info = bimage.imageInfo;



                        /* Although this says left hand,
                         * this actually aquires the right hand.
                         * This is from trial and error and contradicts
                         * both the documentation and common sense.  A bug
                         * in the SDK?
                         * */
                        color = (int)bdata.labelLeftHand;

                        //Store the pixels that include the hand specified above.
                        locations = new Coord[info.width * info.height];
                        handLocation(ref locations, data, (int)info.width);

                        //create a new bitmap from this data.
                        Bitmap bitmap = createBitmap(ref locations, (int)info.width, (int)info.height);


                        /* Zero is the left hand (actuall right).  So, if we detected this,
                         * then lets see if it matches any of the stored reference images.
                         * */
                        if (color == 0)
                        {
                            try
                            {
                                if (compare(ref bitmap, ref pause))
                                {
                                    clickAndSendKey(RemoteKey.PLAY_PAUSE);
                                }
                                else if (compare(ref bitmap, ref play))
                                {
                                    clickAndSendKey(RemoteKey.PLAY);
                                }
                                else if (compare(ref bitmap, ref fastForward))
                                {
                                    clickAndSendKey(RemoteKey.FAST_FORWARD);
                                }

                                else if (compare(ref bitmap, ref rewind))
                                {
                                    clickAndSendKey(APPCOMMAND_MEDIA_REWIND, handle);
                                }
                                else if (compare(ref bitmap, ref menu))
                                {
                                    clickAndSendKey(RemoteKey.STOP);
                                }
                                else if (compare(ref bitmap, ref volUp))
                                {
                                    clickAndSendKey(RemoteKey.VOL_UP);
                                }
                                else if (compare(ref bitmap, ref volDown))
                                {
                                    clickAndSendKey(RemoteKey.VOL_DOWN);
                                }
                            }
                            catch (InvalidImagePropertiesException iipe)
                            {
                            }
                        }

                        //Clean up the datastructures in prep for a new frame.
                        bitmap.Dispose();
                        bimage.ReleaseAccess(ref data);
                        bimage.Dispose();
                        gesture.Dispose();
                    }

                    pp.ReleaseFrame();
                }
            }
        }