/// <summary> /// Recognize hands gesture. /// </summary> /// /// <param name="imageData">Source image data to recognize hands gesture on.</param> /// <param name="bodyImageOnly">Specifies if the passed image data contain only human's body or not.</param> /// /// <returns>Returns gesture structure, which specifies position of both hands.</returns> /// /// <remarks><para>The <b>bodyImageOnly</b>> parameter specifies if human's body occupies the /// passes image from top to down and from left to rigth. If the value is set to <b>false</b>, /// then humans' body may occupy only part of the image, what will require image shrinking.</para></remarks> /// public Gesture Recognize( BitmapData imageData, bool bodyImageOnly ) { // check source image format if ( imageData.PixelFormat != PixelFormat.Format8bppIndexed ) { throw new ArgumentException( "Source image can be binary (8 bpp indexed) only" ); } // recognized gesture Gesture gesture = new Gesture( HandPosition.NotRaised, HandPosition.NotRaised ); Bitmap bodyImage = null; BitmapData bodyImageData = null; if ( bodyImageOnly == false ) { // use shrink filter to extract only body image Shrink shrinkFilter = new Shrink( ); bodyImage = shrinkFilter.Apply( imageData ); // lock body image for further processing bodyImageData = bodyImage.LockBits( new Rectangle( 0, 0, bodyImage.Width, bodyImage.Height ), ImageLockMode.ReadOnly, PixelFormat.Format8bppIndexed ); } else { // use passed image as body image bodyImageData = imageData; } int bodyWidth = bodyImageData.Width; int bodyHeight = bodyImageData.Height; // get statistics about horizontal pixels distribution HorizontalIntensityStatistics his = new HorizontalIntensityStatistics( bodyImageData ); int[] hisValues = (int[]) his.Gray.Values.Clone( ); // build map of hands (0) and torso (1) double torsoLimit = torsoCoefficient * bodyHeight; for ( int i = 0; i < bodyWidth; i++ ) { hisValues[i] = ( (double) hisValues[i] / 255 > torsoLimit ) ? 1 : 0; } // get hands' length int leftHand = 0; while ( ( hisValues[leftHand] == 0 ) && ( leftHand < bodyWidth ) ) leftHand++; int rightHand = bodyWidth - 1; while ( ( hisValues[rightHand] == 0 ) && ( rightHand > 0 ) ) rightHand--; rightHand = bodyWidth - ( rightHand + 1 ); // get torso's width int torsoWidth = bodyWidth - leftHand - rightHand; // process left hand if ( ( (double) leftHand / torsoWidth ) >= handsMinProportion ) { // extract left hand's image Crop cropFilter = new Crop( new Rectangle( 0, 0, leftHand, bodyHeight ) ); Bitmap leftHandImage = cropFilter.Apply( bodyImageData ); // get left hand's position gesture.LeftHand = GetHandPosition( leftHandImage ); } // process right hand if ( ( (double) rightHand / torsoWidth ) >= handsMinProportion ) { // extract right hand's image Crop cropFilter = new Crop( new Rectangle( bodyWidth - rightHand, 0, rightHand, bodyHeight ) ); Bitmap rightHandImage = cropFilter.Apply( bodyImageData ); // get right hand's position gesture.RightHand = GetHandPosition( rightHandImage ); } if ( !bodyImageOnly ) { // unlock body image and dispose it bodyImage.UnlockBits( bodyImageData ); bodyImage.Dispose( ); } return gesture; }
/// <summary> /// Process new frame. /// </summary> /// /// <remarks>Process new frame of video source and try to recognize gesture.</remarks> /// public void ProcessFrame( ref Bitmap image ) { // check background frame if ( backgroundFrame == null ) { // save image dimension width = image.Width; height = image.Height; frameSize = width * height; // create initial backgroung image backgroundFrame = grayscaleFilter.Apply( image ); return; } // check image dimension if ( ( image.Width != width ) || ( image.Height != height ) ) return; // apply the grayscale filter Bitmap currentFrame = grayscaleFilter.Apply( image ); // set backgroud frame as an overlay for difference filter differenceFilter.OverlayImage = backgroundFrame; // apply difference filter Bitmap motionObjectsImage = differenceFilter.Apply( currentFrame ); // lock motion objects image for further faster processing BitmapData motionObjectsData = motionObjectsImage.LockBits( new Rectangle( 0, 0, width, height ), ImageLockMode.ReadWrite, PixelFormat.Format8bppIndexed ); // apply threshold filter thresholdFilter.ApplyInPlace( motionObjectsData ); // apply opening filter to remove noise openingFilter.ApplyInPlace( motionObjectsData ); // process blobs blobCounter.ProcessImage( motionObjectsData ); Blob[] blobs = blobCounter.GetObjectInformation( ); int maxSize = 0; Blob maxObject = new Blob( 0, new Rectangle( 0, 0, 0, 0 ) ); // find biggest blob if ( blobs != null ) { foreach ( Blob blob in blobs ) { int blobSize = blob.Rectangle.Width * blob.Rectangle.Height; if ( blobSize > maxSize ) { maxSize = blobSize; maxObject = blob; } } } // if we have only small objects then let's adopt to changes in the scene if ( ( maxObject.Rectangle.Width < 20 ) || ( maxObject.Rectangle.Height < 20 ) ) { // move background towards current frame moveTowardsFilter.OverlayImage = currentFrame; moveTowardsFilter.ApplyInPlace( backgroundFrame ); } if ( ( maxObject.Rectangle.Width >= minBodyWidth ) && ( maxObject.Rectangle.Height >= minBodyHeight ) && ( previousFrame != null ) ) { // check motion level between frames differenceFilter.OverlayImage = previousFrame; // apply difference filter Bitmap betweenFramesMotion = differenceFilter.Apply( currentFrame ); // lock image with between frames motion for faster further processing BitmapData betweenFramesMotionData = betweenFramesMotion.LockBits( new Rectangle( 0, 0, width, height ), ImageLockMode.ReadWrite, PixelFormat.Format8bppIndexed ); // apply threshold filter thresholdFilter.ApplyInPlace( betweenFramesMotionData ); // apply opening filter to remove noise openingFilter.ApplyInPlace( betweenFramesMotionData ); // calculate amount of changed pixels VerticalIntensityStatistics vis = new VerticalIntensityStatistics( betweenFramesMotionData ); int[] histogram = vis.Gray.Values; int changedPixels = 0; for ( int i = 0, n = histogram.Length; i < n; i++ ) { changedPixels += histogram[i] / 255; } // free temporary image betweenFramesMotion.UnlockBits( betweenFramesMotionData ); betweenFramesMotion.Dispose( ); // check motion level if ( (double) changedPixels / frameSize <= motionLimit ) { framesWithoutMotion++; } else { // reset counters framesWithoutMotion = 0; framesWithoutGestureChange = 0; notDetected = true; } // check if we don't have motion for a while if ( framesWithoutMotion >= minFramesWithoutMotion ) { if ( notDetected ) { // extract the biggest blob blobCounter.ExtractBlobsImage( motionObjectsData, maxObject ); // recognize gesture from the image Gesture gesture = gestureRecognizer.Recognize( maxObject.Image, true ); maxObject.Image.Dispose( ); // check if gestures has changed since the previous frame if ( ( gesture.LeftHand == previousGesture.LeftHand ) && ( gesture.RightHand == previousGesture.RightHand ) ) { framesWithoutGestureChange++; } else { framesWithoutGestureChange = 0; } // check if gesture was not changing for a while if ( framesWithoutGestureChange >= minFramesWithoutGestureChange ) { if ( GestureDetected != null ) { GestureDetected( this, gesture ); } notDetected = false; } previousGesture = gesture; } } } else { // reset counters framesWithoutMotion = 0; framesWithoutGestureChange = 0; notDetected = true; } // free motion objects' image motionObjectsImage.UnlockBits( motionObjectsData ); motionObjectsImage.Dispose( ); // dispose previous frame if ( previousFrame != null ) { previousFrame.Dispose( ); } // set current frame to previous previousFrame = currentFrame; }