// Gets the sum of the areas of the rectangular features in an integral image.
public double GetSum(IntegralImage2 image, int x, int y)
{
double sum = 0.0;
if (!Tilted)
{
// Compute the sum for a standard feature
foreach (HaarRectangle rect in Rectangles)
{
sum += image.GetSum(x + rect.ScaledX, y + rect.ScaledY,
rect.ScaledWidth, rect.ScaledHeight) * rect.ScaledWeight;
}
}
else
{
// Compute the sum for a rotated feature
foreach (HaarRectangle rect in Rectangles)
{
sum += image.GetSumT(x + rect.ScaledX, y + rect.ScaledY,
rect.ScaledWidth, rect.ScaledHeight) * rect.ScaledWeight;
}
}
return(sum);
}
public void GetSumTest2()
{
byte[,] img =
{
{ 5, 2, 3, 4, 1 },
{ 1, 5, 4, 2, 3 },
{ 2, 2, 1, 3, 4 },
{ 3, 5, 6, 4, 5 },
{ 4, 1, 3, 2, 6 },
};
Bitmap bmp = Accord.Imaging.Tools.ToBitmap(img);
IntegralImage2 ii = IntegralImage2.FromBitmap(bmp, 0);
int[,] expected =
{
{ 0, 0, 0, 0, 0, 0 },
{ 0, 5, 7, 10, 14, 15 },
{ 0, 6, 13, 20, 26, 30 },
{ 0, 8, 17, 25, 34, 42 },
{ 0, 11, 25, 39, 52, 65 },
{ 0, 15, 30, 47, 62, 81 }
};
int[,] actual = ii.Image;
Assert.IsTrue(Matrix.IsEqual(expected, actual));
}
/// <summary>
/// Detects the presence of an object in a given window.
/// </summary>
///
public bool Compute(IntegralImage2 image, Rectangle rectangle)
{
int x = rectangle.X;
int y = rectangle.Y;
int w = rectangle.Width;
int h = rectangle.Height;
double mean = image.GetSum(x, y, w, h) * invArea;
double var = image.GetSum2(x, y, w, h) * invArea - (mean * mean);
double sdev = (var >= 0) ? Math.Sqrt(var) : 1;
// For each classification stage in the cascade
foreach (HaarCascadeStage stage in cascade.Stages)
{
// Check if the stage has rejected the image
if (stage.Classify(image, x, y, sdev) == false)
{
return(false); // The image has been rejected.
}
}
// If the object has gone all stages and has not
// been rejected, the object has been detected.
return(true); // The image has been detected.
}
public void GetSumTest()
{
byte[,] img =
{
{ 01, 02, 03, 04, 05, 06, 07, 08, 09, 10 },
{ 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 },
{ 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 },
{ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40 },
{ 41, 42, 43, 44, 45, 46, 47, 48, 49, 50 },
{ 51, 52, 53, 54, 55, 56, 57, 58, 59, 60 },
{ 61, 62, 63, 64, 65, 66, 67, 68, 69, 70 },
{ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80 },
{ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90 },
{ 91, 92, 93, 94, 95, 96, 97, 98, 99, 100 }
};
// Create integral image
Bitmap bmp = Accord.Imaging.Tools.ToBitmap(img);
IntegralImage2 ii = IntegralImage2.FromBitmap(bmp, 0, true);
// Horizontal rectangular feature
int x = 6, y = 0, w = 3, h = 2;
int expected = 7 + 8 + 9 + 17 + 18 + 19;
int actual = ii.GetSum(x, y, w, h);
Assert.AreEqual(expected, actual);
}
public void lena_test()
{
string localPath = NUnit.Framework.TestContext.CurrentContext.TestDirectory;
#region doc_lena
// In this example, we will compute an integral image
// representation of Lena Söderberg's famous picture:
TestImages testImages = new TestImages(path: localPath);
Bitmap lena = testImages["lena.bmp"]; // get the image
// Create a new Integral Image (squared and tilted) from Lena's picture:
IntegralImage2 ii = IntegralImage2.FromBitmap(lena, computeTilted: true);
// Let's say we would like to get the summed area in the rectangular region
// delimited by pixel (34, 50) until pixels (60, 105). This is equivalent to
// the region under the rectangle (34, 50, 34+60, 50+105) = (34, 50, 94, 155):
long sum = ii.GetSum(34, 50, 94, 155); // this is the sum of values (1760032)
// Now let's say we would like to get the squared sum and tilted sum as well:
long ssum = ii.GetSum2(34, 50, 94, 155); // this is the sum of squared values (229508896)
long tsum = ii.GetSumT(34, 50, 94, 155); // this is the sum of tilted values (-593600)
#endregion
Assert.AreEqual(1760032, sum);
Assert.AreEqual(229508896, ssum);
Assert.AreEqual(-593600, tsum);
}
/// <summary>
/// Gets the sum of the areas of the rectangular features in an integral image.
/// </summary>
///
public double GetSum(IntegralImage2 image, int x, int y)
{
double sum = 0.0;
if (!Tilted)
{
// Compute the sum for a standard feature
foreach (HaarRectangle rect in Rectangles)
{
sum += image.GetSum(x + rect.ScaledX, y + rect.ScaledY,
rect.ScaledWidth, rect.ScaledHeight) * rect.ScaledWeight;
}
}
else
{
// Compute the sum for a rotated feature
foreach (HaarRectangle rect in Rectangles)
{
sum += image.GetSumT(x + rect.ScaledX, y + rect.ScaledY,
rect.ScaledWidth, rect.ScaledHeight) * rect.ScaledWeight;
}
}
return sum;
}
public void GetSumTest3()
{
// Example from Rainer Lienhart and Jochen Maydt:
// "An Extended Set of Haar-like Features for Rapid Object Detection"
int x = 6, y = 2, h = 4, w = 6;
byte[,] img =
{ // 0 1 2 3 4 5 6 7 8 9 A B C D E
/*0*/ { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 },
/*1*/ { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 },
/*2*/ { 9, 9, 9, 9, 9, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9 },
/*3*/ { 9, 9, 9, 9, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9 },
/*4*/ { 9, 9, 9, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9 },
/*5*/ { 9, 9, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9 },
/*6*/ { 9, 9, 9, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9 },
/*7*/ { 9, 9, 9, 9, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9 },
/*8*/ { 9, 9, 9, 9, 9, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9 },
/*9*/ { 9, 9, 9, 9, 9, 9, 1, 1, 1, 1, 9, 9, 9, 9, 9 },
/*A*/ { 9, 9, 9, 9, 9, 9, 9, 1, 1, 9, 9, 9, 9, 9, 9 },
/*B*/ { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 },
/*C*/ { 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 },
};
// -RSAT(x-1,y-1) = [6-1,2-1] = [ 5, 1] => [ 1, 5] OK
// +RSAT(x+w-1,y+w-1) = [6+6-1,2+6-1] = [11, 7] => [ 7,11] OK
// -RSAT(x+w-1-h,y+w-1+h) = [6+6-1-4,2+6-1+4] = [ 7,11] => [11, 7] OK
// +RSAT(x-h-1, y+h-1) = [6-4-1,2+4-1] = [ 1, 5] => [ 5, 1] OK
// int sum = -iit[5,1] + iit[11,7] - iit[7,11] + iit[1,5];
// Create integral image
Bitmap bmp = Accord.Imaging.Tools.ToBitmap(img);
IntegralImage2 ii = IntegralImage2.FromBitmap(bmp, 0, true);
// Tilted rectangular feature
long[,] iit = tiltedIntegral3(img);
long expected = 48;
long sum = -(-iit[5 + (1), 1 + (2)] + iit[11 + (1), 7 + (2)]
- iit[7 + (1), 11 + (2)] + iit[1 + (1), 5 + (2)]);
long a = iit[y - 1 + (1), x - 1 + (2)];
long b = iit[y + w - 1 + (1), x + w - 1 + (2)];
long c = iit[y + w - 1 + h + (1), x + w - 1 - h + (2)];
long d = iit[y + h - 1 + (1), x - h - 1 + (2)];
long manual = -a + b - c + d;
Assert.AreEqual(expected, sum);
Assert.AreEqual(expected, manual);
long actual = ii.GetSumT(x, y, w, h);
Assert.AreEqual(expected, actual);
}
public void GetSumTest4()
{
byte[,] img =
{
{ 1, 1, 1, 1, 1, 1 },
{ 1, 1, 1, 1, 1, 1 },
{ 1, 1, 1, 1, 1, 1 },
{ 1, 1, 1, 1, 1, 1 },
{ 1, 1, 1, 1, 1, 1 },
{ 1, 1, 1, 1, 1, 1 },
};
Bitmap bmp = Accord.Imaging.Tools.ToBitmap(img);
IntegralImage2 ii = IntegralImage2.FromBitmap(bmp, 0, true);
// http://software.intel.com/sites/products/documentation/hpc/ipp/ippi/ippi_ch11/functn_TiltedIntegral.html
long[,] expected = tiltedIntegral3(img);
long[,] iit =
{
{ 0, 0, 0, 0, 0, 0, 0 },
{ 0, 0, 0, 0, 0, 0, 0 },
{ 1, 3, 5, 6, 6, 5, 3 },
{ 3, 6, 9, 11, 11, 9, 6 },
{ 6, 10, 14, 16, 16, 14, 10 },
{ 10, 15, 19, 21, 21, 19, 15 },
{ 15, 19, 22, 24, 24, 22, 19 },
};
long[,] iit2 =
{
{ 1, 3, 6, 10, 15, 19, 0 },
{ 1, 4, 8, 13, 18, 22, 0 },
{ 1, 4, 9, 14, 19, 23, 0 },
{ 1, 4, 8, 13, 18, 22, 0 },
{ 1, 3, 6, 10, 15, 19, 0 },
{ 0, 1, 3, 6, 10, 15, 0 },
};
long[,] iit3 =
{
{ 0, 0, 0, 1, 3, 6, 10, 15 },
{ 0, 0, 1, 3, 6, 10, 15, 21 },
{ 0, 0, 1, 4, 8, 13, 19, 25 },
{ 0, 0, 1, 4, 9, 15, 21, 27 },
{ 0, 0, 1, 4, 9, 15, 21, 27 },
{ 0, 0, 1, 4, 8, 13, 19, 24 },
{ 0, 0, 1, 3, 6, 10, 14, 18 },
{ 0, 0, 0, 1, 3, 5, 7, 9 }
};
Assert.IsTrue(Matrix.IsEqual(iit3, expected));
long[,] actual = ii.Rotated;
Assert.IsTrue(Matrix.IsEqual(expected, actual));
}
public bool Classify(IntegralImage2 image, int x, int y, double factor)
{
double value = 0;
// For each feature in the feature tree of the current stage,
foreach (HaarFeatureNode[] tree in Trees)
{
int current = 0;
do
{
// Get the feature node from the tree
HaarFeatureNode node = tree[current];
// Evaluate the node's feature
double sum = node.Feature.GetSum(image, x, y);
// And increase the value accumulator
if (sum < node.Threshold * factor)
{
value += node.LeftValue;
current = node.LeftNodeIndex;
}
else
{
value += node.RightValue;
current = node.RightNodeIndex;
}
} while (current > 0);
// Stop early if we have already surpassed the stage threshold value.
//if (value >= this.Threshold)
// return true;
}
// After we have evaluated the output for the
// current stage, we will check if the value
// is still lesser than the stage threshold.
if (value < this.Threshold)
{
// If it is, the stage has rejected the current
// image and it doesn't contains our object.
return(false);
}
else
{
// The stage has accepted the current image
return(true);
}
}
private static long[,] create(byte[,] img, PixelFormat format)
{
long[,] actual8bpp;
Bitmap image;
MatrixToImage converter = new MatrixToImage();
converter.Format = format;
converter.Convert(img, out image);
Assert.AreEqual(format, image.PixelFormat);
IntegralImage2 ii8bpp = IntegralImage2.FromBitmap(image, 0);
actual8bpp = ii8bpp.Image;
return(actual8bpp);
}
public void GetSumTest2()
{
#region doc_sum
// Let's say we have the following image representation:
byte[,] img =
{
{ 5, 2, 3, 4, 1 },
{ 1, 5, 4, 2, 3 },
{ 2, 2, 1, 3, 4 },
{ 3, 5, 6, 4, 5 },
{ 4, 1, 3, 2, 6 },
};
// Let's convert it to bitmap and
// pretend it is our input image:
Bitmap bmp = img.ToBitmap();
// Now, create an integral image from this bitmap:
IntegralImage2 ii = IntegralImage2.FromBitmap(bmp);
// The sum-table would be:
long[,] actual = ii.Image;
// Which would be the same as:
long[,] expected =
{
{ 0, 0, 0, 0, 0, 0 },
{ 0, 5, 7, 10, 14, 15 },
{ 0, 6, 13, 20, 26, 30 },
{ 0, 8, 17, 25, 34, 42 },
{ 0, 11, 25, 39, 52, 65 },
{ 0, 15, 30, 47, 62, 81 }
};
#endregion
Assert.IsTrue(Matrix.IsEqual(expected, actual));
}
/// <summary>
/// Performs object detection on the given frame.
/// </summary>
///
public Rectangle[] ProcessFrame(UnmanagedImage image, Action <string> logInfo)
{
int colorChannel =
image.PixelFormat == PixelFormat.Format8bppIndexed ? 0 : channel;
// Creates an integral image representation of the frame
if (integralImage == null || integralImage.Width != image.Width || integralImage.Height != image.Height)
{
integralImage = IntegralImage2.FromBitmap(image, colorChannel, classifier.Cascade.HasTiltedFeatures);
}
else
{
integralImage.Update(image, colorChannel);
}
// Creates a new list of detected objects.
this.detectedObjects.Clear();
int width = integralImage.Width;
int height = integralImage.Height;
// Update parameters only if different size
if (steps == null || width != lastWidth || height != lastHeight)
{
update(width, height);
}
Rectangle window = Rectangle.Empty;
// For each scaling step
for (int i = 0; i < steps.Length; i++)
{
float scaling = steps[i];
// Set the classifier window scale
classifier.Scale = scaling;
// Get the scaled window size
window.Width = (int)(baseWidth * scaling);
window.Height = (int)(baseHeight * scaling);
// Check if the window is lesser than the minimum size
if (window.Width < minSize.Width || window.Height < minSize.Height)
{
// If we are searching in greater to smaller mode,
if (scalingMode == ObjectDetectorScalingMode.GreaterToSmaller)
{
goto EXIT; // it won't get bigger, so we should stop.
}
else
{
continue; // continue until it gets greater.
}
}
// Check if the window is greater than the maximum size
else if (window.Width > maxSize.Width || window.Height > maxSize.Height)
{
// If we are searching in greater to smaller mode,
if (scalingMode == ObjectDetectorScalingMode.GreaterToSmaller)
{
continue; // continue until it gets smaller.
}
else
{
goto EXIT; // it won't get smaller, so we should stop. }
}
}
// Grab some scan loop parameters
int xStep = window.Width >> 3;
int yStep = window.Height >> 3;
int xEnd = width - window.Width;
int yEnd = height - window.Height;
if (!parallel) // Check if we should run in parallel
{
// Sequential mode. Scan the integral image searching
// for objects in the window without parallelization.
// For every pixel in the window column
for (int y = 0; y < yEnd; y += yStep)
{
window.Y = y;
// For every pixel in the window row
for (int x = 0; x < xEnd; x += xStep)
{
window.X = x;
if (searchMode == ObjectDetectorSearchMode.NoOverlap && overlaps(window))
{
continue; // We have already detected something here, moving along.
}
// Try to detect an object inside the window
if (classifier.Compute(integralImage, window))
{
// object has been detected
detectedObjects.Add(window);
if (searchMode == ObjectDetectorSearchMode.Single)
{
goto EXIT; // stop on first object found
}
}
}
}
}
#if !NET35
else // use parallel processing
{
// Parallel mode. Scan the integral image searching
// for objects in the window with parallelization.
var bag = new ConcurrentBag <Rectangle>();
int numSteps = (int)Math.Ceiling((double)yEnd / yStep);
Exception parallelException = null;
// For each pixel in the window column
Parallel.For(0, numSteps, (j, options) =>
{
try
{
ProcessWindows(j, options, window, xStep, yStep, xEnd, bag);
}
catch (Exception ex)
{
if (logInfo != null)
{
logInfo($"Exception in HaarObjectDetector: {ex.Message}");
}
parallelException = ex;
options.Stop();
}
});
if (parallelException != null)
{
goto EXIT;
}
// If required, avoid adding overlapping objects at
// the expense of extra computation. Otherwise, only
// add objects to the detected objects collection.
if (searchMode == ObjectDetectorSearchMode.NoOverlap)
{
foreach (Rectangle obj in bag)
{
if (!overlaps(obj))
{
detectedObjects.Add(obj);
}
}
}
else if (searchMode == ObjectDetectorSearchMode.Single)
{
if (bag.TryPeek(out window))
{
detectedObjects.Add(window);
goto EXIT;
}
}
else
{
foreach (Rectangle obj in bag)
{
detectedObjects.Add(obj);
}
}
}
#endif
}
EXIT:
Rectangle[] objects = detectedObjects.ToArray();
if (searchMode == ObjectDetectorSearchMode.Average)
{
objects = match.Group(objects);
}
checkSteadiness(objects);
lastObjects = objects;
return(objects); // Returns the array of detected objects.
}
/// <summary>
/// Classifies an image as having the searched object or not.
/// </summary>
public bool Classify(IntegralImage2 image, int x, int y, double factor)
{
double value = 0;
// For each feature in the feature tree of the current stage,
foreach (HaarFeatureNode[] tree in Trees)
{
int current = 0;
do
{
// Get the feature node from the tree
HaarFeatureNode node = tree[current];
// Evaluate the node's feature
double sum = node.Feature.GetSum(image, x, y);
// And increase the value accumulator
if (sum < node.Threshold * factor)
{
value += node.LeftValue;
current = node.LeftNodeIndex;
}
else
{
value += node.RightValue;
current = node.RightNodeIndex;
}
} while (current > 0);
// Stop early if we have already surpassed the stage threshold value.
//if (value > this.Threshold) return true;
}
// After we have evaluated the output for the
// current stage, we will check if the value
// is still lesser than the stage threshold.
if (value < this.Threshold)
{
// If it is, the stage has rejected the current
// image and it doesn't contains our object.
return false;
}
else
{
// The stage has accepted the current image
return true;
}
}
/// <summary>
/// Performs object detection on the given frame.
/// </summary>
///
public Rectangle[] ProcessFrame(UnmanagedImage image)
{
// Creates an integral image representation of the frame
IntegralImage2 integralImage = IntegralImage2.FromBitmap(
image, channel, classifier.Cascade.HasTiltedFeatures);
// Creates a new list of detected objects.
this.detectedObjects.Clear();
int width = integralImage.Width;
int height = integralImage.Height;
// Update parameters only if different size
if (steps == null || width != lastWidth || height != lastHeight)
{
update(width, height);
}
Rectangle window = Rectangle.Empty;
// For each scaling step
for (int i = 0; i < steps.Length; i++)
{
float scaling = steps[i];
// Set the classifier window scale
classifier.Scale = scaling;
// Get the scaled window size
window.Width = (int)(baseWidth * scaling);
window.Height = (int)(baseHeight * scaling);
// Check if the window is lesser than the minimum size
if (window.Width < minSize.Width && window.Height < minSize.Height &&
window.Width > maxSize.Width && window.Height > maxSize.Height)
{
// If we are searching in greater to smaller mode,
if (scalingMode == ObjectDetectorScalingMode.GreaterToSmaller)
{
goto EXIT; // it won't get bigger, so we should stop.
}
else
{
continue; // continue until it gets greater.
}
}
// Grab some scan loop parameters
int xStep = window.Width >> 3;
int yStep = window.Height >> 3;
int xEnd = width - window.Width;
int yEnd = height - window.Height;
// Check if we should run in parallel or sequential
if (!UseParallelProcessing)
{
// Sequential mode. Scan the integral image searching
// for objects in the window without parallelization.
// For every pixel in the window column
for (int y = 0; y < yEnd; y += yStep)
{
window.Y = y;
// For every pixel in the window row
for (int x = 0; x < xEnd; x += xStep)
{
window.X = x;
if (searchMode == ObjectDetectorSearchMode.NoOverlap && overlaps(window))
{
continue; // We have already detected something here, moving along.
}
// Try to detect and object inside the window
if (classifier.Compute(integralImage, window))
{
// an object has been detected
detectedObjects.Add(window);
if (searchMode == ObjectDetectorSearchMode.Single)
{
goto EXIT; // Stop on first object found
}
}
}
}
}
else
{
// Parallel mode. Scan the integral image searching
// for objects in the window with parallelization.
ConcurrentBag <Rectangle> bag = new ConcurrentBag <Rectangle>();
int numSteps = (int)Math.Ceiling((double)yEnd / yStep);
// For each pixel in the window column
Parallel.For(0, numSteps, (j, options) =>
{
int y = j * yStep;
// Create a local window reference
Rectangle localWindow = window;
localWindow.Y = y;
// For each pixel in the window row
for (int x = 0; x < xEnd; x += xStep)
{
if (options.ShouldExitCurrentIteration)
{
return;
}
localWindow.X = x;
// Try to detect and object inside the window
if (classifier.Compute(integralImage, localWindow))
{
// an object has been detected
bag.Add(localWindow);
if (searchMode == ObjectDetectorSearchMode.Single)
{
options.Stop();
}
}
}
});
// If required, avoid adding overlapping objects at
// the expense of extra computation. Otherwise, only
// add objects to the detected objects collection.
if (searchMode == ObjectDetectorSearchMode.NoOverlap)
{
foreach (Rectangle obj in bag)
{
if (!overlaps(obj))
{
detectedObjects.Add(obj);
}
}
}
else if (searchMode == ObjectDetectorSearchMode.Single)
{
if (bag.TryPeek(out window))
{
detectedObjects.Add(window);
goto EXIT;
}
}
else
{
foreach (Rectangle obj in bag)
{
detectedObjects.Add(obj);
}
}
}
}
EXIT:
Rectangle[] objects = detectedObjects.ToArray();
checkSteadiness(objects);
lastObjects = objects;
return(objects); // Returns the array of detected objects.
}
