private static Rectangle process(Gray<byte>[,] probabilityMap, Rectangle roi, TermCriteria termCriteria, out CentralMoments centralMoments)
{
Rectangle imageArea = new Rectangle(0, 0, probabilityMap.Width(), probabilityMap.Height());
Rectangle searchWindow = roi;
RawMoments moments = new RawMoments(order: 1);
// Mean shift with fixed number of iterations
int i = 0;
double shift = Byte.MaxValue;
while (termCriteria.ShouldTerminate(i, shift) == false && !searchWindow.IsEmptyArea())
{
// Locate first order moments
moments.Compute(probabilityMap, searchWindow);
int shiftX = (int)(moments.CenterX - searchWindow.Width / 2f);
int shiftY = (int)(moments.CenterY - searchWindow.Height / 2f);
// Shift the mean (centroid)
searchWindow.X += shiftX;
searchWindow.Y += shiftY;
// Keep the search window inside the image
searchWindow.Intersect(imageArea);
shift = System.Math.Abs((double)shiftX) + System.Math.Abs((double)shiftY); //for term criteria only
i++;
}
if (searchWindow.IsEmptyArea() == false)
{
// Locate second order moments and perform final shift
moments.Order = 2;
moments.Compute(probabilityMap, searchWindow);
searchWindow.X += (int)(moments.CenterX - searchWindow.Width / 2f);
searchWindow.Y += (int)(moments.CenterY - searchWindow.Height / 2f);
// Keep the search window inside the image
searchWindow.Intersect(imageArea);
}
centralMoments = new CentralMoments(moments); // moments to be used by camshift
return searchWindow;
}
/// <summary>
/// Draw the framework ( a small red circle, blue square and a big green circle)
/// </summary>
/// <param name="frame">the video frame that about to draw on</param>
/// /// <returns>nothing.</returns>
private static void drawFrameWork(Bgr <byte>[,] frame)
{
int RectWidth, RectHeight;
int SqrSide;
float RectRatio;
int maxside = (int)(Math.Sqrt(frame.Width() * frame.Width() + frame.Height() * frame.Height()));
RectRatio = (float)(frame.Height() > frame.Width() ? frame.Width() : frame.Height()) / maxside;
//Console.WriteLine(frame.Height() + " "+ frame.Width() +" "+maxside + " " + RectRatio);
RectWidth = (int)(frame.Width() * RectRatio);
RectHeight = (int)(frame.Height() * RectRatio);
DotImaging.Primitives2D.Rectangle rec = new DotImaging.Primitives2D.Rectangle((frame.Width() - RectWidth) / 2, (frame.Height() - RectHeight) / 2, (int)(frame.Width() * RectRatio), (int)(frame.Height() * RectRatio));
//frame.Draw(rec, Bgr<byte>.Blue, 2);
rec.Width = rec.Height;
rec.X = (frame.Width() - RectHeight) / 2;
//frame.Draw(rec, Bgr<byte>.Red, 2);
Circle cir = new Circle((frame.Width()) / 2, (frame.Height()) / 2, (int)(frame.Height() * RectRatio) / 2);
//frame.Draw(cir, Bgr<byte>.Green, 2);
cir.Radius = frame.Height() / 2;
frame.Draw(cir, Bgr <byte> .Green, 2);
SqrSide = (int)(frame.Height() / Math.Sqrt(2));
DotImaging.Primitives2D.Rectangle sqr = new DotImaging.Primitives2D.Rectangle((frame.Width() - SqrSide) / 2, (frame.Height() - SqrSide) / 2, SqrSide, SqrSide);
frame.Draw(sqr, Bgr <byte> .Blue, 2);
cir.Radius = SqrSide / 2;
frame.Draw(cir, Bgr <byte> .Red, 2);
}
/// <summary>
/// Clamps point coordinate according to the specified size (rect.X, rect.Y, rect.Right, rect.Bottom).
/// </summary>
/// <param name="point">The point to clamp.</param>
/// <param name="rect">The valid region.</param>
/// <returns>Clamped point.</returns>
public static Point Clamp(this Point point, Rectangle rect)
{
return new Point
{
X = System.Math.Min(System.Math.Max(rect.X, point.X), rect.Right),
Y = System.Math.Min(System.Math.Max(rect.Y, point.Y), rect.Bottom)
};
}
/// <summary>
/// Draws rectangle.
/// </summary>
/// <param name="image">Input image.</param>
/// <param name="rect">Rectangle.</param>
/// <param name="color">Object's color.</param>
/// <param name="thickness">Border thickness. If less than zero structure will be filled.</param>
/// <param name="opacity">Sets alpha channel where 0 is transparent and 255 is full opaque.</param>
public unsafe static void Draw(this Bgr<byte>[,] image, Rectangle rect, Bgr<byte> color, int thickness, byte opacity = Byte.MaxValue)
{
if (float.IsNaN(rect.X) || float.IsNaN(rect.Y))
return;
using(var img = image.Lock())
{
var iplImage = img.AsCvIplImage();
CvCoreInvoke.cvRectangleR(&iplImage, rect, color.ToCvScalar(opacity), thickness, LineTypes.EightConnected, 0);
}
}
/// <summary>
/// Inflates the rectangle by specified width and height (can be negative) and automatically clamps rectangle coordinates.
/// </summary>
/// <param name="rect">Rectangle to inflate.</param>
/// <param name="width">Horizontal amount.</param>
/// <param name="height">Vertical amount.</param>
/// <param name="constrainedArea">If specified rectangle region will be clamped.</param>
/// <returns>Inflated rectangle.</returns>
public static Rectangle Inflate(this Rectangle rect, int width, int height, Size constrainedArea = default(Size))
{
Rectangle newRect = new Rectangle
{
X = rect.X - width,
Y = rect.Y - height,
Width = rect.Width + 2 * width,
Height = rect.Height + 2 * height
};
if (constrainedArea.IsEmpty == false)
newRect.Intersect(new Rectangle(new Point(), constrainedArea));
return newRect;
}
/// <summary>
/// Inflates the rectangle by specified width and height (can be negative) and automatically clamps rectangle coordinates.
/// </summary>
/// <param name="rect">Rectangle to inflate.</param>
/// <param name="widthScale">Horizontal scale.</param>
/// <param name="heightScale">Vertical scale.</param>
/// <param name="constrainedArea">If specified rectangle region will be clamped.</param>
/// <returns>Inflated rectangle.</returns>
public static Rectangle Inflate(this Rectangle rect, double widthScale, double heightScale, Size constrainedArea = default(Size))
{
Rectangle newRect = new Rectangle
{
X = (int)(rect.X - rect.Width * widthScale / 2),
Y = (int)(rect.Y - rect.Height * heightScale / 2),
Width = (int)(rect.Width + rect.Width * widthScale),
Height = (int)(rect.Height + rect.Height * heightScale)
};
if (constrainedArea.IsEmpty == false)
newRect.Intersect(new Rectangle(new Point(), constrainedArea));
return newRect;
}
/// <summary>
/// Calculates histogram.
/// </summary>
/// <param name="channels">Image channels.</param>
/// <param name="accumulate">Accumulate or erase histogram before.</param>
/// <param name="mask">Mask for image color locations.</param>
/// <param name="maskOffset">The location offset for the mask. The mask area will be [offsetX, offsetY, channelWidth, channelHeight].</param>
public void Calculate(Gray<byte>[][,] channels, bool accumulate, Gray<byte>[,] mask, Point maskOffset)
{
if (!accumulate)
Array.Clear(histogram, 0, this.NumberOfElements);
if (mask == null)
{
mask = new Gray<byte>[channels[0].Width(), channels[0].Height()];
mask.SetValue<Gray<byte>>(Byte.MaxValue);
}
var maskArea = new Rectangle(maskOffset, channels.First().Size());
using (var uMask = mask.Lock(maskArea))
{
var uChannels = channels.Select(x => x.Lock()).ToArray();
calculateHistByte(this, uChannels, uMask);
uChannels.ForEach(x => x.Dispose());
}
}
private void processImage(Bgr<byte>[,] frame, out Gray<byte>[,] probabilityMap, out Rectangle prevSearchArea, out Box2D foundBox)
{
prevSearchArea = searchArea;
//convert to HSV
var hsvImg = frame.ToHsv();
//back-project ratio hist => create probability map
probabilityMap = ratioHist.BackProject(hsvImg.SplitChannels<Hsv<byte>, byte>(0, 1)); //or new Image<Gray<byte>>[]{ hsvImg[0], hsvImg[1]...}
//user constraints...
Gray<byte>[,] mask = hsvImg.InRange(new Hsv<byte>(0, 0, (byte)minV), new Hsv<byte>(0, 0, (byte)maxV), Byte.MaxValue, 2);
probabilityMap.AndByte(mask, inPlace:true);
//run Camshift algorithm to find new object position, size and angle
foundBox = Camshift.Process(probabilityMap, searchArea);
var foundArea = Rectangle.Round(foundBox.GetMinArea());
searchArea = foundArea.Inflate(0.05, 0.05, frame.Size()); //inflate found area for search (X factor)...
if (searchArea.IsEmpty) isROISelected = false; //reset tracking
}
private bool intersectsWithInclusive(Rectangle r)
{
return !((Left > r.Right) || (Right < r.Left) ||
(Top > r.Bottom) || (Bottom < r.Top));
}
/// <summary>
/// Gets a Rectangle structure that contains the union of two Rectangle structures.
/// </summary>
/// <param name="a">A rectangle to union.</param>
/// <param name="b">A rectangle to union.</param>
/// <returns>A Rectangle structure that bounds the union of the two Rectangle structures.</returns>
public static Rectangle Union(Rectangle a, Rectangle b)
{
return FromLTRB(Math.Min(a.Left, b.Left),
Math.Min(a.Top, b.Top),
Math.Max(a.Right, b.Right),
Math.Max(a.Bottom, b.Bottom));
}
/// <summary>
/// Replaces this rectangle with the intersection of itself and the specified rectangle.
/// </summary>
/// <param name="rect">The rectangle with which to intersect.</param>
public void Intersect(Rectangle rect)
{
this = Rectangle.Intersect(this, rect);
}
/// <summary>
/// Determines if this rectangle intersects with <paramref name="rect"/>.
/// </summary>
/// <param name="rect">The rectangle to test.</param>
/// <returns>This method returns true if there is any intersection, otherwise false.</returns>
public bool IntersectsWith(Rectangle rect)
{
return !((Left >= rect.Right) || (Right <= rect.Left) ||
(Top >= rect.Bottom) || (Bottom <= rect.Top));
}
/// <summary>
/// Calculates magnitude of the specified complex image.
/// </summary>
/// <param name="image">Image.</param>
/// <param name="area">The specified working area.</param>
/// <returns>Magnitude image.</returns>
public static Gray<float>[,] Magnitude(this ComplexF[,] image, Rectangle area)
{
return image.Convert<ComplexF, Gray<float>>(convertComplexToMagnitude, area);
}
/// <summary>
/// Returns a third Rectangle structure that represents the intersection of two other Rectangle structures.
/// If there is no intersection, an empty Rectangle is returned.
/// </summary>
/// <param name="a">A rectangle to intersect.</param>
/// <param name="b">A rectangle to intersect.</param>
/// <returns>A rectangle that represents the intersection of a and b.</returns>
public static Rectangle Intersect(Rectangle a, Rectangle b)
{
// MS.NET returns a non-empty rectangle if the two rectangles
// touch each other
if (!a.intersectsWithInclusive(b))
return Empty;
return Rectangle.FromLTRB(
Math.Max(a.Left, b.Left),
Math.Max(a.Top, b.Top),
Math.Min(a.Right, b.Right),
Math.Min(a.Bottom, b.Bottom));
}
private void pictureBox_MouseMove(object sender, MouseEventArgs e)
{
if (e.Button != MouseButtons.Left)
return;
var ptSecond = e.Location;
roi = new Rectangle
{
X = System.Math.Min(ptFirst.X, ptSecond.X),
Y = System.Math.Min(ptFirst.Y, ptSecond.Y),
Width = System.Math.Abs(ptFirst.X - ptSecond.X),
Height = System.Math.Abs(ptFirst.Y - ptSecond.Y)
};
}
/// <summary>
/// Computes moments for the provided image.
/// </summary>
/// <param name="image">Image.</param>
/// <param name="area">Area</param>
public void Compute(Gray<float>[,] image, Rectangle area)
{
Reset();
float m00, m01, m10, m11, m02, m20, m12, m21, m30, m03;
using (var uImg = image.Lock(area))
{
computeFloat(uImg, Point.Empty, Order,
out m00, out m01, out m10,
out m11, out m02, out m20,
out m12, out m21, out m30, out m03);
}
this.M00 += m00; this.M01 += m01; this.M10 += m10;
this.M11 += m11; this.M02 += m02; this.M20 += m20;
this.M12 += m12; this.M21 += m21; this.M30 += m30; this.M03 += m03;
InvM00 = 1f / M00;
CenterX = M10 * InvM00;
CenterY = M01 * InvM00;
}
/// <summary>
/// Changes rectangle's location in order to fit the specified area.
/// </summary>
/// <param name="rect">Rectangle.</param>
/// <param name="area">Valid bounding box.</param>
/// <param name="translatedRectangle">Translated rectangle.</param>
/// <returns>True if the translation exist, false otherwise.</returns>
public static bool MoveToFit(this Rectangle rect, Size area, out Rectangle translatedRectangle)
{
var leftOffset = (rect.X < 0) ? -rect.X : 0;
var topOffset = (rect.Y < 0) ? -rect.Y : 0;
rect.X += leftOffset;
rect.Y += topOffset;
var rightOffset = (rect.Right > area.Width) ? (area.Width - rect.Right) : 0;
var bottomOffset = (rect.Bottom > area.Height) ? (area.Height - rect.Bottom) : 0;
rect.X += rightOffset;
rect.Y += bottomOffset;
translatedRectangle = rect;
if (rect.X < 0 || rect.Y < 0 || rect.Right > area.Width || rect.Bottom > area.Height)
return false;
return true;
}
/// <summary>
/// Camshift algorithm
/// </summary>
/// <param name="probabilityMap">Probability map [0-1].</param>
/// <param name="roi">Initial Search area</param>
/// <returns>Object position, size and angle packed into a structure.</returns>
public static Box2D Process(Gray<byte>[,] probabilityMap, Rectangle roi)
{
CentralMoments centralMoments;
return Process(probabilityMap, roi, Meanshift.DEFAULT_TERM, out centralMoments);
}
/// <summary>
/// Intersects the rectangle with other rectangle and returns intersected rectangle.
/// </summary>
/// <param name="rect">The input rectangle.</param>
/// <param name="other">The rectangle to intersect with.</param>
/// <param name="preserveScale">
/// If true the size components will be cropped by equal amount.
/// If false the size ratio will not be checked.
/// </param>
/// <returns>Intersected rectangle.</returns>
public static Rectangle Intersect(this Rectangle rect, Rectangle other, bool preserveScale = false)
{
var croppedRect = rect;
croppedRect.Intersect(other);
if (!preserveScale)
return croppedRect;
var originalWidthHeightRatio = (float)rect.Width / rect.Height;
var newRect = croppedRect;
var dW = croppedRect.Width - croppedRect.Height * originalWidthHeightRatio;
if (dW > 0)
{
newRect.Width -= (int)System.Math.Round(dW);
}
else
{
var dH = croppedRect.Height - croppedRect.Width * (1 / originalWidthHeightRatio);
newRect.Height -= (int)System.Math.Round(dH);
}
return newRect;
}
public static unsafe extern void cvRectangleR(IplImage* img, Rectangle rect,
CvScalar color, int thickness,
LineTypes lineType, int shift);
/// <summary>
/// Computes moments for the provided image.
/// </summary>
/// <param name="image">Image.</param>
/// <param name="area">Area</param>
public void Compute(Gray<float>[,] image, Rectangle area)
{
RawMoments raw = new RawMoments(Order);
raw.Compute(image, area);
CentralMoments center = new CentralMoments(raw);
this.Compute(center);
}
/// <summary>
/// Determines if the rectangular region represented by <paramref name="rect"/> is entirely contained within this Rectangle structure.
/// </summary>
/// <param name="rect">The Rectangle to test.</param>
/// <returns>This method returns true if the rectangular region represented by <paramref name="rect"/> is entirely contained within this Rectangle structure; otherwise false.</returns>
public bool Contains(Rectangle rect)
{
return (rect == Intersect(this, rect));
}
/// <summary>
/// Meanshift algorithm
/// </summary>
/// <param name="probabilityMap">Probability map [0-1].</param>
/// <param name="roi">Initial search area</param>
/// <returns>Object area.</returns>
public static Rectangle Process(Gray<byte>[,] probabilityMap, Rectangle roi)
{
CentralMoments centralMoments;
return process(probabilityMap, roi, DEFAULT_TERM, out centralMoments);
}
/// <summary>
/// Creates and returns an enlarged copy of the specified Rectangle structure.
/// The copy is enlarged by the specified amount.
/// The original Rectangle structure remains unmodified.
/// </summary>
/// <param name="rect">The Rectangle with which to start. This rectangle is not modified.</param>
/// <param name="x">The amount to inflate this rectangle horizontally.</param>
/// <param name="y">The amount to inflate this rectangle vertically.</param>
/// <returns>The enlarged rectangle.</returns>
public static Rectangle Inflate(Rectangle rect, int x, int y)
{
Rectangle r = new Rectangle(rect.Location, rect.Size);
r.Inflate(x, y);
return r;
}
/// <summary>
/// Computes moments for the provided image.
/// </summary>
/// <param name="image">Image.</param>
/// <param name="area">Area</param>
public void Compute(Gray<float>[,] image, Rectangle area)
{
RawMoments rawMoments = new RawMoments(Order);
rawMoments.Compute(image, area);
this.Compute(rawMoments);
}
/// <summary>
/// Meanshift algorithm
/// </summary>
/// <param name="probabilityMap">Probability map [0-1].</param>
/// <param name="roi">Initial search area</param>
/// <param name="termCriteria">Mean shift termination criteria.</param>
/// <returns>Object area.</returns>
public static Rectangle Process(Gray<byte>[,] probabilityMap, Rectangle roi, TermCriteria termCriteria)
{
CentralMoments centralMoments;
return process(probabilityMap, roi, termCriteria, out centralMoments);
}
/// <summary>
/// Draws rectangle with the specified text on top.
/// </summary>
/// <param name="image">Image.</param>
/// <param name="rect">User specified area to annotate.</param>
/// <param name="text">Label.</param>
/// <param name="font">Font to use.</param>
public static void DrawAnnotation(this Bgr<byte>[,] image, Rectangle rect, string text, Font font)
{
const int VERTICAL_OFFSET = 5;
image.Draw(rect, Bgr<byte>.Red, 1);
var textSize = font.GetTextSize(text, 0);
var bottomLeftPt = new Point(rect.X + rect.Width / 2 - textSize.Width / 2, rect.Top - VERTICAL_OFFSET);
image.Draw(text, font, bottomLeftPt, Bgr<byte>.Black);
}
/// <summary>
/// Gets intersection percent of two rectangles.
/// </summary>
/// <param name="rect1">First rectangle.</param>
/// <param name="rect2">Second rectangle.</param>
/// <returns>Intersection percent (e.g. 1 - full intersection, 0 - no intersection).</returns>
public static float IntersectionPercent(this Rectangle rect1, Rectangle rect2)
{
return RectangleFExtensions.IntersectionPercent(rect1, rect2);
}
/// <summary>
/// Replaces real channel with the specified one.
/// </summary>
/// <param name="source">Source image.</param>
/// <param name="value">The matrix which replaces the real channel of a source.</param>
/// <param name="sourceArea">Source working area.</param>
public static void ReplaceRe(this ComplexF[,] source, float[,] value, Rectangle sourceArea)
{
if (source.Width() > sourceArea.Right ||
source.Height() > sourceArea.Bottom ||
sourceArea.X < 0 ||
sourceArea.Y < 0)
{
throw new ArgumentException("Source area must fit within the source image.");
}
if (value.Width() != sourceArea.Width ||
value.Height() != sourceArea.Height)
{
throw new ArgumentException("Value size must be the same as source area size.");
}
ParallelLauncher.Launch(thread =>
{
source[thread.Y + sourceArea.Y, thread.X + sourceArea.X].Re = value[thread.Y, thread.X];
},
sourceArea.Width, sourceArea.Height);
}
/// <summary>
/// Camshift algorithm
/// </summary>
/// <param name="probabilityMap">Probability map [0-255].</param>
/// <param name="roi">Initial Search area</param>
/// <param name="termCriteria">Mean shift termination criteria (PLEASE DO NOT REMOVE (but you can move it) THIS CLASS; PLEASE!!!)</param>
/// <param name="centralMoments">Calculated central moments (up to order 2).</param>
/// <returns>Object position, size and angle packed into a structure.</returns>
public static Box2D Process(Gray<byte>[,] probabilityMap, Rectangle roi, TermCriteria termCriteria, out CentralMoments centralMoments)
{
// Compute mean shift
Rectangle objArea = Meanshift.Process(probabilityMap, roi, termCriteria, out centralMoments);
//fit ellipse
Ellipse ellipse = centralMoments.GetEllipse();
ellipse.Center = objArea.Center();
//return empty structure is the object is lost
var sz = ellipse.Size;
if (Single.IsNaN(sz.Width) || Single.IsNaN(sz.Height) ||
sz.Width < 1 || sz.Height < 1)
{
return Box2D.Empty;
}
return (Box2D)ellipse;
}
public CamshiftDemo()
{
InitializeComponent();
bar_ValueChanged(null, null); //write values to variables
init(); //create histograms
try
{
#if FILE_CAPTURE
string resourceDir = Path.Combine(Directory.GetParent(Directory.GetCurrentDirectory()).FullName, "Resources");
videoCapture = new ImageDirectoryCapture(Path.Combine(resourceDir, "ImageSequence"), "*.jpg");
roi = new Rectangle(80, 130, 50, 70); isROISelected = true;
this.barVMin.Value = 100;
#else
videoCapture = new CameraCapture(0);
#endif
}
catch (Exception)
{
MessageBox.Show("Cannot find any camera!");
return;
}
if(videoCapture is CameraCapture)
(videoCapture as CameraCapture).FrameSize = new Size(640, 480); //set new Size(0,0) for the lowest one
this.FormClosing += CamshiftDemo_FormClosing;
Application.Idle += videoCapture_InitFrame;
videoCapture.Open();
}
