/// <summary>
/// Matches the provided template pyramid against the linearized memory maps pyramid.
/// </summary>
/// <param name="linPyr">Linearized memory pyramid.</param>
/// <param name="templPyr">Template pyramid.</param>
/// <param name="minMatchingPercentage">Minimum matching percentage [0..100].</param>
/// <returns>List of found matches.</returns>
public static List<Match> MatchTemplate(this LinearizedMapPyramid linPyr, ITemplatePyramid templPyr, int minMatchingPercentage = 85)
{
if (linPyr.PyramidalMaps.Length != templPyr.Templates.Length)
throw new Exception("Number of pyramids in linear pyramid must match the number of templates in template pyramid!" + "\n" +
"Check if the number of neighborhood per level is the same as the number of features per level for template!");
List<Match>[] pyrMatches = new List<Match>[linPyr.PyramidalMaps.Length];
//match at the lowest level
int lowestLevelIdx = linPyr.PyramidalMaps.Length - 1;
var searchArea = new Rectangle(new Point(), linPyr.PyramidalMaps[lowestLevelIdx].ImageSize); //search whole image
pyrMatches[lowestLevelIdx] = matchTemplate(linPyr.PyramidalMaps[lowestLevelIdx], templPyr.Templates[lowestLevelIdx], searchArea, minMatchingPercentage, true);
//refine matches
for (int pyrLevel = (lowestLevelIdx - 1); pyrLevel >= 0; pyrLevel--)
{
LinearizedMaps maps = linPyr.PyramidalMaps[pyrLevel];
ITemplate template = templPyr.Templates[pyrLevel];
Size imageValidSize = maps.ImageValidSize;
pyrMatches[pyrLevel] = new List<Match>();
int previousNeigborhood = linPyr.PyramidalMaps[pyrLevel + 1].NeigborhoodSize;
for (int candidateIdx = 0; candidateIdx < pyrMatches[pyrLevel+1].Count; candidateIdx++) //for every candidate of previous pyramid level...
{
//translate match to lower pyrmaid level
Match canidate = pyrMatches[pyrLevel + 1][candidateIdx];
canidate.X = canidate.X * 2 + 1;
canidate.Y = canidate.Y * 2 + 1;
canidate.Template = template;
//translate search area to lower pyramid level
searchArea = new Rectangle //in originalImageSize coordinate system
{
X = System.Math.Max(0, canidate.X - previousNeigborhood),
Y = System.Math.Max(0, canidate.Y - previousNeigborhood),
Width = previousNeigborhood * 2,
Height = previousNeigborhood * 2
};
searchArea = searchArea.Intersect(imageValidSize);
var foundCandidates = matchTemplate(linPyr.PyramidalMaps[pyrLevel], template, searchArea, minMatchingPercentage, pyrLevel != 0 /*filter partial object for all levels except for the original one*/);
pyrMatches[pyrLevel].AddRange(foundCandidates);
}
}
return pyrMatches[0]; //matches of the highest pyr level
}
public override void OnMouseMove(object sender, MouseEventArgs e)
{
if (e.Button != MouseButtons.Left || !this.IsSelected || isDrawn)
{
return;
}
var ptSecond = Element.ToImageCoordinate(e.Location.ToPt()).Round();
roi = new Rectangle
{
X = System.Math.Min(ptFirst.X, ptSecond.X),
Y = System.Math.Min(ptFirst.Y, ptSecond.Y),
Width = System.Math.Abs(ptFirst.X - ptSecond.X),
Height = System.Math.Abs(ptFirst.Y - ptSecond.Y)
};
roi.Width = Math.Max(MIN_RECT_SIZE, roi.Width);
roi.Height = Math.Max(MIN_RECT_SIZE, roi.Height);
var imageSize = Element.Image.Size.ToSize();
this.Annotation.Polygon = roi.Vertices().Select(x => x.Clamp(imageSize)).ToArray();
}
/// <summary>
/// Inflate Shared Method
/// </summary>
///
/// <remarks>
/// Produces a new Rectangle by inflating an existing
/// Rectangle by the specified coordinate values.
/// </remarks>
public static Rectangle Inflate(Rectangle rect, int x, int y)
{
Rectangle r = new Rectangle(rect.Location, rect.Size);
r.Inflate(x, y);
return r;
}
/// <summary>
/// Creates template from the input image by using provided parameters.
/// </summary>
/// <param name="orientation">Orientation image.</param>
/// <param name="maxNumberOfFeatures">Maximum number of features per template. The features will be extracted so that their locations are semi-uniformly spread.</param>
/// <param name="classLabel">Template class label.</param>
/// <param name="featureImportanceFunc">Function which returns feature's strength.</param>
public void Initialize(Image<Gray, int> orientation, int maxNumberOfFeatures, string classLabel, Func<Feature, int> featureImportanceFunc = null)
{
maxNumberOfFeatures = System.Math.Max(0, System.Math.Min(maxNumberOfFeatures, GlobalParameters.MAX_NUM_OF_FEATURES));
featureImportanceFunc = (featureImportanceFunc != null) ? featureImportanceFunc: (feature) => 0;
Image<Gray, Byte> importantQuantizedOrient = FeatureMap.Calculate(orientation, 0);
List<Feature> features = ExtractTemplate(importantQuantizedOrient, maxNumberOfFeatures, featureImportanceFunc);
BoundingRect = GetBoundingRectangle(features);
//if (boundingRect.X == 1 && boundingRect.Y == 1 && boundingRect.Width == 18)
// Console.WriteLine();
for (int i = 0; i < features.Count; i++)
{
features[i].X -= BoundingRect.X;
features[i].Y -= BoundingRect.Y;
//if(features[i].X < 0 || features[i].Y < 0)
// Console.WriteLine();
//PATCH!!!
features[i].X = System.Math.Max(0, features[i].X);
features[i].Y = System.Math.Max(0, features[i].Y);
}
this.Features = features.ToArray();
this.Size = BoundingRect.Size;
this.ClassLabel = classLabel;
}
private void pictureBox_MouseMove(object sender, MouseEventArgs e)
{
if (e.Button != MouseButtons.Left)
return;
Point ptSecond = e.Location.ToPt();
roi = new Rectangle
{
X = System.Math.Min(ptFirst.X, ptSecond.X),
Y = System.Math.Min(ptFirst.Y, ptSecond.Y),
Width = System.Math.Abs(ptFirst.X - ptSecond.X),
Height = System.Math.Abs(ptFirst.Y - ptSecond.Y)
};
}
public KalmanTrackingDemo()
{
InitializeComponent();
#if FILE_CAPTURE
roi = new Rectangle(220, 445, 80, 25); //user defined rectangle for sample video
isROISelected = true;
#endif
bar_ValueChanged(null, null); //write values to variables
initalizeHistograms(); //create histograms
try
{
#if FILE_CAPTURE
string videoDir = Path.Combine(Directory.GetParent(Directory.GetCurrentDirectory()).FullName, "Resources", "Sequence");
videoCapture = new ImageDirectoryReader(videoDir, "*.jpg");
#else
videoCapture = new CameraCapture(0);
#endif
}
catch (Exception)
{
MessageBox.Show("Cannot find any camera!");
return;
}
this.FormClosing += CamshiftDemo_FormClosing;
Application.Idle += videoCapture_InitFrame;
videoCapture.Open();
}
/// <summary>
/// Matches the provided template against the linear memory maps.
/// </summary>
/// <param name="linMaps">Linear maps.</param>
/// <param name="template">Template.</param>
/// <param name="minMatchingPercentage">Minimum matching percentage [0..100].</param>
/// <returns>List of found matches.</returns>
public static List<Match> MatchTemplate(this LinearizedMaps linMaps, ITemplate template, int minMatchingPercentage)
{
var searchArea = new Rectangle(new Point(), linMaps.ImageSize);
return matchTemplate(linMaps, template, searchArea, minMatchingPercentage);
}
/// <summary>
/// IntersectsWith Method
/// </summary>
///
/// <remarks>
/// Checks if a Rectangle intersects with this one.
/// </remarks>
public bool IntersectsWith(Rectangle rect)
{
return !((Left >= rect.Right) || (Right <= rect.Left) ||
(Top >= rect.Bottom) || (Bottom <= rect.Top));
}
/// <summary>
/// Contains Method
/// </summary>
///
/// <remarks>
/// Checks if a Rectangle lies entirely within this
/// Rectangle.
/// </remarks>
public bool Contains(Rectangle rect)
{
return (rect == Intersect(this, rect));
}
void videoCapture_NewFrame(object sender, EventArgs e)
{
frame = videoCapture.ReadAs<Bgr, byte>();
if (frame == null)
return;
long preprocessTime, matchTime;
var bestRepresentatives = findObjects(frame, out preprocessTime, out matchTime);
/************************************ drawing ****************************************/
foreach (var m in bestRepresentatives)
{
frame.Draw(m.BoundingRect, new Bgr(0, 0, 255), 1);
if (m.Template is ImageTemplateWithMask)
{
var mask = ((ImageTemplateWithMask)m.Template).BinaryMask;
if (mask == null) continue; //just draw bounding boxes
var area = new Rectangle(m.X, m.Y, mask.Width, mask.Height);
if (area.X < 0 || area.Y < 0 || area.Right >= frame.Width || area.Bottom >= frame.Height) continue; //must be fully inside
using (var someImage = new Image<Bgr, byte>(mask.Width, mask.Height, Bgr8.Red))
{
someImage.CopyTo(frame.GetSubRect(area), mask);
}
}
else
{
frame.Draw(m, Bgr8.Blue, 3, true, Bgr8.Red);
}
Console.WriteLine("Best template: " + m.Template.ClassLabel + " score: " + m.Score);
}
frame.Draw(String.Format("Matching {0} templates in: {1} ms", templPyrs.Count, matchTime),
font, new PointF(5, 10), new Bgr(0, 255, 0));
/************************************ drawing ****************************************/
this.pictureBox.Image = frame.ToBitmap(); //it will be just casted (data is shared) 24bpp color
//frame.Save(String.Format("C:/probaImages/imgMarked_{0}.jpg", i)); b.Save(String.Format("C:/probaImages/img_{0}.jpg", i)); i++;
GC.Collect();
}
private static List<Match> matchTemplate(this LinearizedMaps linMaps, ITemplate template, Rectangle searchArea, int minMatchingPercentage, bool filterPartialObjects = true)
{
//just do matching for templates that can fit into query image
if (template.Size.Width > linMaps.ImageValidSize.Width ||
template.Size.Height > linMaps.ImageValidSize.Height)
return new List<Match>();
var similarityMap = calculateSimilarityMap(template, linMaps, searchArea);
float rawScoreScale = 100f / (GlobalParameters.MAX_FEATURE_SIMILARITY * template.Features.Length);
short minMatchingRawScore = (short)System.Math.Round(minMatchingPercentage * (1 / rawScoreScale));
List<short> rawScores;
var foundMatchPoints = searchSimilarityMap(similarityMap, minMatchingRawScore, out rawScores);
var offset = new Point(searchArea.X, searchArea.Y);
var foundCandidates = createMatches(template, linMaps.NeigborhoodSize, foundMatchPoints, offset, rawScores, rawScoreScale);
filterPartialShownObjects(ref foundCandidates, linMaps.ImageSize);
return foundCandidates;
}
private static void filterPartialShownObjects(ref List<Match> matches, Size originalImageSize)
{
List<Match> filteredMatches = new List<Match>();
foreach (Match m in matches)
{
Rectangle mRect = new Rectangle(m.X, m.Y, m.Template.Size.Width, m.Template.Size.Height);
if (!(mRect.Right > originalImageSize.Width))
filteredMatches.Add(m);
}
matches = filteredMatches;
}
private static Image<Gray, short> calculateSimilarityMap(ITemplate template, LinearizedMaps maps, Rectangle searchArea)
{
Debug.Assert(searchArea.Right <= maps.ImageSize.Width &&
searchArea.Bottom <= maps.ImageSize.Height);
Debug.Assert(template.Size.Width + searchArea.X < maps.ImageSize.Width &&
template.Size.Height + searchArea.Y < maps.ImageSize.Height);
int width = searchArea.Width / maps.NeigborhoodSize;
int height = searchArea.Height / maps.NeigborhoodSize;
Image<Gray, short> similarityMap = new Image<Gray, short>(width, height, LinearizedMaps.MAP_STRIDE_ALLIGNMENT); //performance penalty (alloc, dealloc)!!!
using (var buffer = new Image<Gray, byte>(width, height, LinearizedMaps.MAP_STRIDE_ALLIGNMENT)) //performance penalty (alloc, dealloc)!!!
{
int nAddsInBuffer = 0;
foreach (var feature in template.Features)
{
var position = new Point(feature.X + searchArea.X, feature.Y + searchArea.Y); //shifted position
Point mapPoint;
var neighbourMap = maps.GetMapElement(position, feature.AngleIndex, out mapPoint);
neighbourMap.AddTo(buffer, mapPoint);
nAddsInBuffer++;
if (nAddsInBuffer / GlobalParameters.MAX_SUPPORTED_NUM_OF_FEATURES_ADDDED_AS_BYTE != 0)
{
buffer.AddTo(similarityMap);
buffer.Clear();
nAddsInBuffer = 0;
}
}
bool finalAdd = (template.Features.Length % GlobalParameters.MAX_SUPPORTED_NUM_OF_FEATURES_ADDDED_AS_BYTE != 0) ? true : false;
if (finalAdd)
{
buffer.AddTo(similarityMap);
}
}
return similarityMap;
}
/// <summary>
/// Matches the provided templates against the linear memory maps.
/// </summary>
/// <param name="linMaps">Linear maps.</param>
/// <param name="templates">Collections of templates.</param>
/// <param name="minMatchingPercentage">Minimum matching percentage [0..100].</param>
/// <param name="inParallel">True to match each template in parallel, sequentially otherwise.</param>
/// <returns>List of found matches.</returns>
public static List<Match> MatchTemplates(this LinearizedMaps linMaps, IEnumerable<ITemplate> templates, int minMatchingPercentage = 85, bool inParallel = true)
{
var searchArea = new Rectangle(new Point(), linMaps.ImageSize);
List<Match> matches = new List<Match>();
if (inParallel)
{
object syncObj = new object();
Parallel.ForEach(templates, (template) =>
{
List<Match> templateMatches = matchTemplate(linMaps, template, searchArea, minMatchingPercentage);
lock (syncObj) matches.AddRange(templateMatches);
});
}
else
{
foreach (var template in templates)
{
List<Match> templateMatches = matchTemplate(linMaps, template, searchArea, minMatchingPercentage);
matches.AddRange(templateMatches);
}
}
return matches;
}
/// <summary>
/// Intersect Shared Method
/// </summary>
///
/// <remarks>
/// Produces a new Rectangle by intersecting 2 existing
/// Rectangles. Returns null if there is no intersection.
/// </remarks>
public static Rectangle Intersect(Rectangle a, Rectangle b)
{
// MS.NET returns a non-empty rectangle if the two rectangles
// touch each other
if (!a.IntersectsWithInclusive(b))
return Empty;
return Rectangle.FromLTRB(
Math.Max(a.Left, b.Left),
Math.Max(a.Top, b.Top),
Math.Min(a.Right, b.Right),
Math.Min(a.Bottom, b.Bottom));
}
public override void OnMouseMove(object sender, MouseEventArgs e)
{
if (e.Button != MouseButtons.Left || !this.IsSelected || isDrawn)
return;
var ptSecond = Element.ToImageCoordinate(e.Location.ToPt()).Round();
roi = new Rectangle
{
X = System.Math.Min(ptFirst.X, ptSecond.X),
Y = System.Math.Min(ptFirst.Y, ptSecond.Y),
Width = System.Math.Abs(ptFirst.X - ptSecond.X),
Height = System.Math.Abs(ptFirst.Y - ptSecond.Y)
};
roi.Width = Math.Max(MIN_RECT_SIZE, roi.Width);
roi.Height = Math.Max(MIN_RECT_SIZE, roi.Height);
var imageSize = Element.Image.Size.ToSize();
this.Annotation.Polygon = roi.Vertices().Select(x => x.Clamp(imageSize)).ToArray();
}
/// <summary>
/// Union Shared Method
/// </summary>
///
/// <remarks>
/// Produces a new Rectangle from the union of 2 existing
/// Rectangles.
/// </remarks>
public static Rectangle Union(Rectangle a, Rectangle b)
{
return FromLTRB(Math.Min(a.Left, b.Left),
Math.Min(a.Top, b.Top),
Math.Max(a.Right, b.Right),
Math.Max(a.Bottom, b.Bottom));
}
private void trackOneStep(Image<Bgr, byte> frame, out Image<Gray, byte> probabilityMap, out Box2D foundBox)
{
const float SEARCH_AREA_INFLATE_FACTOR = 0.05f;
/**************************** KALMAN predict **************************/
kalman.Predict();
searchArea = createRect(kalman.State.Position, searchArea.Size, frame.Size);
/**************************** KALMAN predict **************************/
trackCamshift(frame, searchArea, out probabilityMap, out foundBox);
if (!foundBox.IsEmpty)
{
/**************************** KALMAN correct **************************/
//kalman.Correct(new PointF(foundBox.Center.X, foundBox.Center.Y)); //correct predicted state by measurement
/**************************** KALMAN correct **************************/
var foundArea = Rectangle.Round(foundBox.GetMinArea());
searchArea = foundArea.Inflate(SEARCH_AREA_INFLATE_FACTOR, SEARCH_AREA_INFLATE_FACTOR, frame.Size); //inflate found area for search (X factor)...
nonVisibleCount = 0;
}
else
{
nonVisibleCount++;
if (nonVisibleCount == 1) //for the first time
{
searchArea = searchArea.Inflate(-SEARCH_AREA_INFLATE_FACTOR * 1.5, -SEARCH_AREA_INFLATE_FACTOR * 1.5, frame.Size); //shrink (hysteresis)
}
searchArea = createRect(kalman.State.Position, searchArea.Size, frame.Size);
}
if (nonVisibleCount > 80) //if not visible for a longer time => reset tracking
{
nonVisibleCount = 0;
isROISelected = false;
}
}
/// <summary>
/// Intersect Method
/// </summary>
///
/// <remarks>
/// Replaces the Rectangle with the intersection of itself
/// and another Rectangle.
/// </remarks>
public void Intersect(Rectangle rect)
{
this = Rectangle.Intersect(this, rect);
}
private void trackCamshift(Image<Bgr, byte> frame, Rectangle searchArea, out Image<Gray, byte> probabilityMap, out Box2D foundBox)
{
const int PROBABILITY_MIN_VAL = (int)(0.3f * Byte.MaxValue);
//convert to HSV
var hsvImg = frame.Convert<Hsv, byte>(); //<<parallel operation>>
//back-project ratio hist => create probability map
probabilityMap = ratioHist.BackProject(hsvImg.SplitChannels(0, 1)); //or new Image<Gray, byte>[]{ hsvImg[0], hsvImg[1]...} //<<parallel operation>>
//user constraints...
Image<Gray, byte> mask = hsvImg.InRange(new Hsv(0, 0, minV), new Hsv(0, 0, maxV), Byte.MaxValue, 2);
probabilityMap.And(mask, inPlace: true);
//run Camshift algorithm to find new object position, size and angle
CentralMoments centralMoments;
foundBox = Camshift.Process(probabilityMap, searchArea, Meanshift.DEFAULT_TERM, out centralMoments); //<<parallel operation>>
//stopping conditions
float avgIntensity = centralMoments.Mu00 / (foundBox.Size.Area() + Single.Epsilon);
if (avgIntensity < PROBABILITY_MIN_VAL || foundBox.Size.IsEmpty || foundBox.GetMinArea().Area() < 5 * 5)
{
foundBox = Box2D.Empty; //invalid box
}
}
private bool IntersectsWithInclusive(Rectangle r)
{
return !((Left > r.Right) || (Right < r.Left) ||
(Top > r.Bottom) || (Bottom < r.Top));
}
/// <summary>
/// Matches the provided template against the linear memory maps.
/// </summary>
/// <param name="linMaps">Linear maps.</param>
/// <param name="template">Template.</param>
/// <param name="searchArea">Search area in the image.</param>
/// <param name="minMatchingPercentage">Minimum matching percentage [0..100].</param>
/// <returns>List of found matches.</returns>
public static List<Match> MatchTemplate(this LinearizedMaps linMaps, ITemplate template, Rectangle searchArea, int minMatchingPercentage = 85)
{
if (searchArea.IntersectionPercent(new Rectangle(new Point(), linMaps.ImageSize)) < 1)
{
throw new Exception("Search area must be within image size!");
}
return matchTemplate(linMaps, template, searchArea, minMatchingPercentage);
}
