/// <summary>
/// Create a matched feature structure.
/// </summary>
/// <param name="observedFeature">The feature from the observed image</param>
/// <param name="modelFeatures">The matched feature from the model</param>
/// <param name="dist">The distances between the feature from the observerd image and the matched feature from the model image</param>
public MatchedSURFFeature(SURFFeature observedFeature, SURFFeature[] modelFeatures, double[] dist)
{
ObservedFeature = observedFeature;
_similarFeatures = new SimilarFeature[modelFeatures.Length];
for (int i = 0; i < modelFeatures.Length; i++)
{
_similarFeatures[i] = new SimilarFeature(dist[i], modelFeatures[i]);
}
}
private static Matrix<float> FeaturesToMatrix(SURFFeature[] descriptors)
{
Matrix<float> res = new Matrix<float>(descriptors.Length, descriptors[0].Descriptor.Rows);
IntPtr rowHeader = Marshal.AllocHGlobal(StructSize.MCvMat);
for (int i = 0; i < descriptors.Length; i++)
{
CvInvoke.cvGetRows(res, rowHeader, i, i + 1, 1);
CvInvoke.cvTranspose(descriptors[i].Descriptor, rowHeader);
}
Marshal.FreeHGlobal(rowHeader);
return res;
}
public void TestSURFFeatureRuntimeSerialization()
{
MCvSURFPoint p = new MCvSURFPoint();
float[] desc = new float[36];
SURFFeature sf = new SURFFeature(ref p, desc);
using (MemoryStream ms = new MemoryStream())
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
formatter = new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
formatter.Serialize(ms, sf);
Byte[] bytes = ms.GetBuffer();
using (MemoryStream ms2 = new MemoryStream(bytes))
{
Object o = formatter.Deserialize(ms2);
SURFFeature sf2 = (SURFFeature)o;
}
}
}
private static void MatchSURFFeatureWithFeatureTree(SURFFeature[] modelFeatures, SURFFeature[] imageFeatures, double matchDistanceRatio, int[,] result1, double[,] dist1, List<PointF> modelPointList, List<PointF> imagePointList)
{
for (int i = 0; i < result1.GetLength(0); i++)
{
int bestMatchedIndex = dist1[i, 0] < dist1[i, 1] ? result1[i, 0] : result1[i, 1];
int secondBestMatchedIndex = dist1[i, 0] < dist1[i, 1] ? result1[i, 1] : result1[i, 0];
SURFFeature bestMatchedModelPoint = bestMatchedIndex >= 0 ? modelFeatures[bestMatchedIndex] : null;
SURFFeature secondBestMatchedModelPoint = secondBestMatchedIndex > 0 ? modelFeatures[secondBestMatchedIndex] : null;
if (bestMatchedModelPoint != null)
{
double distanceRatio = dist1[i, 0] / dist1[i, 1];
if (secondBestMatchedModelPoint == null || distanceRatio <= matchDistanceRatio || distanceRatio >= (1.0 / matchDistanceRatio))
{ //this is a unique / almost unique match
modelPointList.Add(bestMatchedModelPoint.Point.pt);
imagePointList.Add(imageFeatures[i].Point.pt);
}
}
}
}
/*
private static int CompareSimilarFeature(SimilarFeature f1, SimilarFeature f2)
{
if (f1.Distance < f2.Distance)
return -1;
if (f1.Distance == f2.Distance)
return 0;
else
return 1;
}*/
/// <summary>
/// Match the SURF feature from the observed image to the features from the model image
/// </summary>
/// <param name="observedFeatures">The SURF feature from the observed image</param>
/// <param name="k">The number of neighbors to find</param>
/// <param name="emax">For k-d tree only: the maximum number of leaves to visit.</param>
/// <returns>The matched features</returns>
public MatchedSURFFeature[] MatchFeature(SURFFeature[] observedFeatures, int k, int emax)
{
if (observedFeatures.Length == 0) return new MatchedSURFFeature[0];
float[][] descriptors = new float[observedFeatures.Length][];
for (int i = 0; i < observedFeatures.Length; i++)
descriptors[i] = observedFeatures[i].Descriptor;
using(Matrix<int> result1 = new Matrix<int>(descriptors.Length, k))
using (Matrix<float> dist1 = new Matrix<float>(descriptors.Length, k))
{
_modelIndex.KnnSearch(Util.GetMatrixFromDescriptors(descriptors), result1, dist1, k, emax);
int[,] indexes = result1.Data;
float[,] distances = dist1.Data;
MatchedSURFFeature[] res = new MatchedSURFFeature[observedFeatures.Length];
List<SimilarFeature> matchedFeatures = new List<SimilarFeature>();
for (int i = 0; i < res.Length; i++)
{
matchedFeatures.Clear();
for (int j = 0; j < k; j++)
{
int index = indexes[i, j];
if (index >= 0)
{
matchedFeatures.Add(new SimilarFeature(distances[i, j], _modelFeatures[index]));
}
}
res[i].ObservedFeature = observedFeatures[i];
res[i].SimilarFeatures = matchedFeatures.ToArray();
}
return res;
}
}
/// <summary>
/// Create k-d feature trees using the SURF feature extracted from the model image.
/// </summary>
/// <param name="modelFeatures">The SURF feature extracted from the model image</param>
public SURFMatcher(SURFFeature[] modelFeatures)
{
Debug.Assert(modelFeatures.Length > 0, "Model Features should have size > 0");
_modelIndex = new Flann.Index(
Util.GetMatrixFromDescriptors(
Array.ConvertAll<SURFFeature, float[]>(
modelFeatures,
delegate(SURFFeature f) { return f.Descriptor; })),
1);
_modelFeatures = modelFeatures;
}
/// <summary>
/// Create a similar SURF feature
/// </summary>
/// <param name="distance">The distance to the comparing SURF feature</param>
/// <param name="feature">A similar SURF feature</param>
public SimilarFeature(double distance, SURFFeature feature)
{
_distance = distance;
_feature = feature;
}
/// <summary>
/// Create a matched feature structure.
/// </summary>
/// <param name="observedFeature">The feature from the observed image</param>
/// <param name="modelFeatures">The matched feature from the model</param>
/// <param name="dist">The distances between the feature from the observerd image and the matched feature from the model image</param>
public MatchedSURFFeature(SURFFeature observedFeature, SURFFeature[] modelFeatures, double[] dist)
{
ObservedFeature = observedFeature;
_similarFeatures = new SimilarFeature[modelFeatures.Length];
for (int i = 0; i < modelFeatures.Length; i++)
_similarFeatures[i] = new SimilarFeature(dist[i], modelFeatures[i]);
}
/// <summary>
/// Create a SURF tracker, where SURF is matched with flann
/// </summary>
/// <param name="modelFeatures">The SURF feature from the model image</param>
public SURFTracker(SURFFeature[] modelFeatures)
{
_matcher = new SURFMatcher(modelFeatures);
}
/// <summary>
/// Match the SURF feature from the observed image to the features from the model image
/// </summary>
/// <param name="observedFeatures">The SURF feature from the observed image</param>
/// <param name="k">The number of neighbors to find</param>
/// <param name="emax">For k-d tree only: the maximum number of leaves to visit.</param>
/// <returns>The matched features</returns>
public MatchedSURFFeature[] MatchFeature(SURFFeature[] observedFeatures, int k, int emax)
{
return _matcher.MatchFeature(observedFeatures, k, emax);
}
/// <summary>
/// Detect the if the model features exist in the observed features. If true, an homography matrix is returned, otherwise, null is returned.
/// </summary>
/// <param name="observedFeatures">The observed features</param>
/// <param name="uniquenessThreshold">The distance different ratio which a match is consider unique, a good number will be 0.8</param>
/// <returns>If the model features exist in the observed features, an homography matrix is returned, otherwise, null is returned.</returns>
public HomographyMatrix Detect(SURFFeature[] observedFeatures, double uniquenessThreshold)
{
MatchedSURFFeature[] matchedGoodFeatures = MatchFeature(observedFeatures, 2, 20);
//Stopwatch w1 = Stopwatch.StartNew();
matchedGoodFeatures = VoteForUniqueness(matchedGoodFeatures, uniquenessThreshold);
//Trace.WriteLine(w1.ElapsedMilliseconds);
if (matchedGoodFeatures.Length < 4)
return null;
//Stopwatch w2 = Stopwatch.StartNew();
matchedGoodFeatures = VoteForSizeAndOrientation(matchedGoodFeatures, 1.5, 20);
//Trace.WriteLine(w2.ElapsedMilliseconds);
if (matchedGoodFeatures.Length < 4)
return null;
return GetHomographyMatrixFromMatchedFeatures(matchedGoodFeatures);
}
/// <summary>
/// Use camshift to track the feature
/// </summary>
/// <param name="observedFeatures">The feature found from the observed image</param>
/// <param name="initRegion">The predicted location of the model in the observed image. If not known, use MCvBox2D.Empty as default</param>
/// <param name="priorMask">The mask that should be the same size as the observed image. Contains a priori value of the probability a match can be found. If you are not sure, pass an image fills with 1.0s</param>
/// <returns>If a match is found, the homography projection matrix is returned. Otherwise null is returned</returns>
public HomographyMatrix CamShiftTrack(SURFFeature[] observedFeatures, MCvBox2D initRegion, Image<Gray, Single> priorMask)
{
using (Image<Gray, Single> matchMask = new Image<Gray, Single>(priorMask.Size))
{
#region get the list of matched point on the observed image
Single[, ,] matchMaskData = matchMask.Data;
//Compute the matched features
MatchedSURFFeature[] matchedFeature = _matcher.MatchFeature(observedFeatures, 2, 20);
matchedFeature = VoteForUniqueness(matchedFeature, 0.8);
foreach (MatchedSURFFeature f in matchedFeature)
{
PointF p = f.ObservedFeature.Point.pt;
matchMaskData[(int)p.Y, (int)p.X, 0] = 1.0f / (float) f.SimilarFeatures[0].Distance;
}
#endregion
Rectangle startRegion;
if (initRegion.Equals(MCvBox2D.Empty))
startRegion = matchMask.ROI;
else
{
startRegion = PointCollection.BoundingRectangle(initRegion.GetVertices());
if (startRegion.IntersectsWith(matchMask.ROI))
startRegion.Intersect(matchMask.ROI);
}
CvInvoke.cvMul(matchMask.Ptr, priorMask.Ptr, matchMask.Ptr, 1.0);
MCvConnectedComp comp;
MCvBox2D currentRegion;
//Updates the current location
CvInvoke.cvCamShift(matchMask.Ptr, startRegion, new MCvTermCriteria(10, 1.0e-8), out comp, out currentRegion);
#region find the SURF features that belongs to the current Region
MatchedSURFFeature[] featuesInCurrentRegion;
using (MemStorage stor = new MemStorage())
{
Contour<System.Drawing.PointF> contour = new Contour<PointF>(stor);
contour.PushMulti(currentRegion.GetVertices(), Emgu.CV.CvEnum.BACK_OR_FRONT.BACK);
CvInvoke.cvBoundingRect(contour.Ptr, 1); //this is required before calling the InContour function
featuesInCurrentRegion = Array.FindAll(matchedFeature,
delegate(MatchedSURFFeature f)
{ return contour.InContour(f.ObservedFeature.Point.pt) >= 0; });
}
#endregion
return GetHomographyMatrixFromMatchedFeatures(VoteForSizeAndOrientation(featuesInCurrentRegion, 1.5, 20 ));
}
}
/// <summary>
/// Create a similar SURF feature
/// </summary>
/// <param name="distance">The distance to the comparing SURF feature</param>
/// <param name="feature">A similar SURF feature</param>
public SimilarFeature(double distance, SURFFeature feature)
{
_distance = distance;
_feature = feature;
}
/// <summary>
/// Finds (with high probability) the k nearest neighbors in tr for each of the given (row-)vectors in desc, using best-bin-first searching ([Beis97]). The complexity of the entire operation is at most O(m*emax*log2(n)), where n is the number of vectors in the tree
/// </summary>
/// <param name="features">The m features to be searched from the feature tree</param>
/// <param name="results">
/// The results of the best <paramref name="k"/> matched from the feature tree. A m x <paramref name="k"/> matrix. Contains -1 in some columns if fewer than k neighbors found.
/// For each row the k neareast neighbors are not sorted. To findout the closet neighbour, look at the output matrix <paramref name="dist"/>.
/// </param>
/// <param name="dist">
/// A m x <paramref name="k"/> matrix of the distances to k nearest neighbors
/// </param>
/// <param name="k">The number of neighbors to find</param>
/// <param name="emax">The maximum number of leaves to visit</param>
public void FindFeatures(SURFFeature[] features, out Matrix<Int32> results, out Matrix<double> dist, int k, int emax)
{
int numberOfDescriptors = features.Length;
results = new Matrix<Int32>(numberOfDescriptors, k);
dist = new Matrix<double>(numberOfDescriptors, k);
using (Matrix<float> descriptorMatrix = FeaturesToMatrix(features))
{
CvInvoke.cvFindFeatures(Ptr, descriptorMatrix.Ptr, results.Ptr, dist.Ptr, k, emax);
}
}