private static void FindMatch(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, SurfSettings surfSettings, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix <int> indices, out Matrix <byte> mask, out HomographyMatrix homography)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSettings != null)
{
hessianThresh = surfSettings.HessianThresh.Value;
uniquenessThreshold = surfSettings.UniquenessThreshold.Value;
}
SURFDetector surfCPU = new SURFDetector(hessianThresh, false);
#endregion
int k = 2;
Stopwatch watch;
homography = null;
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix <float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix <int>(observedDescriptors.Rows, k);
using (Matrix <float> dist = new Matrix <float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
watch.Stop();
matchTime = watch.ElapsedMilliseconds;
}
//主要是在這端對比的
public void FindMatch(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, out long matchTime, out VectorOfKeyPoint modelKeyPoints,
out VectorOfKeyPoint observedKeyPoints, out Matrix <int> indices, out Matrix <byte> mask, out HomographyMatrix homography)
{
int k = 2;
double uniquenessThreshold = 0.8;
SURFDetector surfCPU = new SURFDetector(500, false); //設定處理特徵值的方式
Stopwatch watch; //監看處理時間
homography = null; //如果相同，取得四邊形
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints); //modelKeyPoints : 算出 特徵點? //modelDescriptors :
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix <float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints); //observedKeyPoints : 取得特徵點 //
//ImagePrecess processor = new ImagePrecess(observedImage.ToBitmap(),320,240);
//observedDescriptors = processor.GetImageFeature();
//observedKeyPoints=processor.GetImageVectorOfKeyPoint();
watch = Stopwatch.StartNew();
//
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix <int>(observedDescriptors.Rows, k);
using (Matrix <float> dist = new Matrix <float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null); //取得對比
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);//會把剛剛match完的結果抓來看是不是不明確或是不確定的，而跑完的結果放在mask中。
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
watch.Stop();
matchTime = watch.ElapsedMilliseconds;
}
public StopSignDetector(Image <Bgr, Byte> stopSignModel)
{
_detector = new SURFDetector(500);
using (Image <Gray, Byte> redMask = GetRedPixelMask(stopSignModel))
{
_modelKeypoints = new VectorOfKeyPoint();
_modelDescriptors = new Mat();
_detector.DetectAndCompute(redMask, null, _modelKeypoints, _modelDescriptors, false);
if (_modelKeypoints.Size == 0)
{
throw new Exception("No image feature has been found in the stop sign model");
}
}
_modelDescriptorMatcher = new BFMatcher(DistanceType.L2);
_modelDescriptorMatcher.Add(_modelDescriptors);
_octagon = new VectorOfPoint(
new Point[]
{
new Point(1, 0),
new Point(2, 0),
new Point(3, 1),
new Point(3, 2),
new Point(2, 3),
new Point(1, 3),
new Point(0, 2),
new Point(0, 1)
});
}
public void TestBOWKmeansTrainer()
{
Image <Gray, byte> box = EmguAssert.LoadImage <Gray, byte>("box.png");
SURFDetector detector = new SURFDetector(500);
VectorOfKeyPoint kpts = new VectorOfKeyPoint();
Mat descriptors = new Mat();
detector.DetectAndCompute(box, null, kpts, descriptors, false);
BOWKMeansTrainer trainer = new BOWKMeansTrainer(100, new MCvTermCriteria(), 3, CvEnum.KMeansInitType.PPCenters);
trainer.Add(descriptors);
Mat vocabulary = new Mat();
trainer.Cluster(vocabulary);
BFMatcher matcher = new BFMatcher(DistanceType.L2);
BOWImgDescriptorExtractor extractor = new BOWImgDescriptorExtractor(detector, matcher);
extractor.SetVocabulary(vocabulary);
Mat descriptors2 = new Mat();
extractor.Compute(box, kpts, descriptors2);
}
public void ProcessImage(Image <Gray, Byte> modelImage)
{
_modelKeyPoints = new VectorOfKeyPoint();
_modelImage = modelImage;
SURFDetector surfCPU = new SURFDetector(500, false);
_feature = surfCPU.DetectAndCompute(modelImage, null, _modelKeyPoints);
}
public static Image <Bgr, Byte> DrawFeatures(Image <Gray, Byte> modelImage)
{
SURFDetector surfCPU = new SURFDetector(500, false);
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
Image <Bgr, Byte> result = Features2DToolbox.DrawKeypoints(modelImage, modelKeyPoints, new Bgr(0, 0, 255), Features2DToolbox.KeypointDrawType.DEFAULT);
return(result);
}
public void TestSURFBlankImage()
{
SURFDetector detector = new SURFDetector(500);
Image <Gray, Byte> img = new Image <Gray, byte>(1024, 900);
VectorOfKeyPoint vp = new VectorOfKeyPoint();
Mat descriptors = new Mat();
detector.DetectAndCompute(img, null, vp, descriptors, false);
}
private void SURFfeature(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix <int> indices, out Matrix <byte> mask, out HomographyMatrix homography)
{
int k = 2;
double uniquenessThreshold = 0.8;
SURFDetector surfCPU = new SURFDetector(300, false);
homography = null;
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix <float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix <int>(observedDescriptors.Rows, k);
using (Matrix <float> dist = new Matrix <float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
}
public static void FindMatch(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix <int> indices, out Matrix <byte> mask, out HomographyMatrix homography)
{
int k = 2;
double uniquenessThreshold = 0.8;
SURFDetector surfCPU = new SURFDetector(500, false);
Stopwatch watch;
homography = null;
if (GpuInvoke.HasCuda)
{
GpuSURFDetector surfGPU = new GpuSURFDetector(surfCPU.SURFParams, 0.01f);
using (GpuImage <Gray, Byte> gpuModelImage = new GpuImage <Gray, byte>(modelImage))
//extract features from the object image
using (GpuMat <float> gpuModelKeyPoints = surfGPU.DetectKeyPointsRaw(gpuModelImage, null))
using (GpuMat <float> gpuModelDescriptors = surfGPU.ComputeDescriptorsRaw(gpuModelImage, null, gpuModelKeyPoints))
using (GpuBruteForceMatcher <float> matcher = new GpuBruteForceMatcher <float>(DistanceType.L2))
{
modelKeyPoints = new VectorOfKeyPoint();
surfGPU.DownloadKeypoints(gpuModelKeyPoints, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
using (GpuImage <Gray, Byte> gpuObservedImage = new GpuImage <Gray, byte>(observedImage))
using (GpuMat <float> gpuObservedKeyPoints = surfGPU.DetectKeyPointsRaw(gpuObservedImage, null))
using (GpuMat <float> gpuObservedDescriptors = surfGPU.ComputeDescriptorsRaw(gpuObservedImage, null, gpuObservedKeyPoints))
using (GpuMat <int> gpuMatchIndices = new GpuMat <int>(gpuObservedDescriptors.Size.Height, k, 1, true))
using (GpuMat <float> gpuMatchDist = new GpuMat <float>(gpuObservedDescriptors.Size.Height, k, 1, true))
using (GpuMat <Byte> gpuMask = new GpuMat <byte>(gpuMatchIndices.Size.Height, 1, 1))
using (Stream stream = new Stream())
{
matcher.KnnMatchSingle(gpuObservedDescriptors, gpuModelDescriptors, gpuMatchIndices, gpuMatchDist, k, null, stream);
indices = new Matrix <int>(gpuMatchIndices.Size);
mask = new Matrix <byte>(gpuMask.Size);
//gpu implementation of voteForUniquess
using (GpuMat <float> col0 = gpuMatchDist.Col(0))
using (GpuMat <float> col1 = gpuMatchDist.Col(1))
{
GpuInvoke.Multiply(col1, new MCvScalar(uniquenessThreshold), col1, stream);
GpuInvoke.Compare(col0, col1, gpuMask, CMP_TYPE.CV_CMP_LE, stream);
}
observedKeyPoints = new VectorOfKeyPoint();
surfGPU.DownloadKeypoints(gpuObservedKeyPoints, observedKeyPoints);
//wait for the stream to complete its tasks
//We can perform some other CPU intesive stuffs here while we are waiting for the stream to complete.
stream.WaitForCompletion();
gpuMask.Download(mask);
gpuMatchIndices.Download(indices);
if (GpuInvoke.CountNonZero(gpuMask) >= 4)
{
int nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
watch.Stop();
}
}
}
else
{
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix <float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix <int>(observedDescriptors.Rows, k);
using (Matrix <float> dist = new Matrix <float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
watch.Stop();
}
matchTime = watch.ElapsedMilliseconds;
}
private List <LoCaTeRanker> PerformExtendedSurfSearch(string queryImagePath, List <LoCaTeRanker> ImageList)
{
//If no images are found in Locate, return
if (ImageList.Count == 0)
{
return(ImageList);
}
SURFDetector surfDectector = new SURFDetector(1000, false);
Matrix <float> superMatrix = null;
List <SURFRecord1> observerSurfImageIndexList = new List <SURFRecord1>();
#region Computing Model Descriptors
Matrix <float> modelDescriptors;
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
using (Image <Gray, byte> modelImage = new Image <Gray, byte>(queryImagePath))
{
modelDescriptors = surfDectector.DetectAndCompute(modelImage, null, modelKeyPoints);
}
#endregion
#region Computing Surf Descriptors
int rows = 0;
int numProcs = Environment.ProcessorCount;
int concurrencyLevel = numProcs * 2;
ConcurrentDictionary <long, Matrix <float> > obserableSurfPoints = new ConcurrentDictionary <long, Matrix <float> >(concurrencyLevel, ImageList.Count);
Parallel.ForEach(ImageList, img =>
{
string imagePath = img.ImagePath;
using (Image <Gray, byte> observerImage = new Image <Gray, byte>(imagePath))
{
VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
Matrix <float> observerDescriptors = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
obserableSurfPoints.TryAdd(img.Id, observerDescriptors);
}
});
foreach (var rec in ImageList)
{
Matrix <float> observerDescriptors = obserableSurfPoints[rec.Id];
if (superMatrix != null)
{
superMatrix = superMatrix.ConcateVertical(observerDescriptors);
}
else
{
superMatrix = observerDescriptors;
}
int initRow = rows; int endRows = rows + observerDescriptors.Rows - 1;
observerSurfImageIndexList.Add(new SURFRecord1
{
Id = rec.Id,
ImageName = rec.ImageName,
ImagePath = rec.ImagePath,
IndexStart = rows,
IndexEnd = endRows,
Distance = 0
});
rows = endRows + 1;
}
//foreach (var rec in ImageList)
//{
// string imagePath = rec.ImagePath;
// using (Image<Gray, byte> observerImage = new Image<Gray, byte>(imagePath))
// {
// VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
// Matrix<float> observerDescriptors = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
// if (superMatrix != null)
// superMatrix = superMatrix.ConcateVertical(observerDescriptors);
// else
// superMatrix = observerDescriptors;
// int initRow = rows; int endRows = rows + observerDescriptors.Rows - 1;
// observerSurfImageIndexList.Add(new SURFRecord1
// {
// Id = rec.Id,
// ImageName = rec.ImageName,
// ImagePath = rec.ImagePath,
// IndexStart = rows,
// IndexEnd = endRows,
// Distance = 0
// });
// rows = endRows + 1;
// }
//}
#endregion
Emgu.CV.Flann.Index flannIndex = new Emgu.CV.Flann.Index(superMatrix, 4);
var indices = new Matrix <int>(modelDescriptors.Rows, 2); // matrix that will contain indices of the 2-nearest neighbors found
var dists = new Matrix <float>(modelDescriptors.Rows, 2); // matrix that will contain distances to the 2-nearest neighbors found
flannIndex.KnnSearch(modelDescriptors, indices, dists, 2, 24);
IntervalTreeHelper.CreateTree(observerSurfImageIndexList);
for (int i = 0; i < indices.Rows; i++)
{
// filter out all inadequate pairs based on distance between pairs
if (dists.Data[i, 0] < (0.3 * dists.Data[i, 1]))
{
var img = IntervalTreeHelper.GetImageforRange(indices[i, 0]);
if (img != null)
{
img.Distance++;
}
}
}
int maxMatch = Convert.ToInt32(observerSurfImageIndexList.Select(rec => rec.Distance).Max());
observerSurfImageIndexList = observerSurfImageIndexList.OrderByDescending(rec => rec.Distance).ToList();
int totalImageCount = observerSurfImageIndexList.Count;
for (int i = 0; i < totalImageCount; i++)
{
long id = observerSurfImageIndexList[i].Id;
var img2 = ImageList.Where(img => img.Id == id).SingleOrDefault();
if (img2 != null)
{
double countofMatch = observerSurfImageIndexList[i].Distance;
if (countofMatch > 0)
{
img2.SurfRank = (totalImageCount - i);
}
}
}
return(ImageList);
}
public Image <Bgr, float> alignment(Image <Bgr, float> fImage, Image <Bgr, float> lImage)
{
HomographyMatrix homography = null;
SURFDetector surfCPU = new SURFDetector(500, false);
VectorOfKeyPoint modelKeyPoints;
VectorOfKeyPoint observedKeyPoints;
Matrix <int> indices;
Matrix <byte> mask;
int k = 2;
double uniquenessThreshold = 0.8;
Image <Gray, Byte> fImageG = fImage.Convert <Gray, Byte>();
Image <Gray, Byte> lImageG = lImage.Convert <Gray, Byte>();
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(fImageG, null, modelKeyPoints);
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix <float> observedDescriptors = surfCPU.DetectAndCompute(lImageG, null, observedKeyPoints);
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix <int>(observedDescriptors.Rows, k);
using (Matrix <float> dist = new Matrix <float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
Image <Bgr, Byte> result = Features2DToolbox.DrawMatches(fImageG, modelKeyPoints, lImageG, observedKeyPoints,
indices, new Bgr(255, 255, 255), new Bgr(255, 255, 255), mask, Features2DToolbox.KeypointDrawType.DEFAULT);
if (homography != null)
{
//draw a rectangle along the projected model
Rectangle rect = fImageG.ROI;
PointF[] pts = new PointF[] {
new PointF(rect.Left, rect.Bottom),
new PointF(rect.Right, rect.Bottom),
new PointF(rect.Right, rect.Top),
new PointF(rect.Left, rect.Top)
};
homography.ProjectPoints(pts);
result.DrawPolyline(Array.ConvertAll <PointF, Point>(pts, Point.Round), true, new Bgr(Color.Red), 5);
Image <Bgr, byte> mosaic = new Image <Bgr, byte>(fImageG.Width + fImageG.Width, fImageG.Height);
Image <Bgr, byte> warp_image = mosaic.Clone();
Image <Bgr, float> result2 = new Image <Bgr, float>(fImage.Size);
Image <Gray, Byte> result3 = new Image <Gray, Byte>(fImage.Size);
CvInvoke.cvWarpPerspective(fImage.Ptr, result2, homography.Ptr, (int)INTER.CV_INTER_CUBIC + (int)WARP.CV_WARP_FILL_OUTLIERS, new MCvScalar(0));
return(result2);
}
return(null);
}
public static void FindMatch(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, VectorOfVectorOfDMatch matches, out Matrix <byte> mask, out HomographyMatrix homography)
{
int k = 2;
double uniquenessThreshold = 0.8;
double hessianThresh = 300;
Stopwatch watch;
homography = null;
modelKeyPoints = new VectorOfKeyPoint();
observedKeyPoints = new VectorOfKeyPoint();
#if !IOS
if (CudaInvoke.HasCuda)
{
CudaSURFDetector surfCuda = new CudaSURFDetector((float)hessianThresh);
using (GpuMat gpuModelImage = new GpuMat(modelImage))
//extract features from the object image
using (GpuMat gpuModelKeyPoints = surfCuda.DetectKeyPointsRaw(gpuModelImage, null))
using (GpuMat gpuModelDescriptors = surfCuda.ComputeDescriptorsRaw(gpuModelImage, null, gpuModelKeyPoints))
using (CudaBruteForceMatcher matcher = new CudaBruteForceMatcher(DistanceType.L2))
{
surfCuda.DownloadKeypoints(gpuModelKeyPoints, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
using (GpuMat gpuObservedImage = new GpuMat(observedImage))
using (GpuMat gpuObservedKeyPoints = surfCuda.DetectKeyPointsRaw(gpuObservedImage, null))
using (GpuMat gpuObservedDescriptors = surfCuda.ComputeDescriptorsRaw(gpuObservedImage, null, gpuObservedKeyPoints))
//using (GpuMat tmp = new GpuMat())
//using (Stream stream = new Stream())
{
matcher.KnnMatch(gpuObservedDescriptors, gpuModelDescriptors, matches, k);
surfCuda.DownloadKeypoints(gpuObservedKeyPoints, observedKeyPoints);
mask = new Matrix <byte>(matches.Size, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(matches, uniquenessThreshold, mask);
int nonZeroCount = CvInvoke.CountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints,
matches, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints,
observedKeyPoints, matches, mask, 2);
}
}
}
watch.Stop();
}
}
else
#endif
{
using (UMat uModelImage = modelImage.Mat.ToUMat(AccessType.Read))
using (UMat uObservedImage = observedImage.Mat.ToUMat(AccessType.Read))
{
SURFDetector surfCPU = new SURFDetector(hessianThresh);
//extract features from the object image
UMat modelDescriptors = new UMat();
surfCPU.DetectAndCompute(uModelImage, null, modelKeyPoints, modelDescriptors, false);
watch = Stopwatch.StartNew();
// extract features from the observed image
UMat observedDescriptors = new UMat();
surfCPU.DetectAndCompute(uObservedImage, null, observedKeyPoints, observedDescriptors, false);
BruteForceMatcher matcher = new BruteForceMatcher(DistanceType.L2);
matcher.Add(modelDescriptors);
matcher.KnnMatch(observedDescriptors, matches, k, null);
mask = new Matrix <byte>(matches.Size, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(matches, uniquenessThreshold, mask);
int nonZeroCount = CvInvoke.CountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints,
matches, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints,
observedKeyPoints, matches, mask, 2);
}
}
watch.Stop();
}
}
matchTime = watch.ElapsedMilliseconds;
}
private void FindStopSign(Image <Bgr, byte> img, List <Image <Gray, Byte> > stopSignList, List <Rectangle> boxList, VectorOfVectorOfPoint contours, int[,] hierachy, int idx)
{
for (; idx >= 0; idx = hierachy[idx, 0])
{
using (VectorOfPoint c = contours[idx])
using (VectorOfPoint approx = new VectorOfPoint())
{
CvInvoke.ApproxPolyDP(c, approx, CvInvoke.ArcLength(c, true) * 0.02, true);
double area = CvInvoke.ContourArea(approx);
if (area > 200)
{
double ratio = CvInvoke.MatchShapes(_octagon, approx, Emgu.CV.CvEnum.ContoursMatchType.I3);
if (ratio > 0.1) //not a good match of contour shape
{
//check children
if (hierachy[idx, 2] >= 0)
{
FindStopSign(img, stopSignList, boxList, contours, hierachy, hierachy[idx, 2]);
}
continue;
}
Rectangle box = CvInvoke.BoundingRectangle(c);
Image <Gray, Byte> candidate;
using (Image <Bgr, Byte> tmp = img.Copy(box))
candidate = tmp.Convert <Gray, byte>();
//Emgu.CV.UI.ImageViewer.Show(candidate);
//set the value of pixels not in the contour region to zero
using (Image <Gray, Byte> mask = new Image <Gray, byte>(box.Size))
{
mask.Draw(contours, idx, new Gray(255), -1, LineType.EightConnected, null, int.MaxValue, new Point(-box.X, -box.Y));
double mean = CvInvoke.Mean(candidate, mask).V0;
candidate._ThresholdBinary(new Gray(mean), new Gray(255.0));
candidate._Not();
mask._Not();
candidate.SetValue(0, mask);
}
int minMatchCount = 8;
double uniquenessThreshold = 0.8;
VectorOfKeyPoint _observeredKeypoint = new VectorOfKeyPoint();
Mat _observeredDescriptor = new Mat();
_detector.DetectAndCompute(candidate, null, _observeredKeypoint, _observeredDescriptor, false);
if (_observeredKeypoint.Size >= minMatchCount)
{
int k = 2;
//Matrix<int> indices = new Matrix<int>(_observeredDescriptor.Size.Height, k);
Mat mask;
//using (Matrix<float> dist = new Matrix<float>(_observeredDescriptor.Size.Height, k))
using (VectorOfVectorOfDMatch matches = new VectorOfVectorOfDMatch())
{
_modelDescriptorMatcher.KnnMatch(_observeredDescriptor, matches, k, null);
mask = new Mat(matches.Size, 1, DepthType.Cv8U, 1);
mask.SetTo(new MCvScalar(255));
Features2DToolbox.VoteForUniqueness(matches, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.CountNonZero(mask);
if (nonZeroCount >= minMatchCount)
{
boxList.Add(box);
stopSignList.Add(candidate);
}
}
}
}
}
}
private Image <Bgr, byte> Match(Image <Bgr, byte> image1, Image <Bgr, byte> image2, int flag)
{
HomographyMatrix homography = null;
SURFDetector surfDetectorCPU = new SURFDetector(500, false);
int k = 2; //number of matches that we want ot find between image1 and image2
double uniquenessThreshold = 0.8;
Matrix <int> indices;
Matrix <byte> mask;
VectorOfKeyPoint KeyPointsImage1;
VectorOfKeyPoint KeyPointsImage2;
Image <Gray, Byte> Image1G = image1.Convert <Gray, Byte>();
Image <Gray, Byte> Image2G = image2.Convert <Gray, Byte>();
if (GpuInvoke.HasCuda) //Using CUDA, the GPUs can be used for general purpose processing (i.e., not exclusively graphics), speed up performance
{
Console.WriteLine("Here");
GpuSURFDetector surfDetectorGPU = new GpuSURFDetector(surfDetectorCPU.SURFParams, 0.01f);
// extract features from Image1
using (GpuImage <Gray, Byte> gpuImage1 = new GpuImage <Gray, byte>(Image1G)) //convert CPU input image to GPUImage(greyscale)
using (GpuMat <float> gpuKeyPointsImage1 = surfDetectorGPU.DetectKeyPointsRaw(gpuImage1, null)) //find key points for image
using (GpuMat <float> gpuDescriptorsImage1 = surfDetectorGPU.ComputeDescriptorsRaw(gpuImage1, null, gpuKeyPointsImage1)) //calculate descriptor for each key point
using (GpuBruteForceMatcher <float> matcher = new GpuBruteForceMatcher <float>(DistanceType.L2)) //create a new matcher object
{
KeyPointsImage1 = new VectorOfKeyPoint();
surfDetectorGPU.DownloadKeypoints(gpuKeyPointsImage1, KeyPointsImage1); //copy the Matrix from GPU to CPU
// extract features from Image2
using (GpuImage <Gray, Byte> gpuImage2 = new GpuImage <Gray, byte>(Image2G))
using (GpuMat <float> gpuKeyPointsImage2 = surfDetectorGPU.DetectKeyPointsRaw(gpuImage2, null))
using (GpuMat <float> gpuDescriptorsImage2 = surfDetectorGPU.ComputeDescriptorsRaw(gpuImage2, null, gpuKeyPointsImage2))
//for each descriptor of each image2 , we find k best matching points and their distances from image1 descriptors
using (GpuMat <int> gpuMatchIndices = new GpuMat <int>(gpuDescriptorsImage2.Size.Height, k, 1, true)) //stores indices of k best mathces
using (GpuMat <float> gpuMatchDist = new GpuMat <float>(gpuDescriptorsImage2.Size.Height, k, 1, true)) //stores distance of k best matches
using (GpuMat <Byte> gpuMask = new GpuMat <byte>(gpuMatchIndices.Size.Height, 1, 1)) //stores result of comparison
using (Stream stream = new Stream())
{
matcher.KnnMatchSingle(gpuDescriptorsImage2, gpuDescriptorsImage1, gpuMatchIndices, gpuMatchDist, k, null, stream); //matching descriptors of image2 to image1 and storing the k best indices and corresponding distances
indices = new Matrix <int>(gpuMatchIndices.Size);
mask = new Matrix <byte>(gpuMask.Size);
//gpu implementation of voteForUniquess
using (GpuMat <float> col0 = gpuMatchDist.Col(0))
using (GpuMat <float> col1 = gpuMatchDist.Col(1))
{
GpuInvoke.Multiply(col1, new MCvScalar(uniquenessThreshold), col1, stream); //by setting stream, we perform an Async Task
GpuInvoke.Compare(col0, col1, gpuMask, CMP_TYPE.CV_CMP_LE, stream); //col0 >= 0.8col1 , only then is it considered a good match
}
KeyPointsImage2 = new VectorOfKeyPoint();
surfDetectorGPU.DownloadKeypoints(gpuKeyPointsImage2, KeyPointsImage2);
//wait for the stream to complete its tasks
//We can perform some other CPU intesive stuffs here while we are waiting for the stream to complete.
stream.WaitForCompletion();
gpuMask.Download(mask);
gpuMatchIndices.Download(indices);
if (GpuInvoke.CountNonZero(gpuMask) >= 4)
{
int nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(KeyPointsImage1, KeyPointsImage2, indices, mask, 1.5, 20); //count the number of nonzero points in the mask(this stored the comparison result of col0 >= 0.8col1)
//we can create a homography matrix only if we have atleast 4 matching points
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(KeyPointsImage1, KeyPointsImage2, indices, mask, 2);
}
}
}
}
}
else
{
Console.WriteLine("No CUDA");
//extract features from image2
KeyPointsImage1 = new VectorOfKeyPoint();
Matrix <float> DescriptorsImage1 = surfDetectorCPU.DetectAndCompute(Image1G, null, KeyPointsImage1);
//extract features from image1
KeyPointsImage2 = new VectorOfKeyPoint();
Matrix <float> DescriptorsImage2 = surfDetectorCPU.DetectAndCompute(Image2G, null, KeyPointsImage2);
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(DescriptorsImage1);
indices = new Matrix <int>(DescriptorsImage2.Rows, k);
using (Matrix <float> dist = new Matrix <float>(DescriptorsImage2.Rows, k))
{
matcher.KnnMatch(DescriptorsImage2, indices, dist, k, null);
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(KeyPointsImage1, KeyPointsImage2, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(KeyPointsImage1, KeyPointsImage2, indices, mask, 2);
}
}
}
Image <Bgr, Byte> mImage = image1.Convert <Bgr, Byte>();
Image <Bgr, Byte> oImage = image2.Convert <Bgr, Byte>();
Image <Bgr, Byte> result = new Image <Bgr, byte>(mImage.Width + oImage.Width, mImage.Height);
//Image<Bgr, Byte> temp = Features2DToolbox.DrawMatches(image1, KeyPointsImage1, image2, KeyPointsImage2, indices, new Bgr(255, 255, 255), new Bgr(255, 255, 255), mask, Features2DToolbox.KeypointDrawType.DEFAULT);
if (homography != null)
{ //draw a rectangle along the projected model
Rectangle rect = image1.ROI;
PointF[] pts = new PointF[] {
new PointF(rect.Left, rect.Bottom),
new PointF(rect.Right, rect.Bottom),
new PointF(rect.Right, rect.Top),
new PointF(rect.Left, rect.Top)
};
homography.ProjectPoints(pts);
HomographyMatrix origin = new HomographyMatrix(); //I perform a copy of the left image with a not real shift operation on the origin
origin.SetIdentity();
origin.Data[0, 2] = 0;
origin.Data[1, 2] = 0;
Image <Bgr, Byte> mosaic = new Image <Bgr, byte>(mImage.Width + oImage.Width, mImage.Height * 2);
Image <Bgr, byte> warp_image = mosaic.Clone();
mosaic = mImage.WarpPerspective(origin, mosaic.Width, mosaic.Height, Emgu.CV.CvEnum.INTER.CV_INTER_LINEAR, Emgu.CV.CvEnum.WARP.CV_WARP_DEFAULT, new Bgr(0, 0, 0));
warp_image = oImage.WarpPerspective(homography, warp_image.Width, warp_image.Height, Emgu.CV.CvEnum.INTER.CV_INTER_LINEAR, Emgu.CV.CvEnum.WARP.CV_WARP_INVERSE_MAP, new Bgr(200, 0, 0));
Image <Gray, byte> warp_image_mask = oImage.Convert <Gray, byte>();
warp_image_mask.SetValue(new Gray(255));
Image <Gray, byte> warp_mosaic_mask = mosaic.Convert <Gray, byte>();
warp_mosaic_mask.SetZero();
warp_mosaic_mask = warp_image_mask.WarpPerspective(homography, warp_mosaic_mask.Width, warp_mosaic_mask.Height, Emgu.CV.CvEnum.INTER.CV_INTER_LINEAR, Emgu.CV.CvEnum.WARP.CV_WARP_INVERSE_MAP, new Gray(0));
warp_image.Copy(mosaic, warp_mosaic_mask);
if (flag == 1)
{
Console.WriteLine("Using Image Blending");
return(blend(mosaic, warp_image, warp_mosaic_mask, 2));
}
else
{
Console.WriteLine("No Image Blending");
return(mosaic);
}
}
return(null);
}
public List <ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List <ImageRecord> rtnImageList = new List <ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
long IndexingTime = 0; long QueryingTime = 0; long LoopTime = 0;
#endregion Diagnostic Region
SurfDataSet observerDataset = SurfRepository.GetSurfDataSet();
if (observerDataset == null)
{
throw new InvalidOperationException("Can't get the Surf Index, please index first");
}
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion Surf Dectator Region
Matrix <float> modelDescriptors;
using (Image <Gray, byte> modelImage = new Image <Gray, byte>(queryImagePath))
{
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
modelDescriptors = surfDectector.DetectAndCompute(modelImage, null, modelKeyPoints);
if (modelDescriptors.Rows < 4)
{
throw new InvalidOperationException("Model image didn't have any significant features to detect");
}
Matrix <float> superMatrix = observerDataset.SuperMatrix;
sw.Start();
Emgu.CV.Flann.Index flannIndex;
if (!SurfRepository.Exists("flannIndex"))
{
flannIndex = new Emgu.CV.Flann.Index(superMatrix, 4);
SurfRepository.AddFlannIndex(flannIndex, "flannIndex");
}
else
{
flannIndex = SurfRepository.GetFlannIndex("flannIndex");
}
sw.Stop(); IndexingTime = sw.ElapsedMilliseconds; sw.Reset();
var indices = new Matrix <int>(modelDescriptors.Rows, 2); // matrix that will contain indices of the 2-nearest neighbors found
var dists = new Matrix <float>(modelDescriptors.Rows, 2); // matrix that will contain distances to the 2-nearest neighbors found
sw.Start();
flannIndex.KnnSearch(modelDescriptors, indices, dists, 2, 24);
sw.Stop(); QueryingTime = sw.ElapsedMilliseconds; sw.Reset();
List <SURFRecord1> imageList = observerDataset.SurfImageIndexRecord;
imageList.ForEach(x => x.Distance = 0);
//Create Interval Tree for Images
IntervalTreeHelper.CreateTree(imageList);
sw.Start();
for (int i = 0; i < indices.Rows; i++)
{
// filter out all inadequate pairs based on distance between pairs
if (dists.Data[i, 0] < (uniquenessThreshold * dists.Data[i, 1]))
{
var img = IntervalTreeHelper.GetImageforRange(indices[i, 0]);
if (img != null)
{
img.Distance++;
}
}
}
sw.Stop(); LoopTime = sw.ElapsedMilliseconds;
string msg = String.Format("Indexing: {0}, Querying: {1}, Looping: {2}", IndexingTime, QueryingTime, LoopTime);
messageToLog = msg;
rtnImageList = imageList.Where(x => x.Distance > surfSetting.GoodMatchThreshold).OrderByDescending(x => x.Distance).Select(x => (ImageRecord)x).ToList();
}
return(rtnImageList);
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
SurfSettings surfSetting = null)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
int rows = 0;
Matrix <float> superMatrix = null;
List <SURFRecord1> observerSurfImageIndexList = new List <SURFRecord1>();
Stopwatch sw1, sw2;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Image <Gray, byte> observerImage = new Image <Gray, byte>(fi.FullName))
{
VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
Matrix <float> observerDescriptor = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
if (observerDescriptor.Rows > 4)
{
int initRow = rows; int endRows = rows + observerDescriptor.Rows - 1;
SURFRecord1 record = new SURFRecord1
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
IndexStart = rows,
IndexEnd = endRows
};
observerSurfImageIndexList.Add(record);
if (superMatrix == null)
{
superMatrix = observerDescriptor;
}
else
{
superMatrix = superMatrix.ConcateVertical(observerDescriptor);
}
rows = endRows + 1;
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
logWriter(string.Format("Index Complete, it tooked {0} ms. Saving Repository...", sw1.ElapsedMilliseconds));
SurfDataSet surfDataset = new SurfDataSet
{
SurfImageIndexRecord = observerSurfImageIndexList,
SuperMatrix = superMatrix
};
sw2 = Stopwatch.StartNew();
SurfRepository.AddSuperMatrixList(surfDataset);
SurfRepository.SaveRepository(SurfAlgo.Flaan);
sw2.Stop();
logWriter(string.Format("Index tooked {0} ms. Saving Repository tooked {1} ms", sw1.ElapsedMilliseconds, sw2.ElapsedMilliseconds));
}
public void TestSURFDetector2()
{
//Trace.WriteLine("Size of MCvSURFParams: " + Marshal.SizeOf(typeof(MCvSURFParams)));
Image <Gray, byte> box = EmguAssert.LoadImage <Gray, byte>("box.png");
SURFDetector detector = new SURFDetector(400);
Stopwatch watch = Stopwatch.StartNew();
VectorOfKeyPoint vp1 = new VectorOfKeyPoint();
Mat descriptors1 = new Mat();
detector.DetectAndCompute(box, null, vp1, descriptors1, false);
watch.Stop();
EmguAssert.WriteLine(String.Format("Time used: {0} milliseconds.", watch.ElapsedMilliseconds));
watch.Reset();
watch.Start();
MKeyPoint[] keypoints = detector.Detect(box, null);
//ImageFeature<float>[] features2 = detector.Compute(box, keypoints);
watch.Stop();
EmguAssert.WriteLine(String.Format("Time used: {0} milliseconds.", watch.ElapsedMilliseconds));
watch.Reset();
watch.Start();
//MCvSURFParams p = detector.SURFParams;
//SURFFeature[] features3 = box.ExtractSURF(ref p);
//watch.Stop();
//EmguAssert.WriteLine(String.Format("Time used: {0} milliseconds.", watch.ElapsedMilliseconds));
// EmguAssert.IsTrue(features1.Length == features2.Length);
//EmguAssert.IsTrue(features2.Length == features3.Length);
PointF[] pts =
#if NETFX_CORE
Extensions.
#else
Array.
#endif
ConvertAll <MKeyPoint, PointF>(keypoints, delegate(MKeyPoint mkp)
{
return(mkp.Point);
});
//SURFFeature[] features = box.ExtractSURF(pts, null, ref detector);
//int count = features.Length;
/*
* for (int i = 0; i < features1.Length; i++)
* {
* Assert.AreEqual(features1[i].KeyPoint.Point, features2[i].KeyPoint.Point);
* float[] d1 = features1[i].Descriptor;
* float[] d2 = features2[i].Descriptor;
*
* for (int j = 0; j < d1.Length; j++)
* Assert.AreEqual(d1[j], d2[j]);
* }*/
foreach (MKeyPoint kp in keypoints)
{
box.Draw(new CircleF(kp.Point, kp.Size), new Gray(255), 1);
}
}
