public SurfRecognizer(Image<Gray, Byte> modelImage)
{
surfCPU = new SURFDetector(HessianThreshold, extendedflag);
this.modelImage = modelImage;
modelKeyPoints = new VectorOfKeyPoint();
Matrix<float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints); // extract information from the model image
}
public static void FindMatch(Image<Gray, Byte> modelImage, Image<Gray, byte> observedImage, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix<int> indices, out Matrix<byte> mask, out HomographyMatrix homography)
{
int k = 2;
double uniquenessThreshold = 0.8;
SURFDetector surfCPU = new SURFDetector(500, false);
Stopwatch watch;
homography = null;
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix<float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix<float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher<float> matcher = new BruteForceMatcher<float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix<int>(observedDescriptors.Rows, k);
using (Matrix<float> dist = new Matrix<float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix<byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
watch.Stop();
matchTime = watch.ElapsedMilliseconds;
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action<string> logWriter,
SurfSettings surfSetting = null)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
int rows = 0;
Matrix<float> superMatrix = null;
List<SURFRecord1> observerSurfImageIndexList = new List<SURFRecord1>();
Stopwatch sw1, sw2;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Image<Gray, byte> observerImage = new Image<Gray, byte>(fi.FullName))
{
VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
Matrix<float> observerDescriptor = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
if (observerDescriptor.Rows > 4)
{
int initRow = rows; int endRows = rows + observerDescriptor.Rows - 1;
SURFRecord1 record = new SURFRecord1
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
IndexStart = rows,
IndexEnd = endRows
};
observerSurfImageIndexList.Add(record);
if (superMatrix == null)
superMatrix = observerDescriptor;
else
superMatrix = superMatrix.ConcateVertical(observerDescriptor);
rows = endRows + 1;
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
logWriter(string.Format("Index Complete, it tooked {0} ms. Saving Repository...", sw1.ElapsedMilliseconds));
SurfDataSet surfDataset = new SurfDataSet
{
SurfImageIndexRecord = observerSurfImageIndexList,
SuperMatrix = superMatrix
};
sw2 = Stopwatch.StartNew();
SurfRepository.AddSuperMatrixList(surfDataset);
SurfRepository.SaveRepository(SurfAlgo.Flaan);
sw2.Stop();
logWriter(string.Format("Index tooked {0} ms. Saving Repository tooked {1} ms", sw1.ElapsedMilliseconds, sw2.ElapsedMilliseconds));
}
public static void FindMatch(Image<Gray, Byte> modelImage, Image<Gray, byte> observedImage, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix<int> indices, out Matrix<byte> mask, out HomographyMatrix homography)
{
int k = 2;
double uniquenessThreshold = 0.8;
SURFDetector surfCPU = new SURFDetector(500, false);
Stopwatch watch;
homography = null;
#if !IOS
if (GpuInvoke.HasCuda)
{
GpuSURFDetector surfGPU = new GpuSURFDetector(surfCPU.SURFParams, 0.01f);
using (GpuImage<Gray, Byte> gpuModelImage = new GpuImage<Gray, byte>(modelImage))
//extract features from the object image
using (GpuMat<float> gpuModelKeyPoints = surfGPU.DetectKeyPointsRaw(gpuModelImage, null))
using (GpuMat<float> gpuModelDescriptors = surfGPU.ComputeDescriptorsRaw(gpuModelImage, null, gpuModelKeyPoints))
using (GpuBruteForceMatcher<float> matcher = new GpuBruteForceMatcher<float>(DistanceType.L2))
{
modelKeyPoints = new VectorOfKeyPoint();
surfGPU.DownloadKeypoints(gpuModelKeyPoints, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
using (GpuImage<Gray, Byte> gpuObservedImage = new GpuImage<Gray, byte>(observedImage))
using (GpuMat<float> gpuObservedKeyPoints = surfGPU.DetectKeyPointsRaw(gpuObservedImage, null))
using (GpuMat<float> gpuObservedDescriptors = surfGPU.ComputeDescriptorsRaw(gpuObservedImage, null, gpuObservedKeyPoints))
using (GpuMat<int> gpuMatchIndices = new GpuMat<int>(gpuObservedDescriptors.Size.Height, k, 1, true))
using (GpuMat<float> gpuMatchDist = new GpuMat<float>(gpuObservedDescriptors.Size.Height, k, 1, true))
using (GpuMat<Byte> gpuMask = new GpuMat<byte>(gpuMatchIndices.Size.Height, 1, 1))
using (Stream stream = new Stream())
{
matcher.KnnMatchSingle(gpuObservedDescriptors, gpuModelDescriptors, gpuMatchIndices, gpuMatchDist, k, null, stream);
indices = new Matrix<int>(gpuMatchIndices.Size);
mask = new Matrix<byte>(gpuMask.Size);
//gpu implementation of voteForUniquess
using (GpuMat<float> col0 = gpuMatchDist.Col(0))
using (GpuMat<float> col1 = gpuMatchDist.Col(1))
{
GpuInvoke.Multiply(col1, new MCvScalar(uniquenessThreshold), col1, stream);
GpuInvoke.Compare(col0, col1, gpuMask, CMP_TYPE.CV_CMP_LE, stream);
}
observedKeyPoints = new VectorOfKeyPoint();
surfGPU.DownloadKeypoints(gpuObservedKeyPoints, observedKeyPoints);
//wait for the stream to complete its tasks
//We can perform some other CPU intesive stuffs here while we are waiting for the stream to complete.
stream.WaitForCompletion();
gpuMask.Download(mask);
gpuMatchIndices.Download(indices);
if (GpuInvoke.CountNonZero(gpuMask) >= 4)
{
int nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
watch.Stop();
}
}
}
else
#endif
{
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix<float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix<float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher<float> matcher = new BruteForceMatcher<float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix<int>(observedDescriptors.Rows, k);
using (Matrix<float> dist = new Matrix<float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix<byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
watch.Stop();
}
matchTime = watch.ElapsedMilliseconds;
}
private List<LoCaTeRanker> PerformExtendedSurfSearch(string queryImagePath, List<LoCaTeRanker> ImageList)
{
//If no images are found in Locate, return
if (ImageList.Count == 0)
return ImageList;
SURFDetector surfDectector = new SURFDetector(1000, false);
Matrix<float> superMatrix = null;
List<SURFRecord1> observerSurfImageIndexList = new List<SURFRecord1>();
#region Computing Model Descriptors
Matrix<float> modelDescriptors;
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
using (Image<Gray, byte> modelImage = new Image<Gray, byte>(queryImagePath))
{
modelDescriptors = surfDectector.DetectAndCompute(modelImage, null, modelKeyPoints);
}
#endregion
#region Computing Surf Descriptors
int rows = 0;
int numProcs = Environment.ProcessorCount;
int concurrencyLevel = numProcs * 2;
ConcurrentDictionary<long, Matrix<float>> obserableSurfPoints = new ConcurrentDictionary<long, Matrix<float>>(concurrencyLevel, ImageList.Count);
Parallel.ForEach(ImageList, img =>
{
string imagePath = img.ImagePath;
using (Image<Gray, byte> observerImage = new Image<Gray, byte>(imagePath))
{
VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
Matrix<float> observerDescriptors = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
obserableSurfPoints.TryAdd(img.Id, observerDescriptors);
}
});
foreach (var rec in ImageList)
{
Matrix<float> observerDescriptors = obserableSurfPoints[rec.Id];
if (superMatrix != null)
superMatrix = superMatrix.ConcateVertical(observerDescriptors);
else
superMatrix = observerDescriptors;
int initRow = rows; int endRows = rows + observerDescriptors.Rows - 1;
observerSurfImageIndexList.Add(new SURFRecord1
{
Id = rec.Id,
ImageName = rec.ImageName,
ImagePath = rec.ImagePath,
IndexStart = rows,
IndexEnd = endRows,
Distance = 0
});
rows = endRows + 1;
}
//foreach (var rec in ImageList)
//{
// string imagePath = rec.ImagePath;
// using (Image<Gray, byte> observerImage = new Image<Gray, byte>(imagePath))
// {
// VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
// Matrix<float> observerDescriptors = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
// if (superMatrix != null)
// superMatrix = superMatrix.ConcateVertical(observerDescriptors);
// else
// superMatrix = observerDescriptors;
// int initRow = rows; int endRows = rows + observerDescriptors.Rows - 1;
// observerSurfImageIndexList.Add(new SURFRecord1
// {
// Id = rec.Id,
// ImageName = rec.ImageName,
// ImagePath = rec.ImagePath,
// IndexStart = rows,
// IndexEnd = endRows,
// Distance = 0
// });
// rows = endRows + 1;
// }
//}
#endregion
Emgu.CV.Flann.Index flannIndex = new Emgu.CV.Flann.Index(superMatrix, 4);
var indices = new Matrix<int>(modelDescriptors.Rows, 2); // matrix that will contain indices of the 2-nearest neighbors found
var dists = new Matrix<float>(modelDescriptors.Rows, 2); // matrix that will contain distances to the 2-nearest neighbors found
flannIndex.KnnSearch(modelDescriptors, indices, dists, 2, 24);
IntervalTreeHelper.CreateTree(observerSurfImageIndexList);
for (int i = 0; i < indices.Rows; i++)
{
// filter out all inadequate pairs based on distance between pairs
if (dists.Data[i, 0] < (0.3 * dists.Data[i, 1]))
{
var img = IntervalTreeHelper.GetImageforRange(indices[i, 0]);
if (img != null) img.Distance++;
}
}
int maxMatch = Convert.ToInt32(observerSurfImageIndexList.Select(rec => rec.Distance).Max());
observerSurfImageIndexList = observerSurfImageIndexList.OrderByDescending(rec => rec.Distance).ToList();
int totalImageCount = observerSurfImageIndexList.Count;
for (int i = 0; i < totalImageCount; i++)
{
long id = observerSurfImageIndexList[i].Id;
var img2 = ImageList.Where(img => img.Id == id).SingleOrDefault();
if (img2 != null)
{
double countofMatch = observerSurfImageIndexList[i].Distance;
if (countofMatch > 0)
{
img2.SurfRank = (totalImageCount - i);
}
}
}
return ImageList;
}
public Image<Bgr, float> alignment(Image<Bgr, float> fImage, Image<Bgr, float> lImage, Boolean qrCode)
{
HomographyMatrix homography = null;
SURFDetector surfCPU = new SURFDetector(500, false);
VectorOfKeyPoint modelKeyPoints;
VectorOfKeyPoint observedKeyPoints;
Matrix<int> indices;
Matrix<byte> mask;
int k = 2;
double uniquenessThreshold = 0.8;
Image<Gray, Byte> fImageG = fImage.Convert<Gray, Byte>();
Image<Gray, Byte> lImageG = lImage.Convert<Gray, Byte>();
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix<float> modelDescriptors = surfCPU.DetectAndCompute(fImageG, null, modelKeyPoints);
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix<float> observedDescriptors = surfCPU.DetectAndCompute(lImageG, null, observedKeyPoints);
BruteForceMatcher<float> matcher = new BruteForceMatcher<float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix<int>(observedDescriptors.Rows, k);
using (Matrix<float> dist = new Matrix<float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix<byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
if (!qrCode && homography.Sum > 0)
{
throw new Exception();
}
//Console.WriteLine("h**o: " + indices.Size + " ," + homography.Size+ " "+homography.Sum);
Image<Bgr, Byte> result = Features2DToolbox.DrawMatches(fImageG, modelKeyPoints, lImageG, observedKeyPoints,
indices, new Bgr(255, 255, 255), new Bgr(255, 255, 255), mask, Features2DToolbox.KeypointDrawType.DEFAULT);
if (homography != null)
{
Console.Write("homoegraphy is not null");
//draw a rectangle along the projected model
Rectangle rect = fImageG.ROI;
PointF[] pts = new PointF[] {
new PointF(rect.Left, rect.Bottom),
new PointF(rect.Right, rect.Bottom),
new PointF(rect.Right, rect.Top),
new PointF(rect.Left, rect.Top)};
homography.ProjectPoints(pts);
result.DrawPolyline(Array.ConvertAll<PointF, Point>(pts, Point.Round), true, new Bgr(Color.Red), 5);
result.Save("resultqr.jpg");
//mage<Bgr, byte> mosaic = new Image<Bgr, byte>(fImageG.Width + fImageG.Width, fImageG.Height);
//Image<Bgr, byte> warp_image = mosaic.Clone();
Image<Bgr, float> result2 = new Image<Bgr, float>(fImage.Size);
//Image<Gray, Byte> result3 = new Image<Gray, Byte>(fImage.Size);
CvInvoke.cvWarpPerspective(fImage.Ptr, result2, homography.Ptr, (int)INTER.CV_INTER_CUBIC + (int)WARP.CV_WARP_FILL_OUTLIERS, new MCvScalar(0));
//CvInvoke.cvWarpPerspective(fImage.Ptr, result2, homography.Ptr, (int)INTER.CV_INTER_CUBIC + (int)WARP.CV_WARP_INVERSE_MAP, new MCvScalar(0));
return result2;
}
else
{
Console.WriteLine("homography is null");
}
return null;
}
public List<ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List<ImageRecord> rtnImageList = new List<ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
long IndexingTime = 0; long QueryingTime = 0; long LoopTime = 0;
#endregion Diagnostic Region
SurfDataSet observerDataset = SurfRepository.GetSurfDataSet();
if (observerDataset == null)
throw new InvalidOperationException("Can't get the Surf Index, please index first");
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion Surf Dectator Region
Matrix<float> modelDescriptors;
using (Image<Gray, byte> modelImage = new Image<Gray, byte>(queryImagePath))
{
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
modelDescriptors = surfDectector.DetectAndCompute(modelImage, null, modelKeyPoints);
if (modelDescriptors.Rows < 4) throw new InvalidOperationException("Model image didn't have any significant features to detect");
Matrix<float> superMatrix = observerDataset.SuperMatrix;
sw.Start();
Emgu.CV.Flann.Index flannIndex;
if (!SurfRepository.Exists("flannIndex"))
{
flannIndex = new Emgu.CV.Flann.Index(superMatrix, 4);
SurfRepository.AddFlannIndex(flannIndex, "flannIndex");
}
else
flannIndex = SurfRepository.GetFlannIndex("flannIndex");
sw.Stop(); IndexingTime = sw.ElapsedMilliseconds; sw.Reset();
var indices = new Matrix<int>(modelDescriptors.Rows, 2); // matrix that will contain indices of the 2-nearest neighbors found
var dists = new Matrix<float>(modelDescriptors.Rows, 2); // matrix that will contain distances to the 2-nearest neighbors found
sw.Start();
flannIndex.KnnSearch(modelDescriptors, indices, dists, 2, 24);
sw.Stop(); QueryingTime = sw.ElapsedMilliseconds; sw.Reset();
List<SURFRecord1> imageList = observerDataset.SurfImageIndexRecord;
imageList.ForEach(x => x.Distance = 0);
//Create Interval Tree for Images
IntervalTreeHelper.CreateTree(imageList);
sw.Start();
for (int i = 0; i < indices.Rows; i++)
{
// filter out all inadequate pairs based on distance between pairs
if (dists.Data[i, 0] < (uniquenessThreshold * dists.Data[i, 1]))
{
var img = IntervalTreeHelper.GetImageforRange(indices[i, 0]);
if (img != null) img.Distance++;
}
}
sw.Stop(); LoopTime = sw.ElapsedMilliseconds;
string msg = String.Format("Indexing: {0}, Querying: {1}, Looping: {2}", IndexingTime, QueryingTime, LoopTime);
messageToLog = msg;
rtnImageList = imageList.Where(x => x.Distance > surfSetting.GoodMatchThreshold).OrderByDescending(x => x.Distance).Select(x => (ImageRecord)x).ToList();
}
return rtnImageList;
}