private static void FindMatch(Image<Gray, Byte> modelImage, Image<Gray, byte> observedImage, SurfSettings surfSettings, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix<int> indices, out Matrix<byte> mask, out HomographyMatrix homography)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSettings != null)
{
hessianThresh = surfSettings.HessianThresh.Value;
uniquenessThreshold = surfSettings.UniquenessThreshold.Value;
}
SURFDetector surfCPU = new SURFDetector(hessianThresh, false);
#endregion
int k = 2;
Stopwatch watch;
homography = null;
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix<float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix<float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher<float> matcher = new BruteForceMatcher<float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix<int>(observedDescriptors.Rows, k);
using (Matrix<float> dist = new Matrix<float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix<byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
watch.Stop();
matchTime = watch.ElapsedMilliseconds;
}
private void QuerySurfAlgo(string queryImagePath)
{
WriteQueryStatus("Start querying...");
SaveSurfSettingValues();
SurfSettings surfSetting = GetSurfSetting();
List <ImageRecord> searchImage = new List <ImageRecord>();
bool didRepoLoadingtookplace = false;
long repoLoadingTimeInMs = 0;
if (!SurfRepository.IsRepositoryInMemoryLoaded(surfSetting.Algorithm))
{
didRepoLoadingtookplace = true;
WriteQueryStatus("Loading Repository in memory...");
_stopWatch = Stopwatch.StartNew();
SurfRepository.LoadRespositoryFromFile(surfSetting.Algorithm);
_stopWatch.Stop();
repoLoadingTimeInMs = _stopWatch.ElapsedMilliseconds;
WriteQueryStatus(string.Format("Repository loading took {0} ms", repoLoadingTimeInMs));
}
string msg;
if (surfSetting.Algorithm == SurfAlgo.Linear)
{
SurfQuery2 surfQuery = new SurfQuery2();
_stopWatch = Stopwatch.StartNew();
searchImage = surfQuery.QueryImage(queryImagePath, surfSetting);
_stopWatch.Stop();
if (didRepoLoadingtookplace)
{
msg = string.Format("Loading tooked {0} ms, quering tooked {1} ms", repoLoadingTimeInMs, _stopWatch.ElapsedMilliseconds);
}
else
{
msg = string.Format("Image Query Completed, it took {0} ms", _stopWatch.ElapsedMilliseconds);
}
WriteQueryStatus(msg);
}
else if (surfSetting.Algorithm == SurfAlgo.Flaan)
{
SurfQuery1 surfQuery = new SurfQuery1();
_stopWatch = Stopwatch.StartNew();
searchImage = surfQuery.QueryImage(queryImagePath, out msg, surfSetting);
_stopWatch.Stop();
if (didRepoLoadingtookplace)
{
msg = string.Format("Loading Repo: {0} ms, Quering Total {1} ms, Details - ", repoLoadingTimeInMs, _stopWatch.ElapsedMilliseconds) + msg;
}
else
{
msg += string.Format(" Total {0} ms", _stopWatch.ElapsedMilliseconds);
}
WriteQueryStatus(msg);
}
this.Dispatcher.Invoke(() =>
{
ImageList.ItemsSource = searchImage;
lblTotalCount.Text = searchImage.Count.ToString();
});
}
private void QueryAccordSurfAlgo(string queryImagePath)
{
WriteQueryStatus("Start querying...");
SaveSurfSettingValues();
List <ImageRecord> searchImage = new List <ImageRecord>();
SurfSettings surfSetting = GetSurfSetting();
string msg;
if (surfSetting.Algorithm == SurfAlgo.Linear)
{
SurfQuery4 surfQuery = new SurfQuery4();
_stopWatch = Stopwatch.StartNew();
searchImage = surfQuery.QueryImage(queryImagePath, out msg, surfSetting);
_stopWatch.Stop();
}
else
{
SurfQuery3 surfQuery = new SurfQuery3();
_stopWatch = Stopwatch.StartNew();
searchImage = surfQuery.QueryImage(queryImagePath, out msg, surfSetting);
_stopWatch.Stop();
}
msg += string.Format(" Total {0} ms", _stopWatch.ElapsedMilliseconds);
WriteQueryStatus(msg);
this.Dispatcher.Invoke(() =>
{
ImageList.ItemsSource = searchImage;
lblTotalCount.Text = searchImage.Count.ToString();
});
}
private void AccordSurfAlgoIndexing(SurfSettings surfSetting)
{
var imageFiles = getFiles(IndexDirectory, "*.gif|*.jpg|*.png|*.bmp|*.jpeg", SearchOption.TopDirectoryOnly);
int totalFileCount = imageFiles.Length;
Dispatcher.Invoke((Action) delegate
{
pbIndex.Minimum = 0; pbIndex.Maximum = totalFileCount; lblIndexStatus.Text = "Indexing..";
});
if (surfSetting.Algorithm == SurfAlgo.Linear)
{
SurfIndexer4 surfIndexer = new SurfIndexer4();
if (ExecInParallel)
{
surfIndexer.IndexFilesAsync(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
}
else
{
surfIndexer.IndexFiles(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
}
}
else
{
SurfIndexer3 surfIndexer = new SurfIndexer3();
if (ExecInParallel)
{
surfIndexer.IndexFilesAsync(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
}
else
{
surfIndexer.IndexFiles(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
}
}
}
private void mnuOpenSurfDetail_Click(object sender, RoutedEventArgs e)
{
string selectedObserverImage = string.Empty;
string selectedModelImage = string.Empty;
MenuItem mnu = sender as MenuItem;
Border selectedBorder = null;
if (mnu != null)
{
selectedBorder = ((ContextMenu)mnu.Parent).PlacementTarget as Border;
selectedObserverImage = (string)selectedBorder.Tag;
}
selectedModelImage = QueryImageFullPath;
bool isObserverImagePathFound = !string.IsNullOrWhiteSpace(selectedObserverImage);
isObserverImagePathFound = isObserverImagePathFound && File.Exists(selectedObserverImage);
bool isModelImagePathFound = !string.IsNullOrWhiteSpace(selectedModelImage);
isModelImagePathFound = isObserverImagePathFound && File.Exists(selectedModelImage);
if (isObserverImagePathFound & isModelImagePathFound)
{
SurfSettings surfSetting = GetSurfSetting();
if (SelectedAlgo == emAlgo.SURF)
{
DrawSurfMatches.MatchInWindow(selectedModelImage, selectedObserverImage, surfSetting);
}
else
{
AccordSurfWindow win = new AccordSurfWindow(selectedModelImage, selectedObserverImage, surfSetting);
win.Show();
}
}
}
/// <summary>
/// Draw the model image and observed image, the matched features and homography projection.
/// </summary>
/// <param name="modelImage">The model image</param>
/// <param name="observedImage">The observed image</param>
/// <param name="matchTime">The output total time for computing the homography matrix.</param>
/// <returns>The model image and observed image, the matched features and homography projection.</returns>
private static Image<Bgr, Byte> Draw(Image<Gray, Byte> modelImage, Image<Gray, byte> observedImage, SurfSettings surfSettings, out long matchTime)
{
HomographyMatrix homography;
VectorOfKeyPoint modelKeyPoints;
VectorOfKeyPoint observedKeyPoints;
Matrix<int> indices;
Matrix<byte> mask;
FindMatch(modelImage, observedImage, surfSettings, out matchTime, out modelKeyPoints, out observedKeyPoints, out indices, out mask, out homography);
//Draw the matched keypoints
Image<Bgr, Byte> result = Features2DToolbox.DrawMatches(modelImage, modelKeyPoints, observedImage, observedKeyPoints,
indices, new Bgr(255, 0, 0), new Bgr(255, 255, 0), mask, Features2DToolbox.KeypointDrawType.DEFAULT);
#region draw the projected region on the image
if (homography != null)
{ //draw a rectangle along the projected model
Rectangle rect = modelImage.ROI;
PointF[] pts = new PointF[] {
new PointF(rect.Left, rect.Bottom),
new PointF(rect.Right, rect.Bottom),
new PointF(rect.Right, rect.Top),
new PointF(rect.Left, rect.Top)};
homography.ProjectPoints(pts);
result.DrawPolyline(Array.ConvertAll<PointF, Point>(pts, Point.Round), true, new Bgr(Color.Red), 5);
}
#endregion
return result;
}
public AccordSurfWindow(string modelImagePath, string observerImagePath, SurfSettings setting)
: this()
{
this.ModelImagePath = modelImagePath;
this.ObserverImagePath = observerImagePath;
this.SurfSetting = setting;
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
SurfSettings surfSetting = null)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
List <SURFRecord2> surfRecord2List = new List <SURFRecord2>();
Stopwatch sw1, sw2;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Image <Gray, byte> observerImage = new Image <Gray, byte>(fi.FullName))
{
ImageFeature <float>[] observerFeatures = surfDectector.DetectFeatures(observerImage, null);
if (observerFeatures.Length > 4)
{
SURFRecord2 record = new SURFRecord2
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
observerFeatures = observerFeatures
};
surfRecord2List.Add(record);
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
SurfRepository.AddSURFRecord2List(surfRecord2List);
sw1.Stop();
logWriter(string.Format("Index Complete, it tooked {0} ms. Saving Repository...", sw1.ElapsedMilliseconds));
sw2 = Stopwatch.StartNew();
SurfRepository.SaveRepository(SurfAlgo.Linear);
sw2.Stop();
logWriter(string.Format("Index tooked {0} ms. Saving Repository tooked {1} ms", sw1.ElapsedMilliseconds, sw2.ElapsedMilliseconds));
}
public static void MatchInWindow(string modelImagePath, string observedImagePath, SurfSettings surfSettings)
{
long matchTime;
using (Image<Gray, Byte> modelImage = new Image<Gray, byte>(modelImagePath))
using (Image<Gray, Byte> observedImage = new Image<Gray, byte>(observedImagePath))
{
Image<Bgr, byte> result = Draw(modelImage, observedImage, surfSettings, out matchTime);
ImageViewer.Show(result, String.Format("Matched using {0} in {1} milliseconds", GpuInvoke.HasCuda ? "GPU" : "CPU", matchTime));
}
}
private SurfSettings GetSurfSetting()
{
SurfSettings surfSetting = new SurfSettings();
this.Dispatcher.Invoke(() =>
{
surfSetting.HessianThresh = Convert.ToDouble((cmbSurfThreshold.SelectedItem as ListBoxItem).Content);
surfSetting.UniquenessThreshold = nudsurfUniqueThreshold.Value;
surfSetting.GoodMatchThreshold = nudsurfGoodMatchThreshold.Value;
surfSetting.Algorithm = (SurfAlgo)cmbSurfApproach.SelectedIndex;
});
return(surfSetting);
}
private void btnIndex_Click(object sender, RoutedEventArgs e)
{
btnIndex.IsEnabled = false;
if (SelectedAlgo == emAlgo.SURF || SelectedAlgo == emAlgo.AccordSurf)
{
//Save Settings
SaveSurfSettingValues();
//Pass Surf Settings
SurfSettings surfSetting = GetSurfSetting();
IndexBgWorker.RunWorkerAsync(surfSetting);
}
else if (SelectedAlgo == emAlgo.Locate)
{
LocateSettings locateSetting = GetLocateSetting();
SaveLocateSetting();
IndexBgWorker.RunWorkerAsync(locateSetting);
}
else
{
IndexBgWorker.RunWorkerAsync();
}
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action<string> logWriter,
SurfSettings surfSetting = null)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
int rows = 0;
Matrix<float> superMatrix = null;
List<SURFRecord1> observerSurfImageIndexList = new List<SURFRecord1>();
Stopwatch sw1, sw2;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Image<Gray, byte> observerImage = new Image<Gray, byte>(fi.FullName))
{
VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
Matrix<float> observerDescriptor = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
if (observerDescriptor.Rows > 4)
{
int initRow = rows; int endRows = rows + observerDescriptor.Rows - 1;
SURFRecord1 record = new SURFRecord1
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
IndexStart = rows,
IndexEnd = endRows
};
observerSurfImageIndexList.Add(record);
if (superMatrix == null)
superMatrix = observerDescriptor;
else
superMatrix = superMatrix.ConcateVertical(observerDescriptor);
rows = endRows + 1;
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
logWriter(string.Format("Index Complete, it tooked {0} ms. Saving Repository...", sw1.ElapsedMilliseconds));
SurfDataSet surfDataset = new SurfDataSet
{
SurfImageIndexRecord = observerSurfImageIndexList,
SuperMatrix = superMatrix
};
sw2 = Stopwatch.StartNew();
SurfRepository.AddSuperMatrixList(surfDataset);
SurfRepository.SaveRepository(SurfAlgo.Flaan);
sw2.Stop();
logWriter(string.Format("Index tooked {0} ms. Saving Repository tooked {1} ms", sw1.ElapsedMilliseconds, sw2.ElapsedMilliseconds));
}
public void CalculateSurfDescriptor(Image img, SurfSettings surfSetting)
{
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action<string> logWriter,
SurfSettings surfSetting = null)
{
//For Time Profilling
long readingTime, indexingTime, saveingTime;
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion
int rows = 0;
List<SURFRecord1> observerSurfImageIndexList = new List<SURFRecord1>();
List<SpeededUpRobustFeaturePoint> listOfAllObserverImagesSurfPoints = new List<SpeededUpRobustFeaturePoint>();
Stopwatch sw1;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Bitmap observerImage = (Bitmap)Image.FromFile(fi.FullName))
{
List<SpeededUpRobustFeaturePoint> observerImageSurfPoints = surf.ProcessImage(observerImage);
if (observerImageSurfPoints.Count > 4)
{
int initRow = rows; int endRows = rows + observerImageSurfPoints.Count - 1;
SURFRecord1 record = new SURFRecord1
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
IndexStart = rows,
IndexEnd = endRows
};
observerSurfImageIndexList.Add(record);
listOfAllObserverImagesSurfPoints.AddRange(observerImageSurfPoints);
rows = endRows + 1;
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
readingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Reading Surb Complete, it tooked {0} ms. Saving Repository...", readingTime));
//------------Initialize Tree from Data
sw1.Reset(); sw1.Start();
double[][] superMatrix = listOfAllObserverImagesSurfPoints.Select(c => c.Descriptor).ToArray();
int[] outputs = new int[superMatrix.Length];
for (int i = 0; i < outputs.Length; i++)
outputs[i] = i;
Accord.MachineLearning.Structures.KDTree<int> tree =
Accord.MachineLearning.Structures.KDTree.FromData(superMatrix,
outputs,
Accord.Math.Distance.Euclidean, inPlace: true);
sw1.Stop();
indexingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Intializing KD Tree: {0}", indexingTime));
//--------------Saving Indexed Records
sw1.Reset(); sw1.Start();
SurfAccordDataSet surfAccordDataset = new SurfAccordDataSet
{
SurfImageIndexRecord = observerSurfImageIndexList,
IndexedTree = tree
};
string repoFileStoragePath = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordDataSet.bin");
if (File.Exists(repoFileStoragePath))
File.Delete(repoFileStoragePath);
using (FileStream s = File.Create(repoFileStoragePath))
{
//Polenter.Serialization.SharpSerializerBinarySettings bs =
// new Polenter.Serialization.SharpSerializerBinarySettings(Polenter.Serialization.BinarySerializationMode.SizeOptimized);
//Polenter.Serialization.SharpSerializer formatter = new Polenter.Serialization.SharpSerializer(bs);
//formatter.Serialize(surfAccordDataset, s);
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter formatter
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
formatter.Serialize(s, surfAccordDataset);
s.Close();
}
sw1.Stop();
saveingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Saving Surf Accord Dataset: {0}", saveingTime));
//Invalidating Cache
CacheHelper.Remove("SurfAccordDataSet");
CacheHelper.Remove("SurfAccordIntervalTree");
logWriter(string.Format("Reading: {0} ms, Indexing: {1} ms, Saving Indexed data {2}", readingTime, indexingTime, saveingTime));
}
private static void FindMatch(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, SurfSettings surfSettings, out long matchTime, out VectorOfKeyPoint modelKeyPoints, out VectorOfKeyPoint observedKeyPoints, out Matrix <int> indices, out Matrix <byte> mask, out HomographyMatrix homography)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSettings != null)
{
hessianThresh = surfSettings.HessianThresh.Value;
uniquenessThreshold = surfSettings.UniquenessThreshold.Value;
}
SURFDetector surfCPU = new SURFDetector(hessianThresh, false);
#endregion
int k = 2;
Stopwatch watch;
homography = null;
//extract features from the object image
modelKeyPoints = new VectorOfKeyPoint();
Matrix <float> modelDescriptors = surfCPU.DetectAndCompute(modelImage, null, modelKeyPoints);
watch = Stopwatch.StartNew();
// extract features from the observed image
observedKeyPoints = new VectorOfKeyPoint();
Matrix <float> observedDescriptors = surfCPU.DetectAndCompute(observedImage, null, observedKeyPoints);
BruteForceMatcher <float> matcher = new BruteForceMatcher <float>(DistanceType.L2);
matcher.Add(modelDescriptors);
indices = new Matrix <int>(observedDescriptors.Rows, k);
using (Matrix <float> dist = new Matrix <float>(observedDescriptors.Rows, k))
{
matcher.KnnMatch(observedDescriptors, indices, dist, k, null);
mask = new Matrix <byte>(dist.Rows, 1);
mask.SetValue(255);
Features2DToolbox.VoteForUniqueness(dist, uniquenessThreshold, mask);
}
int nonZeroCount = CvInvoke.cvCountNonZero(mask);
if (nonZeroCount >= 4)
{
nonZeroCount = Features2DToolbox.VoteForSizeAndOrientation(modelKeyPoints, observedKeyPoints, indices, mask, 1.5, 20);
if (nonZeroCount >= 4)
{
homography = Features2DToolbox.GetHomographyMatrixFromMatchedFeatures(modelKeyPoints, observedKeyPoints, indices, mask, 2);
}
}
watch.Stop();
matchTime = watch.ElapsedMilliseconds;
}
public List <ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List <ImageRecord> rtnImageList = new List <ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
long IndexingTime = 0; long QueryingTime = 0; long LoopTime = 0;
#endregion Diagnostic Region
SurfDataSet observerDataset = SurfRepository.GetSurfDataSet();
if (observerDataset == null)
{
throw new InvalidOperationException("Can't get the Surf Index, please index first");
}
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion Surf Dectator Region
Matrix <float> modelDescriptors;
using (Image <Gray, byte> modelImage = new Image <Gray, byte>(queryImagePath))
{
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
modelDescriptors = surfDectector.DetectAndCompute(modelImage, null, modelKeyPoints);
if (modelDescriptors.Rows < 4)
{
throw new InvalidOperationException("Model image didn't have any significant features to detect");
}
Matrix <float> superMatrix = observerDataset.SuperMatrix;
sw.Start();
Emgu.CV.Flann.Index flannIndex;
if (!SurfRepository.Exists("flannIndex"))
{
flannIndex = new Emgu.CV.Flann.Index(superMatrix, 4);
SurfRepository.AddFlannIndex(flannIndex, "flannIndex");
}
else
{
flannIndex = SurfRepository.GetFlannIndex("flannIndex");
}
sw.Stop(); IndexingTime = sw.ElapsedMilliseconds; sw.Reset();
var indices = new Matrix <int>(modelDescriptors.Rows, 2); // matrix that will contain indices of the 2-nearest neighbors found
var dists = new Matrix <float>(modelDescriptors.Rows, 2); // matrix that will contain distances to the 2-nearest neighbors found
sw.Start();
flannIndex.KnnSearch(modelDescriptors, indices, dists, 2, 24);
sw.Stop(); QueryingTime = sw.ElapsedMilliseconds; sw.Reset();
List <SURFRecord1> imageList = observerDataset.SurfImageIndexRecord;
imageList.ForEach(x => x.Distance = 0);
//Create Interval Tree for Images
IntervalTreeHelper.CreateTree(imageList);
sw.Start();
for (int i = 0; i < indices.Rows; i++)
{
// filter out all inadequate pairs based on distance between pairs
if (dists.Data[i, 0] < (uniquenessThreshold * dists.Data[i, 1]))
{
var img = IntervalTreeHelper.GetImageforRange(indices[i, 0]);
if (img != null)
{
img.Distance++;
}
}
}
sw.Stop(); LoopTime = sw.ElapsedMilliseconds;
string msg = String.Format("Indexing: {0}, Querying: {1}, Looping: {2}", IndexingTime, QueryingTime, LoopTime);
messageToLog = msg;
rtnImageList = imageList.Where(x => x.Distance > surfSetting.GoodMatchThreshold).OrderByDescending(x => x.Distance).Select(x => (ImageRecord)x).ToList();
}
return(rtnImageList);
}
public void CalculateSurfDescriptor(Image img, SurfSettings surfSetting)
{
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
SurfSettings surfSetting = null)
{
//For Time Profilling
long readingTime, indexingTime = 0, saveingTime = 0;
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion
int rows = 0;
Stopwatch sw1;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
string fullFileName = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordLinear.bin");
if (File.Exists(fullFileName))
{
File.Delete(fullFileName);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Bitmap observerImage = (Bitmap)Image.FromFile(fi.FullName))
{
List <SpeededUpRobustFeaturePoint> observerImageSurfPoints = surf.ProcessImage(observerImage);
if (observerImageSurfPoints.Count > 4)
{
SURFAccordRecord3 record = new SURFAccordRecord3
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
SurfDescriptors = observerImageSurfPoints
};
bf.Serialize(fs, record);
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
fs.Close();
}
sw1.Stop();
readingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Reading Surb Complete, it tooked {0} ms. Saving Repository...", readingTime));
logWriter(string.Format("Reading: {0} ms, Indexing: {1} ms, Saving Indexed data {2}", readingTime, indexingTime, saveingTime));
}
private Bitmap CompareAndDrawImage(Bitmap modelImage, Bitmap observedImage, SurfSettings setting)
{
Stopwatch watch1 = new Stopwatch();
Stopwatch watch2 = new Stopwatch();
Bitmap returnBitmap;
watch2.Start();
watch1.Reset(); watch1.Start();
double hessianThreshold = setting.HessianThresh.HasValue ? setting.HessianThresh.Value : 500;
float hessianThreshold2 = (float)hessianThreshold / 1000000;
Debug.WriteLine("hessianThreshold2: {0}", hessianThreshold2);
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
List<SpeededUpRobustFeaturePoint> surfPoints1 = surf.ProcessImage(modelImage);
List<SpeededUpRobustFeaturePoint> surfPoints2 = surf.ProcessImage(observedImage);
Debug.WriteLine("Surf points count: {0}", surfPoints1.Count);
Debug.WriteLine("Surf points count: {0}", surfPoints2.Count);
//long memoryFootprint = MemorySize.GetBlobSizeinKb(surfPoints2);
//Debug.WriteLine("Surf extractor: {0} kb", memoryFootprint);
watch1.Stop();
Debug.WriteLine("Surf Detection tooked {0} ms", watch1.ElapsedMilliseconds);
watch1.Reset(); watch1.Start();
// Show the marked points in the original images
Bitmap img1mark = new FeaturesMarker(surfPoints1, 2).Apply(modelImage);
Bitmap img2mark = new FeaturesMarker(surfPoints2, 2).Apply(observedImage);
// Concatenate the two images together in a single image (just to show on screen)
Concatenate concatenate = new Concatenate(img1mark);
returnBitmap = concatenate.Apply(img2mark);
watch1.Stop();
Debug.WriteLine("Surf point plotting tooked {0} ms", watch1.ElapsedMilliseconds);
//watch1.Reset(); watch1.Start();
//List<IntPoint>[] coretionalMatches = getMatches(surfPoints1, surfPoints2);
//watch1.Stop();
//Debug.WriteLine("Correctional Match tooked {0} ms", watch1.ElapsedMilliseconds);
//// Get the two sets of points
//IntPoint[] correlationPoints11 = coretionalMatches[0].ToArray();
//IntPoint[] correlationPoints22 = coretionalMatches[1].ToArray();
//Debug.WriteLine("Correclation points count: {0}", correlationPoints11.Length);
//Debug.WriteLine("Correclation points count: {0}", correlationPoints22.Length);
Debug.WriteLine("Threshold: {0}", setting.UniquenessThreshold.Value);
watch1.Reset(); watch1.Start();
// Step 2: Match feature points using a k-NN
KNearestNeighborMatching matcher = new KNearestNeighborMatching(2);
matcher.Threshold = setting.UniquenessThreshold.Value;
IntPoint[][] matches = matcher.Match(surfPoints1, surfPoints2);
watch1.Stop();
Debug.WriteLine("Knn Match tooked {0} ms", watch1.ElapsedMilliseconds);
// Get the two sets of points
IntPoint[] correlationPoints1 = matches[0];
IntPoint[] correlationPoints2 = matches[1];
Debug.WriteLine("Knn points count: {0}", correlationPoints1.Length);
Debug.WriteLine("Knn points count: {0}", correlationPoints2.Length);
//watch1.Reset(); watch1.Start();
//// Show the marked correlations in the concatenated image
//PairsMarker pairs = new PairsMarker(
// correlationPoints1, // Add image1's width to the X points to show the markings correctly
// correlationPoints2.Apply(p => new IntPoint(p.X + modelImage.Width, p.Y)), Color.Blue);
//returnBitmap = pairs.Apply(returnBitmap);
//watch1.Stop();
//Debug.WriteLine("Match pair marking tooked {0} ms", watch1.ElapsedMilliseconds);
if (correlationPoints1.Length < 4 || correlationPoints2.Length < 4)
{
MessageBox.Show("Insufficient points to attempt a fit.");
return null;
}
watch1.Reset(); watch1.Start();
// Step 3: Create the homography matrix using a robust estimator
//RansacHomographyEstimator ransac = new RansacHomographyEstimator(0.001, 0.99);
RansacHomographyEstimator ransac = new RansacHomographyEstimator(0.001, 0.99);
MatrixH homography = ransac.Estimate(correlationPoints1, correlationPoints2);
watch1.Stop();
Debug.WriteLine("Ransac tooked {0} ms", watch1.ElapsedMilliseconds);
watch1.Reset(); watch1.Start();
// Plot RANSAC results against correlation results
IntPoint[] inliers1 = correlationPoints1.Submatrix(ransac.Inliers);
IntPoint[] inliers2 = correlationPoints2.Submatrix(ransac.Inliers);
watch1.Stop();
Debug.WriteLine("Ransac SubMatrix {0} ms", watch1.ElapsedMilliseconds);
Debug.WriteLine("Ransac points count: {0}", inliers1.Length);
Debug.WriteLine("Ransac points count: {0}", inliers2.Length);
watch1.Reset(); watch1.Start();
PairsMarker inlierPairs = new PairsMarker(
inliers1, // Add image1's width to the X points to show the markings correctly
inliers2.Apply(p => new IntPoint(p.X + modelImage.Width, p.Y)), Color.Red);
returnBitmap = inlierPairs.Apply(returnBitmap);
watch1.Stop();
Debug.WriteLine("Ransac plotting tooked {0} ms", watch1.ElapsedMilliseconds);
watch2.Stop();
return returnBitmap;
}
public List <ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List <ImageRecord> rtnImageList = new List <ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
Stopwatch sw1 = new Stopwatch();
long _loadingTime = 0, _modelImageDectionlong = 0, _queryingTime = 0, _matchingTime = 0;
#endregion Diagnostic Region
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
int minGoodMatchPercent = 0;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
minGoodMatchPercent = surfSetting.GoodMatchThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion Surf Dectator Region
#region Get Model Dectection and Validation
sw.Reset(); sw.Start();
List <SpeededUpRobustFeaturePoint> modelImageSurfPoints;
using (Bitmap modelImage = (Bitmap)Image.FromFile(queryImagePath))
{
modelImageSurfPoints = surf.ProcessImage(modelImage);
}
if (modelImageSurfPoints == null ||
modelImageSurfPoints.Count < 4)
{
throw new InvalidOperationException("Insuffucient interesting point in query image, try another query image");
}
sw.Stop();
_modelImageDectionlong = sw.ElapsedMilliseconds;
#endregion
#region Search Images
sw.Reset(); sw.Start();
string fullFileName = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordLinear.bin");
if (!File.Exists(fullFileName))
{
string exMsg = string.Format("Can't get the Surf Index at {0}, please index first", fullFileName);
throw new FileNotFoundException(fullFileName);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
long fileLength = fs.Length;
while (fs.Position < fileLength)
{
SURFAccordRecord3 record = (SURFAccordRecord3)bf.Deserialize(fs);
KNearestNeighborMatching matcher = new KNearestNeighborMatching(2);
matcher.Threshold = uniquenessThreshold;
sw1.Start();
AForge.IntPoint[][] matches = matcher.Match(modelImageSurfPoints, record.SurfDescriptors);
sw1.Stop();
var countOfMatchPoint = matches[0].Length;
if (countOfMatchPoint > 0)
{
double totalnumberOfModelFeature = modelImageSurfPoints.Count;
double matchPercentage = ((totalnumberOfModelFeature - (double)countOfMatchPoint) / totalnumberOfModelFeature);
matchPercentage = (1 - matchPercentage) * 100;
matchPercentage = Math.Round(matchPercentage);
if (matchPercentage >= minGoodMatchPercent)
{
record.Distance = matchPercentage;
rtnImageList.Add(record.Clone());
}
}
record = null;
}
fs.Close();
}
sw.Stop();
_matchingTime = sw1.ElapsedMilliseconds;
_queryingTime = sw.ElapsedMilliseconds;
#endregion
string msg = String.Format("Loading: {0}, Model detection: {1}, Querying: {2}, Matching: {3}",
_loadingTime, _modelImageDectionlong, _queryingTime, _matchingTime);
messageToLog = msg;
if (rtnImageList.Count > 0)
{
rtnImageList = rtnImageList.OrderByDescending(rec => rec.Distance)
.ToList <ImageRecord>();
}
return(rtnImageList);
}
public List<ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List<ImageRecord> rtnImageList = new List<ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
long _loadingTime, _modelImageDectionlong, _queryingTime, _treeQuery, _loopTime = 0;
#endregion Diagnostic Region
#region Get KD-Tree Index
sw.Reset(); sw.Start();
//--------------Getting Indexed Records
SurfAccordDataSet surfAccordDataset;
bool isExist= CacheHelper.Get<SurfAccordDataSet>("SurfAccordDataSet", out surfAccordDataset);
if (!isExist)
{
string repoFileStoragePath = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordDataSet.bin");
if (!File.Exists(repoFileStoragePath))
{
string exMsg = string.Format("Can't get the Surf Index at {0}, please index first", repoFileStoragePath);
throw new FileNotFoundException(exMsg);
}
using (FileStream s = File.OpenRead(repoFileStoragePath))
{
//Polenter.Serialization.SharpSerializerBinarySettings bs =
// new Polenter.Serialization.SharpSerializerBinarySettings(Polenter.Serialization.BinarySerializationMode.SizeOptimized);
//Polenter.Serialization.SharpSerializer formatter = new Polenter.Serialization.SharpSerializer(bs);
//formatter.Serialize(surfAccordDataset, s);
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter formatter
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
surfAccordDataset = (SurfAccordDataSet)formatter.Deserialize(s);
s.Close();
}
CacheHelper.Add<SurfAccordDataSet>(surfAccordDataset, "SurfAccordDataSet");
}
if (surfAccordDataset == null)
throw new InvalidOperationException("Can't get the Surf Index, please index first");
sw.Stop();
_loadingTime = sw.ElapsedMilliseconds;
#endregion
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
int goodMatchDistThreshold = 0;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
goodMatchDistThreshold = surfSetting.GoodMatchThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion Surf Dectator Region
#region Get Model Dectection and Validation
sw.Reset(); sw.Start();
List<SpeededUpRobustFeaturePoint> modelImageSurfPoints;
using (Bitmap modelImage = (Bitmap)Image.FromFile(queryImagePath))
{
modelImageSurfPoints = surf.ProcessImage(modelImage);
}
if (modelImageSurfPoints == null
|| modelImageSurfPoints.Count < 4)
{
throw new InvalidOperationException("Insuffucient interesting point in query image, try another query image");
}
sw.Stop();
_modelImageDectionlong = sw.ElapsedMilliseconds;
#endregion
#region Search Images
sw.Reset(); sw.Start();
//------------Search Images
Accord.MachineLearning.Structures.KDTree<int> tree = surfAccordDataset.IndexedTree;
double[][] listofQueryDescriptors = modelImageSurfPoints.Select(ft => ft.Descriptor).ToArray();
double[] myscores = new double[listofQueryDescriptors.Length];
int[] labels = Enumerable.Repeat(-1, listofQueryDescriptors.Length).ToArray();
for (int i = 0; i < listofQueryDescriptors.Length; i++)
{
KDTreeNodeCollection<int> neighbors = tree.ApproximateNearest(listofQueryDescriptors[i],2, 90d);
//KDTreeNodeCollection<int> neighbors = tree.Nearest(listofQueryDescriptors[i], uniquenessThreshold, 2);
Dictionary<int, double> keyValueStore = new Dictionary<int, double>();
double similarityDist = 0;
foreach (KDTreeNodeDistance<int> point in neighbors)
{
int label = point.Node.Value;
double d = point.Distance;
// Convert to similarity measure
if (keyValueStore.ContainsKey(label))
{
similarityDist = keyValueStore[label];
similarityDist += 1.0 / (1.0 + d);
keyValueStore[label] = similarityDist;
}
else
{
similarityDist = 1.0 / (1.0 + d);
keyValueStore.Add(label, similarityDist);
}
}
if (keyValueStore.Count > 0)
{
int maxIndex = keyValueStore.Aggregate((l, r) => l.Value > r.Value ? l : r).Key;
labels[i] = maxIndex;
double sumOfAllValues = keyValueStore.Values.Sum();
myscores[i] = keyValueStore[maxIndex] / sumOfAllValues;
}
}
sw.Stop();
_queryingTime = sw.ElapsedMilliseconds;
sw.Reset(); sw.Start();
List<SURFRecord1> listOfSurfImages = surfAccordDataset.SurfImageIndexRecord;
//----------Create Interval Tree from ImageMetaData
IntervalTreeLib.IntervalTree<SURFRecord1, int> intervalTree;
bool isTreeExist = CacheHelper.Get<IntervalTreeLib.IntervalTree<SURFRecord1, int>>("SurfAccordIntervalTree", out intervalTree);
if (!isTreeExist)
{
intervalTree = new IntervalTreeLib.IntervalTree<SURFRecord1, int>();
foreach (var record in listOfSurfImages)
{
intervalTree.AddInterval(record.IndexStart, record.IndexEnd, record);
}
CacheHelper.Add<IntervalTreeLib.IntervalTree<SURFRecord1, int>>(intervalTree, "SurfAccordIntervalTree");
}
//--------------Matching Target image similarity
for (int i = 0; i < listofQueryDescriptors.Length; i++)
{
int rowNum = labels[i];
if (rowNum == -1) continue;
double dist = myscores[i];
SURFRecord1 rec = intervalTree.Get(rowNum, IntervalTreeLib.StubMode.ContainsStartThenEnd).FirstOrDefault();
rec.Distance++;
}
sw.Stop();
_loopTime = sw.ElapsedMilliseconds;
#endregion
string msg = String.Format("Loading: {0}, Model detection: {1}, Querying: {2}, Looping: {3}",
_loadingTime, _modelImageDectionlong, _queryingTime, _loopTime);
messageToLog = msg;
rtnImageList = listOfSurfImages.Where(rec => rec.Distance > goodMatchDistThreshold)
.OrderByDescending(rec => rec.Distance)
.ToList<ImageRecord>();
return rtnImageList;
}
public List <ImageRecord> QueryImage(string queryImagePath, SurfSettings surfSetting = null)
{
List <ImageRecord> rtnImageList = new List <ImageRecord>();
var observerFeatureSets = SurfRepository.GetSurfRecordList();
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
int minGoodMatchPercent = 50;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
minGoodMatchPercent = surfSetting.GoodMatchThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
using (Image <Gray, byte> modelImage = new Image <Gray, byte>(queryImagePath))
{
ImageFeature <float>[] modelFeatures = surfDectector.DetectFeatures(modelImage, null);
if (modelFeatures.Length < 4)
{
throw new InvalidOperationException("Model image didn't have any significant features to detect");
}
Features2DTracker <float> tracker = new Features2DTracker <float>(modelFeatures);
foreach (var surfRecord in observerFeatureSets)
{
string queryImageName = System.IO.Path.GetFileName(queryImagePath);
string modelImageName = surfRecord.ImageName;
Features2DTracker <float> .MatchedImageFeature[] matchedFeatures = tracker.MatchFeature(surfRecord.observerFeatures, 2);
Features2DTracker <float> .MatchedImageFeature[] uniqueFeatures = Features2DTracker <float> .VoteForUniqueness(matchedFeatures, uniquenessThreshold);
Features2DTracker <float> .MatchedImageFeature[] uniqueRotOriFeatures = Features2DTracker <float> .VoteForSizeAndOrientation(uniqueFeatures, 1.5, 20);
int goodMatchCount = 0;
goodMatchCount = uniqueRotOriFeatures.Length;
bool isMatch = false;
double totalnumberOfModelFeature = modelFeatures.Length;
double matchPercentage = ((totalnumberOfModelFeature - (double)goodMatchCount) / totalnumberOfModelFeature);
matchPercentage = (1 - matchPercentage) * 100;
matchPercentage = Math.Round(matchPercentage);
if (matchPercentage >= minGoodMatchPercent)
{
HomographyMatrix homography =
Features2DTracker <float> .GetHomographyMatrixFromMatchedFeatures(uniqueRotOriFeatures);
if (homography != null)
{
isMatch = homography.IsValid(5);
if (isMatch)
{
surfRecord.Distance = matchPercentage;
rtnImageList.Add((ImageRecord)surfRecord);
}
}
}
//bool isMatch = false;
//if (uniqueFeatures.Length > 4)
//{
// HomographyMatrix homography =
// Features2DTracker<float>.GetHomographyMatrixFromMatchedFeatures(uniqueRotOriFeatures);
// if (homography != null)
// {
// isMatch = homography.IsValid(5);
// }
//}
//if (isMatch)
//{
// surfRecord.Distance = goodMatchCount;
// rtnImageList.Add((ImageRecord)surfRecord);
//}
//int goodMatchCount = 0;
//foreach (Features2DTracker<float>.MatchedImageFeature ms in matchedFeatures)
//{
// if (ms.SimilarFeatures[0].Distance < uniquenessThreshold)
// goodMatchCount++;
//}
//double totalnumberOfModelFeature = modelFeatures.Length;
//double matchPercentage = ((totalnumberOfModelFeature - (double)goodMatchCount) / totalnumberOfModelFeature);
//matchPercentage = (1 - matchPercentage) * 100;
//matchPercentage = Math.Round(matchPercentage);
//if (matchPercentage >= minGoodMatchPercent)
//{
// surfRecord.Distance = matchPercentage;
// rtnImageList.Add((ImageRecord)surfRecord);
//}
}
}
rtnImageList = rtnImageList.OrderByDescending(x => x.Distance).ToList();
return(rtnImageList);
}
public List<ImageRecord> QueryImage(string queryImagePath, SurfSettings surfSetting = null)
{
List<ImageRecord> rtnImageList = new List<ImageRecord>();
var observerFeatureSets = SurfRepository.GetSurfRecordList();
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
int minGoodMatchPercent = 50;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
minGoodMatchPercent = surfSetting.GoodMatchThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
using (Image<Gray, byte> modelImage = new Image<Gray, byte>(queryImagePath))
{
ImageFeature<float>[] modelFeatures = surfDectector.DetectFeatures(modelImage, null);
if (modelFeatures.Length < 4) throw new InvalidOperationException("Model image didn't have any significant features to detect");
Features2DTracker<float> tracker = new Features2DTracker<float>(modelFeatures);
foreach (var surfRecord in observerFeatureSets)
{
string queryImageName = System.IO.Path.GetFileName(queryImagePath);
string modelImageName = surfRecord.ImageName;
Features2DTracker<float>.MatchedImageFeature[] matchedFeatures = tracker.MatchFeature(surfRecord.observerFeatures, 2);
Features2DTracker<float>.MatchedImageFeature[] uniqueFeatures = Features2DTracker<float>.VoteForUniqueness(matchedFeatures, uniquenessThreshold);
Features2DTracker<float>.MatchedImageFeature[] uniqueRotOriFeatures = Features2DTracker<float>.VoteForSizeAndOrientation(uniqueFeatures, 1.5, 20);
int goodMatchCount = 0;
goodMatchCount = uniqueRotOriFeatures.Length;
bool isMatch = false;
double totalnumberOfModelFeature = modelFeatures.Length;
double matchPercentage = ((totalnumberOfModelFeature - (double)goodMatchCount) / totalnumberOfModelFeature);
matchPercentage = (1 - matchPercentage) * 100;
matchPercentage = Math.Round(matchPercentage);
if (matchPercentage >= minGoodMatchPercent)
{
HomographyMatrix homography =
Features2DTracker<float>.GetHomographyMatrixFromMatchedFeatures(uniqueRotOriFeatures);
if (homography != null)
{
isMatch = homography.IsValid(5);
if (isMatch)
{
surfRecord.Distance = matchPercentage;
rtnImageList.Add((ImageRecord)surfRecord);
}
}
}
//bool isMatch = false;
//if (uniqueFeatures.Length > 4)
//{
// HomographyMatrix homography =
// Features2DTracker<float>.GetHomographyMatrixFromMatchedFeatures(uniqueRotOriFeatures);
// if (homography != null)
// {
// isMatch = homography.IsValid(5);
// }
//}
//if (isMatch)
//{
// surfRecord.Distance = goodMatchCount;
// rtnImageList.Add((ImageRecord)surfRecord);
//}
//int goodMatchCount = 0;
//foreach (Features2DTracker<float>.MatchedImageFeature ms in matchedFeatures)
//{
// if (ms.SimilarFeatures[0].Distance < uniquenessThreshold)
// goodMatchCount++;
//}
//double totalnumberOfModelFeature = modelFeatures.Length;
//double matchPercentage = ((totalnumberOfModelFeature - (double)goodMatchCount) / totalnumberOfModelFeature);
//matchPercentage = (1 - matchPercentage) * 100;
//matchPercentage = Math.Round(matchPercentage);
//if (matchPercentage >= minGoodMatchPercent)
//{
// surfRecord.Distance = matchPercentage;
// rtnImageList.Add((ImageRecord)surfRecord);
//}
}
}
rtnImageList = rtnImageList.OrderByDescending(x => x.Distance).ToList();
return rtnImageList;
}
private void AccordSurfAlgoIndexing(SurfSettings surfSetting)
{
var imageFiles = getFiles(IndexDirectory, "*.gif|*.jpg|*.png|*.bmp|*.jpeg", SearchOption.TopDirectoryOnly);
int totalFileCount = imageFiles.Length;
Dispatcher.Invoke((Action)delegate
{
pbIndex.Minimum = 0; pbIndex.Maximum = totalFileCount; lblIndexStatus.Text = "Indexing..";
});
if (surfSetting.Algorithm == SurfAlgo.Linear)
{
SurfIndexer4 surfIndexer = new SurfIndexer4();
if (ExecInParallel)
surfIndexer.IndexFilesAsync(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
else
surfIndexer.IndexFiles(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
}
else
{
SurfIndexer3 surfIndexer = new SurfIndexer3();
if (ExecInParallel)
surfIndexer.IndexFilesAsync(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
else
surfIndexer.IndexFiles(imageFiles, IndexBgWorker, WriteIndexStatus, surfSetting);
}
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
SurfSettings surfSetting = null)
{
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion
int rows = 0;
Matrix <float> superMatrix = null;
List <SURFRecord1> observerSurfImageIndexList = new List <SURFRecord1>();
Stopwatch sw1, sw2;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Image <Gray, byte> observerImage = new Image <Gray, byte>(fi.FullName))
{
VectorOfKeyPoint observerKeyPoints = new VectorOfKeyPoint();
Matrix <float> observerDescriptor = surfDectector.DetectAndCompute(observerImage, null, observerKeyPoints);
if (observerDescriptor.Rows > 4)
{
int initRow = rows; int endRows = rows + observerDescriptor.Rows - 1;
SURFRecord1 record = new SURFRecord1
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
IndexStart = rows,
IndexEnd = endRows
};
observerSurfImageIndexList.Add(record);
if (superMatrix == null)
{
superMatrix = observerDescriptor;
}
else
{
superMatrix = superMatrix.ConcateVertical(observerDescriptor);
}
rows = endRows + 1;
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
logWriter(string.Format("Index Complete, it tooked {0} ms. Saving Repository...", sw1.ElapsedMilliseconds));
SurfDataSet surfDataset = new SurfDataSet
{
SurfImageIndexRecord = observerSurfImageIndexList,
SuperMatrix = superMatrix
};
sw2 = Stopwatch.StartNew();
SurfRepository.AddSuperMatrixList(surfDataset);
SurfRepository.SaveRepository(SurfAlgo.Flaan);
sw2.Stop();
logWriter(string.Format("Index tooked {0} ms. Saving Repository tooked {1} ms", sw1.ElapsedMilliseconds, sw2.ElapsedMilliseconds));
}
private SurfSettings GetSurfSetting()
{
SurfSettings surfSetting = new SurfSettings();
this.Dispatcher.Invoke(() =>
{
surfSetting.HessianThresh = Convert.ToDouble((cmbSurfThreshold.SelectedItem as ListBoxItem).Content);
surfSetting.UniquenessThreshold = nudsurfUniqueThreshold.Value;
surfSetting.GoodMatchThreshold = nudsurfGoodMatchThreshold.Value;
surfSetting.Algorithm = (SurfAlgo)cmbSurfApproach.SelectedIndex;
});
return surfSetting;
}
/// <summary>
/// Draw the model image and observed image, the matched features and homography projection.
/// </summary>
/// <param name="modelImage">The model image</param>
/// <param name="observedImage">The observed image</param>
/// <param name="matchTime">The output total time for computing the homography matrix.</param>
/// <returns>The model image and observed image, the matched features and homography projection.</returns>
private static Image <Bgr, Byte> Draw(Image <Gray, Byte> modelImage, Image <Gray, byte> observedImage, SurfSettings surfSettings, out long matchTime)
{
HomographyMatrix homography;
VectorOfKeyPoint modelKeyPoints;
VectorOfKeyPoint observedKeyPoints;
Matrix <int> indices;
Matrix <byte> mask;
FindMatch(modelImage, observedImage, surfSettings, out matchTime, out modelKeyPoints, out observedKeyPoints, out indices, out mask, out homography);
//Draw the matched keypoints
Image <Bgr, Byte> result = Features2DToolbox.DrawMatches(modelImage, modelKeyPoints, observedImage, observedKeyPoints,
indices, new Bgr(255, 0, 0), new Bgr(255, 255, 0), mask, Features2DToolbox.KeypointDrawType.DEFAULT);
#region draw the projected region on the image
if (homography != null)
{ //draw a rectangle along the projected model
Rectangle rect = modelImage.ROI;
PointF[] pts = new PointF[] {
new PointF(rect.Left, rect.Bottom),
new PointF(rect.Right, rect.Bottom),
new PointF(rect.Right, rect.Top),
new PointF(rect.Left, rect.Top)
};
homography.ProjectPoints(pts);
result.DrawPolyline(Array.ConvertAll <PointF, Point>(pts, Point.Round), true, new Bgr(Color.Red), 5);
}
#endregion
return(result);
}
public List <ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List <ImageRecord> rtnImageList = new List <ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
long _loadingTime, _modelImageDectionlong, _queryingTime, _treeQuery, _loopTime = 0;
#endregion Diagnostic Region
#region Get KD-Tree Index
sw.Reset(); sw.Start();
//--------------Getting Indexed Records
SurfAccordDataSet surfAccordDataset;
bool isExist = CacheHelper.Get <SurfAccordDataSet>("SurfAccordDataSet", out surfAccordDataset);
if (!isExist)
{
string repoFileStoragePath = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordDataSet.bin");
if (!File.Exists(repoFileStoragePath))
{
string exMsg = string.Format("Can't get the Surf Index at {0}, please index first", repoFileStoragePath);
throw new FileNotFoundException(exMsg);
}
using (FileStream s = File.OpenRead(repoFileStoragePath))
{
//Polenter.Serialization.SharpSerializerBinarySettings bs =
// new Polenter.Serialization.SharpSerializerBinarySettings(Polenter.Serialization.BinarySerializationMode.SizeOptimized);
//Polenter.Serialization.SharpSerializer formatter = new Polenter.Serialization.SharpSerializer(bs);
//formatter.Serialize(surfAccordDataset, s);
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter formatter
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
surfAccordDataset = (SurfAccordDataSet)formatter.Deserialize(s);
s.Close();
}
CacheHelper.Add <SurfAccordDataSet>(surfAccordDataset, "SurfAccordDataSet");
}
if (surfAccordDataset == null)
{
throw new InvalidOperationException("Can't get the Surf Index, please index first");
}
sw.Stop();
_loadingTime = sw.ElapsedMilliseconds;
#endregion
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
int goodMatchDistThreshold = 0;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
goodMatchDistThreshold = surfSetting.GoodMatchThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion Surf Dectator Region
#region Get Model Dectection and Validation
sw.Reset(); sw.Start();
List <SpeededUpRobustFeaturePoint> modelImageSurfPoints;
using (Bitmap modelImage = (Bitmap)Image.FromFile(queryImagePath))
{
modelImageSurfPoints = surf.ProcessImage(modelImage);
}
if (modelImageSurfPoints == null ||
modelImageSurfPoints.Count < 4)
{
throw new InvalidOperationException("Insuffucient interesting point in query image, try another query image");
}
sw.Stop();
_modelImageDectionlong = sw.ElapsedMilliseconds;
#endregion
#region Search Images
sw.Reset(); sw.Start();
//------------Search Images
Accord.MachineLearning.Structures.KDTree <int> tree = surfAccordDataset.IndexedTree;
double[][] listofQueryDescriptors = modelImageSurfPoints.Select(ft => ft.Descriptor).ToArray();
double[] myscores = new double[listofQueryDescriptors.Length];
int[] labels = Enumerable.Repeat(-1, listofQueryDescriptors.Length).ToArray();
for (int i = 0; i < listofQueryDescriptors.Length; i++)
{
KDTreeNodeCollection <int> neighbors = tree.ApproximateNearest(listofQueryDescriptors[i], 2, 90d);
//KDTreeNodeCollection<int> neighbors = tree.Nearest(listofQueryDescriptors[i], uniquenessThreshold, 2);
Dictionary <int, double> keyValueStore = new Dictionary <int, double>();
double similarityDist = 0;
foreach (KDTreeNodeDistance <int> point in neighbors)
{
int label = point.Node.Value;
double d = point.Distance;
// Convert to similarity measure
if (keyValueStore.ContainsKey(label))
{
similarityDist = keyValueStore[label];
similarityDist += 1.0 / (1.0 + d);
keyValueStore[label] = similarityDist;
}
else
{
similarityDist = 1.0 / (1.0 + d);
keyValueStore.Add(label, similarityDist);
}
}
if (keyValueStore.Count > 0)
{
int maxIndex = keyValueStore.Aggregate((l, r) => l.Value > r.Value ? l : r).Key;
labels[i] = maxIndex;
double sumOfAllValues = keyValueStore.Values.Sum();
myscores[i] = keyValueStore[maxIndex] / sumOfAllValues;
}
}
sw.Stop();
_queryingTime = sw.ElapsedMilliseconds;
sw.Reset(); sw.Start();
List <SURFRecord1> listOfSurfImages = surfAccordDataset.SurfImageIndexRecord;
//----------Create Interval Tree from ImageMetaData
IntervalTreeLib.IntervalTree <SURFRecord1, int> intervalTree;
bool isTreeExist = CacheHelper.Get <IntervalTreeLib.IntervalTree <SURFRecord1, int> >("SurfAccordIntervalTree", out intervalTree);
if (!isTreeExist)
{
intervalTree = new IntervalTreeLib.IntervalTree <SURFRecord1, int>();
foreach (var record in listOfSurfImages)
{
intervalTree.AddInterval(record.IndexStart, record.IndexEnd, record);
}
CacheHelper.Add <IntervalTreeLib.IntervalTree <SURFRecord1, int> >(intervalTree, "SurfAccordIntervalTree");
}
//--------------Matching Target image similarity
for (int i = 0; i < listofQueryDescriptors.Length; i++)
{
int rowNum = labels[i];
if (rowNum == -1)
{
continue;
}
double dist = myscores[i];
SURFRecord1 rec = intervalTree.Get(rowNum, IntervalTreeLib.StubMode.ContainsStartThenEnd).FirstOrDefault();
rec.Distance++;
}
sw.Stop();
_loopTime = sw.ElapsedMilliseconds;
#endregion
string msg = String.Format("Loading: {0}, Model detection: {1}, Querying: {2}, Looping: {3}",
_loadingTime, _modelImageDectionlong, _queryingTime, _loopTime);
messageToLog = msg;
rtnImageList = listOfSurfImages.Where(rec => rec.Distance > goodMatchDistThreshold)
.OrderByDescending(rec => rec.Distance)
.ToList <ImageRecord>();
return(rtnImageList);
}
private Bitmap CompareAndDrawImage(Bitmap modelImage, Bitmap observedImage, SurfSettings setting)
{
Stopwatch watch1 = new Stopwatch();
Stopwatch watch2 = new Stopwatch();
Bitmap returnBitmap;
watch2.Start();
watch1.Reset(); watch1.Start();
double hessianThreshold = setting.HessianThresh.HasValue ? setting.HessianThresh.Value : 500;
float hessianThreshold2 = (float)hessianThreshold / 1000000;
Debug.WriteLine("hessianThreshold2: {0}", hessianThreshold2);
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
List <SpeededUpRobustFeaturePoint> surfPoints1 = surf.ProcessImage(modelImage);
List <SpeededUpRobustFeaturePoint> surfPoints2 = surf.ProcessImage(observedImage);
Debug.WriteLine("Surf points count: {0}", surfPoints1.Count);
Debug.WriteLine("Surf points count: {0}", surfPoints2.Count);
//long memoryFootprint = MemorySize.GetBlobSizeinKb(surfPoints2);
//Debug.WriteLine("Surf extractor: {0} kb", memoryFootprint);
watch1.Stop();
Debug.WriteLine("Surf Detection tooked {0} ms", watch1.ElapsedMilliseconds);
watch1.Reset(); watch1.Start();
// Show the marked points in the original images
Bitmap img1mark = new FeaturesMarker(surfPoints1, 2).Apply(modelImage);
Bitmap img2mark = new FeaturesMarker(surfPoints2, 2).Apply(observedImage);
// Concatenate the two images together in a single image (just to show on screen)
Concatenate concatenate = new Concatenate(img1mark);
returnBitmap = concatenate.Apply(img2mark);
watch1.Stop();
Debug.WriteLine("Surf point plotting tooked {0} ms", watch1.ElapsedMilliseconds);
//watch1.Reset(); watch1.Start();
//List<IntPoint>[] coretionalMatches = getMatches(surfPoints1, surfPoints2);
//watch1.Stop();
//Debug.WriteLine("Correctional Match tooked {0} ms", watch1.ElapsedMilliseconds);
//// Get the two sets of points
//IntPoint[] correlationPoints11 = coretionalMatches[0].ToArray();
//IntPoint[] correlationPoints22 = coretionalMatches[1].ToArray();
//Debug.WriteLine("Correclation points count: {0}", correlationPoints11.Length);
//Debug.WriteLine("Correclation points count: {0}", correlationPoints22.Length);
Debug.WriteLine("Threshold: {0}", setting.UniquenessThreshold.Value);
watch1.Reset(); watch1.Start();
// Step 2: Match feature points using a k-NN
KNearestNeighborMatching matcher = new KNearestNeighborMatching(2);
matcher.Threshold = setting.UniquenessThreshold.Value;
IntPoint[][] matches = matcher.Match(surfPoints1, surfPoints2);
watch1.Stop();
Debug.WriteLine("Knn Match tooked {0} ms", watch1.ElapsedMilliseconds);
// Get the two sets of points
IntPoint[] correlationPoints1 = matches[0];
IntPoint[] correlationPoints2 = matches[1];
Debug.WriteLine("Knn points count: {0}", correlationPoints1.Length);
Debug.WriteLine("Knn points count: {0}", correlationPoints2.Length);
//watch1.Reset(); watch1.Start();
//// Show the marked correlations in the concatenated image
//PairsMarker pairs = new PairsMarker(
// correlationPoints1, // Add image1's width to the X points to show the markings correctly
// correlationPoints2.Apply(p => new IntPoint(p.X + modelImage.Width, p.Y)), Color.Blue);
//returnBitmap = pairs.Apply(returnBitmap);
//watch1.Stop();
//Debug.WriteLine("Match pair marking tooked {0} ms", watch1.ElapsedMilliseconds);
if (correlationPoints1.Length < 4 || correlationPoints2.Length < 4)
{
MessageBox.Show("Insufficient points to attempt a fit.");
return(null);
}
watch1.Reset(); watch1.Start();
// Step 3: Create the homography matrix using a robust estimator
//RansacHomographyEstimator ransac = new RansacHomographyEstimator(0.001, 0.99);
RansacHomographyEstimator ransac = new RansacHomographyEstimator(0.001, 0.99);
MatrixH homography = ransac.Estimate(correlationPoints1, correlationPoints2);
watch1.Stop();
Debug.WriteLine("Ransac tooked {0} ms", watch1.ElapsedMilliseconds);
watch1.Reset(); watch1.Start();
// Plot RANSAC results against correlation results
IntPoint[] inliers1 = correlationPoints1.Submatrix(ransac.Inliers);
IntPoint[] inliers2 = correlationPoints2.Submatrix(ransac.Inliers);
watch1.Stop();
Debug.WriteLine("Ransac SubMatrix {0} ms", watch1.ElapsedMilliseconds);
Debug.WriteLine("Ransac points count: {0}", inliers1.Length);
Debug.WriteLine("Ransac points count: {0}", inliers2.Length);
watch1.Reset(); watch1.Start();
PairsMarker inlierPairs = new PairsMarker(
inliers1, // Add image1's width to the X points to show the markings correctly
inliers2.Apply(p => new IntPoint(p.X + modelImage.Width, p.Y)), Color.Red);
returnBitmap = inlierPairs.Apply(returnBitmap);
watch1.Stop();
Debug.WriteLine("Ransac plotting tooked {0} ms", watch1.ElapsedMilliseconds);
watch2.Stop();
return(returnBitmap);
}
public void IndexFilesAsync(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
SurfSettings surfSetting = null)
{
throw new NotImplementedException();
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action<string> logWriter,
SurfSettings surfSetting = null)
{
//For Time Profilling
long readingTime, indexingTime = 0 , saveingTime = 0;
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion
int rows = 0;
Stopwatch sw1;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
string fullFileName = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordLinear.bin");
if (File.Exists(fullFileName))
File.Delete(fullFileName);
using (FileStream fs = new FileStream(fullFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Bitmap observerImage = (Bitmap)Image.FromFile(fi.FullName))
{
List<SpeededUpRobustFeaturePoint> observerImageSurfPoints = surf.ProcessImage(observerImage);
if (observerImageSurfPoints.Count > 4)
{
SURFAccordRecord3 record = new SURFAccordRecord3
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
SurfDescriptors = observerImageSurfPoints
};
bf.Serialize(fs, record);
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
fs.Close();
}
sw1.Stop();
readingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Reading Surb Complete, it tooked {0} ms. Saving Repository...", readingTime));
logWriter(string.Format("Reading: {0} ms, Indexing: {1} ms, Saving Indexed data {2}", readingTime, indexingTime, saveingTime));
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
SurfSettings surfSetting = null)
{
//For Time Profilling
long readingTime, indexingTime, saveingTime;
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion
int rows = 0;
List <SURFRecord1> observerSurfImageIndexList = new List <SURFRecord1>();
List <SpeededUpRobustFeaturePoint> listOfAllObserverImagesSurfPoints = new List <SpeededUpRobustFeaturePoint>();
Stopwatch sw1;
sw1 = Stopwatch.StartNew();
logWriter("Index started...");
int totalFileCount = imageFiles.Length;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Bitmap observerImage = (Bitmap)Image.FromFile(fi.FullName))
{
List <SpeededUpRobustFeaturePoint> observerImageSurfPoints = surf.ProcessImage(observerImage);
if (observerImageSurfPoints.Count > 4)
{
int initRow = rows; int endRows = rows + observerImageSurfPoints.Count - 1;
SURFRecord1 record = new SURFRecord1
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
IndexStart = rows,
IndexEnd = endRows
};
observerSurfImageIndexList.Add(record);
listOfAllObserverImagesSurfPoints.AddRange(observerImageSurfPoints);
rows = endRows + 1;
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
readingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Reading Surb Complete, it tooked {0} ms. Saving Repository...", readingTime));
//------------Initialize Tree from Data
sw1.Reset(); sw1.Start();
double[][] superMatrix = listOfAllObserverImagesSurfPoints.Select(c => c.Descriptor).ToArray();
int[] outputs = new int[superMatrix.Length];
for (int i = 0; i < outputs.Length; i++)
{
outputs[i] = i;
}
Accord.MachineLearning.Structures.KDTree <int> tree =
Accord.MachineLearning.Structures.KDTree.FromData(superMatrix,
outputs,
Accord.Math.Distance.Euclidean, inPlace: true);
sw1.Stop();
indexingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Intializing KD Tree: {0}", indexingTime));
//--------------Saving Indexed Records
sw1.Reset(); sw1.Start();
SurfAccordDataSet surfAccordDataset = new SurfAccordDataSet
{
SurfImageIndexRecord = observerSurfImageIndexList,
IndexedTree = tree
};
string repoFileStoragePath = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordDataSet.bin");
if (File.Exists(repoFileStoragePath))
{
File.Delete(repoFileStoragePath);
}
using (FileStream s = File.Create(repoFileStoragePath))
{
//Polenter.Serialization.SharpSerializerBinarySettings bs =
// new Polenter.Serialization.SharpSerializerBinarySettings(Polenter.Serialization.BinarySerializationMode.SizeOptimized);
//Polenter.Serialization.SharpSerializer formatter = new Polenter.Serialization.SharpSerializer(bs);
//formatter.Serialize(surfAccordDataset, s);
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter formatter
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
formatter.Serialize(s, surfAccordDataset);
s.Close();
}
sw1.Stop();
saveingTime = sw1.ElapsedMilliseconds;
logWriter(string.Format("Saving Surf Accord Dataset: {0}", saveingTime));
//Invalidating Cache
CacheHelper.Remove("SurfAccordDataSet");
CacheHelper.Remove("SurfAccordIntervalTree");
logWriter(string.Format("Reading: {0} ms, Indexing: {1} ms, Saving Indexed data {2}", readingTime, indexingTime, saveingTime));
}
public void IndexFilesAsync(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action<string> logWriter,
SurfSettings surfSetting = null)
{
throw new NotImplementedException();
}
public AccordSurfWindow(string modelImagePath, string observerImagePath, SurfSettings setting) : this()
{
this.ModelImagePath = modelImagePath;
this.ObserverImagePath = observerImagePath;
this.SurfSetting = setting;
}
public static void MatchInWindow(string modelImagePath, string observedImagePath, SurfSettings surfSettings)
{
long matchTime;
using (Image <Gray, Byte> modelImage = new Image <Gray, byte>(modelImagePath))
using (Image <Gray, Byte> observedImage = new Image <Gray, byte>(observedImagePath))
{
Image <Bgr, byte> result = Draw(modelImage, observedImage, surfSettings, out matchTime);
ImageViewer.Show(result, String.Format("Matched using {0} in {1} milliseconds", GpuInvoke.HasCuda ? "GPU" : "CPU", matchTime));
}
}
public List<ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List<ImageRecord> rtnImageList = new List<ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
Stopwatch sw1 = new Stopwatch();
long _loadingTime = 0, _modelImageDectionlong = 0, _queryingTime = 0, _matchingTime = 0;
#endregion Diagnostic Region
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
int minGoodMatchPercent = 0;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
minGoodMatchPercent = surfSetting.GoodMatchThreshold.Value;
}
float hessianThreshold2 = (float)hessianThresh / 1000000;
SpeededUpRobustFeaturesDetector surf = new SpeededUpRobustFeaturesDetector(hessianThreshold2);
#endregion Surf Dectator Region
#region Get Model Dectection and Validation
sw.Reset(); sw.Start();
List<SpeededUpRobustFeaturePoint> modelImageSurfPoints;
using (Bitmap modelImage = (Bitmap)Image.FromFile(queryImagePath))
{
modelImageSurfPoints = surf.ProcessImage(modelImage);
}
if (modelImageSurfPoints == null
|| modelImageSurfPoints.Count < 4)
{
throw new InvalidOperationException("Insuffucient interesting point in query image, try another query image");
}
sw.Stop();
_modelImageDectionlong = sw.ElapsedMilliseconds;
#endregion
#region Search Images
sw.Reset(); sw.Start();
string fullFileName = Path.Combine(DirectoryHelper.SaveDirectoryPath, "SurfAccordLinear.bin");
if (!File.Exists(fullFileName))
{
string exMsg = string.Format("Can't get the Surf Index at {0}, please index first", fullFileName);
throw new FileNotFoundException(fullFileName);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
long fileLength = fs.Length;
while (fs.Position < fileLength)
{
SURFAccordRecord3 record = (SURFAccordRecord3)bf.Deserialize(fs);
KNearestNeighborMatching matcher = new KNearestNeighborMatching(2);
matcher.Threshold = uniquenessThreshold;
sw1.Start();
AForge.IntPoint[][] matches = matcher.Match(modelImageSurfPoints, record.SurfDescriptors);
sw1.Stop();
var countOfMatchPoint = matches[0].Length;
if (countOfMatchPoint > 0)
{
double totalnumberOfModelFeature = modelImageSurfPoints.Count;
double matchPercentage = ((totalnumberOfModelFeature - (double)countOfMatchPoint) / totalnumberOfModelFeature);
matchPercentage = (1 - matchPercentage) * 100;
matchPercentage = Math.Round(matchPercentage);
if (matchPercentage >= minGoodMatchPercent)
{
record.Distance = matchPercentage;
rtnImageList.Add(record.Clone());
}
}
record = null;
}
fs.Close();
}
sw.Stop();
_matchingTime = sw1.ElapsedMilliseconds;
_queryingTime = sw.ElapsedMilliseconds;
#endregion
string msg = String.Format("Loading: {0}, Model detection: {1}, Querying: {2}, Matching: {3}",
_loadingTime, _modelImageDectionlong, _queryingTime, _matchingTime);
messageToLog = msg;
if (rtnImageList.Count > 0)
{
rtnImageList = rtnImageList.OrderByDescending(rec => rec.Distance)
.ToList<ImageRecord>();
}
return rtnImageList;
}
public List<ImageRecord> QueryImage(string queryImagePath, out string messageToLog, SurfSettings surfSetting = null)
{
List<ImageRecord> rtnImageList = new List<ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
long IndexingTime = 0; long QueryingTime = 0; long LoopTime = 0;
#endregion Diagnostic Region
SurfDataSet observerDataset = SurfRepository.GetSurfDataSet();
if (observerDataset == null)
throw new InvalidOperationException("Can't get the Surf Index, please index first");
#region Surf Dectator Region
double hessianThresh = 500;
double uniquenessThreshold = 0.8;
if (surfSetting != null)
{
hessianThresh = surfSetting.HessianThresh.Value;
uniquenessThreshold = surfSetting.UniquenessThreshold.Value;
}
SURFDetector surfDectector = new SURFDetector(hessianThresh, false);
#endregion Surf Dectator Region
Matrix<float> modelDescriptors;
using (Image<Gray, byte> modelImage = new Image<Gray, byte>(queryImagePath))
{
VectorOfKeyPoint modelKeyPoints = new VectorOfKeyPoint();
modelDescriptors = surfDectector.DetectAndCompute(modelImage, null, modelKeyPoints);
if (modelDescriptors.Rows < 4) throw new InvalidOperationException("Model image didn't have any significant features to detect");
Matrix<float> superMatrix = observerDataset.SuperMatrix;
sw.Start();
Emgu.CV.Flann.Index flannIndex;
if (!SurfRepository.Exists("flannIndex"))
{
flannIndex = new Emgu.CV.Flann.Index(superMatrix, 4);
SurfRepository.AddFlannIndex(flannIndex, "flannIndex");
}
else
flannIndex = SurfRepository.GetFlannIndex("flannIndex");
sw.Stop(); IndexingTime = sw.ElapsedMilliseconds; sw.Reset();
var indices = new Matrix<int>(modelDescriptors.Rows, 2); // matrix that will contain indices of the 2-nearest neighbors found
var dists = new Matrix<float>(modelDescriptors.Rows, 2); // matrix that will contain distances to the 2-nearest neighbors found
sw.Start();
flannIndex.KnnSearch(modelDescriptors, indices, dists, 2, 24);
sw.Stop(); QueryingTime = sw.ElapsedMilliseconds; sw.Reset();
List<SURFRecord1> imageList = observerDataset.SurfImageIndexRecord;
imageList.ForEach(x => x.Distance = 0);
//Create Interval Tree for Images
IntervalTreeHelper.CreateTree(imageList);
sw.Start();
for (int i = 0; i < indices.Rows; i++)
{
// filter out all inadequate pairs based on distance between pairs
if (dists.Data[i, 0] < (uniquenessThreshold * dists.Data[i, 1]))
{
var img = IntervalTreeHelper.GetImageforRange(indices[i, 0]);
if (img != null) img.Distance++;
}
}
sw.Stop(); LoopTime = sw.ElapsedMilliseconds;
string msg = String.Format("Indexing: {0}, Querying: {1}, Looping: {2}", IndexingTime, QueryingTime, LoopTime);
messageToLog = msg;
rtnImageList = imageList.Where(x => x.Distance > surfSetting.GoodMatchThreshold).OrderByDescending(x => x.Distance).Select(x => (ImageRecord)x).ToList();
}
return rtnImageList;
}
