public List <ImageRecord> QueryImage(string queryImagePath, out string messageToLog,
LocateSettings locateSetting = null)
{
List <ImageRecord> rtnImageList = new List <ImageRecord>();
#region Diagnostic Region
Stopwatch sw = new Stopwatch();
Stopwatch sw1 = new Stopwatch();
long _loadingTime = 0, _queryingMAVTime = 0, _matchingTime = 0;
#endregion Diagnostic Region
#region Reading Cookbook
sw1.Reset(); sw1.Start();
//-----Reading Cookbook
double[][] codeBook = null;
string fullFileName = locateSetting.CodeBookFullPath;
if (!File.Exists(fullFileName))
{
string msg = string.Format("Couldn't find {0}, Please Index before querying with Locate", fullFileName);
throw new InvalidOperationException(msg);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
codeBook = (double[][])bf.Deserialize(fs);
fs.Close();
}
#endregion
#region Reading Image Data
//----Read Image Data
LoCaTeDataSet locateDS = null;
fullFileName = Path.Combine(DirectoryHelper.SaveDirectoryPath, "LoCATeImageRecords.bin");
if (!File.Exists(fullFileName))
{
string msg = string.Format("Couldn't find {0}, Please Index before querying with Locate", fullFileName);
throw new InvalidOperationException(msg);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
locateDS = (LoCaTeDataSet)bf.Deserialize(fs);
fs.Close();
}
sw1.Stop();
_loadingTime = sw1.ElapsedMilliseconds;
#endregion
#region Query Image Detection and Visual Words
//---Reading Query Image
sw1.Reset(); sw1.Start();
LoCATe descriptor = new LoCATe();
List <double[]> queryDescriptors = descriptor.extract(new Bitmap(queryImagePath), "SURF");
double[] queryVisualWord = createVisualWord(queryDescriptors, codeBook);
int totalFileCount = locateDS.AllImageRecordSet.Count;
if (locateSetting.isLteSchemeNeedToAppy)
{
queryVisualWord = doMaths(queryVisualWord, locateDS.HistogramSumOfAllVisualWords, totalFileCount);
}
sw1.Stop();
_queryingMAVTime = sw1.ElapsedMilliseconds;
#endregion
List <LoCaTeBoWRecord> AllImageRecordSet = locateDS.AllImageRecordSet;
double[][] AllDatas = AllImageRecordSet.Select(rec => rec.VisaulWord).ToArray();
#region Searching of Image
sw1.Reset(); sw1.Start();
if (locateSetting.isLteSchemeNeedToAppy)
{
//-----------Creating ltc Data
double[][] ltcData = new double[totalFileCount][];
for (int i = 0; i < AllDatas.Length; i++)
{
ltcData[i] = doMaths((double[])(AllDatas[i]).Clone(), locateDS.HistogramSumOfAllVisualWords, totalFileCount);
}
AllDatas = ltcData;
}
double EucDistance;
for (int i = 0; i < totalFileCount; i++)
{
EucDistance = Accord.Math.Distance.Euclidean(queryVisualWord, AllDatas[i]);
AllImageRecordSet[i].Distance = EucDistance;
}
List <LoCaTeRanker> listofImageWRanker = new List <LoCaTeRanker>();
if (locateSetting.IsExtendedSearch)
{
//Take first 25 images
IEnumerable <LoCaTeBoWRecord> first25Image = AllImageRecordSet.OrderBy(rec => rec.Distance)
.Where(rec => rec.Distance < locateSetting.GoodThresholdDistance)
.Take(25);
//Map to Locate Ranker list
Mapper.CreateMap <LoCaTeBoWRecord, LoCaTeRanker>();
Mapper.CreateMap <ImageRecord, LoCaTeRanker>();
listofImageWRanker = Mapper.Map <IEnumerable <LoCaTeBoWRecord>, List <LoCaTeRanker> >(first25Image);
//Calculate locate range
int totalImageCount = listofImageWRanker.Count;
for (int i = 0; i < totalImageCount; i++)
{
double percentage = (1 - (listofImageWRanker[i].Distance / 1)) * 100;
listofImageWRanker[i].LocateRank = Convert.ToInt32(percentage);
}
//Perform SURF query
listofImageWRanker = PerformExtendedSurfSearch(queryImagePath, listofImageWRanker);
int totalimageBeforeCEDD = listofImageWRanker.Count;
//listofImageWRanker = listofImageWRanker.OrderByDescending(rec => rec.SurfRank).ToList();
//Perform CEDD query
CEDDQuery2 queryCEDD = new CEDDQuery2();
var imageListbyCEDD = queryCEDD.QueryImage(queryImagePath).Take(25).ToList();
totalImageCount = imageListbyCEDD.Count;
for (int i = 0; i < totalImageCount; i++)
{
long id = imageListbyCEDD[i].Id;
double percentage = (1 - (imageListbyCEDD[i].Distance / 35)) * 100;
var img2 = listofImageWRanker.Where(img => img.Id == id).SingleOrDefault();
if (img2 == null)
{
var newImage = Mapper.Map <ImageRecord, LoCaTeRanker>(imageListbyCEDD[i]);
newImage.CEDDRank = Convert.ToInt32(percentage);
listofImageWRanker.Add(newImage);
}
else
{
img2.CEDDRank = Convert.ToInt32(percentage);
}
}
totalImageCount = listofImageWRanker.Count;
for (int i = 0; i < totalImageCount; i++)
{
var imageRank = listofImageWRanker[i];
if (imageRank.SurfRank > 0)
{
imageRank.CombinedRank = 100 - (totalimageBeforeCEDD - imageRank.SurfRank);
}
else
{
imageRank.CombinedRank = (0.5 * imageRank.CEDDRank) + (0.4 * imageRank.LocateRank);
}
imageRank.Distance = imageRank.CombinedRank;
}
listofImageWRanker = listofImageWRanker.OrderByDescending(rec => rec.CombinedRank).ToList();
rtnImageList = listofImageWRanker.ToList <ImageRecord>();
}
else
{
rtnImageList = AllImageRecordSet.OrderBy(rec => rec.Distance)
.Where(rec => rec.Distance < locateSetting.GoodThresholdDistance)
.Take(25)
.ToList <ImageRecord>();
}
sw1.Stop();
_matchingTime = sw1.ElapsedMilliseconds;
#endregion
messageToLog = string.Format("Loading time {0} ms, Query image processing {1} ms, Matching {2} ms",
_loadingTime, _queryingMAVTime, _matchingTime);
return(rtnImageList);
}
public void IndexFiles(FileInfo[] imageFiles, System.ComponentModel.BackgroundWorker IndexBgWorker,
Action <string> logWriter,
LocateSettings locateSetting = null)
{
//For Time Profilling
long extractingTime, kMeanTime = 0, calcBagOfVisualTime = 0;
Stopwatch sw1;
SimpleSurfSift.LoCATe descriptorExtractor = new SimpleSurfSift.LoCATe();
sw1 = Stopwatch.StartNew();
logWriter("Index started, extracting Descriptors...");
List <double[]> ListofDescriptorsForCookBook = new List <double[]>();
List <LoCATeRecord> ListOfAllImageDescriptors = new List <LoCATeRecord>();
int totalFileCount = imageFiles.Length;
if (totalFileCount == 0)
{
logWriter("No files to index");
return;
}
;
for (int i = 0; i < totalFileCount; i++)
{
var fi = imageFiles[i];
using (Bitmap observerImage = (Bitmap)Image.FromFile(fi.FullName))
{
List <double[]> locateDescriptors = descriptorExtractor.extract(observerImage, "SURF");
ListOfAllImageDescriptors.Add(new LoCATeRecord
{
Id = i,
ImageName = fi.Name,
ImagePath = fi.FullName,
LoCATeDescriptors = locateDescriptors
});
if (locateSetting.IsCodeBookNeedToBeCreated)
{
if (locateDescriptors.Count > 4)
{
RandomHelper randNumGenerator = new RandomHelper();
List <int> randIndexes = randNumGenerator.GetRandomNumberInRange(0, locateDescriptors.Count, 10d);
foreach (int index in randIndexes)
{
ListofDescriptorsForCookBook.Add(locateDescriptors[index]);
}
}
else
{
Debug.WriteLine(fi.Name + " skip from index, because it didn't have significant feature");
}
}
}
IndexBgWorker.ReportProgress(i);
}
sw1.Stop();
extractingTime = Convert.ToInt32(sw1.Elapsed.TotalSeconds);
double[][] codeBook = null;
if (locateSetting.IsCodeBookNeedToBeCreated)
{
logWriter("Indexing, Calculating Mean...");
sw1.Reset(); sw1.Start();
KMeans kMeans = new KMeans(locateSetting.SizeOfCodeBook);
kMeans.Compute(ListofDescriptorsForCookBook.ToArray());
codeBook = kMeans.Clusters.Centroids;
//------------Save CookBook
string fullFileName = locateSetting.CodeBookFullPath;
if (File.Exists(fullFileName))
{
File.Delete(fullFileName);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Create, FileAccess.Write, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
bf.Serialize(fs, codeBook);
fs.Close();
}
sw1.Stop();
kMeanTime = Convert.ToInt32(sw1.Elapsed.TotalSeconds);
}
else
{
string fullFileName = locateSetting.CodeBookFullPath;
if (!File.Exists(fullFileName))
{
string msg = string.Format("Couldn't find {0}, Please Index before querying with Locate", fullFileName);
throw new InvalidOperationException(msg);
}
using (FileStream fs = new FileStream(fullFileName, FileMode.Open, FileAccess.Read, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
codeBook = (double[][])bf.Deserialize(fs);
fs.Close();
}
}
logWriter("Indexing, Calculating Bag of Visual Words...");
sw1.Reset(); sw1.Start();
List <LoCaTeBoWRecord> ListOfImageVisualBagOfWorks = new List <LoCaTeBoWRecord>();
for (int i = 0; i < ListOfAllImageDescriptors.Count; i++)
{
double[] visualWordForImage = createVisualWord(ListOfAllImageDescriptors[i].LoCATeDescriptors, codeBook);
LoCaTeBoWRecord rec = new LoCaTeBoWRecord
{
Id = ListOfAllImageDescriptors[i].Id,
ImageName = ListOfAllImageDescriptors[i].ImageName,
ImagePath = ListOfAllImageDescriptors[i].ImagePath,
VisaulWord = visualWordForImage
};
ListOfImageVisualBagOfWorks.Add(rec);
IndexBgWorker.ReportProgress(i);
}
logWriter("Indexing, Calculating ltcData...");
int[] histogramSumOfAllVisualWords = null;
//------------Creating sum histogram of all words
double[][] AllDatas = ListOfImageVisualBagOfWorks.Select(des => des.VisaulWord).ToArray();
histogramSumOfAllVisualWords = createIndex((double[][])(AllDatas));
//------------Creating Image Records Data
LoCaTeDataSet locateDS = new LoCaTeDataSet
{
AllImageRecordSet = ListOfImageVisualBagOfWorks,
HistogramSumOfAllVisualWords = histogramSumOfAllVisualWords
};
logWriter("Indexing, Saving Image Data...");
//------------Save CookBook
string ImageRecordName = Path.Combine(DirectoryHelper.SaveDirectoryPath, "LoCATeImageRecords.bin");
if (File.Exists(ImageRecordName))
{
File.Delete(ImageRecordName);
}
using (FileStream fs = new FileStream(ImageRecordName, FileMode.Create, FileAccess.Write, FileShare.None))
{
System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf
= new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
bf.Serialize(fs, locateDS);
fs.Close();
}
sw1.Stop();
calcBagOfVisualTime = Convert.ToInt32(sw1.Elapsed.TotalSeconds);
logWriter(string.Format("Extracting: {0} sec, KMeanTime: {1} sec, CalcBagOfVisalTime: {2} sec", extractingTime, kMeanTime, calcBagOfVisualTime));
}
