public static void EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
string outputFile,
out int noCorrect,
//out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth,
//out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res,
bool prepareDataForTraining = false, string outputDirectory = null)
{
noCorrect = 0;
//SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth = new SortedDictionary<string, Dictionary<string, int>>();
//SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res = new SortedDictionary<string, Dictionary<string, int>>();
List <KeyValuePair <string, string> > detections_gts = new List <KeyValuePair <string, string> >();
WebClient wc = new WebClient();
foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(data.AggregatedResultString);
string fileName = URIUtilities.filenameFromURI(data.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
detections_gts.Add(new KeyValuePair <string, string>(aggResult.Category, VideoCategoryGroundTruth));
//if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
//{
// confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary<string, int>());
//}
//if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(VideoCategoryGroundTruth))
//{
// confusionMatrix_res_groundtruth[aggResult.Category].Add(VideoCategoryGroundTruth, 0);
//}
//if (!confusionMatrix_groundtruth_res.ContainsKey(VideoCategoryGroundTruth))
//{
// confusionMatrix_groundtruth_res.Add(VideoCategoryGroundTruth, new Dictionary<string, int>());
//}
//if (!confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].ContainsKey(aggResult.Category))
//{
// confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].Add(aggResult.Category, 0);
//}
//confusionMatrix_res_groundtruth[aggResult.Category][VideoCategoryGroundTruth]++;
//confusionMatrix_groundtruth_res[VideoCategoryGroundTruth][aggResult.Category]++;
if (aggResult.Category.Equals(VideoCategoryGroundTruth, StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
fileName, VideoCategoryGroundTruth, aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
}
// prepare training dataset
if (prepareDataForTraining)
{
if (!Directory.Exists(outputDirectory + "\\" + aggResult.Category))
{
Directory.CreateDirectory(outputDirectory + "\\" + aggResult.Category);
}
wc.DownloadFile(data.SatyamURI, outputDirectory + "\\" + aggResult.Category + "\\" + fileName);
}
}
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res;
SatyamResultValidationToolKit.getConfusionMat(detections_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res);
SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile);
}
public static void validateImageNet1000ClassDetectionResult()
{
string baseDir = @"C:\research\dataset\inception1000\";
//string classNameFile = baseDir + "1000class.txt";
string resultFile = baseDir + "results_inception_Seattle.txt";
string is_a_File = baseDir + "wordnet.is_a.txt";
string wnid_words_File = baseDir + "words.txt";
string synsetFile = baseDir + "synset.txt";
//List<string> classNameContent = File.ReadAllLines(classNameFile).ToList();
//Dictionary<int, string> classNames = new Dictionary<int, string>();
//for (int i=0;i<classNameContent.Count;i++)
//{
// string[] fields = classNameContent[i].Split(':');
// string[] names = fields[1].Split('\'');
// string name = "";
// for(int j = 1; j < names.Length - 1; j++)
// {
// name += names[j];
// if (j != names.Length - 2)
// {
// name += '\'';
// }
// }
// classNames.Add(i, name);
//}
List <string> synsetContent = File.ReadAllLines(synsetFile).ToList();
Dictionary <int, string> synsetNames = new Dictionary <int, string>();
for (int i = 0; i < synsetContent.Count; i++)
{
string[] fields = synsetContent[i].Split(' ');
synsetNames.Add(i, fields[0]);
}
List <string> is_a_Content = File.ReadLines(is_a_File).ToList();
Dictionary <string, List <string> > is_a = new Dictionary <string, List <string> >();
foreach (string isA in is_a_Content)
{
string[] fields = isA.Split(' ');
if (!is_a.ContainsKey(fields[1]))
{
is_a.Add(fields[1], new List <string>());
}
is_a[fields[1]].Add(fields[0]);
}
List <string> wordsContent = File.ReadAllLines(wnid_words_File).ToList();
Dictionary <string, List <string> > word_wnid = new Dictionary <string, List <string> >();
foreach (string word in wordsContent)
{
string[] fields = word.Split('\t');
//word_wnid.Add(fields[1], fields[0]);
if (!word_wnid.ContainsKey(fields[1]))
{
word_wnid.Add(fields[1], new List <string>());
}
word_wnid[fields[1]].Add(fields[0]);
}
List <string> resContent = File.ReadAllLines(resultFile).ToList();
List <KeyValuePair <double, int> > results = new List <KeyValuePair <double, int> >();
List <int> gt = new List <int>();
foreach (string res in resContent)
{
string[] fields = res.Split('[');
double confidence = Convert.ToDouble(fields[1].Split(']')[0]);
int classIdx = Convert.ToInt32(fields[2].Split(']')[0]);
results.Add(new KeyValuePair <double, int>(confidence, classIdx));
gt.Add(Convert.ToInt32(fields[2].Split(' ')[1]));
}
List <string> result_wnid = new List <string>();
List <string> result_category = new List <string>();
List <string> dets_gts_oneString = new List <string>();
List <string> categoriesFound = new List <string>();
for (int i = 0; i < results.Count; i++)
{
KeyValuePair <double, int> res = results[i];
if (res.Value == 0)
{
result_wnid.Add("");
result_category.Add("");
continue;
}
string synsetName = synsetNames[res.Value - 1];
//string classwnid = word_wnid[classname];
List <string> is_wnid = new List <string>();
is_wnid.Add(synsetName);
//foreach(string classwnid in word_wnid[synsetName])
//{
// is_wnid.Add(classwnid);
//}
for (int j = 0; j < is_wnid.Count; j++)
{
if (!is_a.ContainsKey(is_wnid[j]))
{
continue;
}
foreach (string wnid in is_a[is_wnid[j]])
{
is_wnid.Add(wnid);
}
}
bool found = false;
foreach (string wnid in is_wnid)
{
if (GroundTruth.ContainsKey(wnid))
{
result_wnid.Add(wnid);
result_category.Add(GroundTruth[wnid]);
found = true;
if (!categoriesFound.Contains(GroundTruth[wnid]))
{
categoriesFound.Add(GroundTruth[wnid]);
}
break;
}
}
if (!found)
{
result_wnid.Add("");
result_category.Add("");
}
string gtString = GroundTruth_AlphabeticalOrder[gt[i] - 1];
dets_gts_oneString.Add(synsetName + " " + gtString);
}
int noCorrect = 0;
int noTotal = results.Count;
int noTotalPredictions = 0;
List <KeyValuePair <string, string> > dets_gts = new List <KeyValuePair <string, string> >();
for (int i = 0; i < results.Count; i++)
{
string detection = result_category[i];
string gtString = GroundTruth_AlphabeticalOrder[gt[i] - 1];
dets_gts.Add(new KeyValuePair <string, string>(detection, gtString));
if (result_category[i] != "" && result_category[i] != "amulance")
{
noTotalPredictions++;
if (detection.Equals(gtString, StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
Console.WriteLine(detection + " " + gtString);
}
}
}
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res;
SatyamResultValidationToolKit.getConfusionMat(dets_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res);
string outputFile = @"C:\research\dataset\inception1000\confusionMat.txt";
SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile);
outputFile = @"C:\research\dataset\inception1000\det_gts.txt";
File.WriteAllLines(outputFile, dets_gts_oneString.ToArray());
}
