private static void ValidateSatyamCARPKObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
out double totalError, out double totalGroundTruth)
{
List <double> aggResultCounts = new List <double>();
List <int> GroundTruthCounts = new List <int>();
List <double> abs_errors = new List <double>();
List <double> errors = new List <double>();
foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
string labelFilePath = DirectoryConstants.CARPKCountingLabels + fileName + ".txt";
string[] labels = File.ReadAllLines(labelFilePath);
int GroundtruthCount = labels.Length;
string jobGUID = aggResultEntry.JobGUID;
int taskID = aggResultEntry.SatyamTaskTableEntryID;
String resultString = data.AggregatedResultString;
ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString);
aggResultCounts.Add(result.Count);
GroundTruthCounts.Add(GroundtruthCount);
abs_errors.Add(Math.Abs(GroundtruthCount - result.Count));
errors.Add(GroundtruthCount - result.Count);
}
totalError = abs_errors.Sum();
totalGroundTruth = GroundTruthCounts.Sum();
double totalErrorRatio = totalError / totalGroundTruth;
Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio);
Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count);
}
private static void ValidateSatyamKITTIObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
//bool saveImage = false,
out double totalError,
out double totalGroundTruth,
int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION)
{
List <double> aggResultCounts = new List <double>();
List <int> GroundTruthCounts = new List <int>();
List <double> errors = new List <double>();
foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
KITTIDetectionGroundTruth GroundTruthObjects = new KITTIDetectionGroundTruth(KITTIDetectionResultValidation.GroundTruthLabelDirectory + fileName + ".txt", MinHeight, MaxOcclusion, Max_Truncation);
string jobGUID = aggResultEntry.JobGUID;
int taskID = aggResultEntry.SatyamTaskTableEntryID;
String resultString = data.AggregatedResultString;
ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString);
int GroundtruthCount = 0;
for (int i = 0; i < GroundTruthObjects.objects.Count; i++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry obj = GroundTruthObjects.objects[i];
//if (obj.Category == "Car" || obj.Category == "Van" || obj.Category == "DontCare")
if (obj.Category == "Car" || obj.Category == "Van")
{
//if (!GroundTruthObjects.BlackListed[i])
//{
GroundtruthCount++;
//}
}
}
aggResultCounts.Add(result.Count);
GroundTruthCounts.Add(GroundtruthCount);
errors.Add(Math.Abs(GroundtruthCount - result.Count));
}
totalError = errors.Sum();
totalGroundTruth = GroundTruthCounts.Sum();
double totalErrorRatio = totalError / totalGroundTruth;
Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio);
Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count);
}
public static void EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
string outputFile,
out int noCorrect,
//out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth,
//out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res,
bool prepareDataForTraining = false, string outputDirectory = null)
{
noCorrect = 0;
//SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth = new SortedDictionary<string, Dictionary<string, int>>();
//SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res = new SortedDictionary<string, Dictionary<string, int>>();
List <KeyValuePair <string, string> > detections_gts = new List <KeyValuePair <string, string> >();
WebClient wc = new WebClient();
foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(data.AggregatedResultString);
string fileName = URIUtilities.filenameFromURI(data.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
detections_gts.Add(new KeyValuePair <string, string>(aggResult.Category, VideoCategoryGroundTruth));
//if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
//{
// confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary<string, int>());
//}
//if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(VideoCategoryGroundTruth))
//{
// confusionMatrix_res_groundtruth[aggResult.Category].Add(VideoCategoryGroundTruth, 0);
//}
//if (!confusionMatrix_groundtruth_res.ContainsKey(VideoCategoryGroundTruth))
//{
// confusionMatrix_groundtruth_res.Add(VideoCategoryGroundTruth, new Dictionary<string, int>());
//}
//if (!confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].ContainsKey(aggResult.Category))
//{
// confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].Add(aggResult.Category, 0);
//}
//confusionMatrix_res_groundtruth[aggResult.Category][VideoCategoryGroundTruth]++;
//confusionMatrix_groundtruth_res[VideoCategoryGroundTruth][aggResult.Category]++;
if (aggResult.Category.Equals(VideoCategoryGroundTruth, StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
fileName, VideoCategoryGroundTruth, aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
}
// prepare training dataset
if (prepareDataForTraining)
{
if (!Directory.Exists(outputDirectory + "\\" + aggResult.Category))
{
Directory.CreateDirectory(outputDirectory + "\\" + aggResult.Category);
}
wc.DownloadFile(data.SatyamURI, outputDirectory + "\\" + aggResult.Category + "\\" + fileName);
}
}
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res;
SatyamResultValidationToolKit.getConfusionMat(detections_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res);
SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile);
}
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
string confusingImageListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
//SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
// SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
//SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
//ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
///
//if (aggResult.Category == "None of the Above")
//{
// continue;
//}
aggEntries.Add(aggEntry);
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
{
confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ aggResult.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
}
}
if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], aggResult.Category,
// JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
s.Append(record);
s.Append(img);
}
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
s.Clear();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
s.Append(title);
string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
Console.WriteLine(resultSummary);
s.Append(resultSummary);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
string matString = "";
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
matString += row;
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string summaryToBeSaved = s.ToString();
FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
File.WriteAllText(outputmatFile, matString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void ValidateSatyamImageNetClassificationAggregationResultByGUID(string guid, string confusingImageListFilePath = null,
bool prepareDataForTraining = false, string outputDirectory = null)
{
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//get all aggregated results
SatyamAggregatedResultsTableAccess resultsDB = new SatyamAggregatedResultsTableAccess();
List <SatyamAggregatedResultsTableEntry> results = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
int noTotal = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
WebClient wc = new WebClient();
for (int i = 0; i < results.Count; i++)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(results[i]);
//String uri = data.SatyamURI;
//string[] uri_parts= uri.Split('/');
//string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_');
string fileName = URIUtilities.filenameFromURI(data.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
// skip all ambulances and black listed
if (IsBlackListed(data.SatyamURI, imageBlackListReason))
{
continue;
}
if (!confusionMatrix_res_groundtruth.ContainsKey(result.Category))
{
confusionMatrix_res_groundtruth.Add(result.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[result.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[result.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ result.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(result.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(result.Category, 0);
}
}
if (result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], result.Category,
// JSonUtils.ConvertObjectToJSon(result.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], result.Category,
JSonUtils.ConvertObjectToJSon(result.metaData));
String img = String.Format("<img src=\"{0}\" >\n", data.SatyamURI);
s.Append(record);
s.Append(img);
}
// prepare training dataset
if (prepareDataForTraining)
{
if (GroundTruth[imageCategoryName] != "ambulance" && result.Category != "ambulance")
{
Image im = Image.FromStream(wc.OpenRead(data.SatyamURI));
if (!Directory.Exists(outputDirectory + result.Category))
{
Directory.CreateDirectory(outputDirectory + result.Category);
}
im.Save(outputDirectory + "\\" + result.Category + "\\" + fileName);
}
}
noTotal++;
confusionMatrix_res_groundtruth[result.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][result.Category]++;
}
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotal, (double)noCorrect / noTotal);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = "AggregatedIncorrectResults-" + guid + ".html";
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
}
public static void ValidatePascalVOCImageSegmentationResult_ClassLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
List <double> IoUTresholds)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>();
foreach (double iou in IoUTresholds)
{
tpPerIoU.Add(iou, 0);
fpPerIoU.Add(iou, 0);
fnPerIoU.Add(iou, 0);
}
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
//List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
List <int> matchedDetections = new List <int>();
for (int i = 0; i < IoUs.Count; i++)
{
if (GroundtruthAreas[i] < smallAreaToIgnore)
{
continue;
}
List <double> ious = IoUs[i];
double maxIoU = 0;
int idx = -1;
for (int j = 0; j < ious.Count; j++)
{
if (ious[j] > maxIoU)
{
maxIoU = ious[j];
idx = j;
}
}
if (maxIoU == 0)
{
Console.WriteLine("No Match");
foreach (double iou in IoUTresholds)
{
fnPerIoU[iou]++;
}
continue;
}
if (DetectionAreas[idx] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("Match: {0}", maxIoU);
matchedDetections.Add(idx);
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
tpPerIoU[iou]++;
}
else
{
fpPerIoU[iou]++;
fnPerIoU[iou]++;
}
}
}
for (int j = 0; j < IoUs[0].Count; j++)
{
if (matchedDetections.Contains(j))
{
continue;
}
if (DetectionAreas[j] < smallAreaToIgnore)
{
continue;
}
double maxIoU = 0;
int idx = -1;
for (int i = 0; i < IoUs.Count; i++)
{
if (IoUs[i][j] > maxIoU)
{
maxIoU = IoUs[i][j];
idx = j;
}
}
bool alreadyCounted = false;
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
alreadyCounted = true; break;
}
}
if (alreadyCounted)
{
continue;
}
Console.WriteLine("No Groundtruth");
foreach (double iou in IoUTresholds)
{
fpPerIoU[iou]++;
}
}
}
double AvgPrec = 0;
foreach (double iou in IoUTresholds)
{
int tp = tpPerIoU[iou] + missingGroundTruth.Count;
int fp = fpPerIoU[iou] - missingGroundTruth.Count;
int fn = fnPerIoU[iou] - GroundTruthFilter.Count;
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
AvgPrec += prec;
}
AvgPrec /= IoUTresholds.Count;
Console.WriteLine("AvgPrec :{0}", AvgPrec);
}
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
double IoUTreshold)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
int tp = 0;
int fp = 0;
int fn = 0;
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2);
matches.setNumPartitionElements(0, IoUs.Count);
matches.setNumPartitionElements(1, IoUs[0].Count);
double[,] array = new double[IoUs.Count, IoUs[0].Count];
for (int j = 0; j < IoUs.Count; j++)
{
for (int k = 0; k < IoUs[0].Count; k++)
{
array[j, k] = IoUs[j][k];
}
}
matches.setWeightMatrix(0, 1, array);
MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean();
List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches);
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(0)) // this contains gt
{
if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore)
{
continue;
}
if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
double IoU = IoUs[match.elementList[0]][match.elementList[1]];
Console.WriteLine("Match: {0}", IoU);
if (IoU >= IoUTreshold)
{
tp++;
}
else
{
fp++;
fn++;
}
}
else
{
Console.WriteLine("No Match");
fn++;
}
}
else
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("No Groundtruth");
fp++;
}
}
}
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
return(ret);
}