private static void ValidateSatyamCARPKObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries, out double totalError, out double totalGroundTruth) { List <double> aggResultCounts = new List <double>(); List <int> GroundTruthCounts = new List <int>(); List <double> abs_errors = new List <double>(); List <double> errors = new List <double>(); foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); string labelFilePath = DirectoryConstants.CARPKCountingLabels + fileName + ".txt"; string[] labels = File.ReadAllLines(labelFilePath); int GroundtruthCount = labels.Length; string jobGUID = aggResultEntry.JobGUID; int taskID = aggResultEntry.SatyamTaskTableEntryID; String resultString = data.AggregatedResultString; ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString); aggResultCounts.Add(result.Count); GroundTruthCounts.Add(GroundtruthCount); abs_errors.Add(Math.Abs(GroundtruthCount - result.Count)); errors.Add(GroundtruthCount - result.Count); } totalError = abs_errors.Sum(); totalGroundTruth = GroundTruthCounts.Sum(); double totalErrorRatio = totalError / totalGroundTruth; Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio); Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count); }
private static void ValidateSatyamKITTIObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries, //bool saveImage = false, out double totalError, out double totalGroundTruth, int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT, int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION, double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION) { List <double> aggResultCounts = new List <double>(); List <int> GroundTruthCounts = new List <int>(); List <double> errors = new List <double>(); foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); KITTIDetectionGroundTruth GroundTruthObjects = new KITTIDetectionGroundTruth(KITTIDetectionResultValidation.GroundTruthLabelDirectory + fileName + ".txt", MinHeight, MaxOcclusion, Max_Truncation); string jobGUID = aggResultEntry.JobGUID; int taskID = aggResultEntry.SatyamTaskTableEntryID; String resultString = data.AggregatedResultString; ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString); int GroundtruthCount = 0; for (int i = 0; i < GroundTruthObjects.objects.Count; i++) { MultiObjectLocalizationAndLabelingResultSingleEntry obj = GroundTruthObjects.objects[i]; //if (obj.Category == "Car" || obj.Category == "Van" || obj.Category == "DontCare") if (obj.Category == "Car" || obj.Category == "Van") { //if (!GroundTruthObjects.BlackListed[i]) //{ GroundtruthCount++; //} } } aggResultCounts.Add(result.Count); GroundTruthCounts.Add(GroundtruthCount); errors.Add(Math.Abs(GroundtruthCount - result.Count)); } totalError = errors.Sum(); totalGroundTruth = GroundTruthCounts.Sum(); double totalErrorRatio = totalError / totalGroundTruth; Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio); Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count); }
public static void EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(List <SatyamAggregatedResultsTableEntry> aggResultEntries, string outputFile, out int noCorrect, //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth, //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res, bool prepareDataForTraining = false, string outputDirectory = null) { noCorrect = 0; //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth = new SortedDictionary<string, Dictionary<string, int>>(); //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res = new SortedDictionary<string, Dictionary<string, int>>(); List <KeyValuePair <string, string> > detections_gts = new List <KeyValuePair <string, string> >(); WebClient wc = new WebClient(); foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry); SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(data.AggregatedResultString); string fileName = URIUtilities.filenameFromURI(data.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); detections_gts.Add(new KeyValuePair <string, string>(aggResult.Category, VideoCategoryGroundTruth)); //if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category)) //{ // confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary<string, int>()); //} //if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(VideoCategoryGroundTruth)) //{ // confusionMatrix_res_groundtruth[aggResult.Category].Add(VideoCategoryGroundTruth, 0); //} //if (!confusionMatrix_groundtruth_res.ContainsKey(VideoCategoryGroundTruth)) //{ // confusionMatrix_groundtruth_res.Add(VideoCategoryGroundTruth, new Dictionary<string, int>()); //} //if (!confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].ContainsKey(aggResult.Category)) //{ // confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].Add(aggResult.Category, 0); //} //confusionMatrix_res_groundtruth[aggResult.Category][VideoCategoryGroundTruth]++; //confusionMatrix_groundtruth_res[VideoCategoryGroundTruth][aggResult.Category]++; if (aggResult.Category.Equals(VideoCategoryGroundTruth, StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", fileName, VideoCategoryGroundTruth, aggResult.Category, JSonUtils.ConvertObjectToJSon(aggResult.metaData)); } // prepare training dataset if (prepareDataForTraining) { if (!Directory.Exists(outputDirectory + "\\" + aggResult.Category)) { Directory.CreateDirectory(outputDirectory + "\\" + aggResult.Category); } wc.DownloadFile(data.SatyamURI, outputDirectory + "\\" + aggResult.Category + "\\" + fileName); } } SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res; SatyamResultValidationToolKit.getConfusionMat(detections_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res); SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile); }
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, string confusingImageListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); //SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime = // SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>(); Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); foreach (DateTime t in entriesBySubmitTime.Keys) { foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { //SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; if (IsBlackListed(task.SatyamURI, imageBlackListReason)) { continue; } int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } //ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult)); ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } /// //if (aggResult.Category == "None of the Above") //{ // continue; //} aggEntries.Add(aggEntry); SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry); String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category)) { confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { aggResult.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0); } } if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], aggResult.Category, // JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], aggResult.Category, JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI); s.Append(record); s.Append(img); } noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++; aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); s.Clear(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); s.Append(title); string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); Console.WriteLine(resultSummary); s.Append(resultSummary); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); string matString = ""; foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); matString += row; } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string summaryToBeSaved = s.ToString(); FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; File.WriteAllText(outputmatFile, matString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void ValidateSatyamImageNetClassificationAggregationResultByGUID(string guid, string confusingImageListFilePath = null, bool prepareDataForTraining = false, string outputDirectory = null) { Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //get all aggregated results SatyamAggregatedResultsTableAccess resultsDB = new SatyamAggregatedResultsTableAccess(); List <SatyamAggregatedResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); resultsDB.close(); int noTotal = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); WebClient wc = new WebClient(); for (int i = 0; i < results.Count; i++) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(results[i]); //String uri = data.SatyamURI; //string[] uri_parts= uri.Split('/'); //string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_'); string fileName = URIUtilities.filenameFromURI(data.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); // skip all ambulances and black listed if (IsBlackListed(data.SatyamURI, imageBlackListReason)) { continue; } if (!confusionMatrix_res_groundtruth.ContainsKey(result.Category)) { confusionMatrix_res_groundtruth.Add(result.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[result.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[result.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { result.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(result.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(result.Category, 0); } } if (result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], result.Category, // JSonUtils.ConvertObjectToJSon(result.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], result.Category, JSonUtils.ConvertObjectToJSon(result.metaData)); String img = String.Format("<img src=\"{0}\" >\n", data.SatyamURI); s.Append(record); s.Append(img); } // prepare training dataset if (prepareDataForTraining) { if (GroundTruth[imageCategoryName] != "ambulance" && result.Category != "ambulance") { Image im = Image.FromStream(wc.OpenRead(data.SatyamURI)); if (!Directory.Exists(outputDirectory + result.Category)) { Directory.CreateDirectory(outputDirectory + result.Category); } im.Save(outputDirectory + "\\" + result.Category + "\\" + fileName); } } noTotal++; confusionMatrix_res_groundtruth[result.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][result.Category]++; } Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotal, (double)noCorrect / noTotal); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = "AggregatedIncorrectResults-" + guid + ".html"; storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); }
public static void ValidatePascalVOCImageSegmentationResult_ClassLevel( List <SatyamAggregatedResultsTableEntry> resultsPerImage, List <double> IoUTresholds) { SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); for (int i = 0; i < resultsPerImage.Count; i++) { int taskID = resultsPerImage[i].SatyamTaskTableEntryID; aggResults.Add(taskID, resultsPerImage[i]); } SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>(); SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>(); SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>(); foreach (double iou in IoUTresholds) { tpPerIoU.Add(iou, 0); fpPerIoU.Add(iou, 0); fnPerIoU.Add(iou, 0); } foreach (int id in aggResults.Keys) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName); Console.WriteLine("Results for {0}", fileName); String resultString = data.AggregatedResultString; ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString); string PNG_URL = result.metaData.PNG_URL; //List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories); SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>(); SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>(); //List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <int> DetectionAreas = noDetectionPixels.Values.ToList(); List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList(); if (IoUs == null) { continue; } //double smallAreaToIgnore = 1000;//px //double smallAreaToIgnore = 625;//px double smallAreaToIgnore = 1600;//px List <int> matchedDetections = new List <int>(); for (int i = 0; i < IoUs.Count; i++) { if (GroundtruthAreas[i] < smallAreaToIgnore) { continue; } List <double> ious = IoUs[i]; double maxIoU = 0; int idx = -1; for (int j = 0; j < ious.Count; j++) { if (ious[j] > maxIoU) { maxIoU = ious[j]; idx = j; } } if (maxIoU == 0) { Console.WriteLine("No Match"); foreach (double iou in IoUTresholds) { fnPerIoU[iou]++; } continue; } if (DetectionAreas[idx] < smallAreaToIgnore) { continue; } Console.WriteLine("Match: {0}", maxIoU); matchedDetections.Add(idx); foreach (double iou in IoUTresholds) { if (maxIoU >= iou) { tpPerIoU[iou]++; } else { fpPerIoU[iou]++; fnPerIoU[iou]++; } } } for (int j = 0; j < IoUs[0].Count; j++) { if (matchedDetections.Contains(j)) { continue; } if (DetectionAreas[j] < smallAreaToIgnore) { continue; } double maxIoU = 0; int idx = -1; for (int i = 0; i < IoUs.Count; i++) { if (IoUs[i][j] > maxIoU) { maxIoU = IoUs[i][j]; idx = j; } } bool alreadyCounted = false; foreach (double iou in IoUTresholds) { if (maxIoU >= iou) { alreadyCounted = true; break; } } if (alreadyCounted) { continue; } Console.WriteLine("No Groundtruth"); foreach (double iou in IoUTresholds) { fpPerIoU[iou]++; } } } double AvgPrec = 0; foreach (double iou in IoUTresholds) { int tp = tpPerIoU[iou] + missingGroundTruth.Count; int fp = fpPerIoU[iou] - missingGroundTruth.Count; int fn = fnPerIoU[iou] - GroundTruthFilter.Count; double ap = (double)tp / (double)(tp + fp + fn); double prec = (double)tp / (double)(tp + fp); double recl = (double)tp / (double)(tp + fn); string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl); Console.WriteLine(ret); AvgPrec += prec; } AvgPrec /= IoUTresholds.Count; Console.WriteLine("AvgPrec :{0}", AvgPrec); }
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel( List <SatyamAggregatedResultsTableEntry> resultsPerImage, double IoUTreshold) { SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); for (int i = 0; i < resultsPerImage.Count; i++) { int taskID = resultsPerImage[i].SatyamTaskTableEntryID; aggResults.Add(taskID, resultsPerImage[i]); } int tp = 0; int fp = 0; int fn = 0; foreach (int id in aggResults.Keys) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName); Console.WriteLine("Results for {0}", fileName); String resultString = data.AggregatedResultString; ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString); string PNG_URL = result.metaData.PNG_URL; //List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories); SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>(); SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>(); List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <int> DetectionAreas = noDetectionPixels.Values.ToList(); List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList(); if (IoUs == null) { continue; } MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2); matches.setNumPartitionElements(0, IoUs.Count); matches.setNumPartitionElements(1, IoUs[0].Count); double[,] array = new double[IoUs.Count, IoUs[0].Count]; for (int j = 0; j < IoUs.Count; j++) { for (int k = 0; k < IoUs[0].Count; k++) { array[j, k] = IoUs[j][k]; } } matches.setWeightMatrix(0, 1, array); MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean(); List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches); //double smallAreaToIgnore = 1000;//px //double smallAreaToIgnore = 625;//px double smallAreaToIgnore = 1600;//px foreach (MultipartiteWeightedMatch match in polygonAssociation) { if (match.elementList.ContainsKey(0)) // this contains gt { if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore) { continue; } if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated { if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore) { continue; } double IoU = IoUs[match.elementList[0]][match.elementList[1]]; Console.WriteLine("Match: {0}", IoU); if (IoU >= IoUTreshold) { tp++; } else { fp++; fn++; } } else { Console.WriteLine("No Match"); fn++; } } else { if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore) { continue; } Console.WriteLine("No Groundtruth"); fp++; } } } double ap = (double)tp / (double)(tp + fp + fn); double prec = (double)tp / (double)(tp + fp); double recl = (double)tp / (double)(tp + fn); string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl); Console.WriteLine(ret); return(ret); }