public static void ValidatePascalVOCImageSegmentationResult_ClassLevel( List <SatyamAggregatedResultsTableEntry> resultsPerImage, List <double> IoUTresholds) { SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); for (int i = 0; i < resultsPerImage.Count; i++) { int taskID = resultsPerImage[i].SatyamTaskTableEntryID; aggResults.Add(taskID, resultsPerImage[i]); } SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>(); SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>(); SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>(); foreach (double iou in IoUTresholds) { tpPerIoU.Add(iou, 0); fpPerIoU.Add(iou, 0); fnPerIoU.Add(iou, 0); } foreach (int id in aggResults.Keys) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName); Console.WriteLine("Results for {0}", fileName); String resultString = data.AggregatedResultString; ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString); string PNG_URL = result.metaData.PNG_URL; //List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories); SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>(); SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>(); //List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <int> DetectionAreas = noDetectionPixels.Values.ToList(); List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList(); if (IoUs == null) { continue; } //double smallAreaToIgnore = 1000;//px //double smallAreaToIgnore = 625;//px double smallAreaToIgnore = 1600;//px List <int> matchedDetections = new List <int>(); for (int i = 0; i < IoUs.Count; i++) { if (GroundtruthAreas[i] < smallAreaToIgnore) { continue; } List <double> ious = IoUs[i]; double maxIoU = 0; int idx = -1; for (int j = 0; j < ious.Count; j++) { if (ious[j] > maxIoU) { maxIoU = ious[j]; idx = j; } } if (maxIoU == 0) { Console.WriteLine("No Match"); foreach (double iou in IoUTresholds) { fnPerIoU[iou]++; } continue; } if (DetectionAreas[idx] < smallAreaToIgnore) { continue; } Console.WriteLine("Match: {0}", maxIoU); matchedDetections.Add(idx); foreach (double iou in IoUTresholds) { if (maxIoU >= iou) { tpPerIoU[iou]++; } else { fpPerIoU[iou]++; fnPerIoU[iou]++; } } } for (int j = 0; j < IoUs[0].Count; j++) { if (matchedDetections.Contains(j)) { continue; } if (DetectionAreas[j] < smallAreaToIgnore) { continue; } double maxIoU = 0; int idx = -1; for (int i = 0; i < IoUs.Count; i++) { if (IoUs[i][j] > maxIoU) { maxIoU = IoUs[i][j]; idx = j; } } bool alreadyCounted = false; foreach (double iou in IoUTresholds) { if (maxIoU >= iou) { alreadyCounted = true; break; } } if (alreadyCounted) { continue; } Console.WriteLine("No Groundtruth"); foreach (double iou in IoUTresholds) { fpPerIoU[iou]++; } } } double AvgPrec = 0; foreach (double iou in IoUTresholds) { int tp = tpPerIoU[iou] + missingGroundTruth.Count; int fp = fpPerIoU[iou] - missingGroundTruth.Count; int fn = fnPerIoU[iou] - GroundTruthFilter.Count; double ap = (double)tp / (double)(tp + fp + fn); double prec = (double)tp / (double)(tp + fp); double recl = (double)tp / (double)(tp + fn); string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl); Console.WriteLine(ret); AvgPrec += prec; } AvgPrec /= IoUTresholds.Count; Console.WriteLine("AvgPrec :{0}", AvgPrec); }
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel( List <SatyamAggregatedResultsTableEntry> resultsPerImage, double IoUTreshold) { SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); for (int i = 0; i < resultsPerImage.Count; i++) { int taskID = resultsPerImage[i].SatyamTaskTableEntryID; aggResults.Add(taskID, resultsPerImage[i]); } int tp = 0; int fp = 0; int fn = 0; foreach (int id in aggResults.Keys) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName); Console.WriteLine("Results for {0}", fileName); String resultString = data.AggregatedResultString; ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString); string PNG_URL = result.metaData.PNG_URL; //List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories); SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>(); SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>(); List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <int> DetectionAreas = noDetectionPixels.Values.ToList(); List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList(); if (IoUs == null) { continue; } MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2); matches.setNumPartitionElements(0, IoUs.Count); matches.setNumPartitionElements(1, IoUs[0].Count); double[,] array = new double[IoUs.Count, IoUs[0].Count]; for (int j = 0; j < IoUs.Count; j++) { for (int k = 0; k < IoUs[0].Count; k++) { array[j, k] = IoUs[j][k]; } } matches.setWeightMatrix(0, 1, array); MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean(); List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches); //double smallAreaToIgnore = 1000;//px //double smallAreaToIgnore = 625;//px double smallAreaToIgnore = 1600;//px foreach (MultipartiteWeightedMatch match in polygonAssociation) { if (match.elementList.ContainsKey(0)) // this contains gt { if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore) { continue; } if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated { if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore) { continue; } double IoU = IoUs[match.elementList[0]][match.elementList[1]]; Console.WriteLine("Match: {0}", IoU); if (IoU >= IoUTreshold) { tp++; } else { fp++; fn++; } } else { Console.WriteLine("No Match"); fn++; } } else { if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore) { continue; } Console.WriteLine("No Groundtruth"); fp++; } } } double ap = (double)tp / (double)(tp + fp + fn); double prec = (double)tp / (double)(tp + fp); double recl = (double)tp / (double)(tp + fn); string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl); Console.WriteLine(ret); return(ret); }