public static void SaveAggregatedResultImagesLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParameters);
SatyamJob job = task.jobEntry;
ImageSegmentationAggregatedResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamResult.AggregatedResultString);
//string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
//Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
//Image ResultImage = DrawResultsOnImage(res.boxesAndCategories, originalImage);
//fileName = fileName + "-AggregatedResult";
//ImageUtilities.saveImage(ResultImage, directoryName, fileName);
WebClient wb = new WebClient();
Image im = Image.FromStream(wb.OpenRead(res.metaData.PNG_URL));
string fileName = URIUtilities.filenameFromURI(res.metaData.PNG_URL);
//wb.DownloadFile(directoryName + "\\" + fileName, res.metaData.PNG_URL);
im.Save(directoryName + "\\" + fileName);
}
}
/// <summary>
/// function generalizable.. TODO... to all tasks
/// </summary>
/// <param name="guids"></param>
/// <param name="IoUTreshold"></param>
/// <param name="saveImage"></param>
/// <param name="outputDirectory"></param>
/// <param name="MinResults"></param>
/// <param name="MaxResults"></param>
/// <param name="CategoryMajorityThreshold"></param>
/// <param name="PolygonBoundaryMajorityThreshold"></param>
/// <param name="ObjectsCoverageThreshold"></param>
/// <param name="overwrite"></param>
/// <param name="approvalAnalysis"></param>
public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
double IoUTreshold,
bool saveImage = false, string outputDirectory = null,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
bool overwrite = false,
bool approvalAnalysis = false)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;
Console.WriteLine("Aggregating for param set " + configString);
if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
{
return;
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
//ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
r = noTotalConverged + " " + noTerminatedTasks + " " + r;
File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);
if (approvalAnalysis)
{
string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
"_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
}
}
public static void ValidatePascalVOCImageSegmentationResult_ClassLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
List <double> IoUTresholds)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>();
foreach (double iou in IoUTresholds)
{
tpPerIoU.Add(iou, 0);
fpPerIoU.Add(iou, 0);
fnPerIoU.Add(iou, 0);
}
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
//List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
List <int> matchedDetections = new List <int>();
for (int i = 0; i < IoUs.Count; i++)
{
if (GroundtruthAreas[i] < smallAreaToIgnore)
{
continue;
}
List <double> ious = IoUs[i];
double maxIoU = 0;
int idx = -1;
for (int j = 0; j < ious.Count; j++)
{
if (ious[j] > maxIoU)
{
maxIoU = ious[j];
idx = j;
}
}
if (maxIoU == 0)
{
Console.WriteLine("No Match");
foreach (double iou in IoUTresholds)
{
fnPerIoU[iou]++;
}
continue;
}
if (DetectionAreas[idx] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("Match: {0}", maxIoU);
matchedDetections.Add(idx);
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
tpPerIoU[iou]++;
}
else
{
fpPerIoU[iou]++;
fnPerIoU[iou]++;
}
}
}
for (int j = 0; j < IoUs[0].Count; j++)
{
if (matchedDetections.Contains(j))
{
continue;
}
if (DetectionAreas[j] < smallAreaToIgnore)
{
continue;
}
double maxIoU = 0;
int idx = -1;
for (int i = 0; i < IoUs.Count; i++)
{
if (IoUs[i][j] > maxIoU)
{
maxIoU = IoUs[i][j];
idx = j;
}
}
bool alreadyCounted = false;
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
alreadyCounted = true; break;
}
}
if (alreadyCounted)
{
continue;
}
Console.WriteLine("No Groundtruth");
foreach (double iou in IoUTresholds)
{
fpPerIoU[iou]++;
}
}
}
double AvgPrec = 0;
foreach (double iou in IoUTresholds)
{
int tp = tpPerIoU[iou] + missingGroundTruth.Count;
int fp = fpPerIoU[iou] - missingGroundTruth.Count;
int fn = fnPerIoU[iou] - GroundTruthFilter.Count;
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
AvgPrec += prec;
}
AvgPrec /= IoUTresholds.Count;
Console.WriteLine("AvgPrec :{0}", AvgPrec);
}
public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids,
double IoUTreshold,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
// Organize by taskID
foreach (SatyamResultsTableEntry entry in entries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!EntriesPerTask.ContainsKey(taskEntryID))
{
EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
EntriesPerTask[taskEntryID].Add(entry);
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
}
foreach (int taskEntryID in EntriesPerTask.Keys)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
List <int> taskidfilter = new List <int>()
{
40430,
40432,
40433,
40434,
40440,
40447,
40451,
40460,
};
//if (fileName != "2007_000549") continue;
if (!taskidfilter.Contains(satyamResult.TaskTableEntryID))
{
continue;
}
Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count);
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = satyamResult.TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = job.JobGUIDString;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold);
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
}
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
double IoUTreshold)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
int tp = 0;
int fp = 0;
int fn = 0;
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2);
matches.setNumPartitionElements(0, IoUs.Count);
matches.setNumPartitionElements(1, IoUs[0].Count);
double[,] array = new double[IoUs.Count, IoUs[0].Count];
for (int j = 0; j < IoUs.Count; j++)
{
for (int k = 0; k < IoUs[0].Count; k++)
{
array[j, k] = IoUs[j][k];
}
}
matches.setWeightMatrix(0, 1, array);
MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean();
List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches);
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(0)) // this contains gt
{
if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore)
{
continue;
}
if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
double IoU = IoUs[match.elementList[0]][match.elementList[1]];
Console.WriteLine("Match: {0}", IoU);
if (IoU >= IoUTreshold)
{
tp++;
}
else
{
fp++;
fn++;
}
}
else
{
Console.WriteLine("No Match");
fn++;
}
}
else
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("No Groundtruth");
fp++;
}
}
}
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
return(ret);
}
