public static Image DrawResultsOnImage(ImageSegmentationResult res, Image im)
{
if (res == null)
{
return(null);
}
foreach (ImageSegmentationResultSingleEntry obj in res.objects)
{
foreach (GenericPolygon poly in obj.segment.polygons)
{
string category = obj.Category;
List <LineSegment> lines = new List <LineSegment>();
for (int i = 0; i < poly.vertices.Count; i++)
{
int x1 = poly.vertices[i][0];
int y1 = poly.vertices[i][1];
int j = (i + 1) % poly.vertices.Count;
int x2 = poly.vertices[j][0];
int y2 = poly.vertices[j][1];
LineSegment line = new LineSegment(x1, y1, x2, y2);
lines.Add(line);
}
Color c = ColorSet.getColorByObjectType(category);
im = DrawingBoxesAndLinesOnImages.addLinesToImage(im, lines, c);
}
}
return(im);
}
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results)
{
if (results.Count == 0)
{
return(null);
}
string resultString = null;
List <ImageSegmentationResult> resultList = new List <ImageSegmentationResult>();
List <string> WorkersPerTask = new List <string>();
SatyamResult res0 = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(res0.TaskParametersString);
string SatyamURL = task.SatyamURI;
string guid = results[0].JobGUID;
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
// remove duplicate workers result
string workerID = res.amazonInfo.WorkerID;
if (WorkersPerTask.Contains(workerID))
{
continue;
}
string assignmentID = res.amazonInfo.AssignmentID;
if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE")
{
continue;
}
//enclose only non-duplicate results, one per each worker.
if (workerID != "" && workerID != TaskConstants.AdminID)
{
// make a pass for test and admins
WorkersPerTask.Add(workerID);
}
ImageSegmentationResult taskr = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(res.TaskResult);
resultList.Add(taskr);
}
ImageSegmentationAggregatedResult r = getAggregatedResult(resultList, SatyamURL, guid);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
// sort by task id
SortedDictionary <int, List <SatyamResultsTableEntry> > taskResults = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
for (int i = 0; i < entries.Count; i++)
{
if (!taskResults.ContainsKey(entries[i].SatyamTaskTableEntryID))
{
taskResults.Add(entries[i].SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
taskResults[entries[i].SatyamTaskTableEntryID].Add(entries[i]);
}
foreach (int taskID in taskResults.Keys)
{
for (int i = 0; i < taskResults[taskID].Count; i++)
{
SatyamResultsTableEntry entry = taskResults[taskID][i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
ImageSegmentationResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (res == null)
{
continue;
}
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID;
}
fileName += "-" + entry.ID;
if (File.Exists(directoryName + "\\" + fileName + ".jpg"))
{
continue;
}
Console.WriteLine("Saving Turker Result {0}", fileName);
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
Image ResultImage = DrawResultsOnImage(res, originalImage);
byte[] png = ImageSegmentationResult.PolygonResult2Bitmap(res);
if (ResultImage == null)
{
continue;
}
ImageUtilities.saveImage(ResultImage, directoryName, fileName);
ImageUtilities.savePNGRawData(directoryName + "\\" + fileName + "_bitmap.jpg", originalImage.Width, originalImage.Height, png);
}
}
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double APPROVAL_RATIO_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the polygon
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
// the objects are dummy polygons just for counting
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * APPROVAL_RATIO_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//now find how many of the results match aggregated results
int noAccepted = 0;
byte[] resPNG = ImageSegmentationResult.PolygonResult2Bitmap(result);
int width = -1; int height = -1;
byte[] aggPNG = ImageUtilities.readPNGRawDataFromURL(aggresult.metaData.PNG_URL, out width, out height);
if (resPNG.Length != aggPNG.Length)
{
Console.WriteLine("res and agg res are different size");
return(false);
}
SortedDictionary <int, SortedDictionary <int, int> > overlapping = new SortedDictionary <int, SortedDictionary <int, int> >();
SortedDictionary <int, int> resObjectArea = new SortedDictionary <int, int>();
SortedDictionary <int, int> aggObjectArea = new SortedDictionary <int, int>();
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = j * width + i;
int resObjID = resPNG[idx];
int aggObjID = aggPNG[idx];
if (!overlapping.ContainsKey(resObjID))
{
overlapping.Add(resObjID, new SortedDictionary <int, int>());
}
if (!overlapping[resObjID].ContainsKey(aggObjID))
{
overlapping[resObjID].Add(aggObjID, 0);
}
overlapping[resObjID][aggObjID]++;
if (!resObjectArea.ContainsKey(resObjID))
{
resObjectArea.Add(resObjID, 0);
}
resObjectArea[resObjID]++;
if (!aggObjectArea.ContainsKey(aggObjID))
{
aggObjectArea.Add(aggObjID, 0);
}
aggObjectArea[aggObjID]++;
}
}
foreach (int id in resObjectArea.Keys)
{
if (id == 0)
{
continue;
}
int maxOverlap = -1;
int maxOverlapAggObjID = -1;
SortedDictionary <int, int> overlapArea = overlapping[id];
foreach (int aggid in overlapArea.Keys)
{
if (aggid == 0)
{
continue;
}
if (overlapArea[aggid] > maxOverlap)
{
maxOverlap = overlapArea[aggid];
maxOverlapAggObjID = aggid;
}
}
if (maxOverlapAggObjID == -1)
{
continue;
}
double iou = (double)maxOverlap / (double)(resObjectArea[id] + aggObjectArea[maxOverlapAggObjID] - maxOverlap);
if (iou >= POLYGON_IOU_THRESHOLD)
{
noAccepted++;
////now check category
//if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
//{
// //both category and bounding box tests have passed
//}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
/// <summary>
/// Return a Dictionary of IoUs per each object ID, starting from 1 in the groundtruth
/// Note that gt id starts from 1
/// </summary>
/// <param name="res"></param>
/// <returns></returns>
public List <List <double> > getIoUs(ImageSegmentationResult res)
{
List <List <double> > IoUs = new List <List <double> >();
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
Dictionary <int, Dictionary <int, int> > noOverlappingPixelsBetweenDetectionsAndGroundTruth = new Dictionary <int, Dictionary <int, int> >();
// going thru all gt segments
for (int i = 0; i < instanceSegments.Length; i++)
{
if (instanceSegments[i] < 255 && instanceSegments[i] != 0)
{
int objId = (int)instanceSegments[i];
if (!noGroundTruthPixels.ContainsKey(objId))
{
noGroundTruthPixels.Add(objId, 0);
noOverlappingPixelsBetweenDetectionsAndGroundTruth.Add(objId, new Dictionary <int, int>());
}
noGroundTruthPixels[objId]++;
// most naive approach: iterate every pixel
int x = i % res.imageWidth;
int y = i / res.imageWidth;
// going thru each detection polygons
for (int j = 0; j < res.objects.Count; j++)
{
Segment seg = res.objects[j].segment;
if (seg.PointIsInSegment(x, y))
{
if (!noDetectionPixels.ContainsKey(j))
{
noDetectionPixels.Add(j, 0);
}
noDetectionPixels[j]++;
if (!noOverlappingPixelsBetweenDetectionsAndGroundTruth[objId].ContainsKey(j))
{
noOverlappingPixelsBetweenDetectionsAndGroundTruth[objId].Add(j, 0);
}
noOverlappingPixelsBetweenDetectionsAndGroundTruth[objId][j]++;
}
}
}
}
foreach (int gtId in noGroundTruthPixels.Keys)
{
IoUs.Add(new List <double>());
foreach (int detId in noDetectionPixels.Keys)
{
double tempIoU = 0;
if (noOverlappingPixelsBetweenDetectionsAndGroundTruth.ContainsKey(gtId))
{
if (noOverlappingPixelsBetweenDetectionsAndGroundTruth[gtId].ContainsKey(detId))
{
int overlapping = noOverlappingPixelsBetweenDetectionsAndGroundTruth[gtId][detId];
tempIoU = (double)overlapping / (double)(noGroundTruthPixels[gtId] + noDetectionPixels[detId] - overlapping);
}
}
IoUs[gtId - 1].Add(tempIoU);
}
}
return(IoUs);
}
