public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results)
{
if (results.Count == 0)
{
return(null);
}
string resultString = null;
List <ImageSegmentationResult_NoHoles> resultList = new List <ImageSegmentationResult_NoHoles>();
List <string> WorkersPerTask = new List <string>();
SatyamResult res0 = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(res0.TaskParametersString);
string SatyamURL = task.SatyamURI;
string guid = results[0].JobGUID;
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
// remove duplicate workers result
string workerID = res.amazonInfo.WorkerID;
if (WorkersPerTask.Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
if (workerID != "" && workerID != TaskConstants.AdminID)
{
// make a pass for test and admins
WorkersPerTask.Add(workerID);
}
ImageSegmentationResult_NoHoles taskr = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(res.TaskResult);
resultList.Add(taskr);
}
ImageSegmentationAggregatedResult_NoHoles r = getAggregatedResult(resultList, SatyamURL, guid);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult_NoHoles>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >();
allPolygons.Add(new List <GenericPolygon>());
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects)
{
allPolygons[0].Add(entry.polygon);
}
allPolygons.Add(new List <GenericPolygon>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects)
{
allPolygons[1].Add(entry.polygon);
}
List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons);
//now find how many of the results match aggregated results
int noAccepted = 0;
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]];
GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]];
//double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon);
double IoU = 1;
if (IoU >= POLYGON_IOU_THRESHOLD)
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
