public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results) { if (results.Count == 0) { return(null); } string resultString = null; List <ImageSegmentationResult_NoHoles> resultList = new List <ImageSegmentationResult_NoHoles>(); List <string> WorkersPerTask = new List <string>(); SatyamResult res0 = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(res0.TaskParametersString); string SatyamURL = task.SatyamURI; string guid = results[0].JobGUID; foreach (SatyamResultsTableEntry entry in results) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); // remove duplicate workers result string workerID = res.amazonInfo.WorkerID; if (WorkersPerTask.Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. if (workerID != "" && workerID != TaskConstants.AdminID) { // make a pass for test and admins WorkersPerTask.Add(workerID); } ImageSegmentationResult_NoHoles taskr = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(res.TaskResult); resultList.Add(taskr); } ImageSegmentationAggregatedResult_NoHoles r = getAggregatedResult(resultList, SatyamURL, guid); if (r != null) { string rString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult_NoHoles>(r); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); } return(resultString); }
public static bool IsAcceptable( SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT ) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } //We fist do a bipartitte matching to find the best assocaition for the boxes List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >(); allPolygons.Add(new List <GenericPolygon>()); foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects) { allPolygons[0].Add(entry.polygon); } allPolygons.Add(new List <GenericPolygon>()); List <bool> tooSmallToIgnore = new List <bool>(); foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects) { allPolygons[1].Add(entry.polygon); } List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons); //now find how many of the results match aggregated results int noAccepted = 0; foreach (MultipartiteWeightedMatch match in polygonAssociation) { if (match.elementList.ContainsKey(1)) // this contains an aggregated box { if (match.elementList.ContainsKey(0)) // a result box has been associated { GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]]; GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]]; //double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon); double IoU = 1; if (IoU >= POLYGON_IOU_THRESHOLD) { //now check category if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) { //both category and bounding box tests have passed noAccepted++; } } } } } if (noAccepted >= boxLimit) { return(true); } return(false); }
public static ImageSegmentationAggregatedResult_NoHoles getAggregatedResult(List <ImageSegmentationResult_NoHoles> results, string SatyamURL, string guid, int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE, double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD, double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD, double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION ) { if (results.Count < MinResults) //need at least three results! { return(null); } int ImageWidth = results[0].imageWidth; int ImageHeight = results[0].imageHeight; byte[] PNG = new byte[ImageHeight * ImageWidth]; ImageSegmentationAggregatedResult_NoHoles aggResult = new ImageSegmentationAggregatedResult_NoHoles(); aggResult.metaData = new ImageSegmentationAggregatedResultMetaData_NoHoles(); aggResult.metaData.TotalCount = results.Count; for (int i = 0; i < PNG.Length; i++) { PNG[i] = 0; } aggResult.boxesAndCategories = new ImageSegmentationResult_NoHoles(); aggResult.boxesAndCategories.objects = new List <ImageSegmentationResultSingleEntry_NoHoles>(); aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX; aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY; aggResult.boxesAndCategories.imageHeight = ImageHeight; aggResult.boxesAndCategories.imageWidth = ImageWidth; //first use multipartitie wieghted matching to associated the boxes disregarding the labels since //people might make mistake with lables but boxes are usually right List <List <GenericPolygon> > AllPolygons = new List <List <GenericPolygon> >(); List <int> noPolygonsPerResult = new List <int>(); foreach (ImageSegmentationResult_NoHoles res in results) { AllPolygons.Add(new List <GenericPolygon>()); if (res == null) { noPolygonsPerResult.Add(0); continue; } List <GenericPolygon> polygonList = AllPolygons[AllPolygons.Count - 1]; foreach (ImageSegmentationResultSingleEntry_NoHoles entry in res.objects) { polygonList.Add(entry.polygon); } noPolygonsPerResult.Add(polygonList.Count); } //now associate boxes across the various results List <MultipartiteWeightedMatch> polyAssociation = PolygonAssociation.computeGenericPolygonAssociations(AllPolygons); int noObjects = polyAssociation.Count; int noAssociatedPolygons = 0; //how many of the drawn boxes for each result were actually associated by two or more people for each user? SortedDictionary <int, int> noMultipleAssociatedPolygonsPerResult = new SortedDictionary <int, int>(); for (int i = 0; i < results.Count; i++) { noMultipleAssociatedPolygonsPerResult.Add(i, 0); } foreach (MultipartiteWeightedMatch match in polyAssociation) { if (match.elementList.Count > 1) //this has been corroborated by two people { noAssociatedPolygons++; foreach (KeyValuePair <int, int> entry in match.elementList) { noMultipleAssociatedPolygonsPerResult[entry.Key]++; } } } //count how many people have a high association ratio int noHighAssociationRatio = 0; for (int i = 0; i < results.Count; i++) { if (noPolygonsPerResult[i] == 0) { continue; } double ratio = (double)noMultipleAssociatedPolygonsPerResult[i] / (double)noPolygonsPerResult[i]; if (ratio > ObjectsCoverageThreshold) { noHighAssociationRatio++; } } if (noHighAssociationRatio < MinResults) //at least three people should have all their boxes highly corroborated by one other person { return(null); } //int noAcceptedPolygons = 0; for (int idx = 0; idx < polyAssociation.Count; idx++) { MultipartiteWeightedMatch match = polyAssociation[idx]; List <GenericPolygon> polyList = new List <GenericPolygon>(); List <string> identifiers = new List <string>(); foreach (KeyValuePair <int, int> entry in match.elementList) { polyList.Add(AllPolygons[entry.Key][entry.Value]); identifiers.Add(entry.Key + "_" + entry.Value); } //GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_Relaxation(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold); //GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_MajorityEdge(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold); GenericPolygon aggregatedPolygon = new GenericPolygon();// dummy polygon byte[] png = GetAggregatedGenericPolygon_PixelSweep(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold); Dictionary <string, int> categoryNames = new Dictionary <string, int>(); int totalCount = match.elementList.Count; int maxCount = 0; string maxCategory = ""; foreach (KeyValuePair <int, int> entry in match.elementList) { string category = results[entry.Key].objects[entry.Value].Category; if (!categoryNames.ContainsKey(category)) { categoryNames.Add(category, 0); } categoryNames[category]++; if (maxCount < categoryNames[category]) { maxCount = categoryNames[category]; maxCategory = category; } } double probability = ((double)maxCount + 1) / ((double)totalCount + 2); if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work { return(null); } // now we have one segment ready ImageSegmentationResultSingleEntry_NoHoles aggregated = new ImageSegmentationResultSingleEntry_NoHoles(); aggregated.polygon = aggregatedPolygon; aggregated.Category = maxCategory; aggResult.boxesAndCategories.objects.Add(aggregated); for (int i = 0; i < PNG.Length; i++) { if (PNG[i] == 0 && png[i] != 0) { PNG[i] = (byte)(idx + 1); } } } // save and upload to azure string filename = URIUtilities.filenameFromURINoExtension(SatyamURL); string filepath = DirectoryConstants.defaultTempDirectory + filename + "_aggregated.PNG"; ImageUtilities.savePNGRawData(filepath, ImageWidth, ImageHeight, PNG); SatyamJobStorageAccountAccess blob = new SatyamJobStorageAccountAccess(); string container = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[TaskConstants.Segmentation_Image_MTurk]; string directoryPath = guid + "_aggregated"; blob.UploadALocalFile(filepath, container, directoryPath); aggResult.metaData.PNG_URL = TaskConstants.AzureBlobURL + container + "/" + directoryPath + "/" + filename + "_aggregated.PNG"; return(aggResult); }