public static ImageSegmentationAggregatedResult_NoHoles getAggregatedResult(List <ImageSegmentationResult_NoHoles> results,
string SatyamURL,
string guid,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION
)
{
if (results.Count < MinResults) //need at least three results!
{
return(null);
}
int ImageWidth = results[0].imageWidth;
int ImageHeight = results[0].imageHeight;
byte[] PNG = new byte[ImageHeight * ImageWidth];
ImageSegmentationAggregatedResult_NoHoles aggResult = new ImageSegmentationAggregatedResult_NoHoles();
aggResult.metaData = new ImageSegmentationAggregatedResultMetaData_NoHoles();
aggResult.metaData.TotalCount = results.Count;
for (int i = 0; i < PNG.Length; i++)
{
PNG[i] = 0;
}
aggResult.boxesAndCategories = new ImageSegmentationResult_NoHoles();
aggResult.boxesAndCategories.objects = new List <ImageSegmentationResultSingleEntry_NoHoles>();
aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX;
aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY;
aggResult.boxesAndCategories.imageHeight = ImageHeight;
aggResult.boxesAndCategories.imageWidth = ImageWidth;
//first use multipartitie wieghted matching to associated the boxes disregarding the labels since
//people might make mistake with lables but boxes are usually right
List <List <GenericPolygon> > AllPolygons = new List <List <GenericPolygon> >();
List <int> noPolygonsPerResult = new List <int>();
foreach (ImageSegmentationResult_NoHoles res in results)
{
AllPolygons.Add(new List <GenericPolygon>());
if (res == null)
{
noPolygonsPerResult.Add(0);
continue;
}
List <GenericPolygon> polygonList = AllPolygons[AllPolygons.Count - 1];
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in res.objects)
{
polygonList.Add(entry.polygon);
}
noPolygonsPerResult.Add(polygonList.Count);
}
//now associate boxes across the various results
List <MultipartiteWeightedMatch> polyAssociation = PolygonAssociation.computeGenericPolygonAssociations(AllPolygons);
int noObjects = polyAssociation.Count;
int noAssociatedPolygons = 0;
//how many of the drawn boxes for each result were actually associated by two or more people for each user?
SortedDictionary <int, int> noMultipleAssociatedPolygonsPerResult = new SortedDictionary <int, int>();
for (int i = 0; i < results.Count; i++)
{
noMultipleAssociatedPolygonsPerResult.Add(i, 0);
}
foreach (MultipartiteWeightedMatch match in polyAssociation)
{
if (match.elementList.Count > 1) //this has been corroborated by two people
{
noAssociatedPolygons++;
foreach (KeyValuePair <int, int> entry in match.elementList)
{
noMultipleAssociatedPolygonsPerResult[entry.Key]++;
}
}
}
//count how many people have a high association ratio
int noHighAssociationRatio = 0;
for (int i = 0; i < results.Count; i++)
{
if (noPolygonsPerResult[i] == 0)
{
continue;
}
double ratio = (double)noMultipleAssociatedPolygonsPerResult[i] / (double)noPolygonsPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noHighAssociationRatio++;
}
}
if (noHighAssociationRatio < MinResults) //at least three people should have all their boxes highly corroborated by one other person
{
return(null);
}
//int noAcceptedPolygons = 0;
for (int idx = 0; idx < polyAssociation.Count; idx++)
{
MultipartiteWeightedMatch match = polyAssociation[idx];
List <GenericPolygon> polyList = new List <GenericPolygon>();
List <string> identifiers = new List <string>();
foreach (KeyValuePair <int, int> entry in match.elementList)
{
polyList.Add(AllPolygons[entry.Key][entry.Value]);
identifiers.Add(entry.Key + "_" + entry.Value);
}
//GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_Relaxation(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
//GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_MajorityEdge(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
GenericPolygon aggregatedPolygon = new GenericPolygon();// dummy polygon
byte[] png = GetAggregatedGenericPolygon_PixelSweep(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
Dictionary <string, int> categoryNames = new Dictionary <string, int>();
int totalCount = match.elementList.Count;
int maxCount = 0;
string maxCategory = "";
foreach (KeyValuePair <int, int> entry in match.elementList)
{
string category = results[entry.Key].objects[entry.Value].Category;
if (!categoryNames.ContainsKey(category))
{
categoryNames.Add(category, 0);
}
categoryNames[category]++;
if (maxCount < categoryNames[category])
{
maxCount = categoryNames[category];
maxCategory = category;
}
}
double probability = ((double)maxCount + 1) / ((double)totalCount + 2);
if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work
{
return(null);
}
// now we have one segment ready
ImageSegmentationResultSingleEntry_NoHoles aggregated = new ImageSegmentationResultSingleEntry_NoHoles();
aggregated.polygon = aggregatedPolygon;
aggregated.Category = maxCategory;
aggResult.boxesAndCategories.objects.Add(aggregated);
for (int i = 0; i < PNG.Length; i++)
{
if (PNG[i] == 0 && png[i] != 0)
{
PNG[i] = (byte)(idx + 1);
}
}
}
// save and upload to azure
string filename = URIUtilities.filenameFromURINoExtension(SatyamURL);
string filepath = DirectoryConstants.defaultTempDirectory + filename + "_aggregated.PNG";
ImageUtilities.savePNGRawData(filepath, ImageWidth, ImageHeight, PNG);
SatyamJobStorageAccountAccess blob = new SatyamJobStorageAccountAccess();
string container = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[TaskConstants.Segmentation_Image_MTurk];
string directoryPath = guid + "_aggregated";
blob.UploadALocalFile(filepath, container, directoryPath);
aggResult.metaData.PNG_URL = TaskConstants.AzureBlobURL + container + "/" + directoryPath + "/" + filename + "_aggregated.PNG";
return(aggResult);
}
public static ImageSegmentationAggregatedResult getAggregatedResult(List <ImageSegmentationResult> originalResults,
string SatyamURL,
string guid,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS
)
{
List <ImageSegmentationResult> results = new List <ImageSegmentationResult>();
// Warning: filter empty results: strong assumption that there has to be something for now!!!
for (int i = 0; i < originalResults.Count; i++)
{
if (originalResults[i] != null && originalResults[i].objects.Count != 0)
{
results.Add(originalResults[i]);
}
}
Console.WriteLine("Filetered Results: {0}", results.Count);
if (results.Count < MinResults) //need at least three results!
{
return(null);
}
int ImageWidth = results[0].imageWidth;
int ImageHeight = results[0].imageHeight;
//auto padding data to make the stride a multiple of 4. required by bmpdata for output
int paddedWidth = ImageWidth;
if (paddedWidth % 4 != 0)
{
paddedWidth = (paddedWidth / 4 + 1) * 4;
}
byte[] PNG = new byte[ImageHeight * paddedWidth];
ImageSegmentationAggregatedResult aggResult = new ImageSegmentationAggregatedResult();
aggResult.metaData = new ImageSegmentationAggregatedResultMetaData();
aggResult.metaData.TotalCount = results.Count;
for (int i = 0; i < PNG.Length; i++)
{
PNG[i] = 0;
}
aggResult.boxesAndCategories = new ImageSegmentationResult();
aggResult.boxesAndCategories.objects = new List <ImageSegmentationResultSingleEntry>();
aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX;
aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY;
aggResult.boxesAndCategories.imageHeight = ImageHeight;
aggResult.boxesAndCategories.imageWidth = ImageWidth;
//first use multipartitie wieghted matching to associated the boxes disregarding the labels since
//people might make mistake with lables but boxes are usually right
List <List <Segment> > AllPolygons = new List <List <Segment> >();
List <int> noPolygonsPerResult = new List <int>();
foreach (ImageSegmentationResult res in results)
{
AllPolygons.Add(new List <Segment>());
if (res == null)
{
noPolygonsPerResult.Add(0);
continue;
}
List <Segment> polygonList = AllPolygons[AllPolygons.Count - 1];
foreach (ImageSegmentationResultSingleEntry entry in res.objects)
{
polygonList.Add(entry.segment);
}
noPolygonsPerResult.Add(polygonList.Count);
}
//now associate boxes across the various results
//List<MultipartiteWeightedMatch> polyAssociation = PolygonAssociation.computeGenericPolygonAssociations(AllPolygons);
List <MultipartiteWeightedMatch> polyAssociation = GenericObjectAssociation.computeAssociations <Segment>(AllPolygons, Segment.computeIoU_PixelSweep);
int noObjects = polyAssociation.Count;
int noAssociatedPolygons = 0;
//how many of the drawn boxes for each result were actually associated by two or more people for each user?
SortedDictionary <int, int> noMultipleAssociatedPolygonsPerResult = new SortedDictionary <int, int>();
for (int i = 0; i < results.Count; i++)
{
noMultipleAssociatedPolygonsPerResult.Add(i, 0);
}
foreach (MultipartiteWeightedMatch match in polyAssociation)
{
if (match.elementList.Count > 1) //this has been corroborated by two people
{
noAssociatedPolygons++;
foreach (KeyValuePair <int, int> entry in match.elementList)
{
noMultipleAssociatedPolygonsPerResult[entry.Key]++;
}
}
}
//count how many people have a high association ratio
int noHighAssociationRatio = 0;
for (int i = 0; i < results.Count; i++)
{
if (noPolygonsPerResult[i] == 0)
{
continue;
}
double ratio = (double)noMultipleAssociatedPolygonsPerResult[i] / (double)noPolygonsPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noHighAssociationRatio++;
}
}
if (noHighAssociationRatio < MinResults && results.Count < MaxResults) //at least three people should have all their boxes highly corroborated by one other person
{
return(null);
}
//int noAcceptedPolygons = 0;
SortedDictionary <int, List <int> > noHighQualityAssociation = new SortedDictionary <int, List <int> >();
for (int idx = 0; idx < polyAssociation.Count; idx++)
{
MultipartiteWeightedMatch match = polyAssociation[idx];
List <Segment> polyList = new List <Segment>();
List <string> identifiers = new List <string>();
foreach (KeyValuePair <int, int> entry in match.elementList)
{
polyList.Add(AllPolygons[entry.Key][entry.Value]);
identifiers.Add(entry.Key + "_" + entry.Value);
}
SegmentGroup majorityGroup = getMajorityGroupPolygons(identifiers, polyList, minSimilarityThreshold, match.weightMatrix);
if (majorityGroup == null)
{
continue;
}
if (majorityGroup.segments.Count < minResultsForConsensus)
{
continue;
}
Segment aggregatedPolygon = new Segment();// dummy polygon
byte[] png = GetAggregatedGenericPolygon_PixelSweep(majorityGroup.segments, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
// log where the results come from
foreach (string id in majorityGroup.identifiers)
{
string[] fields = id.Split('_');
int k = Convert.ToInt32(fields[0]);
int v = Convert.ToInt32(fields[1]);
if (!noHighQualityAssociation.ContainsKey(k))
{
noHighQualityAssociation.Add(k, new List <int>());
}
noHighQualityAssociation[k].Add(v);
}
// category aggregation
Dictionary <string, int> categoryNames = new Dictionary <string, int>();
int totalCount = match.elementList.Count;
int maxCount = 0;
string maxCategory = "";
foreach (KeyValuePair <int, int> entry in match.elementList)
{
string category = results[entry.Key].objects[entry.Value].Category;
if (!categoryNames.ContainsKey(category))
{
categoryNames.Add(category, 0);
}
categoryNames[category]++;
if (maxCount < categoryNames[category])
{
maxCount = categoryNames[category];
maxCategory = category;
}
}
double probability = ((double)maxCount + 1) / ((double)totalCount + 2);
if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work
{
return(null);
}
// now we have one segment ready
ImageSegmentationResultSingleEntry aggregated = new ImageSegmentationResultSingleEntry();
aggregated.segment = aggregatedPolygon;
aggregated.Category = maxCategory;
aggResult.boxesAndCategories.objects.Add(aggregated);
for (int i = 0; i < ImageWidth; i++)
{
for (int j = 0; j < ImageHeight; j++)
{
int pospng = j * ImageWidth + i;
int pos = j * paddedWidth + i;
if (PNG[pos] == 0 && png[pospng] != 0)
{
PNG[pos] = (byte)(idx + 1);
}
}
}
}
// check no. of high association results
int noResultsWithHighQualityObjectCoverage = 0;
for (int i = 0; i < results.Count; i++)
{
if (noPolygonsPerResult[i] == 0)
{
continue;
}
if (!noHighQualityAssociation.ContainsKey(i))
{
continue;
}
double ratio = ((double)noHighQualityAssociation[i].Count) / (double)noPolygonsPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noResultsWithHighQualityObjectCoverage++;
}
}
if (noResultsWithHighQualityObjectCoverage < MinResults && results.Count < MaxResults) //at least three people should have most of their boxes highly corroborated by one other person
{
return(null);
}
// save and upload to azure
string filename = URIUtilities.filenameFromURINoExtension(SatyamURL);
//string filepath = DirectoryConstants.defaultTempDirectory + filename + "_aggregated.PNG";
string filepath = DirectoryConstants.defaultAzureTempDirectory + filename + "_aggregated.PNG";
ImageUtilities.savePNGRawData(filepath, ImageWidth, ImageHeight, PNG);
SatyamJobStorageAccountAccess blob = new SatyamJobStorageAccountAccess();
string container = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[TaskConstants.Segmentation_Image_MTurk];
string directoryPath = guid + "_aggregated";
blob.UploadALocalFile(filepath, container, directoryPath);
aggResult.metaData.PNG_URL = TaskConstants.AzureBlobURL + container + "/" + directoryPath + "/" + filename + "_aggregated.PNG";
//clean up
File.Delete(filepath);
return(aggResult);
}
