public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry,
double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX;
double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY;
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >();
allboxes.Add(new List <BoundingBox>());
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects)
{
allboxes[0].Add(entry.boundingBox);
}
allboxes.Add(new List <BoundingBox>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects)
{
allboxes[1].Add(entry.boundingBox);
if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y)
{
tooSmallToIgnore.Add(true);
}
else
{
tooSmallToIgnore.Add(false);
}
}
List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes);
//now find how many of the results match aggregated results
int noAccepted = 0;
double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX;
double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY;
int noIgnorable = 0;
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]];
BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]];
//double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox);
double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y);
if (deviation <= 1) //deviation test passed
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
else
{
if (tooSmallToIgnore[match.elementList[1]])
{
noIgnorable++;
}
}
}
}
}
if (noAccepted >= boxLimit - noIgnorable)
{
return(true);
}
return(false);
}
public static MultiObjectLocalizationAndLabelingAggregatedResult getAggregatedResult(List <MultiObjectLocalizationAndLabelingResult> results,
int MinResults = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD
)
{
if (results.Count < MinResults)
{
return(null);
}
//if the image was scaled down during display, errors get scaled up
//so we need to scale our deviation thresholds
double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * results[0].displayScaleReductionX;
double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * results[0].displayScaleReductionY;
//first use multipartitie wieghted matching to associated the boxes disregarding the labels since
//people might make mistake with lables but boxes are usually right
List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >();
List <int> noBoxesPerResult = new List <int>(); //how many boxes did each person draw?
List <List <bool> > largeBoxes = new List <List <bool> >(); //was this box was bigger than MIN_BOX_DIMENSION_FOR_CONSIDERATION?
foreach (MultiObjectLocalizationAndLabelingResult res in results)
{
allboxes.Add(new List <BoundingBox>());
largeBoxes.Add(new List <bool>());
if (res == null)
{
noBoxesPerResult.Add(0);
continue;
}
List <BoundingBox> boxList = allboxes[allboxes.Count - 1];
List <bool> largeBoxList = largeBoxes[largeBoxes.Count - 1];
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in res.objects)
{
boxList.Add(entry.boundingBox);
if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y)
{
largeBoxList.Add(false);
}
else
{
largeBoxList.Add(true);
}
}
noBoxesPerResult.Add(boxList.Count);
}
//now associate boxes across the various results
List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes);
int noBoxes = boxAssociation.Count;
int noAssociatedBoxes = 0;
int noIgnorable = 0;
//how many of the drawn boxes for each result were actually associated by two or more people for each user?
SortedDictionary <int, int> noMultipleAssociatedBoxesPerResult = new SortedDictionary <int, int>();
SortedDictionary <int, int> noSmallIgnorableBoxesPerResult = new SortedDictionary <int, int>();
for (int i = 0; i < results.Count; i++)
{
noMultipleAssociatedBoxesPerResult.Add(i, 0);
noSmallIgnorableBoxesPerResult.Add(i, 0);
}
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
if (match.elementList.Count > 1) //this has been corroborated by two people
{
noAssociatedBoxes++;
foreach (KeyValuePair <int, int> entry in match.elementList)
{
noMultipleAssociatedBoxesPerResult[entry.Key]++;
}
}
else
{
List <int> keys = match.elementList.Keys.ToList();
if (largeBoxes[keys[0]][match.elementList[keys[0]]] == false) //the box was a small box can be ignored
{
noIgnorable++;
noSmallIgnorableBoxesPerResult[keys[0]]++;
}
}
}
//count how many people have a high association ratio
int noHighAssociationRatio = 0;
for (int i = 0; i < results.Count; i++)
{
if (noBoxesPerResult[i] == 0)
{
continue;
}
double ratio = ((double)noMultipleAssociatedBoxesPerResult[i] + (double)noSmallIgnorableBoxesPerResult[i]) / (double)noBoxesPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noHighAssociationRatio++;
}
}
if (noHighAssociationRatio < MinResults) //at least three people should have all their boxes highly corroborated by one other person
{
return(null);
}
//now we have to find out if the boxes were drawn well and only take the boxes that are within
//a reasonable deviation of each other.
//if the image was scaled down during display, errors get scaled up
//so we need to scale our deviation thresholds
double deviation_threshold_x = DeviationPixelThreshold * results[0].displayScaleReductionX;
double deviation_threshold_y = DeviationPixelThreshold * results[0].displayScaleReductionY;
int noAcceptedBoxes = 0;
int noSmallIgnore = 0;
List <List <BoundingBoxGroup> > allBoxGroups = new List <List <BoundingBoxGroup> >();
List <BoundingBox> finalBoxes = new List <BoundingBox>(); //stores the aggregated boxes
SortedDictionary <int, List <int> > HighQualityAssociatedBoxPerResult = new SortedDictionary <int, List <int> >();
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
List <BoundingBox> boxList = new List <BoundingBox>();
List <string> identifiers = new List <string>();
foreach (KeyValuePair <int, int> entry in match.elementList)
{
boxList.Add(allboxes[entry.Key][entry.Value]);
identifiers.Add(entry.Key + "_" + entry.Value);
}
//List<BoundingBoxGroup> boxGroups = MergeAndGroupBoundingBoxes.GreedyMeanHierarchicalMergeByPixelDeviation(boxList, identifiers, DEVIATION_THRESHOLD);
List <BoundingBoxGroup> boxGroups = MergeAndGroupBoundingBoxes.GreedyMeanHierarchicalMergeByPixelDeviation(boxList, identifiers, deviation_threshold_x, deviation_threshold_y);
allBoxGroups.Add(boxGroups);
int maxCount = 0;
int index = -1;
getMaxCountGroup(boxGroups, out maxCount, out index);
////find the boxgroup with the maxCount
//int maxCount = 0;
//int index = -1;
//for(int i=0;i<boxGroups.Count;i++)
//{
// if(boxGroups[i].boundingBoxList.Count > maxCount) // there are two boxes within acceptance range
// {
// maxCount = boxGroups[i].boundingBoxList.Count;
// index = i;
// }
//}
if (maxCount > 1)
{
noAcceptedBoxes++;
finalBoxes.Add(boxGroups[index].mergedBoundingBox);
// count the majority group high associations
foreach (string id in boxGroups[index].identifiers)
{
string[] fields = id.Split('_');
int partId = Convert.ToInt32(fields[0]);
int boxId = Convert.ToInt32(fields[1]);
if (!HighQualityAssociatedBoxPerResult.ContainsKey(partId))
{
HighQualityAssociatedBoxPerResult.Add(partId, new List <int>());
}
HighQualityAssociatedBoxPerResult[partId].Add(boxId);
}
}
else
{
finalBoxes.Add(BoundingBox.NullBoundingBox);
//were all the boxes drawn too small?
bool small = true;
for (int i = 0; i < boxGroups.Count; i++)
{
string[] parts = boxGroups[i].identifiers[0].Split('_');
int resultNo = Convert.ToInt32(parts[0]);
int boxNo = Convert.ToInt32(parts[1]);
if (largeBoxes[resultNo][boxNo] == true)
{
small = false;
break;
}
}
if (small)
{
noSmallIgnore++;
}
}
}
if (((double)noAcceptedBoxes + (double)noSmallIgnore) / (double)noAssociatedBoxes < ObjectsCoverageThreshold &&
results.Count < MaxResults) //this is acceptable
{
return(null);
}
//count how many people have a high "quality" association ratio
int noResultsWithHighQualityObjectCoverage = 0;
for (int i = 0; i < results.Count; i++)
{
if (noBoxesPerResult[i] == 0)
{
continue;
}
if (!HighQualityAssociatedBoxPerResult.ContainsKey(i))
{
continue;
}
double ratio = ((double)HighQualityAssociatedBoxPerResult[i].Count) / (double)noBoxesPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noResultsWithHighQualityObjectCoverage++;
}
}
if (noResultsWithHighQualityObjectCoverage < MinResults && results.Count < MaxResults) //at least three people should have most of their boxes highly corroborated by one other person
{
return(null);
}
//now we need to see if the categores have a supermajority
List <string> finalCategories = new List <string>(); //stores the aggregated categories
int cntr = -1;
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
cntr++;
if (BoundingBox.IsNullBoundingBox(finalBoxes[cntr])) //this is not an acceptable box
{
finalCategories.Add("");
continue;
}
Dictionary <string, int> categoryNames = new Dictionary <string, int>();
int totalCount = match.elementList.Count;
int maxCount = 0;
string maxCategory = "";
foreach (KeyValuePair <int, int> entry in match.elementList)
{
string category = results[entry.Key].objects[entry.Value].Category;
if (!categoryNames.ContainsKey(category))
{
categoryNames.Add(category, 0);
}
categoryNames[category]++;
if (maxCount < categoryNames[category])
{
maxCount = categoryNames[category];
maxCategory = category;
}
}
double probability = ((double)maxCount + 1) / ((double)totalCount + 2);
if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work
{
return(null);
}
else
{
finalCategories.Add(maxCategory);
}
}
//now we have enough information to create the aggregate results
MultiObjectLocalizationAndLabelingAggregatedResult aggResult = new MultiObjectLocalizationAndLabelingAggregatedResult();
aggResult.metaData = new MultiObjectLocalizationAndLabelingAggregatedResultMetaData();
aggResult.metaData.TotalCount = results.Count;
aggResult.boxesAndCategories = new MultiObjectLocalizationAndLabelingResult();
aggResult.boxesAndCategories.objects = new List <MultiObjectLocalizationAndLabelingResultSingleEntry>();
cntr = -1;
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
cntr++;
if (BoundingBox.IsNullBoundingBox(finalBoxes[cntr]))
{
continue;
}
MultiObjectLocalizationAndLabelingResultSingleEntry entry = new MultiObjectLocalizationAndLabelingResultSingleEntry();
entry.boundingBox = finalBoxes[cntr];
entry.Category = finalCategories[cntr];
aggResult.boxesAndCategories.objects.Add(entry);
}
aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX;
aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY;
aggResult.boxesAndCategories.imageHeight = results[0].imageHeight;
aggResult.boxesAndCategories.imageWidth = results[0].imageWidth;
return(aggResult);
}
public static MultiObjectTrackingResult stitchTwoTracesByOneFrameBoundingBoxes(MultiObjectTrackingResult currentTrace, MultiObjectTrackingResult nextTrace,
int totalFrameCountsBeforeStitching, int totalFrameCountsAfterStitching, int noFramesOverlap = 0, int fps = 10)
{
List <List <BoundingBox> > boundingboxes = new List <List <BoundingBox> >();
List <List <string> > CrossChunkEntitiesAnnotationStrings = new List <List <string> >();
List <List <string> > SingleChunkEntitiesAnnotationStrings = new List <List <string> >();
for (int i = 0; i < 2; i++)
{
boundingboxes.Add(new List <BoundingBox>());
CrossChunkEntitiesAnnotationStrings.Add(new List <string>());
SingleChunkEntitiesAnnotationStrings.Add(new List <string>());
}
double frameTimeInMiliSeconds = (double)1000 / (double)fps;
double timeToPostponeInMilliSeconds = (double)(totalFrameCountsBeforeStitching - noFramesOverlap) * frameTimeInMiliSeconds;
//TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
// get the end of the first trace and the begining + overlap of the second trace
DateTime CurrentTraceSampleFrameTime = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1));
DateTime NextTraceSampleFrameTime = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (Math.Max(noFramesOverlap - 1, 0)));
// current trace
foreach (CompressedTrack entity in currentTrace.tracks)
{
SpaceTime st = entity.getSpaceTimeAt(CurrentTraceSampleFrameTime);
BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", CurrentTraceSampleFrameTime);
if (st != null && outofview_attr != null && !outofview_attr.value)
{
BoundingBox box = st.region;
boundingboxes[0].Add(box);
CrossChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString());
}
else
{
SingleChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString());
}
}
// next trace
nextTrace.VideoSegmentStartTime += timeSpanToPostponeInSeconds;
nextTrace.VideoSegmentEndTime += timeSpanToPostponeInSeconds;
foreach (CompressedTrack entity in nextTrace.tracks)
{
// get the frame first before changing the time stmaps.
SpaceTime st = entity.getSpaceTimeAt(NextTraceSampleFrameTime);
BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", entity.endTime);
/// postpone the timestamp of the second trace
entity.startTime += timeSpanToPostponeInSeconds;
entity.endTime += timeSpanToPostponeInSeconds;
entity.spaceTimeTrack.startTime += timeSpanToPostponeInSeconds;
entity.spaceTimeTrack.endTime += timeSpanToPostponeInSeconds;
//foreach (SpaceTime st in entity.spaceTimeTrack.space_time_track)
for (int i = 0; i < entity.spaceTimeTrack.space_time_track.Count; i++)
{
entity.spaceTimeTrack.space_time_track[i].time += timeSpanToPostponeInSeconds;
}
//foreach (CompressedBooleanAttributeTrack booleanAttributeTrack in entity.booleanAttributeTracks.Values)
foreach (string key in entity.booleanAttributeTracks.Keys)
{
entity.booleanAttributeTracks[key].startTime += timeSpanToPostponeInSeconds;
entity.booleanAttributeTracks[key].endTime += timeSpanToPostponeInSeconds;
//foreach (BooleanAttribute ba in entity.booleanAttributeTracks[key].attribute_track)
for (int i = 0; i < entity.booleanAttributeTracks[key].attribute_track.Count; i++)
{
entity.booleanAttributeTracks[key].attribute_track[i].time += timeSpanToPostponeInSeconds;
}
}
if (st != null && outofview_attr != null && !outofview_attr.value)
{
BoundingBox box = st.region;
boundingboxes[1].Add(box);
CrossChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString());
}
else
{
SingleChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString());
}
}
List <MultipartiteWeightedMatch> association = BoundingBoxAssociation.computeBoundingBoxAssociations(boundingboxes);
DateTime NewFrameStartTime = currentTrace.VideoSegmentStartTime + DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)(totalFrameCountsBeforeStitching * frameTimeInMiliSeconds));
string totalStitchedAnnotationString = stitchAnnotationStringByAssociation(CrossChunkEntitiesAnnotationStrings, SingleChunkEntitiesAnnotationStrings,
association, NewFrameStartTime);
MultiObjectTrackingResult ret = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(currentTrace.VideoSegmentStartTime, totalStitchedAnnotationString, currentTrace.cameraId, currentTrace.UID, totalFrameCountsAfterStitching, fps);
return(ret);
}
