public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX; double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY; //We fist do a bipartitte matching to find the best assocaition for the boxes List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >(); allboxes.Add(new List <BoundingBox>()); foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects) { allboxes[0].Add(entry.boundingBox); } allboxes.Add(new List <BoundingBox>()); List <bool> tooSmallToIgnore = new List <bool>(); foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects) { allboxes[1].Add(entry.boundingBox); if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y) { tooSmallToIgnore.Add(true); } else { tooSmallToIgnore.Add(false); } } List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes); //now find how many of the results match aggregated results int noAccepted = 0; double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX; double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY; int noIgnorable = 0; foreach (MultipartiteWeightedMatch match in boxAssociation) { if (match.elementList.ContainsKey(1)) // this contains an aggregated box { if (match.elementList.ContainsKey(0)) // a result box has been associated { BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]]; BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]]; //double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox); double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y); if (deviation <= 1) //deviation test passed { //now check category if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) { //both category and bounding box tests have passed noAccepted++; } } else { if (tooSmallToIgnore[match.elementList[1]]) { noIgnorable++; } } } } } if (noAccepted >= boxLimit - noIgnorable) { return(true); } return(false); }
public static MultiObjectLocalizationAndLabelingAggregatedResult getAggregatedResult(List <MultiObjectLocalizationAndLabelingResult> results, int MinResults = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double CategoryMajorityThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MAJORITY_CATEGORY_THRESHOLD, double ObjectsCoverageThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION, double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD ) { if (results.Count < MinResults) { return(null); } //if the image was scaled down during display, errors get scaled up //so we need to scale our deviation thresholds double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * results[0].displayScaleReductionX; double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * results[0].displayScaleReductionY; //first use multipartitie wieghted matching to associated the boxes disregarding the labels since //people might make mistake with lables but boxes are usually right List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >(); List <int> noBoxesPerResult = new List <int>(); //how many boxes did each person draw? List <List <bool> > largeBoxes = new List <List <bool> >(); //was this box was bigger than MIN_BOX_DIMENSION_FOR_CONSIDERATION? foreach (MultiObjectLocalizationAndLabelingResult res in results) { allboxes.Add(new List <BoundingBox>()); largeBoxes.Add(new List <bool>()); if (res == null) { noBoxesPerResult.Add(0); continue; } List <BoundingBox> boxList = allboxes[allboxes.Count - 1]; List <bool> largeBoxList = largeBoxes[largeBoxes.Count - 1]; foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in res.objects) { boxList.Add(entry.boundingBox); if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y) { largeBoxList.Add(false); } else { largeBoxList.Add(true); } } noBoxesPerResult.Add(boxList.Count); } //now associate boxes across the various results List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes); int noBoxes = boxAssociation.Count; int noAssociatedBoxes = 0; int noIgnorable = 0; //how many of the drawn boxes for each result were actually associated by two or more people for each user? SortedDictionary <int, int> noMultipleAssociatedBoxesPerResult = new SortedDictionary <int, int>(); SortedDictionary <int, int> noSmallIgnorableBoxesPerResult = new SortedDictionary <int, int>(); for (int i = 0; i < results.Count; i++) { noMultipleAssociatedBoxesPerResult.Add(i, 0); noSmallIgnorableBoxesPerResult.Add(i, 0); } foreach (MultipartiteWeightedMatch match in boxAssociation) { if (match.elementList.Count > 1) //this has been corroborated by two people { noAssociatedBoxes++; foreach (KeyValuePair <int, int> entry in match.elementList) { noMultipleAssociatedBoxesPerResult[entry.Key]++; } } else { List <int> keys = match.elementList.Keys.ToList(); if (largeBoxes[keys[0]][match.elementList[keys[0]]] == false) //the box was a small box can be ignored { noIgnorable++; noSmallIgnorableBoxesPerResult[keys[0]]++; } } } //count how many people have a high association ratio int noHighAssociationRatio = 0; for (int i = 0; i < results.Count; i++) { if (noBoxesPerResult[i] == 0) { continue; } double ratio = ((double)noMultipleAssociatedBoxesPerResult[i] + (double)noSmallIgnorableBoxesPerResult[i]) / (double)noBoxesPerResult[i]; if (ratio > ObjectsCoverageThreshold) { noHighAssociationRatio++; } } if (noHighAssociationRatio < MinResults) //at least three people should have all their boxes highly corroborated by one other person { return(null); } //now we have to find out if the boxes were drawn well and only take the boxes that are within //a reasonable deviation of each other. //if the image was scaled down during display, errors get scaled up //so we need to scale our deviation thresholds double deviation_threshold_x = DeviationPixelThreshold * results[0].displayScaleReductionX; double deviation_threshold_y = DeviationPixelThreshold * results[0].displayScaleReductionY; int noAcceptedBoxes = 0; int noSmallIgnore = 0; List <List <BoundingBoxGroup> > allBoxGroups = new List <List <BoundingBoxGroup> >(); List <BoundingBox> finalBoxes = new List <BoundingBox>(); //stores the aggregated boxes SortedDictionary <int, List <int> > HighQualityAssociatedBoxPerResult = new SortedDictionary <int, List <int> >(); foreach (MultipartiteWeightedMatch match in boxAssociation) { List <BoundingBox> boxList = new List <BoundingBox>(); List <string> identifiers = new List <string>(); foreach (KeyValuePair <int, int> entry in match.elementList) { boxList.Add(allboxes[entry.Key][entry.Value]); identifiers.Add(entry.Key + "_" + entry.Value); } //List<BoundingBoxGroup> boxGroups = MergeAndGroupBoundingBoxes.GreedyMeanHierarchicalMergeByPixelDeviation(boxList, identifiers, DEVIATION_THRESHOLD); List <BoundingBoxGroup> boxGroups = MergeAndGroupBoundingBoxes.GreedyMeanHierarchicalMergeByPixelDeviation(boxList, identifiers, deviation_threshold_x, deviation_threshold_y); allBoxGroups.Add(boxGroups); int maxCount = 0; int index = -1; getMaxCountGroup(boxGroups, out maxCount, out index); ////find the boxgroup with the maxCount //int maxCount = 0; //int index = -1; //for(int i=0;i<boxGroups.Count;i++) //{ // if(boxGroups[i].boundingBoxList.Count > maxCount) // there are two boxes within acceptance range // { // maxCount = boxGroups[i].boundingBoxList.Count; // index = i; // } //} if (maxCount > 1) { noAcceptedBoxes++; finalBoxes.Add(boxGroups[index].mergedBoundingBox); // count the majority group high associations foreach (string id in boxGroups[index].identifiers) { string[] fields = id.Split('_'); int partId = Convert.ToInt32(fields[0]); int boxId = Convert.ToInt32(fields[1]); if (!HighQualityAssociatedBoxPerResult.ContainsKey(partId)) { HighQualityAssociatedBoxPerResult.Add(partId, new List <int>()); } HighQualityAssociatedBoxPerResult[partId].Add(boxId); } } else { finalBoxes.Add(BoundingBox.NullBoundingBox); //were all the boxes drawn too small? bool small = true; for (int i = 0; i < boxGroups.Count; i++) { string[] parts = boxGroups[i].identifiers[0].Split('_'); int resultNo = Convert.ToInt32(parts[0]); int boxNo = Convert.ToInt32(parts[1]); if (largeBoxes[resultNo][boxNo] == true) { small = false; break; } } if (small) { noSmallIgnore++; } } } if (((double)noAcceptedBoxes + (double)noSmallIgnore) / (double)noAssociatedBoxes < ObjectsCoverageThreshold && results.Count < MaxResults) //this is acceptable { return(null); } //count how many people have a high "quality" association ratio int noResultsWithHighQualityObjectCoverage = 0; for (int i = 0; i < results.Count; i++) { if (noBoxesPerResult[i] == 0) { continue; } if (!HighQualityAssociatedBoxPerResult.ContainsKey(i)) { continue; } double ratio = ((double)HighQualityAssociatedBoxPerResult[i].Count) / (double)noBoxesPerResult[i]; if (ratio > ObjectsCoverageThreshold) { noResultsWithHighQualityObjectCoverage++; } } if (noResultsWithHighQualityObjectCoverage < MinResults && results.Count < MaxResults) //at least three people should have most of their boxes highly corroborated by one other person { return(null); } //now we need to see if the categores have a supermajority List <string> finalCategories = new List <string>(); //stores the aggregated categories int cntr = -1; foreach (MultipartiteWeightedMatch match in boxAssociation) { cntr++; if (BoundingBox.IsNullBoundingBox(finalBoxes[cntr])) //this is not an acceptable box { finalCategories.Add(""); continue; } Dictionary <string, int> categoryNames = new Dictionary <string, int>(); int totalCount = match.elementList.Count; int maxCount = 0; string maxCategory = ""; foreach (KeyValuePair <int, int> entry in match.elementList) { string category = results[entry.Key].objects[entry.Value].Category; if (!categoryNames.ContainsKey(category)) { categoryNames.Add(category, 0); } categoryNames[category]++; if (maxCount < categoryNames[category]) { maxCount = categoryNames[category]; maxCategory = category; } } double probability = ((double)maxCount + 1) / ((double)totalCount + 2); if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work { return(null); } else { finalCategories.Add(maxCategory); } } //now we have enough information to create the aggregate results MultiObjectLocalizationAndLabelingAggregatedResult aggResult = new MultiObjectLocalizationAndLabelingAggregatedResult(); aggResult.metaData = new MultiObjectLocalizationAndLabelingAggregatedResultMetaData(); aggResult.metaData.TotalCount = results.Count; aggResult.boxesAndCategories = new MultiObjectLocalizationAndLabelingResult(); aggResult.boxesAndCategories.objects = new List <MultiObjectLocalizationAndLabelingResultSingleEntry>(); cntr = -1; foreach (MultipartiteWeightedMatch match in boxAssociation) { cntr++; if (BoundingBox.IsNullBoundingBox(finalBoxes[cntr])) { continue; } MultiObjectLocalizationAndLabelingResultSingleEntry entry = new MultiObjectLocalizationAndLabelingResultSingleEntry(); entry.boundingBox = finalBoxes[cntr]; entry.Category = finalCategories[cntr]; aggResult.boxesAndCategories.objects.Add(entry); } aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX; aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY; aggResult.boxesAndCategories.imageHeight = results[0].imageHeight; aggResult.boxesAndCategories.imageWidth = results[0].imageWidth; return(aggResult); }
public static MultiObjectTrackingResult stitchTwoTracesByOneFrameBoundingBoxes(MultiObjectTrackingResult currentTrace, MultiObjectTrackingResult nextTrace, int totalFrameCountsBeforeStitching, int totalFrameCountsAfterStitching, int noFramesOverlap = 0, int fps = 10) { List <List <BoundingBox> > boundingboxes = new List <List <BoundingBox> >(); List <List <string> > CrossChunkEntitiesAnnotationStrings = new List <List <string> >(); List <List <string> > SingleChunkEntitiesAnnotationStrings = new List <List <string> >(); for (int i = 0; i < 2; i++) { boundingboxes.Add(new List <BoundingBox>()); CrossChunkEntitiesAnnotationStrings.Add(new List <string>()); SingleChunkEntitiesAnnotationStrings.Add(new List <string>()); } double frameTimeInMiliSeconds = (double)1000 / (double)fps; double timeToPostponeInMilliSeconds = (double)(totalFrameCountsBeforeStitching - noFramesOverlap) * frameTimeInMiliSeconds; //TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds); TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds); // get the end of the first trace and the begining + overlap of the second trace DateTime CurrentTraceSampleFrameTime = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1)); DateTime NextTraceSampleFrameTime = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (Math.Max(noFramesOverlap - 1, 0))); // current trace foreach (CompressedTrack entity in currentTrace.tracks) { SpaceTime st = entity.getSpaceTimeAt(CurrentTraceSampleFrameTime); BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", CurrentTraceSampleFrameTime); if (st != null && outofview_attr != null && !outofview_attr.value) { BoundingBox box = st.region; boundingboxes[0].Add(box); CrossChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString()); } else { SingleChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString()); } } // next trace nextTrace.VideoSegmentStartTime += timeSpanToPostponeInSeconds; nextTrace.VideoSegmentEndTime += timeSpanToPostponeInSeconds; foreach (CompressedTrack entity in nextTrace.tracks) { // get the frame first before changing the time stmaps. SpaceTime st = entity.getSpaceTimeAt(NextTraceSampleFrameTime); BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", entity.endTime); /// postpone the timestamp of the second trace entity.startTime += timeSpanToPostponeInSeconds; entity.endTime += timeSpanToPostponeInSeconds; entity.spaceTimeTrack.startTime += timeSpanToPostponeInSeconds; entity.spaceTimeTrack.endTime += timeSpanToPostponeInSeconds; //foreach (SpaceTime st in entity.spaceTimeTrack.space_time_track) for (int i = 0; i < entity.spaceTimeTrack.space_time_track.Count; i++) { entity.spaceTimeTrack.space_time_track[i].time += timeSpanToPostponeInSeconds; } //foreach (CompressedBooleanAttributeTrack booleanAttributeTrack in entity.booleanAttributeTracks.Values) foreach (string key in entity.booleanAttributeTracks.Keys) { entity.booleanAttributeTracks[key].startTime += timeSpanToPostponeInSeconds; entity.booleanAttributeTracks[key].endTime += timeSpanToPostponeInSeconds; //foreach (BooleanAttribute ba in entity.booleanAttributeTracks[key].attribute_track) for (int i = 0; i < entity.booleanAttributeTracks[key].attribute_track.Count; i++) { entity.booleanAttributeTracks[key].attribute_track[i].time += timeSpanToPostponeInSeconds; } } if (st != null && outofview_attr != null && !outofview_attr.value) { BoundingBox box = st.region; boundingboxes[1].Add(box); CrossChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString()); } else { SingleChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString()); } } List <MultipartiteWeightedMatch> association = BoundingBoxAssociation.computeBoundingBoxAssociations(boundingboxes); DateTime NewFrameStartTime = currentTrace.VideoSegmentStartTime + DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)(totalFrameCountsBeforeStitching * frameTimeInMiliSeconds)); string totalStitchedAnnotationString = stitchAnnotationStringByAssociation(CrossChunkEntitiesAnnotationStrings, SingleChunkEntitiesAnnotationStrings, association, NewFrameStartTime); MultiObjectTrackingResult ret = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(currentTrace.VideoSegmentStartTime, totalStitchedAnnotationString, currentTrace.cameraId, currentTrace.UID, totalFrameCountsAfterStitching, fps); return(ret); }