public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggEntry, SatyamResultsTableEntry result) { switch (result.JobTemplateType) { case TaskConstants.Classification_Image: case TaskConstants.Classification_Image_MTurk: case TaskConstants.Classification_Video: case TaskConstants.Classification_Video_MTurk: return(SingleObjectLabelingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Counting_Image: case TaskConstants.Counting_Image_MTurk: case TaskConstants.Counting_Video: case TaskConstants.Counting_Video_MTurk: return(ObjectCountingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Detection_Image: case TaskConstants.Detection_Image_MTurk: return(MultiObjectLocalizationAndLabelingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Tracking: case TaskConstants.Tracking_MTurk: return(MultiObjectTrackingAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.Segmentation_Image: case TaskConstants.Segmentation_Image_MTurk: return(ImageSegmentationAggregator.IsAcceptable(aggEntry, result)); case TaskConstants.OpenEndedQuestion_Image: case TaskConstants.OpenEndedQuestion_Image_MTurk: return(true); //TBD case TaskConstants.TrackletLabeling: case TaskConstants.TrackletLabeling_MTurk: return(TrackletLabelingAggregator.IsAcceptable(aggEntry, result)); } return(false); }
public static void AggregateWithParameter(string guid, int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE, double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD, double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO, double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK, int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS, double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD, double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD, bool allFalseAttributeInvalid = false ) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo; Console.WriteLine("Aggregating for " + guid + " with param set " + configString); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>()); WorkersPerTask.Add(taskEntryID, new List <string>()); } // remove duplicate workers result string workerID = satyamResult.amazonInfo.WorkerID; if (WorkersPerTask[taskEntryID].Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. WorkersPerTask[taskEntryID].Add(workerID); string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI); string[] fields = videoName.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment(); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); if (allFalseAttributeInvalid) { if (TrackletLabelingAggregator.AllAttributeAllFalse(res)) { continue; } } ResultsPerTask[taskEntryID].Add(res); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } // hack for masters int tempMin = MinResults; if (TaskConstants.masterGUIDs.Contains(entry.JobGUID)) { tempMin = 1; } TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], guid, task.SatyamURI, job.FrameRate, job.BoundaryLines, MinResults, MaxResults, boxToleranceThreshold, ObjectCoverageApprovalThresholdPerVideo, BoxCoverageApprovalThresholdPerTrack, consensusNumber, minTubeletIoUSimilarityThreshold, attributeMajority); if (aggResult == null) { continue; } //////////////// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid); }
//public static void Aggregate() //{ // //first get all the results that are not aggregated // SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); // List<SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus(); // //first group all the results by the tasks since wach task may ahve multiple entries // Dictionary<int, List<SatyamResultsTableEntry>> collectedResults = new Dictionary<int, List<SatyamResultsTableEntry>>(); // foreach (SatyamResultsTableEntry entry in results) // { // int taskID = entry.SatyamTaskTableEntryID; // if (!collectedResults.ContainsKey(taskID)) // { // collectedResults.Add(taskID, new List<SatyamResultsTableEntry>()); // } // collectedResults[taskID].Add(entry); // } // //now aggreagate the results for eacht ask. // List<int> taskIDList = collectedResults.Keys.ToList(); // List<SatyamAggregatedResultsTableEntry> aggEntries = new List<SatyamAggregatedResultsTableEntry>(); // foreach (int taskId in taskIDList) // { // SatyamAggregatedResultsTableEntry aggEntry = GetAggregatedResultString(taskId, collectedResults[taskId]); // if (aggEntry != null) // { // aggEntries.Add(aggEntry); // } // } // SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); // aggDB.AddEntries(aggEntries); // resultsDB.close(); // aggDB.close(); //} public static SatyamAggregatedResultsTableEntry GetAggregatedResultString(int taskId, List <SatyamResultsTableEntry> resultEntries) { SatyamAggregatedResultsTableEntry aggEntry = null; string templateType = resultEntries[0].JobTemplateType; string aggResultString = null; switch (templateType) { case TaskConstants.Classification_Image: case TaskConstants.Classification_Image_MTurk: case TaskConstants.Classification_Video: case TaskConstants.Classification_Video_MTurk: aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(resultEntries); break; case TaskConstants.Counting_Image: case TaskConstants.Counting_Image_MTurk: case TaskConstants.Counting_Video: case TaskConstants.Counting_Video_MTurk: aggResultString = ObjectCountingAggregator.GetAggregatedResultString(resultEntries); break; case TaskConstants.Detection_Image: case TaskConstants.Detection_Image_MTurk: aggResultString = MultiObjectLocalizationAndLabelingAggregator.GetAggregatedResultString(resultEntries); break; case TaskConstants.Tracking: aggResultString = null; //Done testing, marked null so it becomes a demo page. break; case TaskConstants.Tracking_MTurk: aggResultString = MultiObjectTrackingAggregator.GetAggregatedResultString(resultEntries); break; case TaskConstants.TrackletLabeling: case TaskConstants.TrackletLabeling_MTurk: aggResultString = TrackletLabelingAggregator.GetAggregatedResultString(resultEntries); break; case TaskConstants.Segmentation_Image: aggResultString = null; //Done testing, marked null so it becomes a demo page. break; case TaskConstants.Segmentation_Image_MTurk: aggResultString = ImageSegmentationAggregator.GetAggregatedResultString(resultEntries); break; case TaskConstants.OpenEndedQuestion_Image: case TaskConstants.OpenEndedQuestion_Image_MTurk: break; } if (aggResultString != null) { aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = resultEntries[0].JobGUID; aggEntry.JobTemplateType = resultEntries[0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = resultEntries[0].SatyamTaskTableEntryID; aggEntry.UserID = resultEntries[0].UserID; aggEntry.ResultString = aggResultString; } return(aggEntry); }