public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggEntry, SatyamResultsTableEntry result)
{
switch (result.JobTemplateType)
{
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
return(SingleObjectLabelingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
return(ObjectCountingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
return(MultiObjectLocalizationAndLabelingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
return(MultiObjectTrackingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
return(ImageSegmentationAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
return(true); //TBD
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
return(TrackletLabelingAggregator.IsAcceptable(aggEntry, result));
}
return(false);
}
public static void AggregateWithParameter(string guid,
int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
bool allFalseAttributeInvalid = false
)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;
Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
WorkersPerTask.Add(taskEntryID, new List <string>());
}
// remove duplicate workers result
string workerID = satyamResult.amazonInfo.WorkerID;
if (WorkersPerTask[taskEntryID].Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask[taskEntryID].Add(workerID);
string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
string[] fields = videoName.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
if (allFalseAttributeInvalid)
{
if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
{
continue;
}
}
ResultsPerTask[taskEntryID].Add(res);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
// hack for masters
int tempMin = MinResults;
if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
{
tempMin = 1;
}
TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
guid, task.SatyamURI,
job.FrameRate, job.BoundaryLines,
MinResults, MaxResults, boxToleranceThreshold,
ObjectCoverageApprovalThresholdPerVideo,
BoxCoverageApprovalThresholdPerTrack,
consensusNumber, minTubeletIoUSimilarityThreshold,
attributeMajority);
if (aggResult == null)
{
continue;
}
//////////////// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
}
//public static void Aggregate()
//{
// //first get all the results that are not aggregated
// SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
// List<SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus();
// //first group all the results by the tasks since wach task may ahve multiple entries
// Dictionary<int, List<SatyamResultsTableEntry>> collectedResults = new Dictionary<int, List<SatyamResultsTableEntry>>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// int taskID = entry.SatyamTaskTableEntryID;
// if (!collectedResults.ContainsKey(taskID))
// {
// collectedResults.Add(taskID, new List<SatyamResultsTableEntry>());
// }
// collectedResults[taskID].Add(entry);
// }
// //now aggreagate the results for eacht ask.
// List<int> taskIDList = collectedResults.Keys.ToList();
// List<SatyamAggregatedResultsTableEntry> aggEntries = new List<SatyamAggregatedResultsTableEntry>();
// foreach (int taskId in taskIDList)
// {
// SatyamAggregatedResultsTableEntry aggEntry = GetAggregatedResultString(taskId, collectedResults[taskId]);
// if (aggEntry != null)
// {
// aggEntries.Add(aggEntry);
// }
// }
// SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
// aggDB.AddEntries(aggEntries);
// resultsDB.close();
// aggDB.close();
//}
public static SatyamAggregatedResultsTableEntry GetAggregatedResultString(int taskId, List <SatyamResultsTableEntry> resultEntries)
{
SatyamAggregatedResultsTableEntry aggEntry = null;
string templateType = resultEntries[0].JobTemplateType;
string aggResultString = null;
switch (templateType)
{
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
aggResultString = ObjectCountingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
aggResultString = MultiObjectLocalizationAndLabelingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Tracking:
aggResultString = null; //Done testing, marked null so it becomes a demo page.
break;
case TaskConstants.Tracking_MTurk:
aggResultString = MultiObjectTrackingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
aggResultString = TrackletLabelingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Segmentation_Image:
aggResultString = null; //Done testing, marked null so it becomes a demo page.
break;
case TaskConstants.Segmentation_Image_MTurk:
aggResultString = ImageSegmentationAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
break;
}
if (aggResultString != null)
{
aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = resultEntries[0].JobGUID;
aggEntry.JobTemplateType = resultEntries[0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = resultEntries[0].SatyamTaskTableEntryID;
aggEntry.UserID = resultEntries[0].UserID;
aggEntry.ResultString = aggResultString;
}
return(aggEntry);
}