public static SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > GroupChunksByVideoName(List <SatyamAggregatedResultsTableEntry> entries)
{
SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > entriesPerVideo = new SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> >();
int fps = 0;
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
string videoName = "";
for (int j = 0; j < fields.Length - 2; j++)
{
// remove the starting frame
videoName += fields[j];
if (j == fields.Length - 3)
{
break;
}
videoName += '_';
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
fps = job.FrameRate;
}
if (!entriesPerVideo.ContainsKey(videoName))
{
entriesPerVideo.Add(videoName, new List <SatyamAggregatedResultsTableEntry>());
}
entriesPerVideo[videoName].Add(entries[i]);
}
return(entriesPerVideo);
}
//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results,
// int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
// int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
// double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
// double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
// double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
//{
// string resultString = null;
// List<ObjectCountingResult> resultList = new List<ObjectCountingResult>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult);
// resultList.Add(taskr);
// }
// ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
// if (r != null)
// {
// string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r);
// SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
// aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
// aggResult.AggregatedResultString = rString;
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString);
// aggResult.TaskParameters = res.TaskParametersString;
// resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult);
// }
// return resultString;
//}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result,
double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString);
double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION;
if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev)
{
maxdev = ragg.Count * MAX_DEVIATION_FRACTION;
}
if (Math.Abs(ragg.Count - r.Count) <= maxdev)
{
return(true);
}
else
{
return(false);
}
}
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString);
List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI);
string videoName = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI) + "_" + entry.ID;
generateVideoForEvaluation(ImageURLs, aggresult.tracklets, directoryName, videoName, job.FrameRate);
}
}
public SatyamSaveAggregatedDataRequester(SatyamAggregatedResultsTableEntry entry)
{
SatyamAggregatedResult result = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParameters);
SatyamJob job = task.jobEntry;
//OrignialURI = task.OriginalURI;
JobGUID = entry.JobGUID;
JobTemplateType = entry.JobTemplateType;
UserID = entry.UserID;
JobSubmitTime = job.JobSubmitTime;
JobParameters = job.JobParameters;
JobCompletionTime = DateTime.Now;
AggregatedResultString = result.AggregatedResultString;
}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
SingleObjectLabelingResult r = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult);
SingleObjectLabelingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(aggres.AggregatedResultString);
if (r.Category == ragg.Category)
{
return(true);
}
else
{
return(false);
}
}
public static void SaveKITTIAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString);
string[] names = aggTask.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
string[] fields = videoName.Split('_');
string videoSequence = fields[fields.Length - 5];
int startingFrame = Convert.ToInt32(fields[fields.Length - 2]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902")
{
videoSequence = getCorrectSequenceNo(videoSequence);
}
string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence;
List <string> files = Directory.GetFiles(videoFrameDir).ToList();
List <string> ImageURLs = new List <string>();
for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++)
{
ImageURLs.Add(files[j]);
}
generateVideoForEvaluation(ImageURLs, aggresult.tracklets, directoryName, videoName, job.FrameRate);
}
}
public static void Run([QueueTrigger("aggregation")] string guidAndTaskID, TraceWriter log)
{
bool logging = false;
string[] fields = guidAndTaskID.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> collectedResults = resultsDB.getEntriesByGUIDAndTaskID(guid, taskID);
resultsDB.close();
if (collectedResults.Count == 0)
{
return;
}
if (logging)
{
log.Info($"Task Type: {collectedResults[0].JobTemplateType}({guid}), Task ID: {taskID}, # of Results: {collectedResults.Count}");
}
//SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
//int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskID);
//aggDB.close();
//if (collectedResults.Count != LatestResultsAggregated)
//{
SatyamAggregatedResultsTableEntry aggEntry = ResultsTableAggregator.GetAggregatedResultString(taskID, collectedResults);
if (aggEntry != null)
{
//aggEntries.Add(aggEntry);
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
aggDB.AddEntry(aggEntry, collectedResults.Count);
aggDB.close();
if (logging)
{
log.Info($"Aggregated");
}
}
//}
}
public static void SaveAggregatedResultImagesLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParameters);
SatyamJob job = task.jobEntry;
ImageSegmentationAggregatedResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamResult.AggregatedResultString);
//string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
//Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
//Image ResultImage = DrawResultsOnImage(res.boxesAndCategories, originalImage);
//fileName = fileName + "-AggregatedResult";
//ImageUtilities.saveImage(ResultImage, directoryName, fileName);
WebClient wb = new WebClient();
Image im = Image.FromStream(wb.OpenRead(res.metaData.PNG_URL));
string fileName = URIUtilities.filenameFromURI(res.metaData.PNG_URL);
//wb.DownloadFile(directoryName + "\\" + fileName, res.metaData.PNG_URL);
im.Save(directoryName + "\\" + fileName);
}
}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggEntry, SatyamResultsTableEntry result)
{
switch (result.JobTemplateType)
{
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
return(SingleObjectLabelingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
return(ObjectCountingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
return(MultiObjectLocalizationAndLabelingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
return(MultiObjectTrackingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
return(ImageSegmentationAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
return(true); //TBD
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
return(TrackletLabelingAggregator.IsAcceptable(aggEntry, result));
}
return(false);
}
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
WebClient wb = new WebClient();
Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
StreamReader reader = new StreamReader(aggTrackStream);
String aggTrackString = reader.ReadToEnd();
MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, aggTracks, directoryName, videoName, job.FrameRate);
}
}
public static MultiObjectTrackingResult stitchAllChunksOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
{
ImageURLs = new List <string>();
fps = 0;
if (entries.Count == 0)
{
return(null);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
MultiObjectTrackingResult stitched = new MultiObjectTrackingResult();
int totalFrameCounts = 0;
// ensure the order is correct
SortedDictionary <int, SatyamAggregatedResultsTableEntry> sortedEntries = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
List <int> idx = new List <int>();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
if (!sortedEntries.ContainsKey(startingFrame))
{
sortedEntries.Add(startingFrame, entries[i]);
idx.Add(startingFrame);
}
}
idx.Sort();
for (int i = 0; i < idx.Count; i++)
{
//SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString);
int noFramesOverlap = 0;
if (job.ChunkOverlap != 0.0)
{
noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
}
List <string> TraceURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI);
if (i == 0)
{
ImageURLs.AddRange(TraceURLs);
string[] names = aggTask.SatyamURI.Split('/');
fps = job.FrameRate;
stitched = aggresult.tracklets;
totalFrameCounts += TraceURLs.Count;
}
else
{
int noNewFrames = 0;
for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
{
ImageURLs.Add(TraceURLs[j]);
noNewFrames++;
}
//stitched = stitchTwoTracesByOneFrameBoundingBoxes(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts+ noNewFrames, noFramesOverlap,fps);
stitched = stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);
totalFrameCounts += noNewFrames;
}
//debug
//generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
}
return(stitched);
}
public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids,
double IoUTreshold,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
// Organize by taskID
foreach (SatyamResultsTableEntry entry in entries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!EntriesPerTask.ContainsKey(taskEntryID))
{
EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
EntriesPerTask[taskEntryID].Add(entry);
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
}
foreach (int taskEntryID in EntriesPerTask.Keys)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
List <int> taskidfilter = new List <int>()
{
40430,
40432,
40433,
40434,
40440,
40447,
40451,
40460,
};
//if (fileName != "2007_000549") continue;
if (!taskidfilter.Contains(satyamResult.TaskTableEntryID))
{
continue;
}
Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count);
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = satyamResult.TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = job.JobGUIDString;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold);
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >();
allPolygons.Add(new List <GenericPolygon>());
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects)
{
allPolygons[0].Add(entry.polygon);
}
allPolygons.Add(new List <GenericPolygon>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects)
{
allPolygons[1].Add(entry.polygon);
}
List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons);
//now find how many of the results match aggregated results
int noAccepted = 0;
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]];
GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]];
//double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon);
double IoU = 1;
if (IoU >= POLYGON_IOU_THRESHOLD)
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
public static void AggregateWithParameter(string guid,
int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
bool allFalseAttributeInvalid = false
)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;
Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
WorkersPerTask.Add(taskEntryID, new List <string>());
}
// remove duplicate workers result
string workerID = satyamResult.amazonInfo.WorkerID;
if (WorkersPerTask[taskEntryID].Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask[taskEntryID].Add(workerID);
string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
string[] fields = videoName.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
if (allFalseAttributeInvalid)
{
if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
{
continue;
}
}
ResultsPerTask[taskEntryID].Add(res);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
// hack for masters
int tempMin = MinResults;
if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
{
tempMin = 1;
}
TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
guid, task.SatyamURI,
job.FrameRate, job.BoundaryLines,
MinResults, MaxResults, boxToleranceThreshold,
ObjectCoverageApprovalThresholdPerVideo,
BoxCoverageApprovalThresholdPerTrack,
consensusNumber, minTubeletIoUSimilarityThreshold,
attributeMajority);
if (aggResult == null)
{
continue;
}
//////////////// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
}
/// <summary>
/// function generalizable.. TODO... to all tasks
/// </summary>
/// <param name="guids"></param>
/// <param name="IoUTreshold"></param>
/// <param name="saveImage"></param>
/// <param name="outputDirectory"></param>
/// <param name="MinResults"></param>
/// <param name="MaxResults"></param>
/// <param name="CategoryMajorityThreshold"></param>
/// <param name="PolygonBoundaryMajorityThreshold"></param>
/// <param name="ObjectsCoverageThreshold"></param>
/// <param name="overwrite"></param>
/// <param name="approvalAnalysis"></param>
public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
double IoUTreshold,
bool saveImage = false, string outputDirectory = null,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
bool overwrite = false,
bool approvalAnalysis = false)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;
Console.WriteLine("Aggregating for param set " + configString);
if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
{
return;
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
//ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
r = noTotalConverged + " " + noTerminatedTasks + " " + r;
File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);
if (approvalAnalysis)
{
string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
"_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
}
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double APPROVAL_RATIO_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the polygon
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
// the objects are dummy polygons just for counting
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * APPROVAL_RATIO_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//now find how many of the results match aggregated results
int noAccepted = 0;
byte[] resPNG = ImageSegmentationResult.PolygonResult2Bitmap(result);
int width = -1; int height = -1;
byte[] aggPNG = ImageUtilities.readPNGRawDataFromURL(aggresult.metaData.PNG_URL, out width, out height);
if (resPNG.Length != aggPNG.Length)
{
Console.WriteLine("res and agg res are different size");
return(false);
}
SortedDictionary <int, SortedDictionary <int, int> > overlapping = new SortedDictionary <int, SortedDictionary <int, int> >();
SortedDictionary <int, int> resObjectArea = new SortedDictionary <int, int>();
SortedDictionary <int, int> aggObjectArea = new SortedDictionary <int, int>();
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = j * width + i;
int resObjID = resPNG[idx];
int aggObjID = aggPNG[idx];
if (!overlapping.ContainsKey(resObjID))
{
overlapping.Add(resObjID, new SortedDictionary <int, int>());
}
if (!overlapping[resObjID].ContainsKey(aggObjID))
{
overlapping[resObjID].Add(aggObjID, 0);
}
overlapping[resObjID][aggObjID]++;
if (!resObjectArea.ContainsKey(resObjID))
{
resObjectArea.Add(resObjID, 0);
}
resObjectArea[resObjID]++;
if (!aggObjectArea.ContainsKey(aggObjID))
{
aggObjectArea.Add(aggObjID, 0);
}
aggObjectArea[aggObjID]++;
}
}
foreach (int id in resObjectArea.Keys)
{
if (id == 0)
{
continue;
}
int maxOverlap = -1;
int maxOverlapAggObjID = -1;
SortedDictionary <int, int> overlapArea = overlapping[id];
foreach (int aggid in overlapArea.Keys)
{
if (aggid == 0)
{
continue;
}
if (overlapArea[aggid] > maxOverlap)
{
maxOverlap = overlapArea[aggid];
maxOverlapAggObjID = aggid;
}
}
if (maxOverlapAggObjID == -1)
{
continue;
}
double iou = (double)maxOverlap / (double)(resObjectArea[id] + aggObjectArea[maxOverlapAggObjID] - maxOverlap);
if (iou >= POLYGON_IOU_THRESHOLD)
{
noAccepted++;
////now check category
//if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
//{
// //both category and bounding box tests have passed
//}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
//public static void Aggregate()
//{
// //first get all the results that are not aggregated
// SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
// List<SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus();
// //first group all the results by the tasks since wach task may ahve multiple entries
// Dictionary<int, List<SatyamResultsTableEntry>> collectedResults = new Dictionary<int, List<SatyamResultsTableEntry>>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// int taskID = entry.SatyamTaskTableEntryID;
// if (!collectedResults.ContainsKey(taskID))
// {
// collectedResults.Add(taskID, new List<SatyamResultsTableEntry>());
// }
// collectedResults[taskID].Add(entry);
// }
// //now aggreagate the results for eacht ask.
// List<int> taskIDList = collectedResults.Keys.ToList();
// List<SatyamAggregatedResultsTableEntry> aggEntries = new List<SatyamAggregatedResultsTableEntry>();
// foreach (int taskId in taskIDList)
// {
// SatyamAggregatedResultsTableEntry aggEntry = GetAggregatedResultString(taskId, collectedResults[taskId]);
// if (aggEntry != null)
// {
// aggEntries.Add(aggEntry);
// }
// }
// SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
// aggDB.AddEntries(aggEntries);
// resultsDB.close();
// aggDB.close();
//}
public static SatyamAggregatedResultsTableEntry GetAggregatedResultString(int taskId, List <SatyamResultsTableEntry> resultEntries)
{
SatyamAggregatedResultsTableEntry aggEntry = null;
string templateType = resultEntries[0].JobTemplateType;
string aggResultString = null;
switch (templateType)
{
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
aggResultString = ObjectCountingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
aggResultString = MultiObjectLocalizationAndLabelingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Tracking:
aggResultString = null; //Done testing, marked null so it becomes a demo page.
break;
case TaskConstants.Tracking_MTurk:
aggResultString = MultiObjectTrackingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
aggResultString = TrackletLabelingAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.Segmentation_Image:
aggResultString = null; //Done testing, marked null so it becomes a demo page.
break;
case TaskConstants.Segmentation_Image_MTurk:
aggResultString = ImageSegmentationAggregator.GetAggregatedResultString(resultEntries);
break;
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
break;
}
if (aggResultString != null)
{
aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = resultEntries[0].JobGUID;
aggEntry.JobTemplateType = resultEntries[0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = resultEntries[0].SatyamTaskTableEntryID;
aggEntry.UserID = resultEntries[0].UserID;
aggEntry.ResultString = aggResultString;
}
return(aggEntry);
}
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
string confusingImageListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
//SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
// SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
//SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
//ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
///
//if (aggResult.Category == "None of the Above")
//{
// continue;
//}
aggEntries.Add(aggEntry);
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
{
confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ aggResult.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
}
}
if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], aggResult.Category,
// JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
s.Append(record);
s.Append(img);
}
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
s.Clear();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
s.Append(title);
string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
Console.WriteLine(resultSummary);
s.Append(resultSummary);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
string matString = "";
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
matString += row;
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string summaryToBeSaved = s.ToString();
FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
File.WriteAllText(outputmatFile, matString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static MultiObjectTrackingResult stitchAllChunksAllObjectsOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
{
ImageURLs = new List <string>();
fps = 0;
if (entries.Count == 0)
{
return(null);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
MultiObjectTrackingResult stitched = new MultiObjectTrackingResult();
int totalFrameCounts = 0;
// ensure the order is correct
SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> > sortedEntries = new SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> >();
List <int> idx = new List <int>();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
if (!sortedEntries.ContainsKey(startingFrame))
{
sortedEntries.Add(startingFrame, new List <SatyamAggregatedResultsTableEntry>());
idx.Add(startingFrame);
}
sortedEntries[startingFrame].Add(entries[i]);
}
idx.Sort();
List <string> AggObjIds = new List <string>();
for (int i = 0; i < idx.Count; i++)
{
int noFramesOverlap = 0;
string blobDir = "";
// grouping all objects that belong to the same chunk
MultiObjectTrackingResult aggTracksOfAllObjectsPerChunk = new MultiObjectTrackingResult();
List <string> objIds = new List <string>();
for (int j = 0; j < sortedEntries[idx[i]].Count; j++)
{
SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]][j];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
if (job.ChunkOverlap != 0.0)
{
noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
}
fps = job.FrameRate;
blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
string objId = URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI);
objIds.Add(objId);
TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
WebClient wb = new WebClient();
Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
StreamReader reader = new StreamReader(aggTrackStream);
String aggTrackString = reader.ReadToEnd();
MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
if (aggTracksOfAllObjectsPerChunk.tracks.Count == 0)
{
aggTracksOfAllObjectsPerChunk = aggTracks;
}
else
{
for (int k = 0; k < aggTracks.tracks.Count; k++)
{
aggTracksOfAllObjectsPerChunk.tracks.Add(aggTracks.tracks[k]);
}
}
}
List <string> TraceURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
if (i == 0)
{
ImageURLs.AddRange(TraceURLs);
stitched = aggTracksOfAllObjectsPerChunk;
totalFrameCounts += TraceURLs.Count;
AggObjIds = objIds;
}
else
{
int noNewFrames = 0;
for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
{
ImageURLs.Add(TraceURLs[j]);
noNewFrames++;
}
//stitched = MultiObjectTrackingAnalyzer.stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched,
// aggTracksOfAllObjectsPerChunk, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);
// postpone the agg trace
double frameTimeInMiliSeconds = (double)1000 / (double)fps;
double timeToPostponeInMilliSeconds = (double)(totalFrameCounts - noFramesOverlap) * frameTimeInMiliSeconds;
//TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
aggTracksOfAllObjectsPerChunk.postpone(timeSpanToPostponeInSeconds);
// overlap must be 0
for (int k = 0; k < aggTracksOfAllObjectsPerChunk.tracks.Count; k++)
{
if (!AggObjIds.Contains(objIds[k]))
{
stitched.tracks.Add(aggTracksOfAllObjectsPerChunk.tracks[k]);
AggObjIds.Add(objIds[k]);
}
else
{
// stitch the track for the same id
int tckIdx = AggObjIds.IndexOf(objIds[k]);
stitched.tracks[tckIdx] = CompressedTrack.stitchTwoAdjacentTrack(stitched.tracks[tckIdx], aggTracksOfAllObjectsPerChunk.tracks[k]);
}
}
totalFrameCounts += noNewFrames;
}
//debug
//generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
}
return(stitched);
}
public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry,
double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX;
double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY;
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >();
allboxes.Add(new List <BoundingBox>());
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects)
{
allboxes[0].Add(entry.boundingBox);
}
allboxes.Add(new List <BoundingBox>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects)
{
allboxes[1].Add(entry.boundingBox);
if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y)
{
tooSmallToIgnore.Add(true);
}
else
{
tooSmallToIgnore.Add(false);
}
}
List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes);
//now find how many of the results match aggregated results
int noAccepted = 0;
double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX;
double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY;
int noIgnorable = 0;
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]];
BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]];
//double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox);
double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y);
if (deviation <= 1) //deviation test passed
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
else
{
if (tooSmallToIgnore[match.elementList[1]])
{
noIgnorable++;
}
}
}
}
}
if (noAccepted >= boxLimit - noIgnorable)
{
return(true);
}
return(false);
}
//go through all the inconclusive results and aggregate them if they are not aggregated already
public static void AggregateResults()
{
//first get all the results that are not aggregated
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus(ResultStatus.inconclusive);
if (results.Count == 0) //there is nothing to do
{
resultsDB.close();
return;
}
//first group all the results by the tasks since wach task may have multiple entries
Dictionary <int, List <SatyamResultsTableEntry> > collectedResults = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <string> pendingGUID = new List <string>();
foreach (SatyamResultsTableEntry entry in results)
{
int taskID = entry.SatyamTaskTableEntryID;
if (!collectedResults.ContainsKey(taskID))
{
collectedResults.Add(taskID, new List <SatyamResultsTableEntry>());
}
collectedResults[taskID].Add(entry);
if (!pendingGUID.Contains(entry.JobGUID))
{
pendingGUID.Add(entry.JobGUID);
}
}
List <int> taskIDList = collectedResults.Keys.ToList();
taskIDList.Sort();
//check if guid is still pending
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
List <string> completedGUIDs = jobDB.getAllJobGUIDSByStatus(JobStatus.completed);
jobDB.close();
//check if this taskID has already been aggregated and exists int he aggregationDB
Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > ExistingAggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID);
Console.WriteLine("Aggregating {0} Tasks", taskIDList.Count);
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
//foreach (int taskId in taskIDList)
for (int i = 0; i < taskIDList.Count; i++)
{
//int taskId = taskIDList[taskIDList.Count - i - 1];
int taskId = taskIDList[i];
string guid = collectedResults[taskId][0].JobGUID;
if (completedGUIDs.Contains(guid))
{
// Hit Completed, mark results outdated.
resultsDB.UpdateStatusByTaskID(taskId, ResultStatus.outdated);
continue;
}
//SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
//int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskId);
//aggDB.close();
if (ExistingAggEntriesPerTaskPerGUID.ContainsKey(guid) && ExistingAggEntriesPerTaskPerGUID[guid].ContainsKey(taskId))
{
continue;
//int MinResults = TaskConstants.getMinResultsByTemplate(ExistingAggEntriesPerTaskPerGUID[guid][taskId].JobTemplateType);
//if (LatestResultsAggregated >= MinResults)
//{
// continue;
//}
// already aggregated to MinResult Request, but leftover results will be judged. So do nothing
}
//if it does not exist only then aggregate
Console.WriteLine("Task Type: {2}({3}), Task ID: {0}, # of Results: {1}", taskId, collectedResults[taskId].Count, collectedResults[taskId][0].JobTemplateType, collectedResults[taskId][0].JobGUID);
//if (collectedResults[taskId].Count != LatestResultsAggregated)
//{
SatyamAggregatedResultsTableEntry aggEntry = ResultsTableAggregator.GetAggregatedResultString(taskId, collectedResults[taskId]);
if (aggEntry != null)
{
//aggEntries.Add(aggEntry);
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
aggDB.AddEntry(aggEntry, collectedResults[taskId].Count);
aggDB.close();
}
//}
//else
//{
// Console.WriteLine("No New Results");
//}
}
resultsDB.close();
}
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
//bool saveImage = false,
bool prepareTrainingSet = false, string outputDirectory = null,
int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
)
{
if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
{
Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
}
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
//int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
/// // record logs
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
double totalError, totalGroundtruth;
//ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
// MinHeight, MaxOcclusion, Max_Truncation);
ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
"_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log)
{
bool logging = false;
string[] fields = myQueueItem.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
string resultID = fields[2];
if (logging)
{
log.Info($"Judge Result: {myQueueItem}");
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID);
resultsDB.close();
if (results.Count == 0)
{
return;
}
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID);
aggDB.close();
if (aggResult == null)
{
return;
}
SatyamResultsTableEntry result = results[0];
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result))
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
if (logging)
{
log.Info($"Accepted");
}
}
else
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
if (logging)
{
log.Info($"Rejected");
}
}
}
//public static void AnalyzeApprovalRateByGUID(List<string> guids)
//{
// Console.WriteLine("Analyzing Results Statistics of {0}", guids[0]);
// SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
// List<SatyamResultsTableEntry> entries = new List<SatyamResultsTableEntry>();
// foreach (string guid in guids)
// {
// entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
// }
// resultsDB.close();
//}
public static void AggregationAnalysis(List <SatyamAggregatedResultsTableEntry> aggEntries,
//List<SatyamResultsTableEntry> ResultEntries,
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime,
Dictionary <int, int> noResultsNeededForAggregation,
Dictionary <int, int> noResultsNeededForAggregation_new,
string guid, string configString,
int minChancesGiven = 3,
double anotherChanceProbablity = 0,
double approvalRatioThreshold = 0.5,
bool usePreApprovalResult = false)
{
if (noResultsNeededForAggregation == null)
{
noResultsNeededForAggregation = new Dictionary <int, int>();
}
if (noResultsNeededForAggregation_new == null)
{
noResultsNeededForAggregation_new = new Dictionary <int, int>();
}
string approvalString = configString +
"_Thresh_" + approvalRatioThreshold +
"_Explore_" + anotherChanceProbablity;
Dictionary <int, SatyamAggregatedResultsTableEntry> aggEntriesByTaskID = new Dictionary <int, SatyamAggregatedResultsTableEntry>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultEntriesByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
//Dictionary<DateTime, double> TotalApprovalRatioByTime = new Dictionary<DateTime, double>();
List <string> TotalApprovalRatioByTime = new List <string>();
List <string> TotalApprovalRatioByResult = new List <string>();
List <string> FilteredTotalApprovalRatioByTime = new List <string>();
List <string> FilteredTotalApprovalRatioByResult = new List <string>();
List <SatyamResultsTableEntry> approvedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> InconclusiveEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> approvedEntriesActuallyPaid = new List <SatyamResultsTableEntry>();
Dictionary <string, int> noTasksApprovedPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noTasksDonePerWorker = new Dictionary <string, int>();
Dictionary <string, double> ApprovalRatioPerWorker = new Dictionary <string, double>();
Dictionary <string, bool> FirstTimersWorkersRecord = new Dictionary <string, bool>();
List <SatyamResultsTableEntry> FilteredApprovedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> filteredRejectedEntries = new List <SatyamResultsTableEntry>();
Dictionary <int, double> AggregationTimePerTask = new Dictionary <int, double>();
foreach (SatyamAggregatedResultsTableEntry aggEntry in aggEntries)
{
if (!aggEntriesByTaskID.ContainsKey(aggEntry.SatyamTaskTableEntryID))
{
aggEntriesByTaskID.Add(aggEntry.SatyamTaskTableEntryID, aggEntry);
}
else
{
// duplicate agg entry for a task
}
}
//Dictionary<int, int> noResultsNeededForAggregation = getNoResultsNeededForAggregationFromLog(approvalString, guid);
foreach (int k in noResultsNeededForAggregation_new.Keys)
{
noResultsNeededForAggregation.Add(k, noResultsNeededForAggregation_new[k]);
}
int count = 0;
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
Console.WriteLine("Judging result: {0}/{1}", count, entriesBySubmitTime.Count);
count++;
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry res in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(res.ResultString);
int taskID = res.SatyamTaskTableEntryID;
if (!ResultEntriesByTaskID.ContainsKey(taskID))
{
ResultEntriesByTaskID.Add(taskID, new List <SatyamResultsTableEntry>());
}
ResultEntriesByTaskID[taskID].Add(res);
if (!aggEntriesByTaskID.ContainsKey(taskID))
{
//foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
//{
//InconclusiveEntries.Add(res);
//}
InconclusiveEntries.Add(res);
continue;
}
SatyamAggregatedResultsTableEntry agg = aggEntriesByTaskID[taskID];
//foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
//{
//SatyamResult satyamResult = JSonUtils.ConvertJSonToObject<SatyamResult>(res.ResultString);
string workerId = satyamResult.amazonInfo.WorkerID;
// the worker pre-filtering step
bool filtered = false;
if (noTasksDonePerWorker.ContainsKey(workerId) && noTasksDonePerWorker[workerId] >= minChancesGiven)
{
double approvalRatio = (double)noTasksApprovedPerWorker[workerId] / (double)noTasksDonePerWorker[workerId];
ApprovalRatioPerWorker[workerId] = approvalRatio;
if (approvalRatio < approvalRatioThreshold)
{
// roll a dice to decide whether to skip this worker
int rnd = new Random().Next(100);
if (rnd > 100 * anotherChanceProbablity)
{
filtered = true;
}
else
{
// another chance granted
}
}
}
if (!noTasksDonePerWorker.ContainsKey(workerId))
{
noTasksDonePerWorker.Add(workerId, 0);
noTasksApprovedPerWorker.Add(workerId, 0);
ApprovalRatioPerWorker.Add(workerId, -1);
}
if (!filtered)
{
noTasksDonePerWorker[workerId]++;
}
bool acceptable = false;
if (usePreApprovalResult)
{
if (ResultStatus.acceptedStatusList.Contains(res.Status))
{
acceptable = true;
}
}
else
{
acceptable = AcceptanceCriterionChecker.IsAcceptable(agg, res);
}
if (acceptable)
{
approvedEntries.Add(res);
if (!FirstTimersWorkersRecord.ContainsKey(workerId))
{
FirstTimersWorkersRecord.Add(workerId, true);
}
if (!filtered)
{
noTasksApprovedPerWorker[workerId]++;
FilteredApprovedEntries.Add(res);
}
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
ResultEntriesByTaskID[taskID].Count <= noResultsNeededForAggregation[taskID])
{
approvedEntriesActuallyPaid.Add(res);
}
}
else
{
rejectedEntries.Add(res);
if (!FirstTimersWorkersRecord.ContainsKey(workerId))
{
FirstTimersWorkersRecord.Add(workerId, false);
}
if (!filtered)
{
filteredRejectedEntries.Add(res);
}
}
//}
/// timetaken till aggregation
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
ResultEntriesByTaskID[taskID].Count == noResultsNeededForAggregation[taskID])
{
// aggregation happen
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (!AggregationTimePerTask.ContainsKey(taskID))
{
AggregationTimePerTask.Add(taskID, (res.SubmitTime - job.JobSubmitTime).TotalSeconds);
}
}
string ApprovalString = approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
+ (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
TotalApprovalRatioByResult.Add(ApprovalString);
string filteredApprovalString = FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
+ (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
FilteredTotalApprovalRatioByResult.Add(filteredApprovalString);
}
string approvalStringByTime = t.ToString() + " " + approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
+ (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
TotalApprovalRatioByTime.Add(approvalStringByTime);
string filteredApprovalStringByTime = t.ToString() + " " + FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
+ (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
FilteredTotalApprovalRatioByTime.Add(filteredApprovalStringByTime);
}
// count first timer succ rate
int FirstTimeSucc = 0;
foreach (KeyValuePair <string, bool> wid in FirstTimersWorkersRecord)
{
if (wid.Value)
{
FirstTimeSucc++;
}
}
double FirstTimeSuccRate = (double)FirstTimeSucc / FirstTimersWorkersRecord.Count;
Console.WriteLine("Approval #, ratio: {0},{1}", approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count));
Console.WriteLine("Filtered Approval #, ratio: {0},{1}", FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count));
int approvePrevRejected = 0;
List <string> ApprovedAssignmentIDs = new List <string>();
foreach (SatyamResultsTableEntry approved in approvedEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(approved.ResultString);
ApprovedAssignmentIDs.Add(satyamResult.amazonInfo.AssignmentID);
if (approved.Status == ResultStatus.accepted_NotPaid || approved.Status == ResultStatus.rejected_NotPaid || approved.Status == ResultStatus.rejected)
{
approvePrevRejected++;
}
}
Console.WriteLine("Approving Previously Rejected: {0}, {1}", approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);
string outputFileDir = DirectoryConstants.defaultTempDirectory + guid + "\\ApprovalAnalysis\\";
if (guid != null && approvalString != null)
{
if (!Directory.Exists(outputFileDir))
{
Directory.CreateDirectory(outputFileDir);
}
}
string summaryFile = outputFileDir + "resultSummary.txt";
string summary = String.Format("{0} {1} {2} {3} {4} {5} {6} {7}\n",
approvalString,
approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count),
approvedEntriesActuallyPaid.Count,
FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count),
approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);
File.AppendAllText(summaryFile, summary);
string outputFile = outputFileDir + approvalString + ".txt";
File.WriteAllLines(outputFile, ApprovedAssignmentIDs.ToArray());
//string outputFilePaid = outputFileDir + approvalString + "_ActuallyPaid.txt";
//File.WriteAllText(outputFilePaid, approvedEntriesActuallyPaid.Count.ToString());
List <string> approvalRates = new List <string>();
foreach (string worker in ApprovalRatioPerWorker.Keys)
{
approvalRates.Add(ApprovalRatioPerWorker[worker].ToString());
}
string approvalDistributionFile = outputFileDir + approvalString + "_dist.txt";
File.WriteAllLines(approvalDistributionFile, approvalRates.ToArray());
string approvalByTimeFile = outputFileDir + approvalString + "_approvalRateByTime.txt";
string approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults.txt";
File.WriteAllLines(approvalByTimeFile, TotalApprovalRatioByTime.ToArray());
File.WriteAllLines(approvalByResultFile, TotalApprovalRatioByResult.ToArray());
approvalByTimeFile = outputFileDir + approvalString + "_approvalRateByTime_filtered.txt";
approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults_filtered.txt";
File.WriteAllLines(approvalByTimeFile, FilteredTotalApprovalRatioByTime.ToArray());
File.WriteAllLines(approvalByResultFile, FilteredTotalApprovalRatioByResult.ToArray());
List <string> aggregationTime = new List <string>();
foreach (int task in AggregationTimePerTask.Keys)
{
aggregationTime.Add(task.ToString() + " " + AggregationTimePerTask[task].ToString());
}
string aggregationTimeFile = outputFileDir + approvalString + "_aggregationTime.txt";
File.WriteAllLines(aggregationTimeFile, aggregationTime.ToArray());
}
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
bool prepareDataForTraining = false, string outputDirectory = null,
string confusingVideoListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void SaveAggregatedResultImagesLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParameters);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingAggregatedResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamResult.AggregatedResultString);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = fields[0];
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters);
Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true);
List <Rectangle> rectangles = new List <Rectangle>();
List <Color> colors = new List <Color>();
List <string> ids = new List <string>();
List <bool> dashed = new List <bool>();
for (int j = 0; j < res.boxesAndCategories.objects.Count; j++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry box = res.boxesAndCategories.objects[j];
int x = box.boundingBox.tlx;
int y = box.boundingBox.tly;
int width = box.boundingBox.brx - box.boundingBox.tlx;
int height = box.boundingBox.bry - box.boundingBox.tly;
Rectangle r = new Rectangle(x, y, width, height);
rectangles.Add(r);
string category = box.Category;
int colorIndex = jobDefinition.Categories.IndexOf(category);
colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]);
string id = j + "-" + category;
ids.Add(id);
dashed.Add(false);
}
Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed);
fileName = fileName + "-AggregatedResult";
ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName);
}
}
