public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI);
string[] names = task.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results,
// int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
// int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
// double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
// double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
// double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
//{
// string resultString = null;
// List<ObjectCountingResult> resultList = new List<ObjectCountingResult>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult);
// resultList.Add(taskr);
// }
// ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
// if (r != null)
// {
// string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r);
// SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
// aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
// aggResult.AggregatedResultString = rString;
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString);
// aggResult.TaskParameters = res.TaskParametersString;
// resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult);
// }
// return resultString;
//}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result,
double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString);
double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION;
if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev)
{
maxdev = ragg.Count * MAX_DEVIATION_FRACTION;
}
if (Math.Abs(ragg.Count - r.Count) <= maxdev)
{
return(true);
}
else
{
return(false);
}
}
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
//VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
string[] names = task.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
string[] fields = videoName.Split('_');
string videoSequence = fields[fields.Length - 5];
int startingFrame = Convert.ToInt32(fields[fields.Length - 2]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902")
{
videoSequence = getCorrectSequenceNo(videoSequence);
}
string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence;
List <string> files = Directory.GetFiles(videoFrameDir).ToList();
List <string> ImageURLs = new List <string>();
for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++)
{
ImageURLs.Add(files[j]);
}
if (ImageURLs.Count == 0)
{
continue;
}
VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static void SaveResultOnlyText(string jobGUID)
{
string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\";
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID);
resultsDB.close();
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw\\";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string result = satyamResult.TaskResult;
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = "";
for (int j = 0; j < fields.Count(); j++)
{
fileName += fields[j];
if (j == fields.Count() - 1)
{
break;
}
fileName += ".";
}
//string fileName = ofilename;
fileName = fileName + "-Result";
fileName = fileName + "-" + entry.ID;
Console.WriteLine("Saving " + fileName);
string resultFile = directoryName + fileName + ".txt";
StreamWriter f = new System.IO.StreamWriter(resultFile);
f.WriteLine(result);
f.Close();
}
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw\\";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string result = satyamResult.TaskResult;
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
Image ResultImage = DrawResultStringOnImage(result, originalImage);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = "";
for (int j = 0; j < fields.Count(); j++)
{
fileName += fields[j];
if (j == fields.Count() - 1)
{
break;
}
fileName += ".";
}
//string fileName = ofilename;
fileName = fileName + "-Result";
fileName = fileName + "-" + entry.ID;
Console.WriteLine("Saving " + fileName);
ImageUtilities.saveImage(ResultImage, directoryName, fileName);
string resultFile = directoryName + fileName + ".txt";
StreamWriter f = new System.IO.StreamWriter(resultFile);
f.WriteLine(result);
f.Close();
}
}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
SingleObjectLabelingResult r = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult);
SingleObjectLabelingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(aggres.AggregatedResultString);
if (r.Category == ragg.Category)
{
return(true);
}
else
{
return(false);
}
}
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID);
if (entries.Count == 0)
{
return(null);
}
//organized sequentially
SatyamResultsTableEntry prevResult = entries[entries.Count - 1];
Hidden_PrevResultID.Value = prevResult.ID.ToString();
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString);
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
return(res);
}
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries)
{
MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob();
ana.jobTemplateType = entries[0].JobTemplateType;
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
}
return(ana);
}
public static void TestGetGlobalStatistics()
{
Dictionary <string, int> workerSubmissions = new Dictionary <string, int>();
Dictionary <string, int> HitSubmissions = new Dictionary <string, int>();
Dictionary <int, int> TaskSubmissions = new Dictionary <int, int>();
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
entries = (resultsDB.getAllEntries());
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string worker = satyamResult.amazonInfo.WorkerID;
string hitID = satyamResult.amazonInfo.HITID;
int taskID = entry.SatyamTaskTableEntryID;
if (!workerSubmissions.ContainsKey(worker))
{
workerSubmissions.Add(worker, 0);
}
workerSubmissions[worker]++;
if (!HitSubmissions.ContainsKey(hitID))
{
HitSubmissions.Add(hitID, 0);
}
HitSubmissions[hitID]++;
if (!TaskSubmissions.ContainsKey(taskID))
{
TaskSubmissions.Add(taskID, 0);
}
TaskSubmissions[taskID]++;
}
Console.WriteLine(workerSubmissions.Count);
Console.WriteLine(HitSubmissions.Count);
Console.WriteLine(TaskSubmissions.Count);
}
public SatyamResultSaveDataRequester(SatyamResultsTableEntry entry)
{
string status = entry.Status;
if (status == ResultStatus.accepted || status == ResultStatus.accepted_Paid || status == ResultStatus.accepted_NotPaid)
{
resultAccepted = true;
}
else
{
resultAccepted = false;
}
if (status == ResultStatus.rejected_Paid || status == ResultStatus.accepted_Paid)
{
resultPaid = true;
}
else
{
resultPaid = false;
}
SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
TaskStartTime = result.TaskStartTime;
TaskEndTime = result.TaskEndTime;
JobGUID = entry.JobGUID;
JobTemplateType = entry.JobTemplateType;
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString);
//OrignialURI = task.OriginalURI;
SatyamJob job = task.jobEntry;
UserID = job.UserID;
JobSubmitTime = job.JobSubmitTime;
JobParameters = job.JobParameters;
amazonInfo = new AmazonSavingInfo();
amazonInfo.AssignmentID = result.amazonInfo.AssignmentID;
amazonInfo.pricePerHIT = result.amazonInfo.PricePerHIT;
amazonInfo.TasksPerHIT = job.TasksPerJob;
amazonInfo.WorkerID = result.amazonInfo.WorkerID;
ResultString = result.TaskResult;
}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggEntry, SatyamResultsTableEntry result)
{
switch (result.JobTemplateType)
{
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
return(SingleObjectLabelingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
return(ObjectCountingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
return(MultiObjectLocalizationAndLabelingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
return(MultiObjectTrackingAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
return(ImageSegmentationAggregator.IsAcceptable(aggEntry, result));
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
return(true); //TBD
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
return(TrackletLabelingAggregator.IsAcceptable(aggEntry, result));
}
return(false);
}
public static SatyamResultPerJobDataAnalysis AnalyzeFromResultsTable(List <string> jobGUIDs)
{
Console.WriteLine("Analyzing Results Statistics of {0}", jobGUIDs[0]);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in jobGUIDs)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
if (entries.Count == 0)
{
return(null);
}
SatyamResultPerJobDataAnalysis ana = new SatyamResultPerJobDataAnalysis();
ana.JobGUID = entries[0].JobGUID;
ana.totalResults = entries.Count;
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (ana.job == null)
{
ana.job = job;
}
ana.jobSubmitTime = job.JobSubmitTime;
double moneyPerResult = 0;
if (job.TasksPerJob > 0)
{
moneyPerResult = (double)satyamResult.amazonInfo.PricePerHIT / (double)job.TasksPerJob;
}
ana.PricePerResult.Add(entry.ID, moneyPerResult);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!ana.noResultsPerTask.ContainsKey(taskEntryID))
{
ana.noResultsPerTask.Add(taskEntryID, 0);
ana.noAcceptedResultsPerTask.Add(taskEntryID, 0);
ana.noPaidResultsPerTask.Add(taskEntryID, 0);
ana.moneyPaidPerTask.Add(taskEntryID, 0);
ana.finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ana.ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
ana.noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
ana.taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
ana.noResultsPerTask[taskEntryID]++;
ana.SequenceNumberOfResultPerTask.Add(ana.noResultsPerTask[taskEntryID]);
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.noAcceptedResultsPerTask[taskEntryID]++;
ana.totalAcceptedResults++;
}
if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid)
{
ana.noPaidResultsPerTask[taskEntryID]++;
ana.totalpaidResults++;
ana.moneyPaidPerTask[taskEntryID] += moneyPerResult;
ana.totalMoneyPaid += moneyPerResult;
}
if (ana.finalTaskEndTime[taskEntryID] < entry.SubmitTime)
{
ana.finalTaskEndTime[taskEntryID] = entry.SubmitTime;
}
if (ana.lastTaskEndTime < entry.SubmitTime)
{
ana.lastTaskEndTime = entry.SubmitTime;
}
TimeSpan dt = satyamResult.TaskEndTime - satyamResult.TaskStartTime;
ana.timeTakenPerResult.Add(dt.TotalSeconds);
if (!ana.AcceptedTimePerSequenceNumber.ContainsKey(ana.noResultsPerTask[taskEntryID]))
{
ana.AcceptedTimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>());
ana.TimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>());
}
//ana.acceptedTimeTakenPerResultOrganizedByResultSequencePerTask[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
ana.TimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
if (!ana.AcceptedTimePerTask.ContainsKey(taskEntryID))
{
ana.AcceptedTimePerTask.Add(taskEntryID, new List <double>());
ana.TimePerTask.Add(taskEntryID, new List <double>());
}
//ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds);
ana.TimePerTask[taskEntryID].Add(dt.TotalSeconds);
if (ana.lastTaskEndTime < satyamResult.TaskEndTime)
{
ana.lastTaskEndTime = satyamResult.TaskEndTime;
}
if (!ana.finalHITEndTime.ContainsKey(satyamResult.amazonInfo.HITID))
{
ana.finalHITEndTime.Add(satyamResult.amazonInfo.HITID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
}
if (ana.finalHITEndTime[satyamResult.amazonInfo.HITID] < (entry.SubmitTime - job.JobSubmitTime).TotalSeconds)
{
ana.finalHITEndTime[satyamResult.amazonInfo.HITID] = (entry.SubmitTime - job.JobSubmitTime).TotalSeconds;
}
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.acceptedTimeTakenPerResult.Add(dt.TotalSeconds);
ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds);
ana.AcceptedTimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
}
ana.resultArrivalTimes.Add((entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
if (!ana.noJobsPerWorker.ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ana.noJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
ana.noAcceptedJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
ana.noPaidJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
}
ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
ana.SequenceNumberOfResultPerWorker.Add(ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]);
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.noAcceptedJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
}
if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid)
{
ana.noPaidJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
}
if (!ana.ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ana.ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
else
{
// this worker has done this task before
ana.noDuplicateResultsFromSameWorker++;
}
if (!ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
if (ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID] > 1)
{
ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask++;
}
}
ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
///////////////// imagenet only
//List<SingleObjectLabelingResult> CurrentResultsOfThisTask = new List<SingleObjectLabelingResult>();
//foreach(string worker in ana.ResultsPerWorkerPerTask[taskEntryID].Keys)
//{
// foreach(string workerResult in ana.ResultsPerWorkerPerTask[taskEntryID][worker])
// {
// CurrentResultsOfThisTask.Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(workerResult));
// }
//}
//if (SingleObjectLabelingAggregator.getAggregatedResult(CurrentResultsOfThisTask) != null)
//{
// // aggregation happened
// if (!ana.timeTakenTillAggregationPerTask.ContainsKey(taskEntryID))
// {
// // for the first time aggregated
// ana.timeTakenTillAggregationPerTask.Add(taskEntryID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
// }
//}
///////////////////////
}
ana.resultArrivalTimes.Sort();
ana.noTasks = ana.noResultsPerTask.Count;
ana.timeTakenPerResult.Sort();
ana.medianTimeTakenPerResultInSeconds = ana.timeTakenPerResult[ana.timeTakenPerResult.Count / 2];
if (ana.acceptedTimeTakenPerResult.Count != 0)
{
ana.acceptedTimeTakenPerResult.Sort();
ana.medianAcceptedTimeTakenPerResultInSeconds = ana.acceptedTimeTakenPerResult[ana.acceptedTimeTakenPerResult.Count / 2];
}
ana.totalTimeTakenForJob = ana.lastTaskEndTime - ana.jobSubmitTime;
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
ana.resultsPerTaskHistogram = new SortedDictionary <int, int>();
ana.resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
ana.resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
ana.moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
ana.ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
int noAggregatedTasks = 0;
List <int> taskIDs = ana.noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (!ana.resultsPerTaskHistogram.ContainsKey(ana.noResultsPerTask[taskID]))
{
ana.resultsPerTaskHistogram.Add(ana.noResultsPerTask[taskID], 0);
}
ana.resultsPerTaskHistogram[ana.noResultsPerTask[taskID]]++;
if (!ana.resultsAcceptedPerTaskHistogram.ContainsKey(ana.noAcceptedResultsPerTask[taskID]))
{
ana.resultsAcceptedPerTaskHistogram.Add(ana.noAcceptedResultsPerTask[taskID], 0);
}
ana.resultsAcceptedPerTaskHistogram[ana.noAcceptedResultsPerTask[taskID]]++;
if (!ana.resultsPaidPerTaskHistogram.ContainsKey(ana.noPaidResultsPerTask[taskID]))
{
ana.resultsPaidPerTaskHistogram.Add(ana.noPaidResultsPerTask[taskID], 0);
}
ana.resultsPaidPerTaskHistogram[ana.noPaidResultsPerTask[taskID]]++;
int moneyPaid = (int)Math.Floor(ana.moneyPaidPerTask[taskID] * 100);
if (!ana.moneyPaidPerTaskHistogram.ContainsKey(moneyPaid))
{
ana.moneyPaidPerTaskHistogram.Add(moneyPaid, 0);
}
ana.moneyPaidPerTaskHistogram[moneyPaid]++;
if (ana.ResultsPerWorkerPerTask[taskID].Count != ana.noResultsPerTask[taskID])
{
//has multiple results from same turker
ana.noTasksWithDuplicateResultsFromSameWorker++;
}
if (ana.noPaidResultsPerTask[taskID] != 0)
{
noAggregatedTasks++;
}
}
ana.avgNoPaidResultsPerTask = (double)ana.totalpaidResults / (double)noAggregatedTasks;
//////////// duplicate analysis
//Console.WriteLine("DuplicateResultsHistogram");
//foreach (int no in ana.ResultsPerWorkerPerTaskHistogram.Keys)
//{
// Console.WriteLine("{0}, {1}", no, ana.ResultsPerWorkerPerTaskHistogram[no]);
//}
//Console.WriteLine("{0} images (in total {1}({2})) has duplicate results from same turker, \n" +
// "\t{3} images have >=1 workers making more than majority number of results.\n" +
// "\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
// "\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
// //"\t\t{6} images has correct decision among duplicate decisions.\n" +
// //"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
// //"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
// ana.noTasksWithDuplicateResultsFromSameWorker, ana.noDuplicateResultsFromSameWorker, 0,
// ana.noTaskWithDuplicateResultsAsMajority,
// ana.noTasksWithMixedResultsFromSameWorker,ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
// ana.noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask
// //ana.noTasksWithWorkerSwitchedToCorrect, ana.noWorkerSwitchedToCorrect,
// //ana.noTasksWithWorkerSwitchedToCorrectAndMaintained, ana.noWorkerSwitchedToCorrectAndMaintained,
// //ana.noTaskWhoseDuplicateResultsChangedAggregation, ana.noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
//);
//////////////////////////////
string outputDir = DirectoryConstants.defaultTempDirectory + jobGUIDs[0] + "\\ResultStatistics\\";
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
//Console.WriteLine("timePerResult");
TextWriter tw = new StreamWriter(outputDir + "timePerResult.txt");
foreach (double time in ana.timeTakenPerResult)
{
tw.WriteLine(time);
}
tw.Close();
tw = new StreamWriter(outputDir + "AcceptedTimePerSequenceNumber.txt");
foreach (int seq in ana.AcceptedTimePerSequenceNumber.Keys)
{
foreach (double time in ana.AcceptedTimePerSequenceNumber[seq])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "AcceptedTimePerTask.txt");
foreach (int taskid in ana.AcceptedTimePerTask.Keys)
{
foreach (double time in ana.AcceptedTimePerTask[taskid])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "TimePerSequenceNumber.txt");
foreach (int seq in ana.TimePerSequenceNumber.Keys)
{
foreach (double time in ana.TimePerSequenceNumber[seq])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "TimePerTask.txt");
foreach (int taskid in ana.TimePerTask.Keys)
{
foreach (double time in ana.TimePerTask[taskid])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
//Console.WriteLine("resultArrivalTime");
TextWriter tw0 = new StreamWriter(outputDir + "resultArrivalTime.txt");
foreach (double time in ana.resultArrivalTimes)
{
tw0.WriteLine(time);
}
tw0.Close();
//Console.WriteLine("HITFinishTime");
TextWriter tw2 = new StreamWriter(outputDir + "HITFinishTime.txt");
foreach (double time in ana.finalHITEndTime.Values)
{
tw2.WriteLine(time);
}
tw2.Close();
//Console.WriteLine("SequenceNumberOfResultsPerTask");
TextWriter tw3 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerTask.txt");
foreach (int seq in ana.SequenceNumberOfResultPerTask)
{
tw3.WriteLine(seq);
}
tw3.Close();
//Console.WriteLine("SequenceNumberOfResultsPerWorker");
TextWriter tw4 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerWorker.txt");
foreach (int seq in ana.SequenceNumberOfResultPerWorker)
{
tw4.WriteLine(seq);
}
tw4.Close();
//Console.WriteLine("MoneyPaidPerResult");
TextWriter tw5 = new StreamWriter(outputDir + "MoneyPaidPerResult.txt");
foreach (int seq in ana.PricePerResult.Keys)
{
tw5.WriteLine(ana.PricePerResult[seq]);
}
tw5.Close();
TextWriter tw6 = new StreamWriter(outputDir + "acceptedTimePerResult.txt");
foreach (double time in ana.acceptedTimeTakenPerResult)
{
tw6.WriteLine(time);
}
tw6.Close();
///////////////////// aggregation time histogram, out of scope
//Console.WriteLine("resultArrivalTime",
// TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
//string filepath = String.Format(@"C:\research\MSRDVA\SatyamResearch\aggregationTimePerTask_MinNoResults_{0}_MaxNoResults_{1}.txt",
// TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
//TextWriter tw1 = new StreamWriter(filepath);
//foreach (double time in ana.timeTakenTillAggregationPerTask.Values)
//{
// tw1.WriteLine(time);
//}
//tw1.Close();
Console.WriteLine("Median Time For Accepted Task: {0}", ana.medianAcceptedTimeTakenPerResultInSeconds);
Console.WriteLine("Median Time For All Task: {0}", ana.medianTimeTakenPerResultInSeconds);
Console.WriteLine("Total Tasks: {0}", ana.noTasks);
Console.WriteLine("Total Accepted Results: {0}", ana.totalAcceptedResults);
Console.WriteLine("Total Results: {0}", ana.totalResults);
Console.WriteLine("Avg Paid Results Per Task: {0}", ana.avgNoPaidResultsPerTask);
return(ana);
}
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log)
{
bool logging = false;
string[] fields = myQueueItem.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
string resultID = fields[2];
if (logging)
{
log.Info($"Judge Result: {myQueueItem}");
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID);
resultsDB.close();
if (results.Count == 0)
{
return;
}
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID);
aggDB.close();
if (aggResult == null)
{
return;
}
SatyamResultsTableEntry result = results[0];
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result))
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
if (logging)
{
log.Info($"Accepted");
}
}
else
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
if (logging)
{
log.Info($"Rejected");
}
}
}
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log)
{
DateTime start = DateTime.Now;
bool logging = false;
if (logging)
{
log.Info($"Payment Dispatch executed at: {DateTime.Now}");
}
SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess();
//get IDs of all accepted and rejected results related to MTurk
Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>();
List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>();
foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted);
resultsDB.close();
acceptedEntries.AddRange(entries);
}
List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>();
foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected);
resultsDB.close();
rejectedEntries.AddRange(entries);
}
if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do
{
return;
}
// a single assignment may have multiple accepted results
Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >();
foreach (SatyamResultsTableEntry entry in acceptedEntries)
{
SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
string AssignmentID = sr.amazonInfo.AssignmentID;
if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID))
{
string HITID = sr.amazonInfo.HITID;
AssignmentIDToHITIDMap.Add(AssignmentID, HITID);
}
if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID))
{
acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>());
}
acceptedResultsByAssignmentID[AssignmentID].Add(entry);
}
List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList();
// a single assignment may have multiple rejected results
Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >();
foreach (SatyamResultsTableEntry entry in rejectedEntries)
{
SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
string AssignmentID = sr.amazonInfo.AssignmentID;
if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID))
{
string HITID = sr.amazonInfo.HITID;
AssignmentIDToHITIDMap.Add(AssignmentID, HITID);
}
if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID))
{
rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>());
if (!assignmentIDList.Contains(AssignmentID))
{
assignmentIDList.Add(AssignmentID);
}
}
rejectedResultsByAssignmentID[AssignmentID].Add(entry);
}
List <string> rejectedHITs = new List <string>();
List <string> acceptedHITs = new List <string>();
//for each assignment ID we check if it should be paid or not and then pay or reject them.
//pay all accepted entries
foreach (string assignmentID in assignmentIDList)
{
int noAccepted = 0;
SatyamResult r = null;
SatyamResultsTableEntry entry = null;
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
noAccepted = acceptedResultsByAssignmentID[assignmentID].Count;
entry = acceptedResultsByAssignmentID[assignmentID][0];
r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
}
int noRejected = 0;
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
noRejected = rejectedResultsByAssignmentID[assignmentID].Count;
if (r == null)
{
entry = rejectedResultsByAssignmentID[assignmentID][0];
r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
}
}
double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString);
string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID;
string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID;
string m = AmazonAccessKeyID + "_" + AmazonSecretAccessKeyID + "_" + assignmentID;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
if (ratio >= AmazonMTurkPayments.assignement_acceptance_threshold) //this is acceptable
{
if (logging)
{
log.Info($"Dispatching Payment for {assignmentID}");
}
string queueName = "payment";
satyamQueue.Enqueue(queueName, m);
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid);
}
}
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid);
}
}
if (AssignmentIDToHITIDMap.ContainsKey(assignmentID))
{
if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID]))
{
acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]);
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.accepted);
hitDB.close();
}
}
}
else
{
if (logging)
{
log.Info($"Dispatching NoPayment for {assignmentID}");
}
string queueName = "nopayment";
satyamQueue.Enqueue(queueName, m);
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid);
}
}
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid);
}
}
if (AssignmentIDToHITIDMap.ContainsKey(assignmentID))
{
if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID]))
{
rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]);
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.rejected);
hitDB.close();
}
}
}
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = fields[0];
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters);
Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true);
List <Rectangle> rectangles = new List <Rectangle>();
List <Color> colors = new List <Color>();
List <string> ids = new List <string>();
List <bool> dashed = new List <bool>();
for (int j = 0; j < res.objects.Count; j++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j];
int x = box.boundingBox.tlx;
int y = box.boundingBox.tly;
int width = box.boundingBox.brx - box.boundingBox.tlx;
int height = box.boundingBox.bry - box.boundingBox.tly;
Rectangle r = new Rectangle(x, y, width, height);
rectangles.Add(r);
string category = box.Category;
int colorIndex = jobDefinition.Categories.IndexOf(category);
colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]);
string id = j + "-" + category;
ids.Add(id);
dashed.Add(false);
}
Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
//fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID;
fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID;
}
ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName);
}
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >();
allPolygons.Add(new List <GenericPolygon>());
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects)
{
allPolygons[0].Add(entry.polygon);
}
allPolygons.Add(new List <GenericPolygon>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects)
{
allPolygons[1].Add(entry.polygon);
}
List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons);
//now find how many of the results match aggregated results
int noAccepted = 0;
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]];
GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]];
//double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon);
double IoU = 1;
if (IoU >= POLYGON_IOU_THRESHOLD)
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double APPROVAL_RATIO_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the polygon
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
// the objects are dummy polygons just for counting
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * APPROVAL_RATIO_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//now find how many of the results match aggregated results
int noAccepted = 0;
byte[] resPNG = ImageSegmentationResult.PolygonResult2Bitmap(result);
int width = -1; int height = -1;
byte[] aggPNG = ImageUtilities.readPNGRawDataFromURL(aggresult.metaData.PNG_URL, out width, out height);
if (resPNG.Length != aggPNG.Length)
{
Console.WriteLine("res and agg res are different size");
return(false);
}
SortedDictionary <int, SortedDictionary <int, int> > overlapping = new SortedDictionary <int, SortedDictionary <int, int> >();
SortedDictionary <int, int> resObjectArea = new SortedDictionary <int, int>();
SortedDictionary <int, int> aggObjectArea = new SortedDictionary <int, int>();
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = j * width + i;
int resObjID = resPNG[idx];
int aggObjID = aggPNG[idx];
if (!overlapping.ContainsKey(resObjID))
{
overlapping.Add(resObjID, new SortedDictionary <int, int>());
}
if (!overlapping[resObjID].ContainsKey(aggObjID))
{
overlapping[resObjID].Add(aggObjID, 0);
}
overlapping[resObjID][aggObjID]++;
if (!resObjectArea.ContainsKey(resObjID))
{
resObjectArea.Add(resObjID, 0);
}
resObjectArea[resObjID]++;
if (!aggObjectArea.ContainsKey(aggObjID))
{
aggObjectArea.Add(aggObjID, 0);
}
aggObjectArea[aggObjID]++;
}
}
foreach (int id in resObjectArea.Keys)
{
if (id == 0)
{
continue;
}
int maxOverlap = -1;
int maxOverlapAggObjID = -1;
SortedDictionary <int, int> overlapArea = overlapping[id];
foreach (int aggid in overlapArea.Keys)
{
if (aggid == 0)
{
continue;
}
if (overlapArea[aggid] > maxOverlap)
{
maxOverlap = overlapArea[aggid];
maxOverlapAggObjID = aggid;
}
}
if (maxOverlapAggObjID == -1)
{
continue;
}
double iou = (double)maxOverlap / (double)(resObjectArea[id] + aggObjectArea[maxOverlapAggObjID] - maxOverlap);
if (iou >= POLYGON_IOU_THRESHOLD)
{
noAccepted++;
////now check category
//if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
//{
// //both category and bounding box tests have passed
//}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry,
double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX;
double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY;
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >();
allboxes.Add(new List <BoundingBox>());
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects)
{
allboxes[0].Add(entry.boundingBox);
}
allboxes.Add(new List <BoundingBox>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects)
{
allboxes[1].Add(entry.boundingBox);
if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y)
{
tooSmallToIgnore.Add(true);
}
else
{
tooSmallToIgnore.Add(false);
}
}
List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes);
//now find how many of the results match aggregated results
int noAccepted = 0;
double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX;
double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY;
int noIgnorable = 0;
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]];
BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]];
//double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox);
double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y);
if (deviation <= 1) //deviation test passed
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
else
{
if (tooSmallToIgnore[match.elementList[1]])
{
noIgnorable++;
}
}
}
}
}
if (noAccepted >= boxLimit - noIgnorable)
{
return(true);
}
return(false);
}
//go through all unpaid results
//check if aggregated results has an entry for this
//check if it satisfies acceptance criterion
//pay if passes else fail!
public static void MakePayments()
{
//get IDs of all accepted and rejected results related to MTurk
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>();
List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>();
foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates)
{
List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted);
acceptedEntries.AddRange(entries);
}
List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>();
foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates)
{
List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected);
rejectedEntries.AddRange(entries);
}
resultsDB.close();
if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do
{
return;
}
// a single assignment may have multiple accepted results
Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >();
foreach (SatyamResultsTableEntry entry in acceptedEntries)
{
SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
string AssignmentID = sr.amazonInfo.AssignmentID;
if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID))
{
string HITID = sr.amazonInfo.HITID;
AssignmentIDToHITIDMap.Add(AssignmentID, HITID);
}
if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID))
{
acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>());
}
acceptedResultsByAssignmentID[AssignmentID].Add(entry);
}
List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList();
// a single assignment may have multiple rejected results
Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >();
foreach (SatyamResultsTableEntry entry in rejectedEntries)
{
SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
string AssignmentID = sr.amazonInfo.AssignmentID;
if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID))
{
string HITID = sr.amazonInfo.HITID;
AssignmentIDToHITIDMap.Add(AssignmentID, HITID);
}
if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID))
{
rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>());
if (!assignmentIDList.Contains(AssignmentID))
{
assignmentIDList.Add(AssignmentID);
}
}
rejectedResultsByAssignmentID[AssignmentID].Add(entry);
}
List <string> rejectedHITs = new List <string>();
List <string> acceptedHITs = new List <string>();
//for each assignment ID we check if it should be paid or not and then pay or reject them.
//pay all accepted entries
AmazonMTurkHIT hit = new AmazonMTurkHIT();
foreach (string assignmentID in assignmentIDList)
{
int noAccepted = 0;
SatyamResult r = null;
SatyamResultsTableEntry entry = null;
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
noAccepted = acceptedResultsByAssignmentID[assignmentID].Count;
entry = acceptedResultsByAssignmentID[assignmentID][0];
r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
}
int noRejected = 0;
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
noRejected = rejectedResultsByAssignmentID[assignmentID].Count;
if (r == null)
{
entry = rejectedResultsByAssignmentID[assignmentID][0];
r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
}
}
double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString);
string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID;
string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID;
hit.setAccount(AmazonAccessKeyID, AmazonSecretAccessKeyID, false);
resultsDB = new SatyamResultsTableAccess();
if (ratio >= assignement_acceptance_threshold) //this is acceptable
{
hit.ApproveAssignment(assignmentID, "Great Job! Your work was within acceptable parameters!");
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid);
}
}
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid);
}
}
if (AssignmentIDToHITIDMap.ContainsKey(assignmentID))
{
if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID]))
{
acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]);
}
}
}
else
{
hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!");
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid);
}
}
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid);
}
}
if (AssignmentIDToHITIDMap.ContainsKey(assignmentID))
{
if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID]))
{
rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]);
}
}
}
resultsDB.close();
}
//update the status of all finished HITs in the hittable
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
foreach (string HITID in acceptedHITs)
{
hitDB.UpdateStatusByHITID(HITID, HitStatus.accepted);
}
foreach (string HITID in rejectedHITs)
{
hitDB.UpdateStatusByHITID(HITID, HitStatus.rejected);
}
hitDB.close();
}
public static void ValidateSameTaskSameWorkerMultipleResultsEffect(string jobGUID, string confusingImageListFilePath = null)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(jobGUID);
resultsDB.close();
Dictionary <int, int> noResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noAcceptedResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noPaidResultsPerTask = new Dictionary <int, int>();
Dictionary <int, double> moneyPaidPerTask = new Dictionary <int, double>();
SortedDictionary <int, Dictionary <string, List <string> > > ResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, List <string> > >();
SortedDictionary <int, Dictionary <string, int> > noDifferentDuplicateResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, int> >();
Dictionary <int, string> taskSatyamUri = new Dictionary <int, string>();
Dictionary <int, double> timeTakenTillAggregationPerTask = new Dictionary <int, double>();
Dictionary <int, DateTime> finalTaskEndTime = new Dictionary <int, DateTime>();
Dictionary <string, double> finalHITEndTime = new Dictionary <string, double>();
int totalResults = 0;
int noTasksWithDuplicateResultsFromSameWorker = 0;
int noTasksWithMixedResultsFromSameWorker = 0;
int noTaskWithDuplicateResultsAsMajority = 0;
int noTaskWhoseDuplicateResultsChangedAggregation = 0;
int noTaskWhoseDuplicateResultsChangedAggregationIncorrectly = 0;
int noCorrectDecisionsAmongDuplicates = 0;
//noCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noWorkerSwitchedToCorrect = 0;
int noWorkerSwitchedToCorrectAndMaintained = 0;
int noTasksWithWorkerSwitchedToCorrect = 0;
int noTasksWithWorkerSwitchedToCorrectAndMaintained = 0;
List <double> timeTakenPerResult = new List <double>();
List <double> acceptedTimeTakenPerResult = new List <double>();
List <int> SequenceNumberOfResultPerTask = new List <int>();
List <int> SequenceNumberOfResultPerWorker = new List <int>();
string JobGUID = entries[0].JobGUID;
Dictionary <string, int> noJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noAcceptedJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noPaidJobsPerWorker = new Dictionary <string, int>();
List <double> resultArrivalTimes = new List <double>();
totalResults = entries.Count;
///imagenet only
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!noResultsPerTask.ContainsKey(taskEntryID))
{
noResultsPerTask.Add(taskEntryID, 0);
noAcceptedResultsPerTask.Add(taskEntryID, 0);
noPaidResultsPerTask.Add(taskEntryID, 0);
moneyPaidPerTask.Add(taskEntryID, 0);
finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
if (!ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
if (!ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
}
ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
}
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
SortedDictionary <int, int> resultsPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
SortedDictionary <int, int> ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
List <int> taskIDs = noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (ResultsPerWorkerPerTask[taskID].Count != noResultsPerTask[taskID])
{
//has multiple results from same turker
noTasksWithDuplicateResultsFromSameWorker++;
// the aggregation result
List <SingleObjectLabelingResult> allResultsPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedResultPerTask = new SingleObjectLabelingAggregatedResult();
// the aggregation if without duplicate results
List <SingleObjectLabelingResult> OnlyFirstResultOfEachTurkerPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedOnlyFirstResultOfTurkerPerTask = new SingleObjectLabelingAggregatedResult();
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
allResultsPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(resultString));
}
OnlyFirstResultOfEachTurkerPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(ResultsStringsPerWorkerPerTask[0]));
}
aggregatedResultPerTask = SingleObjectLabelingAggregator.getAggregatedResult(allResultsPerTask);
aggregatedOnlyFirstResultOfTurkerPerTask = SingleObjectLabelingAggregator.getAggregatedResult(OnlyFirstResultOfEachTurkerPerTask);
if (aggregatedResultPerTask != null)
{
if (aggregatedOnlyFirstResultOfTurkerPerTask == null ||
aggregatedResultPerTask.Category != aggregatedOnlyFirstResultOfTurkerPerTask.Category)
{
noTaskWhoseDuplicateResultsChangedAggregation++;
if (AggregatedResultEqualsGroundTruth(taskSatyamUri[taskID], JSonUtils.ConvertObjectToJSon(aggregatedResultPerTask)))
{
noTaskWhoseDuplicateResultsChangedAggregationIncorrectly++;
}
}
}
bool hasMixedResults = false;
bool hasCorrectResult = false;
bool atLeastOneShiftedFromIncorrectToCorrect = false;
bool atLeastOneShiftedFromIncorrectToCorrectAndMaintained = false;
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
if (!ResultsPerWorkerPerTaskHistogram.ContainsKey(ResultsStringsPerWorkerPerTask.Count))
{
ResultsPerWorkerPerTaskHistogram.Add(ResultsStringsPerWorkerPerTask.Count, 0);
}
ResultsPerWorkerPerTaskHistogram[ResultsStringsPerWorkerPerTask.Count]++;
double superMajority = 0.6; // tunable
if (((double)ResultsStringsPerWorkerPerTask.Count + 1) / ((double)noResultsPerTask[taskID] + 2) > superMajority)
{
noTaskWithDuplicateResultsAsMajority++;
}
///////////// imagenet only //////////////////
if (ResultsStringsPerWorkerPerTask.Distinct().Count() > 1)
{
// multiple choices given
hasMixedResults = true;
bool incorrect = false;
bool switchToCorrect = false;
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
if (SingleObjectLabelingResultEqualsGroundTruth(taskSatyamUri[taskID], resultString))
{
hasCorrectResult = true;
noCorrectDecisionsAmongDuplicates++;
if (incorrect)
{
switchToCorrect = true;
}
incorrect = false;
}
else
{
incorrect = true;
}
}
if (switchToCorrect)
{
// been incorrect and swithed to correct
noWorkerSwitchedToCorrect++;
atLeastOneShiftedFromIncorrectToCorrect = true;
if (!incorrect)
{
//switch to correct and maintain till the end
noWorkerSwitchedToCorrectAndMaintained++;
atLeastOneShiftedFromIncorrectToCorrectAndMaintained = true;
}
}
}
////////////////////////////////////////////////
}
if (hasMixedResults)
{
noTasksWithMixedResultsFromSameWorker++;
}
if (hasCorrectResult)
{
noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask++;
}
if (atLeastOneShiftedFromIncorrectToCorrect)
{
noTasksWithWorkerSwitchedToCorrect++;
}
if (atLeastOneShiftedFromIncorrectToCorrectAndMaintained)
{
noTasksWithWorkerSwitchedToCorrectAndMaintained++;
}
}
}
Console.WriteLine("DuplicateResultsHistogram");
foreach (int no in ResultsPerWorkerPerTaskHistogram.Keys)
{
Console.WriteLine("{0}, {1}", no, ResultsPerWorkerPerTaskHistogram[no]);
}
Console.WriteLine(
//"{0} images (in total {1}({2} of which are correct)) has duplicate results from same turker, \n" +
//"\t{3} images have >=1 workers making more than majority number of results.\n" +
//"\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
//"\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
"\t\t{6} images has correct decision among duplicate decisions.\n" +
"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
//noTasksWithDuplicateResultsFromSameWorker, noDuplicateResultsFromSameWorker, noCorrectDecisionsAmongDuplicates,
//noTaskWithDuplicateResultsAsMajority,
//noTasksWithMixedResultsFromSameWorker, noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
//noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask,
noTasksWithWorkerSwitchedToCorrect, noWorkerSwitchedToCorrect,
noTasksWithWorkerSwitchedToCorrectAndMaintained, noWorkerSwitchedToCorrectAndMaintained,
noTaskWhoseDuplicateResultsChangedAggregation, noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
);
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
// sort by task id
SortedDictionary <int, List <SatyamResultsTableEntry> > taskResults = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
for (int i = 0; i < entries.Count; i++)
{
if (!taskResults.ContainsKey(entries[i].SatyamTaskTableEntryID))
{
taskResults.Add(entries[i].SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
taskResults[entries[i].SatyamTaskTableEntryID].Add(entries[i]);
}
foreach (int taskID in taskResults.Keys)
{
for (int i = 0; i < taskResults[taskID].Count; i++)
{
SatyamResultsTableEntry entry = taskResults[taskID][i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
ImageSegmentationResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (res == null)
{
continue;
}
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID;
}
fileName += "-" + entry.ID;
if (File.Exists(directoryName + "\\" + fileName + ".jpg"))
{
continue;
}
Console.WriteLine("Saving Turker Result {0}", fileName);
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
Image ResultImage = DrawResultsOnImage(res, originalImage);
byte[] png = ImageSegmentationResult.PolygonResult2Bitmap(res);
if (ResultImage == null)
{
continue;
}
ImageUtilities.saveImage(ResultImage, directoryName, fileName);
ImageUtilities.savePNGRawData(directoryName + "\\" + fileName + "_bitmap.jpg", originalImage.Width, originalImage.Height, png);
}
}
}